Imported Upstream version 1.4.0-2013.03upstream/1.4.0-2013.03

1436 files changed, 49354 insertions, 25416 deletions

diff --git a/.gitignore b/.gitignore
index bd6ba1c..53fe9c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,13 @@
 config-devices.*
 config-all-devices.*
+config-all-disas.*
 config-host.*
 config-target.*
-trace.h
-trace.c
-trace-dtrace.h
-trace-dtrace.dtrace
+trace/generated-tracers.h
+trace/generated-tracers.c
+trace/generated-tracers-dtrace.h
+trace/generated-tracers-dtrace.dtrace
+libcacard/trace/generated-tracers.c
 *-timestamp
 *-softmmu
 *-darwin-user
@@ -47,6 +49,7 @@ test-qmp-output-visitor
 test-string-input-visitor
 test-string-output-visitor
 test-visitor-serialization
+fsdev/virtfs-proxy-helper
 fsdev/virtfs-proxy-helper.1
 fsdev/virtfs-proxy-helper.pod
 .gdbinit
@@ -68,6 +71,7 @@ fsdev/virtfs-proxy-helper.pod
 *.tp
 *.vr
 *.d
+!scripts/qemu-guest-agent/fsfreeze-hook.d
 *.o
 *.lo
 *.la
diff --git a/Changelog.LINARO b/Changelog.LINARO
index 5e5668d..0448cb7 100644
--- a/Changelog.LINARO
+++ b/Changelog.LINARO
@@ -12,6 +12,15 @@ here. For full change detail consult the git history:
 Bug numbers refer to Launchpad qemu-linaro bugs, eg:
 https://bugs.launchpad.net/qemu-linaro/+bug/703094
 
+version 2013.03:
+ - Rebased onto upstream v1.4.0.
+ - Fixes #1082292: warning messages from pflash_write during
+   Linux probe for flash type are now suppressed
+ - The ARM KVM support in this version is in sync with the
+   ABI as committed to the upstream Linux kernel for 3.9.
+   This feature is still under development but will no longer
+   be subject to kernel-vs-userspace ABI breaks.
+
 version 2012.12:
  - Rebased onto upstream v1.3.0.
  - Fixes #1009901: deleting a usb-storage device from the
diff --git a/HACKING b/HACKING
index 89a6b3a..6654d33 100644
--- a/HACKING
+++ b/HACKING
@@ -123,3 +123,23 @@ gcc's printf attribute directive in the prototype.
 This makes it so gcc's -Wformat and -Wformat-security options can do
 their jobs and cross-check format strings with the number and types
 of arguments.
+
+6. C standard, implementation defined and undefined behaviors
+
+C code in QEMU should be written to the C99 language specification. A copy
+of the final version of the C99 standard with corrigenda TC1, TC2, and TC3
+included, formatted as a draft, can be downloaded from:
+ http://www.open-std.org/jtc1/sc22/WG14/www/docs/n1256.pdf
+
+The C language specification defines regions of undefined behavior and
+implementation defined behavior (to give compiler authors enough leeway to
+produce better code).  In general, code in QEMU should follow the language
+specification and avoid both undefined and implementation defined
+constructs. ("It works fine on the gcc I tested it with" is not a valid
+argument...) However there are a few areas where we allow ourselves to
+assume certain behaviors because in practice all the platforms we care about
+behave in the same way and writing strictly conformant code would be
+painful. These are:
+ * you may assume that integers are 2s complement representation
+ * you may assume that right shift of a signed integer duplicates
+   the sign bit (ie it is an arithmetic shift, not a logical shift)
diff --git a/MAINTAINERS b/MAINTAINERS
index 2ede20d..21043e4 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -98,6 +98,7 @@ S: Maintained
 F: target-ppc/
 
 S390
+M: Richard Henderson <rth@twiddle.net>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/
@@ -132,7 +133,7 @@ Guest CPU Cores (KVM):
 ----------------------
 
 Overall
-M: Avi Kivity <avi@redhat.com>
+M: Gleb Natapov <gleb@redhat.com>
 M: Marcelo Tosatti <mtosatti@redhat.com>
 L: kvm@vger.kernel.org
 S: Supported
@@ -150,7 +151,7 @@ S: Maintained
 F: target-s390x/kvm.c
 
 X86
-M: Avi Kivity <avi@redhat.com>
+M: Gleb Natapov <gleb@redhat.com>
 M: Marcelo Tosatti <mtosatti@redhat.com>
 L: kvm@vger.kernel.org
 S: Supported
@@ -379,7 +380,7 @@ New World
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
 S: Maintained
-F: hw/ppc_newworld.c
+F: hw/ppc/mac_newworld.c
 F: hw/unin_pci.c
 F: hw/dec_pci.[hc]
 
@@ -387,15 +388,16 @@ Old World
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
 S: Maintained
-F: hw/ppc_oldworld.c
+F: hw/ppc/mac_oldworld.c
 F: hw/grackle_pci.c
 
 PReP
 M: Andreas Färber <andreas.faerber@web.de>
 L: qemu-ppc@nongnu.org
 S: Odd Fixes
-F: hw/ppc_prep.c
+F: hw/ppc/prep.c
 F: hw/prep_pci.[hc]
+F: hw/pc87312.[hc]
 
 sPAPR
 M: David Gibson <david@gibson.dropbear.id.au>
@@ -490,6 +492,7 @@ F: hw/omap*
 PCI
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
+F: hw/pci/*
 F: hw/pci*
 F: hw/piix*
 
@@ -599,6 +602,7 @@ M: Andreas Färber <afaerber@suse.de>
 S: Supported
 F: qom/cpu.c
 F: include/qemu/cpu.h
+F: target-i386/cpu.c
 
 Device Tree
 M: Peter Crosthwaite <peter.crosthwaite@petalogix.com>
diff --git a/Makefile b/Makefile
index 0be3fba..fdd0439 100644
--- a/Makefile
+++ b/Makefile
@@ -31,12 +31,15 @@ ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 endif
 endif
 
-GENERATED_HEADERS = config-host.h trace.h qemu-options.def
+GENERATED_HEADERS = config-host.h qemu-options.def
+GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h
+GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c
+
+GENERATED_HEADERS += trace/generated-tracers.h
 ifeq ($(TRACE_BACKEND),dtrace)
-GENERATED_HEADERS += trace-dtrace.h
+GENERATED_HEADERS += trace/generated-tracers-dtrace.h
 endif
-GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h
-GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c trace.c
+GENERATED_SOURCES += trace/generated-tracers.c
 
 # Don't try to regenerate Makefile or configure
 # We don't generate any of them
@@ -99,6 +102,18 @@ defconfig:
 	rm -f config-all-devices.mak $(SUBDIR_DEVICES_MAK)
 
 -include config-all-devices.mak
+-include config-all-disas.mak
+CONFIG_SOFTMMU := $(if $(filter %-softmmu,$(TARGET_DIRS)),y)
+CONFIG_USER_ONLY := $(if $(filter %-user,$(TARGET_DIRS)),y)
+CONFIG_ALL=y
+
+ifneq ($(wildcard config-host.mak),)
+include $(SRC_PATH)/Makefile.objs
+include $(SRC_PATH)/tests/Makefile
+endif
+ifeq ($(CONFIG_SMARTCARD_NSS),y)
+include $(SRC_PATH)/libcacard/Makefile
+endif
 
 all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all
 
@@ -112,26 +127,16 @@ SUBDIR_RULES=$(patsubst %,subdir-%, $(TARGET_DIRS))
 subdir-%:
 	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C $* V="$(V)" TARGET_DIR="$*/" all,)
 
-ifneq ($(wildcard config-host.mak),)
-include $(SRC_PATH)/Makefile.objs
-endif
-
-subdir-libcacard: $(oslib-obj-y) $(trace-obj-y) qemu-timer-common.o
-
 subdir-pixman: pixman/Makefile
 	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C pixman V="$(V)" all,)
 
 pixman/Makefile: $(SRC_PATH)/pixman/configure
-	(cd pixman; CFLAGS="$(CFLAGS) -fPIC" $(SRC_PATH)/pixman/configure $(AUTOCONF_HOST) --disable-gtk --disable-shared --enable-static)
+	(cd pixman; CFLAGS="$(CFLAGS) -fPIC $(extra_cflags) $(extra_ldflags)" $(SRC_PATH)/pixman/configure $(AUTOCONF_HOST) --disable-gtk --disable-shared --enable-static)
 
 $(SRC_PATH)/pixman/configure:
 	(cd $(SRC_PATH)/pixman; autoreconf -v --install)
 
-$(SUBDIR_RULES): libqemustub.a
-
-$(filter %-softmmu,$(SUBDIR_RULES)): $(universal-obj-y) $(trace-obj-y) $(common-obj-y) $(extra-obj-y) subdir-libdis
-
-$(filter %-user,$(SUBDIR_RULES)): $(universal-obj-y) $(trace-obj-y) subdir-libdis-user subdir-libuser
+$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y)
 
 ROMSUBDIR_RULES=$(patsubst %,romsubdir-%, $(ROMS))
 romsubdir-%:
@@ -141,66 +146,31 @@ ALL_SUBDIRS=$(TARGET_DIRS) $(patsubst %,pc-bios/%, $(ROMS))
 
 recurse-all: $(SUBDIR_RULES) $(ROMSUBDIR_RULES)
 
-audio/audio.o audio/fmodaudio.o: QEMU_CFLAGS += $(FMOD_CFLAGS)
-
-QEMU_CFLAGS+=$(CURL_CFLAGS)
-
-QEMU_CFLAGS += -I$(SRC_PATH)/include
-
-ui/cocoa.o: ui/cocoa.m
-
-ui/sdl.o audio/sdlaudio.o ui/sdl_zoom.o hw/baum.o: QEMU_CFLAGS += $(SDL_CFLAGS)
-
-ui/vnc.o: QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
-
 bt-host.o: QEMU_CFLAGS += $(BLUEZ_CFLAGS)
 
 version.o: $(SRC_PATH)/version.rc config-host.h
 	$(call quiet-command,$(WINDRES) -I. -o $@ $<,"  RC    $(TARGET_DIR)$@")
 
 version-obj-$(CONFIG_WIN32) += version.o
+Makefile: $(version-obj-y)
 
 ######################################################################
-# Build library with stubs
+# Build libraries
 
 libqemustub.a: $(stub-obj-y)
-
-######################################################################
-# Support building shared library libcacard
-
-.PHONY: libcacard.la install-libcacard
-ifeq ($(LIBTOOL),)
-libcacard.la:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-
-install-libcacard:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-else
-libcacard.la: $(oslib-obj-y) qemu-timer-common.o $(addsuffix .lo, $(basename $(trace-obj-y)))
-	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C libcacard V="$(V)" TARGET_DIR="$*/" libcacard.la,)
-
-install-libcacard: libcacard.la
-	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C libcacard V="$(V)" TARGET_DIR="$*/" install-libcacard,)
-endif
+libqemuutil.a: $(util-obj-y)
 
 ######################################################################
 
 qemu-img.o: qemu-img-cmds.h
 
-tools-obj-y = $(oslib-obj-y) $(trace-obj-y) qemu-tool.o qemu-timer.o \
-	main-loop.o iohandler.o error.o
-tools-obj-$(CONFIG_POSIX) += compatfd.o
-
-qemu-img$(EXESUF): qemu-img.o $(tools-obj-y) $(block-obj-y) libqemustub.a
-qemu-nbd$(EXESUF): qemu-nbd.o $(tools-obj-y) $(block-obj-y) libqemustub.a
-qemu-io$(EXESUF): qemu-io.o cmd.o $(tools-obj-y) $(block-obj-y) libqemustub.a
+qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-io$(EXESUF): qemu-io.o cmd.o $(block-obj-y) libqemuutil.a libqemustub.a
 
 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
 
-vscclient$(EXESUF): $(libcacard-y) $(oslib-obj-y) $(trace-obj-y) libcacard/vscclient.o libqemustub.a
-	$(call quiet-command,$(CC) $(LDFLAGS) -o $@ $^ $(libcacard_libs) $(LIBS),"  LINK  $@")
-
-fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o oslib-posix.o $(trace-obj-y)
+fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o libqemuutil.a libqemustub.a
 fsdev/virtfs-proxy-helper$(EXESUF): LIBS += -lcap
 
 qemu-img-cmds.h: $(SRC_PATH)/qemu-img-cmds.hx
@@ -211,20 +181,16 @@ qemu-ga$(EXESUF): QEMU_CFLAGS += -I qga/qapi-generated
 
 gen-out-type = $(subst .,-,$(suffix $@))
 
-ifneq ($(wildcard config-host.mak),)
-include $(SRC_PATH)/tests/Makefile
-endif
-
 qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
 
 qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h :\
-$(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
+$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
 qga/qapi-generated/qga-qapi-visit.c qga/qapi-generated/qga-qapi-visit.h :\
-$(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
+$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
 qga/qapi-generated/qga-qmp-commands.h qga/qapi-generated/qga-qmp-marshal.c :\
-$(SRC_PATH)/qapi-schema-guest.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
+$(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o qga/qapi-generated -p "qga-" < $<, "  GEN   $@")
 
 qapi-types.c qapi-types.h :\
@@ -240,27 +206,27 @@ $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
 $(qga-obj-y) qemu-ga.o: $(QGALIB_GEN)
 
-qemu-ga$(EXESUF): qemu-ga.o $(qga-obj-y) $(oslib-obj-y) $(trace-obj-y) $(qapi-obj-y) $(qobject-obj-y) $(version-obj-y) libqemustub.a
-
-QEMULIBS=libuser libdis libdis-user
+qemu-ga$(EXESUF): $(qga-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
 
 clean:
 # avoid old build problems by removing potentially incorrect old files
 	rm -f config.mak op-i386.h opc-i386.h gen-op-i386.h op-arm.h opc-arm.h gen-op-arm.h
 	rm -f qemu-options.def
-	find . -name '*.[od]' -exec rm -f {} +
-	rm -f *.a *.lo $(TOOLS) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
+	find . -name '*.[oda]' -type f -exec rm -f {} +
+	find . -name '*.l[oa]' -type f -exec rm -f {} +
+	rm -f $(TOOLS) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
 	rm -Rf .libs
 	rm -f qemu-img-cmds.h
-	rm -f trace-dtrace.dtrace trace-dtrace.dtrace-timestamp
 	@# May not be present in GENERATED_HEADERS
-	rm -f trace-dtrace.h trace-dtrace.h-timestamp
+	rm -f trace/generated-tracers-dtrace.dtrace*
+	rm -f trace/generated-tracers-dtrace.h*
 	rm -f $(foreach f,$(GENERATED_HEADERS),$(f) $(f)-timestamp)
 	rm -f $(foreach f,$(GENERATED_SOURCES),$(f) $(f)-timestamp)
 	rm -rf qapi-generated
 	rm -rf qga/qapi-generated
 	$(MAKE) -C tests/tcg clean
-	for d in $(ALL_SUBDIRS) $(QEMULIBS) libcacard; do \
+	for d in $(ALL_SUBDIRS); do \
 	if test -d $$d; then $(MAKE) -C $$d $@ || exit 1; fi; \
 	rm -f $$d/qemu-options.def; \
         done
@@ -274,7 +240,7 @@ qemu-%.tar.bz2:
 
 distclean: clean
 	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi
-	rm -f config-all-devices.mak
+	rm -f config-all-devices.mak config-all-disas.mak
 	rm -f roms/seabios/config.mak roms/vgabios/config.mak
 	rm -f qemu-doc.info qemu-doc.aux qemu-doc.cp qemu-doc.cps qemu-doc.dvi
 	rm -f qemu-doc.fn qemu-doc.fns qemu-doc.info qemu-doc.ky qemu-doc.kys
@@ -283,7 +249,7 @@ distclean: clean
 	rm -f config.log
 	rm -f linux-headers/asm
 	rm -f qemu-tech.info qemu-tech.aux qemu-tech.cp qemu-tech.dvi qemu-tech.fn qemu-tech.info qemu-tech.ky qemu-tech.log qemu-tech.pdf qemu-tech.pg qemu-tech.toc qemu-tech.tp qemu-tech.vr
-	for d in $(TARGET_DIRS) $(QEMULIBS); do \
+	for d in $(TARGET_DIRS); do \
 	rm -rf $$d || exit 1 ; \
         done
 	if test -f pixman/config.log; then make -C pixman distclean; fi
@@ -296,6 +262,7 @@ bepo
 ifdef INSTALL_BLOBS
 BLOBS=bios.bin sgabios.bin vgabios.bin vgabios-cirrus.bin \
 vgabios-stdvga.bin vgabios-vmware.bin vgabios-qxl.bin \
+acpi-dsdt.aml q35-acpi-dsdt.aml \
 ppc_rom.bin openbios-sparc32 openbios-sparc64 openbios-ppc \
 pxe-e1000.rom pxe-eepro100.rom pxe-ne2k_pci.rom \
 pxe-pcnet.rom pxe-rtl8139.rom pxe-virtio.rom \
diff --git a/Makefile.dis b/Makefile.dis
deleted file mode 100644
index 2cfec6a..0000000
--- a/Makefile.dis
+++ /dev/null
@@ -1,20 +0,0 @@
-# Makefile for disassemblers.
-
-include ../config-host.mak
-include config.mak
-include $(SRC_PATH)/rules.mak
-
-.PHONY: all
-
-$(call set-vpath, $(SRC_PATH))
-
-QEMU_CFLAGS+=-I..
-
-include $(SRC_PATH)/Makefile.objs
-
-all: $(libdis-y)
-# Dummy command so that make thinks it has done something
-	@true
-
-clean:
-	rm -f *.o *.d *.a *~
diff --git a/Makefile.objs b/Makefile.objs
index 3c7abca..21e9c91 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -1,58 +1,31 @@
 #######################################################################
-# Stub library, linked in tools
+# Common libraries for tools and emulators
 stub-obj-y = stubs/
+util-obj-y = util/ qobject/ qapi/ trace/
 
 #######################################################################
-# Target-independent parts used in system and user emulation
-universal-obj-y =
-universal-obj-y += qemu-log.o
-
-#######################################################################
-# QObject
-qobject-obj-y = qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
-qobject-obj-y += qjson.o json-lexer.o json-streamer.o json-parser.o
-qobject-obj-y += qerror.o error.o qemu-error.o
-
-universal-obj-y += $(qobject-obj-y)
-
-#######################################################################
-# QOM
-qom-obj-y = qom/
-
-universal-obj-y += $(qom-obj-y)
+# block-obj-y is code used by both qemu system emulation and qemu-img
 
-#######################################################################
-# oslib-obj-y is code depending on the OS (win32 vs posix)
-oslib-obj-y = osdep.o cutils.o qemu-timer-common.o
-oslib-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o
-oslib-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o
+block-obj-y = async.o thread-pool.o
+block-obj-y += nbd.o block.o blockjob.o
+block-obj-y += main-loop.o iohandler.o qemu-timer.o
+block-obj-$(CONFIG_POSIX) += aio-posix.o
+block-obj-$(CONFIG_WIN32) += aio-win32.o
+block-obj-y += block/
+block-obj-y += qapi-types.o qapi-visit.o
 
-#######################################################################
-# coroutines
-coroutine-obj-y = qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
-coroutine-obj-y += qemu-coroutine-sleep.o
+block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+block-obj-y += qemu-coroutine-sleep.o
 ifeq ($(CONFIG_UCONTEXT_COROUTINE),y)
-coroutine-obj-$(CONFIG_POSIX) += coroutine-ucontext.o
+block-obj-$(CONFIG_POSIX) += coroutine-ucontext.o
 else
 ifeq ($(CONFIG_SIGALTSTACK_COROUTINE),y)
-coroutine-obj-$(CONFIG_POSIX) += coroutine-sigaltstack.o
+block-obj-$(CONFIG_POSIX) += coroutine-sigaltstack.o
 else
-coroutine-obj-$(CONFIG_POSIX) += coroutine-gthread.o
+block-obj-$(CONFIG_POSIX) += coroutine-gthread.o
 endif
 endif
-coroutine-obj-$(CONFIG_WIN32) += coroutine-win32.o
-
-#######################################################################
-# block-obj-y is code used by both qemu system emulation and qemu-img
-
-block-obj-y = iov.o cache-utils.o qemu-option.o module.o async.o
-block-obj-y += nbd.o block.o blockjob.o aes.o qemu-config.o
-block-obj-y += thread-pool.o qemu-progress.o qemu-sockets.o uri.o notify.o
-block-obj-y += $(coroutine-obj-y) $(qobject-obj-y) $(version-obj-y)
-block-obj-$(CONFIG_POSIX) += event_notifier-posix.o aio-posix.o
-block-obj-$(CONFIG_WIN32) += event_notifier-win32.o aio-win32.o
-block-obj-y += block/
-block-obj-y += $(qapi-obj-y) qapi-types.o qapi-visit.o
+block-obj-$(CONFIG_WIN32) += coroutine-win32.o
 
 ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
 # Lots of the fsdev/9pcode is pulled in by vl.c via qemu_fsdev_add.
@@ -61,44 +34,44 @@ CONFIG_REALLY_VIRTFS=y
 endif
 
 ######################################################################
+# smartcard
+
+libcacard-y += libcacard/cac.o libcacard/event.o
+libcacard-y += libcacard/vcard.o libcacard/vreader.o
+libcacard-y += libcacard/vcard_emul_nss.o
+libcacard-y += libcacard/vcard_emul_type.o
+libcacard-y += libcacard/card_7816.o
+
+######################################################################
 # Target independent part of system emulation. The long term path is to
 # suppress *all* target specific code in case of system emulation, i.e. a
 # single QEMU executable should support all CPUs and machines.
 
+ifeq ($(CONFIG_SOFTMMU),y)
 common-obj-y = $(block-obj-y) blockdev.o blockdev-nbd.o block/
-common-obj-y += net.o net/
-common-obj-y += qom/
-common-obj-y += readline.o console.o cursor.o
-common-obj-y += qemu-pixman.o
-common-obj-y += $(oslib-obj-y)
+common-obj-y += net/
+common-obj-y += readline.o
 common-obj-$(CONFIG_WIN32) += os-win32.o
 common-obj-$(CONFIG_POSIX) += os-posix.o
 
 common-obj-$(CONFIG_LINUX) += fsdev/
-extra-obj-$(CONFIG_LINUX) += fsdev/
 
-common-obj-y += tcg-runtime.o host-utils.o main-loop.o
-common-obj-y += input.o
-common-obj-y += buffered_file.o migration.o migration-tcp.o
+common-obj-y += migration.o migration-tcp.o
 common-obj-y += qemu-char.o #aio.o
-common-obj-y += block-migration.o iohandler.o
-common-obj-y += bitmap.o bitops.o
-common-obj-y += page_cache.o
+common-obj-y += block-migration.o
+common-obj-y += page_cache.o xbzrle.o
 
 common-obj-$(CONFIG_POSIX) += migration-exec.o migration-unix.o migration-fd.o
-common-obj-$(CONFIG_WIN32) += version.o
 
 common-obj-$(CONFIG_SPICE) += spice-qemu-char.o
 
 common-obj-y += audio/
 common-obj-y += hw/
+
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
 
 common-obj-y += dma-helpers.o
-common-obj-y += acl.o
-common-obj-$(CONFIG_POSIX) += compatfd.o
-common-obj-y += qemu-timer.o qemu-timer-common.o
 common-obj-y += qtest.o
 common-obj-y += vl.o
 
@@ -106,135 +79,31 @@ common-obj-$(CONFIG_SLIRP) += slirp/
 
 common-obj-y += backends/
 
-######################################################################
-# libseccomp
-ifeq ($(CONFIG_SECCOMP),y)
-common-obj-y += qemu-seccomp.o
-endif
-
-######################################################################
-# libuser
-
-user-obj-y =
-user-obj-y += envlist.o path.o
-user-obj-y += tcg-runtime.o host-utils.o
-user-obj-y += cache-utils.o
-user-obj-y += module.o
-user-obj-y += qemu-user.o
-user-obj-y += $(trace-obj-y)
-user-obj-y += qom/
-
-######################################################################
-# libdis
-# NOTE: the disassembler code is only needed for debugging
-
-libdis-y =
-libdis-$(CONFIG_ALPHA_DIS) += alpha-dis.o
-libdis-$(CONFIG_ARM_DIS) += arm-dis.o
-libdis-$(CONFIG_CRIS_DIS) += cris-dis.o
-libdis-$(CONFIG_HPPA_DIS) += hppa-dis.o
-libdis-$(CONFIG_I386_DIS) += i386-dis.o
-libdis-$(CONFIG_IA64_DIS) += ia64-dis.o
-libdis-$(CONFIG_M68K_DIS) += m68k-dis.o
-libdis-$(CONFIG_MICROBLAZE_DIS) += microblaze-dis.o
-libdis-$(CONFIG_MIPS_DIS) += mips-dis.o
-libdis-$(CONFIG_PPC_DIS) += ppc-dis.o
-libdis-$(CONFIG_S390_DIS) += s390-dis.o
-libdis-$(CONFIG_SH4_DIS) += sh4-dis.o
-libdis-$(CONFIG_SPARC_DIS) += sparc-dis.o
-libdis-$(CONFIG_LM32_DIS) += lm32-dis.o
-
-######################################################################
-# trace
-
-ifeq ($(TRACE_BACKEND),dtrace)
-TRACE_H_EXTRA_DEPS=trace-dtrace.h
-endif
-trace.h: trace.h-timestamp $(TRACE_H_EXTRA_DEPS)
-trace.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
-	$(call quiet-command,$(TRACETOOL) \
-		--format=h \
-		--backend=$(TRACE_BACKEND) \
-		< $< > $@,"  GEN   trace.h")
-	@cmp -s $@ trace.h || cp $@ trace.h
-
-trace.c: trace.c-timestamp
-trace.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
-	$(call quiet-command,$(TRACETOOL) \
-		--format=c \
-		--backend=$(TRACE_BACKEND) \
-		< $< > $@,"  GEN   trace.c")
-	@cmp -s $@ trace.c || cp $@ trace.c
-
-trace.o: trace.c $(GENERATED_HEADERS)
-
-trace-dtrace.h: trace-dtrace.dtrace
-	$(call quiet-command,dtrace -o $@ -h -s $<, "  GEN   trace-dtrace.h")
-
-# Normal practice is to name DTrace probe file with a '.d' extension
-# but that gets picked up by QEMU's Makefile as an external dependency
-# rule file. So we use '.dtrace' instead
-trace-dtrace.dtrace: trace-dtrace.dtrace-timestamp
-trace-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
-	$(call quiet-command,$(TRACETOOL) \
-		--format=d \
-		--backend=$(TRACE_BACKEND) \
-		< $< > $@,"  GEN   trace-dtrace.dtrace")
-	@cmp -s $@ trace-dtrace.dtrace || cp $@ trace-dtrace.dtrace
-
-trace-dtrace.o: trace-dtrace.dtrace $(GENERATED_HEADERS)
-	$(call quiet-command,dtrace -o $@ -G -s $<, "  GEN   trace-dtrace.o")
-
-ifeq ($(LIBTOOL),)
-trace-dtrace.lo: trace-dtrace.dtrace
-	@echo "missing libtool. please install and rerun configure."; exit 1
-else
-trace-dtrace.lo: trace-dtrace.dtrace
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC dtrace -o $@ -G -s $<, "  lt GEN trace-dtrace.o")
-endif
-
-trace/simple.o: trace/simple.c $(GENERATED_HEADERS)
-
-trace-obj-$(CONFIG_TRACE_DTRACE) += trace-dtrace.o
-ifneq ($(TRACE_BACKEND),dtrace)
-trace-obj-y = trace.o
-endif
-
-trace-obj-$(CONFIG_TRACE_DEFAULT) += trace/default.o
-trace-obj-$(CONFIG_TRACE_SIMPLE) += trace/simple.o
-trace-obj-$(CONFIG_TRACE_SIMPLE) += qemu-timer-common.o
-trace-obj-$(CONFIG_TRACE_STDERR) += trace/stderr.o
-trace-obj-y += trace/control.o
-
-$(trace-obj-y): $(GENERATED_HEADERS)
-
-######################################################################
-# smartcard
-
-libcacard-y += libcacard/cac.o libcacard/event.o
-libcacard-y += libcacard/vcard.o libcacard/vreader.o
-libcacard-y += libcacard/vcard_emul_nss.o
-libcacard-y += libcacard/vcard_emul_type.o
-libcacard-y += libcacard/card_7816.o
+common-obj-$(CONFIG_SECCOMP) += qemu-seccomp.o
 
 common-obj-$(CONFIG_SMARTCARD_NSS) += $(libcacard-y)
 
 ######################################################################
 # qapi
 
-qapi-obj-y = qapi/
-qapi-obj-y += qapi-types.o qapi-visit.o
-
 common-obj-y += qmp-marshal.o qapi-visit.o qapi-types.o
 common-obj-y += qmp.o hmp.o
+endif
 
-universal-obj-y += $(qapi-obj-y)
+#######################################################################
+# Target-independent parts used in system and user emulation
+common-obj-y += qemu-log.o
+common-obj-y += tcg-runtime.o
+common-obj-y += hw/
+common-obj-y += qom/
+common-obj-y += disas/
 
 ######################################################################
 # guest agent
 
-qga-obj-y = qga/ qemu-ga.o module.o qemu-tool.o
-qga-obj-$(CONFIG_POSIX) += qemu-sockets.o qemu-option.o
+# FIXME: a few definitions from qapi-types.o/qapi-visit.o are needed
+# by libqemuutil.a.  These should be moved to a separate .json schema.
+qga-obj-y = qga/ qapi-types.o qapi-visit.o
 
 vl.o: QEMU_CFLAGS+=$(GPROF_CFLAGS)
 
@@ -244,11 +113,8 @@ QEMU_CFLAGS+=$(GLIB_CFLAGS)
 
 nested-vars += \
 	stub-obj-y \
+	util-obj-y \
 	qga-obj-y \
-	qom-obj-y \
-	qapi-obj-y \
 	block-obj-y \
-	user-obj-y \
-	common-obj-y \
-	extra-obj-y
+	common-obj-y
 dummy := $(call unnest-vars)
diff --git a/Makefile.target b/Makefile.target
index 927347b..760da1e 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -54,7 +54,7 @@ $(QEMU_PROG).stp: $(SRC_PATH)/trace-events
 		--binary=$(bindir)/$(QEMU_PROG) \
 		--target-arch=$(TARGET_ARCH) \
 		--target-type=$(TARGET_TYPE) \
-		< $< > $@,"  GEN   $(QEMU_PROG).stp")
+		< $< > $@,"  GEN   $(TARGET_DIR)$(QEMU_PROG).stp")
 else
 stap:
 endif
@@ -69,14 +69,12 @@ all: $(PROGS) stap
 obj-y = exec.o translate-all.o cpu-exec.o
 obj-y += tcg/tcg.o tcg/optimize.o
 obj-$(CONFIG_TCG_INTERPRETER) += tci.o
+obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o
 obj-y += fpu/softfloat.o
-obj-y += disas.o
-obj-$(CONFIG_TCI_DIS) += tci-dis.o
 obj-y += target-$(TARGET_BASE_ARCH)/
+obj-y += disas.o
 obj-$(CONFIG_GDBSTUB_XML) += gdbstub-xml.o
 
-tci-dis.o: QEMU_CFLAGS += -I$(SRC_PATH)/tcg -I$(SRC_PATH)/tcg/tci
-
 #########################################################
 # Linux user emulator target
 
@@ -85,7 +83,7 @@ ifdef CONFIG_LINUX_USER
 QEMU_CFLAGS+=-I$(SRC_PATH)/linux-user/$(TARGET_ABI_DIR) -I$(SRC_PATH)/linux-user
 
 obj-y += linux-user/
-obj-y += gdbstub.o thunk.o user-exec.o $(oslib-obj-y)
+obj-y += gdbstub.o thunk.o user-exec.o
 
 endif #CONFIG_LINUX_USER
 
@@ -97,7 +95,7 @@ ifdef CONFIG_BSD_USER
 QEMU_CFLAGS+=-I$(SRC_PATH)/bsd-user -I$(SRC_PATH)/bsd-user/$(TARGET_ARCH)
 
 obj-y += bsd-user/
-obj-y += gdbstub.o user-exec.o $(oslib-obj-y)
+obj-y += gdbstub.o user-exec.o
 
 endif #CONFIG_BSD_USER
 
@@ -121,11 +119,6 @@ obj-$(CONFIG_NO_GET_MEMORY_MAPPING) += memory_mapping-stub.o
 obj-$(CONFIG_NO_CORE_DUMP) += dump-stub.o
 LIBS+=-lz
 
-QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
-QEMU_CFLAGS += $(VNC_SASL_CFLAGS)
-QEMU_CFLAGS += $(VNC_JPEG_CFLAGS)
-QEMU_CFLAGS += $(VNC_PNG_CFLAGS)
-
 # xen support
 obj-$(CONFIG_XEN) += xen-all.o xen-mapcache.o
 obj-$(CONFIG_NO_XEN) += xen-stub.o
@@ -152,25 +145,16 @@ nested-vars += obj-y
 include $(SRC_PATH)/Makefile.objs
 
 all-obj-y = $(obj-y)
-all-obj-y += $(addprefix ../, $(universal-obj-y))
-
-ifdef CONFIG_SOFTMMU
 all-obj-y += $(addprefix ../, $(common-obj-y))
-all-obj-y += $(addprefix ../libdis/, $(libdis-y))
-all-obj-y += $(addprefix ../, $(trace-obj-y))
-else
-all-obj-y += $(addprefix ../libuser/, $(user-obj-y))
-all-obj-y += $(addprefix ../libdis-user/, $(libdis-y))
-endif #CONFIG_LINUX_USER
 
 ifdef QEMU_PROGW
 # The linker builds a windows executable. Make also a console executable.
-$(QEMU_PROGW): $(all-obj-y) ../libqemustub.a
+$(QEMU_PROGW): $(all-obj-y) ../libqemuutil.a ../libqemustub.a
 	$(call LINK,$^)
 $(QEMU_PROG): $(QEMU_PROGW)
 	$(call quiet-command,$(OBJCOPY) --subsystem console $(QEMU_PROGW) $(QEMU_PROG),"  GEN   $(TARGET_DIR)$(QEMU_PROG)")
 else
-$(QEMU_PROG): $(all-obj-y) ../libqemustub.a
+$(QEMU_PROG): $(all-obj-y) ../libqemuutil.a ../libqemustub.a
 	$(call LINK,$^)
 endif
 
diff --git a/Makefile.user b/Makefile.user
deleted file mode 100644
index 9302d33..0000000
--- a/Makefile.user
+++ /dev/null
@@ -1,24 +0,0 @@
-# Makefile for qemu target independent user files.
-
-include ../config-host.mak
-include $(SRC_PATH)/rules.mak
--include config.mak
-
-.PHONY: all
-
-$(call set-vpath, $(SRC_PATH))
-
-QEMU_CFLAGS+=-I..
-QEMU_CFLAGS += -I$(SRC_PATH)/include
-QEMU_CFLAGS += -DCONFIG_USER_ONLY
-
-include $(SRC_PATH)/Makefile.objs
-
-all: $(user-obj-y)
-# Dummy command so that make thinks it has done something
-	@true
-
-clean:
-	for d in . trace; do \
-	rm -f $$d/*.o $$d/*.d $$d/*.a $$d/*~; \
-	done
diff --git a/VERSION b/VERSION
index f0bb29e..88c5fb8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.3.0
+1.4.0
diff --git a/VERSION.LINARO b/VERSION.LINARO
index db3c08b..a45fb97 100644
--- a/VERSION.LINARO
+++ b/VERSION.LINARO
@@ -1 +1 @@
-qemu-linaro 2012.12
+qemu-linaro 2013.03
diff --git a/a.out.h b/a.out.h
deleted file mode 100644
index 33ca7f7..0000000
--- a/a.out.h
+++ /dev/null
@@ -1,430 +0,0 @@
-/* a.out.h
-
-   Copyright 1997, 1998, 1999, 2001 Red Hat, Inc.
-
-This file is part of Cygwin.
-
-This software is a copyrighted work licensed under the terms of the
-Cygwin license.  Please consult the file "CYGWIN_LICENSE" for
-details. */
-
-#ifndef _A_OUT_H_
-#define _A_OUT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#define COFF_IMAGE_WITH_PE
-#define COFF_LONG_SECTION_NAMES
-
-/*** coff information for Intel 386/486.  */
-
-
-/********************** FILE HEADER **********************/
-
-struct external_filehdr {
-  short f_magic;	/* magic number			*/
-  short f_nscns;	/* number of sections		*/
-  host_ulong f_timdat;	/* time & date stamp		*/
-  host_ulong f_symptr;	/* file pointer to symtab	*/
-  host_ulong f_nsyms;	/* number of symtab entries	*/
-  short f_opthdr;	/* sizeof(optional hdr)		*/
-  short f_flags;	/* flags			*/
-};
-
-/* Bits for f_flags:
- *	F_RELFLG	relocation info stripped from file
- *	F_EXEC		file is executable (no unresolved external references)
- *	F_LNNO		line numbers stripped from file
- *	F_LSYMS		local symbols stripped from file
- *	F_AR32WR	file has byte ordering of an AR32WR machine (e.g. vax)
- */
-
-#define F_RELFLG	(0x0001)
-#define F_EXEC		(0x0002)
-#define F_LNNO		(0x0004)
-#define F_LSYMS		(0x0008)
-
-
-
-#define	I386MAGIC	0x14c
-#define I386PTXMAGIC	0x154
-#define I386AIXMAGIC	0x175
-
-/* This is Lynx's all-platform magic number for executables. */
-
-#define LYNXCOFFMAGIC	0415
-
-#define I386BADMAG(x) (((x).f_magic != I386MAGIC) \
-		       && (x).f_magic != I386AIXMAGIC \
-		       && (x).f_magic != I386PTXMAGIC \
-		       && (x).f_magic != LYNXCOFFMAGIC)
-
-#define	FILHDR	struct external_filehdr
-#define	FILHSZ	20
-
-
-/********************** AOUT "OPTIONAL HEADER"=
- **********************/
-
-
-typedef struct
-{
-  unsigned short magic;		/* type of file				*/
-  unsigned short vstamp;	/* version stamp			*/
-  host_ulong	tsize;		/* text size in bytes, padded to FW bdry*/
-  host_ulong	dsize;		/* initialized data "  "		*/
-  host_ulong	bsize;		/* uninitialized data "   "		*/
-  host_ulong	entry;		/* entry pt.				*/
-  host_ulong text_start;	/* base of text used for this file */
-  host_ulong data_start;	/* base of data used for this file=
- */
-}
-AOUTHDR;
-
-#define AOUTSZ 28
-#define AOUTHDRSZ 28
-
-#define OMAGIC          0404    /* object files, eg as output */
-#define ZMAGIC          0413    /* demand load format, eg normal ld output */
-#define STMAGIC		0401	/* target shlib */
-#define SHMAGIC		0443	/* host   shlib */
-
-
-/* define some NT default values */
-/*  #define NT_IMAGE_BASE        0x400000 moved to internal.h */
-#define NT_SECTION_ALIGNMENT 0x1000
-#define NT_FILE_ALIGNMENT    0x200
-#define NT_DEF_RESERVE       0x100000
-#define NT_DEF_COMMIT        0x1000
-
-/********************** SECTION HEADER **********************/
-
-
-struct external_scnhdr {
-  char		s_name[8];	/* section name			*/
-  host_ulong	s_paddr;	/* physical address, offset
-				   of last addr in scn */
-  host_ulong	s_vaddr;	/* virtual address		*/
-  host_ulong	s_size;		/* section size			*/
-  host_ulong	s_scnptr;	/* file ptr to raw data for section */
-  host_ulong	s_relptr;	/* file ptr to relocation	*/
-  host_ulong	s_lnnoptr;	/* file ptr to line numbers	*/
-  unsigned short s_nreloc;	/* number of relocation entries	*/
-  unsigned short s_nlnno;	/* number of line number entries*/
-  host_ulong	s_flags;	/* flags			*/
-};
-
-#define	SCNHDR	struct external_scnhdr
-#define	SCNHSZ	40
-
-/*
- * names of "special" sections
- */
-#define _TEXT	".text"
-#define _DATA	".data"
-#define _BSS	".bss"
-#define _COMMENT ".comment"
-#define _LIB ".lib"
-
-/********************** LINE NUMBERS **********************/
-
-/* 1 line number entry for every "breakpointable" source line in a section.
- * Line numbers are grouped on a per function basis; first entry in a function
- * grouping will have l_lnno = 0 and in place of physical address will be the
- * symbol table index of the function name.
- */
-struct external_lineno {
-  union {
-    host_ulong l_symndx; /* function name symbol index, iff l_lnno 0 */
-    host_ulong l_paddr;	/* (physical) address of line number	*/
-  } l_addr;
-  unsigned short l_lnno;	/* line number		*/
-};
-
-#define	LINENO	struct external_lineno
-#define	LINESZ	6
-
-/********************** SYMBOLS **********************/
-
-#define E_SYMNMLEN	8	/* # characters in a symbol name	*/
-#define E_FILNMLEN	14	/* # characters in a file name		*/
-#define E_DIMNUM	4	/* # array dimensions in auxiliary entry */
-
-struct QEMU_PACKED external_syment
-{
-  union {
-    char e_name[E_SYMNMLEN];
-    struct {
-      host_ulong e_zeroes;
-      host_ulong e_offset;
-    } e;
-  } e;
-  host_ulong e_value;
-  unsigned short e_scnum;
-  unsigned short e_type;
-  char e_sclass[1];
-  char e_numaux[1];
-};
-
-#define N_BTMASK	(0xf)
-#define N_TMASK		(0x30)
-#define N_BTSHFT	(4)
-#define N_TSHIFT	(2)
-
-union external_auxent {
-  struct {
-    host_ulong x_tagndx;	/* str, un, or enum tag indx */
-    union {
-      struct {
-	unsigned short  x_lnno; /* declaration line number */
-	unsigned short  x_size; /* str/union/array size */
-      } x_lnsz;
-      host_ulong x_fsize;	/* size of function */
-    } x_misc;
-    union {
-      struct {			/* if ISFCN, tag, or .bb */
-	host_ulong x_lnnoptr;/* ptr to fcn line # */
-	host_ulong x_endndx;	/* entry ndx past block end */
-      } x_fcn;
-      struct {			/* if ISARY, up to 4 dimen. */
-	char x_dimen[E_DIMNUM][2];
-      } x_ary;
-    } x_fcnary;
-    unsigned short x_tvndx;	/* tv index */
-  } x_sym;
-
-  union {
-    char x_fname[E_FILNMLEN];
-    struct {
-      host_ulong x_zeroes;
-      host_ulong x_offset;
-    } x_n;
-  } x_file;
-
-  struct {
-    host_ulong x_scnlen;	/* section length */
-    unsigned short x_nreloc;	/* # relocation entries */
-    unsigned short x_nlinno;	/* # line numbers */
-    host_ulong x_checksum;	/* section COMDAT checksum */
-    unsigned short x_associated;/* COMDAT associated section index */
-    char x_comdat[1];		/* COMDAT selection number */
-  } x_scn;
-
-  struct {
-    host_ulong x_tvfill;	/* tv fill value */
-    unsigned short x_tvlen;	/* length of .tv */
-    char x_tvran[2][2];		/* tv range */
-  } x_tv;	/* info about .tv section (in auxent of symbol .tv)) */
-
-};
-
-#define	SYMENT	struct external_syment
-#define	SYMESZ	18
-#define	AUXENT	union external_auxent
-#define	AUXESZ	18
-
-#define _ETEXT	"etext"
-
-/********************** RELOCATION DIRECTIVES **********************/
-
-struct external_reloc {
-  char r_vaddr[4];
-  char r_symndx[4];
-  char r_type[2];
-};
-
-#define RELOC struct external_reloc
-#define RELSZ 10
-
-/* end of coff/i386.h */
-
-/* PE COFF header information */
-
-#ifndef _PE_H
-#define _PE_H
-
-/* NT specific file attributes */
-#define IMAGE_FILE_RELOCS_STRIPPED           0x0001
-#define IMAGE_FILE_EXECUTABLE_IMAGE          0x0002
-#define IMAGE_FILE_LINE_NUMS_STRIPPED        0x0004
-#define IMAGE_FILE_LOCAL_SYMS_STRIPPED       0x0008
-#define IMAGE_FILE_BYTES_REVERSED_LO         0x0080
-#define IMAGE_FILE_32BIT_MACHINE             0x0100
-#define IMAGE_FILE_DEBUG_STRIPPED            0x0200
-#define IMAGE_FILE_SYSTEM                    0x1000
-#define IMAGE_FILE_DLL                       0x2000
-#define IMAGE_FILE_BYTES_REVERSED_HI         0x8000
-
-/* additional flags to be set for section headers to allow the NT loader to
-   read and write to the section data (to replace the addresses of data in
-   dlls for one thing); also to execute the section in .text's case=
- */
-#define IMAGE_SCN_MEM_DISCARDABLE 0x02000000
-#define IMAGE_SCN_MEM_EXECUTE     0x20000000
-#define IMAGE_SCN_MEM_READ        0x40000000
-#define IMAGE_SCN_MEM_WRITE       0x80000000
-
-/*
- * Section characteristics added for ppc-nt
- */
-
-#define IMAGE_SCN_TYPE_NO_PAD                0x00000008  /* Reserved.  */
-
-#define IMAGE_SCN_CNT_CODE                   0x00000020  /* Section contains code. */
-#define IMAGE_SCN_CNT_INITIALIZED_DATA       0x00000040  /* Section contains initialized data. */
-#define IMAGE_SCN_CNT_UNINITIALIZED_DATA     0x00000080  /* Section contains uninitialized data. */
-
-#define IMAGE_SCN_LNK_OTHER                  0x00000100  /* Reserved.  */
-#define IMAGE_SCN_LNK_INFO                   0x00000200  /* Section contains comments or some other type of information. */
-#define IMAGE_SCN_LNK_REMOVE                 0x00000800  /* Section contents will not become part of image. */
-#define IMAGE_SCN_LNK_COMDAT                 0x00001000  /* Section contents comdat. */
-
-#define IMAGE_SCN_MEM_FARDATA                0x00008000
-
-#define IMAGE_SCN_MEM_PURGEABLE              0x00020000
-#define IMAGE_SCN_MEM_16BIT                  0x00020000
-#define IMAGE_SCN_MEM_LOCKED                 0x00040000
-#define IMAGE_SCN_MEM_PRELOAD                0x00080000
-
-#define IMAGE_SCN_ALIGN_1BYTES               0x00100000
-#define IMAGE_SCN_ALIGN_2BYTES               0x00200000
-#define IMAGE_SCN_ALIGN_4BYTES               0x00300000
-#define IMAGE_SCN_ALIGN_8BYTES               0x00400000
-#define IMAGE_SCN_ALIGN_16BYTES              0x00500000  /* Default alignment if no others are specified. */
-#define IMAGE_SCN_ALIGN_32BYTES              0x00600000
-#define IMAGE_SCN_ALIGN_64BYTES              0x00700000
-
-
-#define IMAGE_SCN_LNK_NRELOC_OVFL            0x01000000  /* Section contains extended relocations. */
-#define IMAGE_SCN_MEM_NOT_CACHED             0x04000000  /* Section is not cachable.               */
-#define IMAGE_SCN_MEM_NOT_PAGED              0x08000000  /* Section is not pageable.               */
-#define IMAGE_SCN_MEM_SHARED                 0x10000000  /* Section is shareable.                  */
-
-/* COMDAT selection codes.  */
-
-#define IMAGE_COMDAT_SELECT_NODUPLICATES     (1) /* Warn if duplicates.  */
-#define IMAGE_COMDAT_SELECT_ANY		     (2) /* No warning.  */
-#define IMAGE_COMDAT_SELECT_SAME_SIZE	     (3) /* Warn if different size.  */
-#define IMAGE_COMDAT_SELECT_EXACT_MATCH	     (4) /* Warn if different.  */
-#define IMAGE_COMDAT_SELECT_ASSOCIATIVE	     (5) /* Base on other section.  */
-
-/* Magic values that are true for all dos/nt implementations */
-#define DOSMAGIC       0x5a4d
-#define NT_SIGNATURE   0x00004550
-
-/* NT allows long filenames, we want to accommodate this.  This may break
-     some of the bfd functions */
-#undef  FILNMLEN
-#define FILNMLEN	18	/* # characters in a file name		*/
-
-
-#ifdef COFF_IMAGE_WITH_PE
-/* The filehdr is only weired in images */
-
-#undef FILHDR
-struct external_PE_filehdr
-{
-  /* DOS header fields */
-  unsigned short e_magic;	/* Magic number, 0x5a4d */
-  unsigned short e_cblp;	/* Bytes on last page of file, 0x90 */
-  unsigned short e_cp;		/* Pages in file, 0x3 */
-  unsigned short e_crlc;	/* Relocations, 0x0 */
-  unsigned short e_cparhdr;	/* Size of header in paragraphs, 0x4 */
-  unsigned short e_minalloc;	/* Minimum extra paragraphs needed, 0x0 */
-  unsigned short e_maxalloc;	/* Maximum extra paragraphs needed, 0xFFFF */
-  unsigned short e_ss;		/* Initial (relative) SS value, 0x0 */
-  unsigned short e_sp;		/* Initial SP value, 0xb8 */
-  unsigned short e_csum;	/* Checksum, 0x0 */
-  unsigned short e_ip;		/* Initial IP value, 0x0 */
-  unsigned short e_cs;		/* Initial (relative) CS value, 0x0 */
-  unsigned short e_lfarlc;	/* File address of relocation table, 0x40 */
-  unsigned short e_ovno;	/* Overlay number, 0x0 */
-  char e_res[4][2];		/* Reserved words, all 0x0 */
-  unsigned short e_oemid;	/* OEM identifier (for e_oeminfo), 0x0 */
-  unsigned short e_oeminfo;	/* OEM information; e_oemid specific, 0x0 */
-  char e_res2[10][2];		/* Reserved words, all 0x0 */
-  host_ulong e_lfanew;	/* File address of new exe header, 0x80 */
-  char dos_message[16][4];	/* other stuff, always follow DOS header */
-  unsigned int nt_signature;	/* required NT signature, 0x4550 */
-
-  /* From standard header */
-
-  unsigned short f_magic;	/* magic number			*/
-  unsigned short f_nscns;	/* number of sections		*/
-  host_ulong f_timdat;	/* time & date stamp		*/
-  host_ulong f_symptr;	/* file pointer to symtab	*/
-  host_ulong f_nsyms;	/* number of symtab entries	*/
-  unsigned short f_opthdr;	/* sizeof(optional hdr)		*/
-  unsigned short f_flags;	/* flags			*/
-};
-
-
-#define FILHDR struct external_PE_filehdr
-#undef FILHSZ
-#define FILHSZ 152
-
-#endif
-
-typedef struct
-{
-  unsigned short magic;		/* type of file				*/
-  unsigned short vstamp;	/* version stamp			*/
-  host_ulong	tsize;		/* text size in bytes, padded to FW bdry*/
-  host_ulong	dsize;		/* initialized data "  "		*/
-  host_ulong	bsize;		/* uninitialized data "   "		*/
-  host_ulong	entry;		/* entry pt.				*/
-  host_ulong text_start;	/* base of text used for this file */
-  host_ulong data_start;	/* base of all data used for this file */
-
-  /* NT extra fields; see internal.h for descriptions */
-  host_ulong  ImageBase;
-  host_ulong  SectionAlignment;
-  host_ulong  FileAlignment;
-  unsigned short  MajorOperatingSystemVersion;
-  unsigned short  MinorOperatingSystemVersion;
-  unsigned short  MajorImageVersion;
-  unsigned short  MinorImageVersion;
-  unsigned short  MajorSubsystemVersion;
-  unsigned short  MinorSubsystemVersion;
-  char  Reserved1[4];
-  host_ulong  SizeOfImage;
-  host_ulong  SizeOfHeaders;
-  host_ulong  CheckSum;
-  unsigned short Subsystem;
-  unsigned short DllCharacteristics;
-  host_ulong  SizeOfStackReserve;
-  host_ulong  SizeOfStackCommit;
-  host_ulong  SizeOfHeapReserve;
-  host_ulong  SizeOfHeapCommit;
-  host_ulong  LoaderFlags;
-  host_ulong  NumberOfRvaAndSizes;
-  /* IMAGE_DATA_DIRECTORY DataDirectory[IMAGE_NUMBEROF_DIRECTORY_ENTRIES]; */
-  char  DataDirectory[16][2][4]; /* 16 entries, 2 elements/entry, 4 chars */
-
-} PEAOUTHDR;
-
-
-#undef AOUTSZ
-#define AOUTSZ (AOUTHDRSZ + 196)
-
-#undef  E_FILNMLEN
-#define E_FILNMLEN	18	/* # characters in a file name		*/
-#endif
-
-/* end of coff/pe.h */
-
-#define DT_NON		(0)	/* no derived type */
-#define DT_PTR		(1)	/* pointer */
-#define DT_FCN		(2)	/* function */
-#define DT_ARY		(3)	/* array */
-
-#define ISPTR(x)	(((x) & N_TMASK) == (DT_PTR << N_BTSHFT))
-#define ISFCN(x)	(((x) & N_TMASK) == (DT_FCN << N_BTSHFT))
-#define ISARY(x)	(((x) & N_TMASK) == (DT_ARY << N_BTSHFT))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _A_OUT_H_ */
diff --git a/aio-posix.c b/aio-posix.c
index 05cc84e..fe4dbb4 100644
--- a/aio-posix.c
+++ b/aio-posix.c
@@ -14,9 +14,9 @@
  */
 
 #include "qemu-common.h"
-#include "block.h"
-#include "qemu-queue.h"
-#include "qemu_socket.h"
+#include "block/block.h"
+#include "qemu/queue.h"
+#include "qemu/sockets.h"
 
 struct AioHandler
 {
@@ -264,5 +264,6 @@ bool aio_poll(AioContext *ctx, bool blocking)
         }
     }
 
-    return progress;
+    assert(progress || busy);
+    return true;
 }
diff --git a/aio-win32.c b/aio-win32.c
index cec4646..38723bf 100644
--- a/aio-win32.c
+++ b/aio-win32.c
@@ -16,9 +16,9 @@
  */
 
 #include "qemu-common.h"
-#include "block.h"
-#include "qemu-queue.h"
-#include "qemu_socket.h"
+#include "block/block.h"
+#include "qemu/queue.h"
+#include "qemu/sockets.h"
 
 struct AioHandler {
     EventNotifier *e;
@@ -214,5 +214,6 @@ bool aio_poll(AioContext *ctx, bool blocking)
         events[ret - WAIT_OBJECT_0] = events[--count];
     }
 
-    return progress;
+    assert(progress || busy);
+    return true;
 }
diff --git a/arch_init.c b/arch_init.c
index e6effe8..8da868b 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -29,25 +29,26 @@
 #include <sys/mman.h>
 #endif
 #include "config.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "bitops.h"
-#include "bitmap.h"
-#include "arch_init.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
+#include "sysemu/arch_init.h"
 #include "audio/audio.h"
 #include "hw/pc.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/audiodev.h"
-#include "kvm.h"
-#include "migration.h"
-#include "net.h"
-#include "gdbstub.h"
+#include "sysemu/kvm.h"
+#include "migration/migration.h"
+#include "exec/gdbstub.h"
 #include "hw/smbios.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "hw/pcspk.h"
-#include "qemu/page_cache.h"
+#include "migration/page_cache.h"
+#include "qemu/config-file.h"
 #include "qmp-commands.h"
 #include "trace.h"
+#include "exec/cpu-all.h"
 
 #ifdef DEBUG_ARCH_INIT
 #define DPRINTF(fmt, ...) \
@@ -264,16 +265,21 @@ uint64_t xbzrle_mig_pages_overflow(void)
     return acct_info.xbzrle_overflows;
 }
 
-static void save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
-        int cont, int flag)
+static size_t save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
+                             int cont, int flag)
 {
-        qemu_put_be64(f, offset | cont | flag);
-        if (!cont) {
-                qemu_put_byte(f, strlen(block->idstr));
-                qemu_put_buffer(f, (uint8_t *)block->idstr,
-                                strlen(block->idstr));
-        }
+    size_t size;
+
+    qemu_put_be64(f, offset | cont | flag);
+    size = 8;
 
+    if (!cont) {
+        qemu_put_byte(f, strlen(block->idstr));
+        qemu_put_buffer(f, (uint8_t *)block->idstr,
+                        strlen(block->idstr));
+        size += 1 + strlen(block->idstr);
+    }
+    return size;
 }
 
 #define ENCODING_FLAG_XBZRLE 0x1
@@ -320,34 +326,43 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t *current_data,
     }
 
     /* Send XBZRLE based compressed page */
-    save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_XBZRLE);
+    bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_XBZRLE);
     qemu_put_byte(f, ENCODING_FLAG_XBZRLE);
     qemu_put_be16(f, encoded_len);
     qemu_put_buffer(f, XBZRLE.encoded_buf, encoded_len);
-    bytes_sent = encoded_len + 1 + 2;
+    bytes_sent += encoded_len + 1 + 2;
     acct_info.xbzrle_pages++;
     acct_info.xbzrle_bytes += bytes_sent;
 
     return bytes_sent;
 }
 
-static RAMBlock *last_block;
+
+/* This is the last block that we have visited serching for dirty pages
+ */
+static RAMBlock *last_seen_block;
+/* This is the last block from where we have sent data */
+static RAMBlock *last_sent_block;
 static ram_addr_t last_offset;
 static unsigned long *migration_bitmap;
 static uint64_t migration_dirty_pages;
+static uint32_t last_version;
 
-static inline bool migration_bitmap_test_and_reset_dirty(MemoryRegion *mr,
-                                                         ram_addr_t offset)
+static inline
+ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
+                                                 ram_addr_t start)
 {
-    bool ret;
-    int nr = (mr->ram_addr + offset) >> TARGET_PAGE_BITS;
+    unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
+    unsigned long nr = base + (start >> TARGET_PAGE_BITS);
+    unsigned long size = base + (int128_get64(mr->size) >> TARGET_PAGE_BITS);
 
-    ret = test_and_clear_bit(nr, migration_bitmap);
+    unsigned long next = find_next_bit(migration_bitmap, size, nr);
 
-    if (ret) {
+    if (next < size) {
+        clear_bit(next, migration_bitmap);
         migration_dirty_pages--;
     }
-    return ret;
+    return (next - base) << TARGET_PAGE_BITS;
 }
 
 static inline bool migration_bitmap_set_dirty(MemoryRegion *mr,
@@ -381,15 +396,14 @@ static void migration_bitmap_sync(void)
     trace_migration_bitmap_sync_start();
     memory_global_sync_dirty_bitmap(get_system_memory());
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) {
-            if (memory_region_get_dirty(block->mr, addr, TARGET_PAGE_SIZE,
-                                        DIRTY_MEMORY_MIGRATION)) {
+            if (memory_region_test_and_clear_dirty(block->mr,
+                                                   addr, TARGET_PAGE_SIZE,
+                                                   DIRTY_MEMORY_MIGRATION)) {
                 migration_bitmap_set_dirty(block->mr, addr);
             }
         }
-        memory_region_reset_dirty(block->mr, 0, block->length,
-                                  DIRTY_MEMORY_MIGRATION);
     }
     trace_migration_bitmap_sync_end(migration_dirty_pages
                                     - num_dirty_pages_init);
@@ -405,39 +419,54 @@ static void migration_bitmap_sync(void)
     }
 }
 
-
 /*
  * ram_save_block: Writes a page of memory to the stream f
  *
- * Returns:  0: if the page hasn't changed
- *          -1: if there are no more dirty pages
- *           n: the amount of bytes written in other case
+ * Returns:  The number of bytes written.
+ *           0 means no dirty pages
  */
 
 static int ram_save_block(QEMUFile *f, bool last_stage)
 {
-    RAMBlock *block = last_block;
+    RAMBlock *block = last_seen_block;
     ram_addr_t offset = last_offset;
-    int bytes_sent = -1;
+    bool complete_round = false;
+    int bytes_sent = 0;
     MemoryRegion *mr;
     ram_addr_t current_addr;
 
     if (!block)
-        block = QLIST_FIRST(&ram_list.blocks);
+        block = QTAILQ_FIRST(&ram_list.blocks);
 
-    do {
+    while (true) {
         mr = block->mr;
-        if (migration_bitmap_test_and_reset_dirty(mr, offset)) {
+        offset = migration_bitmap_find_and_reset_dirty(mr, offset);
+        if (complete_round && block == last_seen_block &&
+            offset >= last_offset) {
+            break;
+        }
+        if (offset >= block->length) {
+            offset = 0;
+            block = QTAILQ_NEXT(block, next);
+            if (!block) {
+                block = QTAILQ_FIRST(&ram_list.blocks);
+                complete_round = true;
+            }
+        } else {
             uint8_t *p;
-            int cont = (block == last_block) ? RAM_SAVE_FLAG_CONTINUE : 0;
+            int cont = (block == last_sent_block) ?
+                RAM_SAVE_FLAG_CONTINUE : 0;
 
             p = memory_region_get_ram_ptr(mr) + offset;
 
+            /* In doubt sent page as normal */
+            bytes_sent = -1;
             if (is_dup_page(p)) {
                 acct_info.dup_pages++;
-                save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_COMPRESS);
+                bytes_sent = save_block_hdr(f, block, offset, cont,
+                                            RAM_SAVE_FLAG_COMPRESS);
                 qemu_put_byte(f, *p);
-                bytes_sent = 1;
+                bytes_sent += 1;
             } else if (migrate_use_xbzrle()) {
                 current_addr = block->offset + offset;
                 bytes_sent = save_xbzrle_page(f, p, current_addr, block,
@@ -447,30 +476,22 @@ static int ram_save_block(QEMUFile *f, bool last_stage)
                 }
             }
 
-            /* either we didn't send yet (we may have had XBZRLE overflow) */
+            /* XBZRLE overflow or normal page */
             if (bytes_sent == -1) {
-                save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE);
+                bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE);
                 qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
-                bytes_sent = TARGET_PAGE_SIZE;
+                bytes_sent += TARGET_PAGE_SIZE;
                 acct_info.norm_pages++;
             }
 
             /* if page is unmodified, continue to the next */
-            if (bytes_sent != 0) {
+            if (bytes_sent > 0) {
+                last_sent_block = block;
                 break;
             }
         }
-
-        offset += TARGET_PAGE_SIZE;
-        if (offset >= block->length) {
-            offset = 0;
-            block = QLIST_NEXT(block, next);
-            if (!block)
-                block = QLIST_FIRST(&ram_list.blocks);
-        }
-    } while (block != last_block || offset != last_offset);
-
-    last_block = block;
+    }
+    last_seen_block = block;
     last_offset = offset;
 
     return bytes_sent;
@@ -498,46 +519,21 @@ uint64_t ram_bytes_total(void)
     RAMBlock *block;
     uint64_t total = 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next)
+    QTAILQ_FOREACH(block, &ram_list.blocks, next)
         total += block->length;
 
     return total;
 }
 
-static int block_compar(const void *a, const void *b)
-{
-    RAMBlock * const *ablock = a;
-    RAMBlock * const *bblock = b;
-
-    return strcmp((*ablock)->idstr, (*bblock)->idstr);
-}
-
-static void sort_ram_list(void)
-{
-    RAMBlock *block, *nblock, **blocks;
-    int n;
-    n = 0;
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
-        ++n;
-    }
-    blocks = g_malloc(n * sizeof *blocks);
-    n = 0;
-    QLIST_FOREACH_SAFE(block, &ram_list.blocks, next, nblock) {
-        blocks[n++] = block;
-        QLIST_REMOVE(block, next);
-    }
-    qsort(blocks, n, sizeof *blocks, block_compar);
-    while (--n >= 0) {
-        QLIST_INSERT_HEAD(&ram_list.blocks, blocks[n], next);
-    }
-    g_free(blocks);
-}
-
 static void migration_end(void)
 {
-    memory_global_dirty_log_stop();
+    if (migration_bitmap) {
+        memory_global_dirty_log_stop();
+        g_free(migration_bitmap);
+        migration_bitmap = NULL;
+    }
 
-    if (migrate_use_xbzrle()) {
+    if (XBZRLE.cache) {
         cache_fini(XBZRLE.cache);
         g_free(XBZRLE.cache);
         g_free(XBZRLE.encoded_buf);
@@ -552,12 +548,12 @@ static void ram_migration_cancel(void *opaque)
     migration_end();
 }
 
-
 static void reset_ram_globals(void)
 {
-    last_block = NULL;
+    last_seen_block = NULL;
+    last_sent_block = NULL;
     last_offset = 0;
-    sort_ram_list();
+    last_version = ram_list.version;
 }
 
 #define MAX_WAIT 50 /* ms, half buffered_file limit */
@@ -568,9 +564,10 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
 
     migration_bitmap = bitmap_new(ram_pages);
-    bitmap_set(migration_bitmap, 1, ram_pages);
+    bitmap_set(migration_bitmap, 0, ram_pages);
     migration_dirty_pages = ram_pages;
 
+    qemu_mutex_lock_ramlist();
     bytes_transferred = 0;
     reset_ram_globals();
 
@@ -592,12 +589,13 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
 
     qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         qemu_put_byte(f, strlen(block->idstr));
         qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
         qemu_put_be64(f, block->length);
     }
 
+    qemu_mutex_unlock_ramlist();
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
     return 0;
@@ -605,26 +603,28 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
 
 static int ram_save_iterate(QEMUFile *f, void *opaque)
 {
-    uint64_t bytes_transferred_last;
-    double bwidth = 0;
     int ret;
     int i;
-    uint64_t expected_downtime;
-    MigrationState *s = migrate_get_current();
+    int64_t t0;
+    int total_sent = 0;
 
-    bytes_transferred_last = bytes_transferred;
-    bwidth = qemu_get_clock_ns(rt_clock);
+    qemu_mutex_lock_ramlist();
 
+    if (ram_list.version != last_version) {
+        reset_ram_globals();
+    }
+
+    t0 = qemu_get_clock_ns(rt_clock);
     i = 0;
     while ((ret = qemu_file_rate_limit(f)) == 0) {
         int bytes_sent;
 
         bytes_sent = ram_save_block(f, false);
         /* no more blocks to sent */
-        if (bytes_sent < 0) {
+        if (bytes_sent == 0) {
             break;
         }
-        bytes_transferred += bytes_sent;
+        total_sent += bytes_sent;
         acct_info.iterations++;
         /* we want to check in the 1st loop, just in case it was the 1st time
            and we had to sync the dirty bitmap.
@@ -632,7 +632,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
            iterations
         */
         if ((i & 63) == 0) {
-            uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000;
+            uint64_t t1 = (qemu_get_clock_ns(rt_clock) - t0) / 1000000;
             if (t1 > MAX_WAIT) {
                 DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
                         t1, i);
@@ -642,38 +642,23 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
         i++;
     }
 
+    qemu_mutex_unlock_ramlist();
+
     if (ret < 0) {
+        bytes_transferred += total_sent;
         return ret;
     }
 
-    bwidth = qemu_get_clock_ns(rt_clock) - bwidth;
-    bwidth = (bytes_transferred - bytes_transferred_last) / bwidth;
-
-    /* if we haven't transferred anything this round, force
-     * expected_downtime to a very high value, but without
-     * crashing */
-    if (bwidth == 0) {
-        bwidth = 0.000001;
-    }
-
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+    total_sent += 8;
+    bytes_transferred += total_sent;
 
-    expected_downtime = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
-    DPRINTF("ram_save_live: expected(%" PRIu64 ") <= max(" PRIu64 ")?\n",
-            expected_downtime, migrate_max_downtime());
-
-    if (expected_downtime <= migrate_max_downtime()) {
-        migration_bitmap_sync();
-        expected_downtime = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
-        s->expected_downtime = expected_downtime / 1000000; /* ns -> ms */
-
-        return expected_downtime <= migrate_max_downtime();
-    }
-    return 0;
+    return total_sent;
 }
 
 static int ram_save_complete(QEMUFile *f, void *opaque)
 {
+    qemu_mutex_lock_ramlist();
     migration_bitmap_sync();
 
     /* try transferring iterative blocks of memory */
@@ -684,21 +669,32 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 
         bytes_sent = ram_save_block(f, true);
         /* no more blocks to sent */
-        if (bytes_sent < 0) {
+        if (bytes_sent == 0) {
             break;
         }
         bytes_transferred += bytes_sent;
     }
-    memory_global_dirty_log_stop();
+    migration_end();
 
+    qemu_mutex_unlock_ramlist();
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
-    g_free(migration_bitmap);
-    migration_bitmap = NULL;
-
     return 0;
 }
 
+static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+{
+    uint64_t remaining_size;
+
+    remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
+
+    if (remaining_size < max_size) {
+        migration_bitmap_sync();
+        remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
+    }
+    return remaining_size;
+}
+
 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
 {
     int ret, rc = 0;
@@ -761,7 +757,7 @@ static inline void *host_from_stream_offset(QEMUFile *f,
     qemu_get_buffer(f, (uint8_t *)id, len);
     id[len] = 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (!strncmp(id, block->idstr, sizeof(id)))
             return memory_region_get_ram_ptr(block->mr) + offset;
     }
@@ -805,7 +801,7 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                     id[len] = 0;
                     length = qemu_get_be64(f);
 
-                    QLIST_FOREACH(block, &ram_list.blocks, next) {
+                    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
                         if (!strncmp(id, block->idstr, sizeof(id))) {
                             if (block->length != length) {
                                 ret =  -EINVAL;
@@ -840,7 +836,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
             memset(host, ch, TARGET_PAGE_SIZE);
 #ifndef _WIN32
             if (ch == 0 &&
-                (!kvm_enabled() || kvm_has_sync_mmu())) {
+                (!kvm_enabled() || kvm_has_sync_mmu()) &&
+                getpagesize() <= TARGET_PAGE_SIZE) {
                 qemu_madvise(host, TARGET_PAGE_SIZE, QEMU_MADV_DONTNEED);
             }
 #endif
@@ -854,9 +851,6 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
 
             qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
         } else if (flags & RAM_SAVE_FLAG_XBZRLE) {
-            if (!migrate_use_xbzrle()) {
-                return -EINVAL;
-            }
             void *host = host_from_stream_offset(f, addr, flags);
             if (!host) {
                 return -EINVAL;
@@ -884,6 +878,7 @@ SaveVMHandlers savevm_ram_handlers = {
     .save_live_setup = ram_save_setup,
     .save_live_iterate = ram_save_iterate,
     .save_live_complete = ram_save_complete,
+    .save_live_pending = ram_save_pending,
     .load_state = ram_load,
     .cancel = ram_migration_cancel,
 };
diff --git a/async.c b/async.c
index 3f0e8f3..72d268a 100644
--- a/async.c
+++ b/async.c
@@ -23,8 +23,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-aio.h"
-#include "main-loop.h"
+#include "block/aio.h"
+#include "qemu/main-loop.h"
 
 /***********************************************************/
 /* bottom halves (can be seen as timers which expire ASAP) */
@@ -215,8 +215,3 @@ void aio_context_unref(AioContext *ctx)
 {
     g_source_unref(&ctx->source);
 }
-
-void aio_flush(AioContext *ctx)
-{
-    while (aio_poll(ctx, true));
-}
diff --git a/audio/Makefile.objs b/audio/Makefile.objs
index 0f2932d..d71a877 100644
--- a/audio/Makefile.objs
+++ b/audio/Makefile.objs
@@ -12,3 +12,6 @@ common-obj-$(CONFIG_WINWAVE) += winwaveaudio.o
 common-obj-$(CONFIG_AUDIO_PT_INT) += audio_pt_int.o
 common-obj-$(CONFIG_AUDIO_WIN_INT) += audio_win_int.o
 common-obj-y += wavcapture.o
+
+$(obj)/audio.o $(obj)/fmodaudio.o: QEMU_CFLAGS += $(FMOD_CFLAGS)
+$(obj)/sdlaudio.o: QEMU_CFLAGS += $(SDL_CFLAGS)
diff --git a/audio/alsaaudio.c b/audio/alsaaudio.c
index cb45b49..e4e5442 100644
--- a/audio/alsaaudio.c
+++ b/audio/alsaaudio.c
@@ -23,7 +23,7 @@
  */
 #include <alsa/asoundlib.h>
 #include "qemu-common.h"
-#include "qemu-char.h"
+#include "qemu/main-loop.h"
 #include "audio.h"
 
 #if QEMU_GNUC_PREREQ(4, 3)
diff --git a/audio/audio.c b/audio/audio.c
index 1c77389..02bb886 100644
--- a/audio/audio.c
+++ b/audio/audio.c
@@ -23,9 +23,9 @@
  */
 #include "hw/hw.h"
 #include "audio.h"
-#include "monitor.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "monitor/monitor.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 #define AUDIO_CAP "audio"
 #include "audio_int.h"
@@ -828,8 +828,9 @@ static int audio_attach_capture (HWVoiceOut *hw)
         QLIST_INSERT_HEAD (&hw_cap->sw_head, sw, entries);
         QLIST_INSERT_HEAD (&hw->cap_head, sc, entries);
 #ifdef DEBUG_CAPTURE
-        asprintf (&sw->name, "for %p %d,%d,%d",
-                  hw, sw->info.freq, sw->info.bits, sw->info.nchannels);
+        sw->name = g_strdup_printf ("for %p %d,%d,%d",
+                                    hw, sw->info.freq, sw->info.bits,
+                                    sw->info.nchannels);
         dolog ("Added %s active = %d\n", sw->name, sw->active);
 #endif
         if (sw->active) {
diff --git a/audio/audio.h b/audio/audio.h
index a70fda9..e7ea397 100644
--- a/audio/audio.h
+++ b/audio/audio.h
@@ -25,7 +25,7 @@
 #define QEMU_AUDIO_H
 
 #include "config-host.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 typedef void (*audio_callback_fn) (void *opaque, int avail);
 
diff --git a/audio/noaudio.c b/audio/noaudio.c
index 54958f8..9f23aa2 100644
--- a/audio/noaudio.c
+++ b/audio/noaudio.c
@@ -23,7 +23,7 @@
  */
 #include "qemu-common.h"
 #include "audio.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #define AUDIO_CAP "noaudio"
 #include "audio_int.h"
diff --git a/audio/ossaudio.c b/audio/ossaudio.c
index df51b7c..00be9c9 100644
--- a/audio/ossaudio.c
+++ b/audio/ossaudio.c
@@ -31,8 +31,8 @@
 #include <sys/soundcard.h>
 #endif
 #include "qemu-common.h"
-#include "host-utils.h"
-#include "qemu-char.h"
+#include "qemu/main-loop.h"
+#include "qemu/host-utils.h"
 #include "audio.h"
 
 #define AUDIO_CAP "oss"
diff --git a/audio/spiceaudio.c b/audio/spiceaudio.c
index 6f15591..bc24557 100644
--- a/audio/spiceaudio.c
+++ b/audio/spiceaudio.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ui/qemu-spice.h"
 
 #define AUDIO_CAP "spice"
diff --git a/audio/wavaudio.c b/audio/wavaudio.c
index a449b51..950fa8f 100644
--- a/audio/wavaudio.c
+++ b/audio/wavaudio.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "audio.h"
 
 #define AUDIO_CAP "wav"
diff --git a/audio/wavcapture.c b/audio/wavcapture.c
index 4f785f5..9d94623 100644
--- a/audio/wavcapture.c
+++ b/audio/wavcapture.c
@@ -1,5 +1,5 @@
 #include "hw/hw.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "audio.h"
 
 typedef struct {
diff --git a/audio/winwaveaudio.c b/audio/winwaveaudio.c
index 72babbf..8dbd145 100644
--- a/audio/winwaveaudio.c
+++ b/audio/winwaveaudio.c
@@ -1,7 +1,7 @@
 /* public domain */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "audio.h"
 
 #define AUDIO_CAP "winwave"
diff --git a/backends/rng-egd.c b/backends/rng-egd.c
index ad84737..5e012e9 100644
--- a/backends/rng-egd.c
+++ b/backends/rng-egd.c
@@ -11,8 +11,8 @@
  */
 
 #include "qemu/rng.h"
-#include "qemu-char.h"
-#include "qerror.h"
+#include "char/char.h"
+#include "qapi/qmp/qerror.h"
 #include "hw/qdev.h" /* just for DEFINE_PROP_CHR */
 
 #define TYPE_RNG_EGD "rng-egd"
@@ -207,7 +207,7 @@ static void rng_egd_class_init(ObjectClass *klass, void *data)
     rbc->opened = rng_egd_opened;
 }
 
-static TypeInfo rng_egd_info = {
+static const TypeInfo rng_egd_info = {
     .name = TYPE_RNG_EGD,
     .parent = TYPE_RNG_BACKEND,
     .instance_size = sizeof(RngEgd),
diff --git a/backends/rng-random.c b/backends/rng-random.c
index 9c9923b..0d11088 100644
--- a/backends/rng-random.c
+++ b/backends/rng-random.c
@@ -12,8 +12,8 @@
 
 #include "qemu/rng-random.h"
 #include "qemu/rng.h"
-#include "qerror.h"
-#include "main-loop.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/main-loop.h"
 
 struct RndRandom
 {
@@ -144,7 +144,7 @@ static void rng_random_class_init(ObjectClass *klass, void *data)
     rbc->opened = rng_random_opened;
 }
 
-static TypeInfo rng_random_info = {
+static const TypeInfo rng_random_info = {
     .name = TYPE_RNG_RANDOM,
     .parent = TYPE_RNG_BACKEND,
     .instance_size = sizeof(RndRandom),
diff --git a/backends/rng.c b/backends/rng.c
index 06f2611..3d33898 100644
--- a/backends/rng.c
+++ b/backends/rng.c
@@ -11,7 +11,7 @@
  */
 
 #include "qemu/rng.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 void rng_backend_request_entropy(RngBackend *s, size_t size,
                                  EntropyReceiveFunc *receive_entropy,
@@ -76,7 +76,7 @@ static void rng_backend_init(Object *obj)
                              NULL);
 }
 
-static TypeInfo rng_backend_info = {
+static const TypeInfo rng_backend_info = {
     .name = TYPE_RNG_BACKEND,
     .parent = TYPE_OBJECT,
     .instance_size = sizeof(RngBackend),
diff --git a/balloon.c b/balloon.c
index e02ab1c..e321f2c 100644
--- a/balloon.c
+++ b/balloon.c
@@ -24,13 +24,13 @@
  * THE SOFTWARE.
  */
 
-#include "monitor.h"
-#include "cpu-common.h"
-#include "kvm.h"
-#include "balloon.h"
+#include "monitor/monitor.h"
+#include "exec/cpu-common.h"
+#include "sysemu/kvm.h"
+#include "sysemu/balloon.h"
 #include "trace.h"
 #include "qmp-commands.h"
-#include "qjson.h"
+#include "qapi/qmp/qjson.h"
 
 static QEMUBalloonEvent *balloon_event_fn;
 static QEMUBalloonStatus *balloon_stat_fn;
diff --git a/block-migration.c b/block-migration.c
index 71b9601..43ab202 100644
--- a/block-migration.c
+++ b/block-migration.c
@@ -14,16 +14,17 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "hw/hw.h"
-#include "qemu-queue.h"
-#include "qemu-timer.h"
-#include "block-migration.h"
-#include "migration.h"
-#include "blockdev.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+#include "migration/block.h"
+#include "migration/migration.h"
+#include "sysemu/blockdev.h"
 #include <assert.h>
 
-#define BLOCK_SIZE (BDRV_SECTORS_PER_DIRTY_CHUNK << BDRV_SECTOR_BITS)
+#define BLOCK_SIZE                       (1 << 20)
+#define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
 
 #define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
 #define BLK_MIG_FLAG_EOS                0x02
@@ -77,9 +78,7 @@ typedef struct BlkMigState {
     int64_t total_sector_sum;
     int prev_progress;
     int bulk_completed;
-    long double total_time;
     long double prev_time_offset;
-    int reads;
 } BlkMigState;
 
 static BlkMigState block_mig_state;
@@ -132,12 +131,6 @@ uint64_t blk_mig_bytes_total(void)
     return sum << BDRV_SECTOR_BITS;
 }
 
-static inline long double compute_read_bwidth(void)
-{
-    assert(block_mig_state.total_time != 0);
-    return (block_mig_state.reads / block_mig_state.total_time) * BLOCK_SIZE;
-}
-
 static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
 {
     int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
@@ -191,8 +184,6 @@ static void blk_mig_read_cb(void *opaque, int ret)
 
     blk->ret = ret;
 
-    block_mig_state.reads++;
-    block_mig_state.total_time += (curr_time - block_mig_state.prev_time_offset);
     block_mig_state.prev_time_offset = curr_time;
 
     QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
@@ -264,7 +255,7 @@ static void set_dirty_tracking(int enable)
     BlkMigDevState *bmds;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        bdrv_set_dirty_tracking(bmds->bs, enable);
+        bdrv_set_dirty_tracking(bmds->bs, enable ? BLOCK_SIZE : 0);
     }
 }
 
@@ -310,8 +301,6 @@ static void init_blk_migration(QEMUFile *f)
     block_mig_state.total_sector_sum = 0;
     block_mig_state.prev_progress = -1;
     block_mig_state.bulk_completed = 0;
-    block_mig_state.total_time = 0;
-    block_mig_state.reads = 0;
 
     bdrv_iterate(init_blk_migration_it, NULL);
 }
@@ -490,33 +479,7 @@ static int64_t get_remaining_dirty(void)
         dirty += bdrv_get_dirty_count(bmds->bs);
     }
 
-    return dirty * BLOCK_SIZE;
-}
-
-static int is_stage2_completed(void)
-{
-    int64_t remaining_dirty;
-    long double bwidth;
-
-    if (block_mig_state.bulk_completed == 1) {
-
-        remaining_dirty = get_remaining_dirty();
-        if (remaining_dirty == 0) {
-            return 1;
-        }
-
-        bwidth = compute_read_bwidth();
-
-        if ((remaining_dirty / bwidth) <=
-            migrate_max_downtime()) {
-            /* finish stage2 because we think that we can finish remaining work
-               below max_downtime */
-
-            return 1;
-        }
-    }
-
-    return 0;
+    return dirty << BDRV_SECTOR_BITS;
 }
 
 static void blk_mig_cleanup(void)
@@ -576,6 +539,7 @@ static int block_save_setup(QEMUFile *f, void *opaque)
 static int block_save_iterate(QEMUFile *f, void *opaque)
 {
     int ret;
+    int64_t last_ftell = qemu_ftell(f);
 
     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
             block_mig_state.submitted, block_mig_state.transferred);
@@ -606,7 +570,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
             }
         }
     }
-    if (ret) {
+    if (ret < 0) {
         blk_mig_cleanup();
         return ret;
     }
@@ -619,7 +583,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
 
     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
 
-    return is_stage2_completed();
+    return qemu_ftell(f) - last_ftell;
 }
 
 static int block_save_complete(QEMUFile *f, void *opaque)
@@ -646,7 +610,7 @@ static int block_save_complete(QEMUFile *f, void *opaque)
     } while (ret == 0);
 
     blk_mig_cleanup();
-    if (ret) {
+    if (ret < 0) {
         return ret;
     }
     /* report completion */
@@ -659,6 +623,22 @@ static int block_save_complete(QEMUFile *f, void *opaque)
     return 0;
 }
 
+static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+{
+    /* Estimate pending number of bytes to send */
+    uint64_t pending = get_remaining_dirty() +
+                       block_mig_state.submitted * BLOCK_SIZE +
+                       block_mig_state.read_done * BLOCK_SIZE;
+
+    /* Report at least one block pending during bulk phase */
+    if (pending == 0 && !block_mig_state.bulk_completed) {
+        pending = BLOCK_SIZE;
+    }
+
+    DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
+    return pending;
+}
+
 static int block_load(QEMUFile *f, void *opaque, int version_id)
 {
     static int banner_printed;
@@ -724,7 +704,7 @@ static int block_load(QEMUFile *f, void *opaque, int version_id)
                    (addr == 100) ? '\n' : '\r');
             fflush(stdout);
         } else if (!(flags & BLK_MIG_FLAG_EOS)) {
-            fprintf(stderr, "Unknown flags\n");
+            fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
             return -EINVAL;
         }
         ret = qemu_file_get_error(f);
@@ -755,6 +735,7 @@ SaveVMHandlers savevm_block_handlers = {
     .save_live_setup = block_save_setup,
     .save_live_iterate = block_save_iterate,
     .save_live_complete = block_save_complete,
+    .save_live_pending = block_save_pending,
     .load_state = block_load,
     .cancel = block_migration_cancel,
     .is_active = block_is_active,
diff --git a/block.c b/block.c
index c05875f..50dab8e 100644
--- a/block.c
+++ b/block.c
@@ -24,16 +24,16 @@
 #include "config-host.h"
 #include "qemu-common.h"
 #include "trace.h"
-#include "monitor.h"
-#include "block_int.h"
-#include "blockjob.h"
-#include "module.h"
-#include "qjson.h"
-#include "sysemu.h"
-#include "notify.h"
-#include "qemu-coroutine.h"
+#include "monitor/monitor.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+#include "qemu/module.h"
+#include "qapi/qmp/qjson.h"
+#include "sysemu/sysemu.h"
+#include "qemu/notify.h"
+#include "block/coroutine.h"
 #include "qmp-commands.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #ifdef CONFIG_BSD
 #include <sys/types.h>
@@ -155,10 +155,6 @@ void bdrv_io_limits_enable(BlockDriverState *bs)
 {
     qemu_co_queue_init(&bs->throttled_reqs);
     bs->block_timer = qemu_new_timer_ns(vm_clock, bdrv_block_timer, bs);
-    bs->slice_time  = 5 * BLOCK_IO_SLICE_TIME;
-    bs->slice_start = qemu_get_clock_ns(vm_clock);
-    bs->slice_end   = bs->slice_start + bs->slice_time;
-    memset(&bs->io_base, 0, sizeof(bs->io_base));
     bs->io_limits_enabled = true;
 }
 
@@ -518,22 +514,16 @@ BlockDriver *bdrv_find_protocol(const char *filename)
     return NULL;
 }
 
-static int find_image_format(const char *filename, BlockDriver **pdrv)
+static int find_image_format(BlockDriverState *bs, const char *filename,
+                             BlockDriver **pdrv)
 {
-    int ret, score, score_max;
+    int score, score_max;
     BlockDriver *drv1, *drv;
     uint8_t buf[2048];
-    BlockDriverState *bs;
-
-    ret = bdrv_file_open(&bs, filename, 0);
-    if (ret < 0) {
-        *pdrv = NULL;
-        return ret;
-    }
+    int ret = 0;
 
     /* Return the raw BlockDriver * to scsi-generic devices or empty drives */
-    if (bs->sg || !bdrv_is_inserted(bs)) {
-        bdrv_delete(bs);
+    if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
         drv = bdrv_find_format("raw");
         if (!drv) {
             ret = -ENOENT;
@@ -543,7 +533,6 @@ static int find_image_format(const char *filename, BlockDriver **pdrv)
     }
 
     ret = bdrv_pread(bs, 0, buf, sizeof(buf));
-    bdrv_delete(bs);
     if (ret < 0) {
         *pdrv = NULL;
         return ret;
@@ -634,10 +623,31 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs)
     bs->copy_on_read--;
 }
 
+static int bdrv_open_flags(BlockDriverState *bs, int flags)
+{
+    int open_flags = flags | BDRV_O_CACHE_WB;
+
+    /*
+     * Clear flags that are internal to the block layer before opening the
+     * image.
+     */
+    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+
+    /*
+     * Snapshots should be writable.
+     */
+    if (bs->is_temporary) {
+        open_flags |= BDRV_O_RDWR;
+    }
+
+    return open_flags;
+}
+
 /*
  * Common part for opening disk images and files
  */
-static int bdrv_open_common(BlockDriverState *bs, const char *filename,
+static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
+    const char *filename,
     int flags, BlockDriver *drv)
 {
     int ret, open_flags;
@@ -665,31 +675,22 @@ static int bdrv_open_common(BlockDriverState *bs, const char *filename,
     bs->opaque = g_malloc0(drv->instance_size);
 
     bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
-    open_flags = flags | BDRV_O_CACHE_WB;
-
-    /*
-     * Clear flags that are internal to the block layer before opening the
-     * image.
-     */
-    open_flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
-
-    /*
-     * Snapshots should be writable.
-     */
-    if (bs->is_temporary) {
-        open_flags |= BDRV_O_RDWR;
-    }
+    open_flags = bdrv_open_flags(bs, flags);
 
     bs->read_only = !(open_flags & BDRV_O_RDWR);
 
     /* Open the image, either directly or using a protocol */
     if (drv->bdrv_file_open) {
-        ret = drv->bdrv_file_open(bs, filename, open_flags);
-    } else {
-        ret = bdrv_file_open(&bs->file, filename, open_flags);
-        if (ret >= 0) {
-            ret = drv->bdrv_open(bs, open_flags);
+        if (file != NULL) {
+            bdrv_swap(file, bs);
+            ret = 0;
+        } else {
+            ret = drv->bdrv_file_open(bs, filename, open_flags);
         }
+    } else {
+        assert(file != NULL);
+        bs->file = file;
+        ret = drv->bdrv_open(bs, open_flags);
     }
 
     if (ret < 0) {
@@ -709,10 +710,7 @@ static int bdrv_open_common(BlockDriverState *bs, const char *filename,
     return 0;
 
 free_and_fail:
-    if (bs->file) {
-        bdrv_delete(bs->file);
-        bs->file = NULL;
-    }
+    bs->file = NULL;
     g_free(bs->opaque);
     bs->opaque = NULL;
     bs->drv = NULL;
@@ -734,7 +732,7 @@ int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags)
     }
 
     bs = bdrv_new("");
-    ret = bdrv_open_common(bs, filename, flags, drv);
+    ret = bdrv_open_common(bs, NULL, filename, flags, drv);
     if (ret < 0) {
         bdrv_delete(bs);
         return ret;
@@ -789,6 +787,7 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
     int ret;
     /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
     char tmp_filename[PATH_MAX + 1];
+    BlockDriverState *file = NULL;
 
     if (flags & BDRV_O_SNAPSHOT) {
         BlockDriverState *bs1;
@@ -848,25 +847,36 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
         bs->is_temporary = 1;
     }
 
+    /* Open image file without format layer */
+    if (flags & BDRV_O_RDWR) {
+        flags |= BDRV_O_ALLOW_RDWR;
+    }
+
+    ret = bdrv_file_open(&file, filename, bdrv_open_flags(bs, flags));
+    if (ret < 0) {
+        return ret;
+    }
+
     /* Find the right image format driver */
     if (!drv) {
-        ret = find_image_format(filename, &drv);
+        ret = find_image_format(file, filename, &drv);
     }
 
     if (!drv) {
         goto unlink_and_fail;
     }
 
-    if (flags & BDRV_O_RDWR) {
-        flags |= BDRV_O_ALLOW_RDWR;
-    }
-
     /* Open the image */
-    ret = bdrv_open_common(bs, filename, flags, drv);
+    ret = bdrv_open_common(bs, file, filename, flags, drv);
     if (ret < 0) {
         goto unlink_and_fail;
     }
 
+    if (bs->file != file) {
+        bdrv_delete(file);
+        file = NULL;
+    }
+
     /* If there is a backing file, use it */
     if ((flags & BDRV_O_NO_BACKING) == 0) {
         ret = bdrv_open_backing_file(bs);
@@ -888,6 +898,9 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
     return 0;
 
 unlink_and_fail:
+    if (file != NULL) {
+        bdrv_delete(file);
+    }
     if (bs->is_temporary) {
         unlink(filename);
     }
@@ -1273,7 +1286,6 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
     bs_dest->iostatus           = bs_src->iostatus;
 
     /* dirty bitmap */
-    bs_dest->dirty_count        = bs_src->dirty_count;
     bs_dest->dirty_bitmap       = bs_src->dirty_bitmap;
 
     /* job */
@@ -1661,10 +1673,10 @@ static void tracked_request_begin(BdrvTrackedRequest *req,
 /**
  * Round a region to cluster boundaries
  */
-static void round_to_clusters(BlockDriverState *bs,
-                              int64_t sector_num, int nb_sectors,
-                              int64_t *cluster_sector_num,
-                              int *cluster_nb_sectors)
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors)
 {
     BlockDriverInfo bdi;
 
@@ -1706,8 +1718,8 @@ static void coroutine_fn wait_for_overlapping_requests(BlockDriverState *bs,
      * CoR read and write operations are atomic and guest writes cannot
      * interleave between them.
      */
-    round_to_clusters(bs, sector_num, nb_sectors,
-                      &cluster_sector_num, &cluster_nb_sectors);
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
 
     do {
         retry = false;
@@ -2022,36 +2034,6 @@ int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
     return ret;
 }
 
-#define BITS_PER_LONG  (sizeof(unsigned long) * 8)
-
-static void set_dirty_bitmap(BlockDriverState *bs, int64_t sector_num,
-                             int nb_sectors, int dirty)
-{
-    int64_t start, end;
-    unsigned long val, idx, bit;
-
-    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
-    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    for (; start <= end; start++) {
-        idx = start / BITS_PER_LONG;
-        bit = start % BITS_PER_LONG;
-        val = bs->dirty_bitmap[idx];
-        if (dirty) {
-            if (!(val & (1UL << bit))) {
-                bs->dirty_count++;
-                val |= 1UL << bit;
-            }
-        } else {
-            if (val & (1UL << bit)) {
-                bs->dirty_count--;
-                val &= ~(1UL << bit);
-            }
-        }
-        bs->dirty_bitmap[idx] = val;
-    }
-}
-
 /* Return < 0 if error. Important errors are:
   -EIO         generic I/O error (may happen for all errors)
   -ENOMEDIUM   No media inserted.
@@ -2203,8 +2185,8 @@ static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
     /* Cover entire cluster so no additional backing file I/O is required when
      * allocating cluster in the image file.
      */
-    round_to_clusters(bs, sector_num, nb_sectors,
-                      &cluster_sector_num, &cluster_nb_sectors);
+    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
+                           &cluster_sector_num, &cluster_nb_sectors);
 
     trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
                                    cluster_sector_num, cluster_nb_sectors);
@@ -2818,7 +2800,9 @@ int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
          *
          * [sector_num+x, nr_sectors] allocated.
          */
-        if (n > pnum_inter) {
+        if (n > pnum_inter &&
+            (intermediate == top ||
+             sector_num + pnum_inter < intermediate->total_sectors)) {
             n = pnum_inter;
         }
 
@@ -2850,8 +2834,9 @@ BlockInfo *bdrv_query_info(BlockDriverState *bs)
     if (bs->dirty_bitmap) {
         info->has_dirty = true;
         info->dirty = g_malloc0(sizeof(*info->dirty));
-        info->dirty->count = bdrv_get_dirty_count(bs) *
-            BDRV_SECTORS_PER_DIRTY_CHUNK * BDRV_SECTOR_SIZE;
+        info->dirty->count = bdrv_get_dirty_count(bs) * BDRV_SECTOR_SIZE;
+        info->dirty->granularity =
+            ((int64_t) BDRV_SECTOR_SIZE << hbitmap_granularity(bs->dirty_bitmap));
     }
 
     if (bs->drv) {
@@ -3028,7 +3013,46 @@ void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event)
     }
 
     drv->bdrv_debug_event(bs, event);
+}
+
+int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
+                          const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
+        return bs->drv->bdrv_debug_breakpoint(bs, event, tag);
+    }
 
+    return -ENOTSUP;
+}
+
+int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_resume) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
+        return bs->drv->bdrv_debug_resume(bs, tag);
+    }
+
+    return -ENOTSUP;
+}
+
+bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
+{
+    while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
+        bs = bs->file;
+    }
+
+    if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
+        return bs->drv->bdrv_debug_is_suspended(bs, tag);
+    }
+
+    return false;
 }
 
 /**************************************************************/
@@ -3282,11 +3306,7 @@ char *get_human_readable_size(char *buf, int buf_size, int64_t size)
 char *bdrv_snapshot_dump(char *buf, int buf_size, QEMUSnapshotInfo *sn)
 {
     char buf1[128], date_buf[128], clock_buf[128];
-#ifdef _WIN32
-    struct tm *ptm;
-#else
     struct tm tm;
-#endif
     time_t ti;
     int64_t secs;
 
@@ -3296,15 +3316,9 @@ char *bdrv_snapshot_dump(char *buf, int buf_size, QEMUSnapshotInfo *sn)
                  "ID", "TAG", "VM SIZE", "DATE", "VM CLOCK");
     } else {
         ti = sn->date_sec;
-#ifdef _WIN32
-        ptm = localtime(&ti);
-        strftime(date_buf, sizeof(date_buf),
-                 "%Y-%m-%d %H:%M:%S", ptm);
-#else
         localtime_r(&ti, &tm);
         strftime(date_buf, sizeof(date_buf),
                  "%Y-%m-%d %H:%M:%S", &tm);
-#endif
         secs = sn->vm_clock_nsec / 1000000000;
         snprintf(clock_buf, sizeof(clock_buf),
                  "%02d:%02d:%02d.%03d",
@@ -3778,12 +3792,20 @@ typedef struct BlockDriverAIOCBCoroutine {
     BlockDriverAIOCB common;
     BlockRequest req;
     bool is_write;
+    bool *done;
     QEMUBH* bh;
 } BlockDriverAIOCBCoroutine;
 
 static void bdrv_aio_co_cancel_em(BlockDriverAIOCB *blockacb)
 {
-    qemu_aio_flush();
+    BlockDriverAIOCBCoroutine *acb =
+        container_of(blockacb, BlockDriverAIOCBCoroutine, common);
+    bool done = false;
+
+    acb->done = &done;
+    while (!done) {
+        qemu_aio_wait();
+    }
 }
 
 static const AIOCBInfo bdrv_em_co_aiocb_info = {
@@ -3796,6 +3818,11 @@ static void bdrv_co_em_bh(void *opaque)
     BlockDriverAIOCBCoroutine *acb = opaque;
 
     acb->common.cb(acb->common.opaque, acb->req.error);
+
+    if (acb->done) {
+        *acb->done = true;
+    }
+
     qemu_bh_delete(acb->bh);
     qemu_aio_release(acb);
 }
@@ -3834,6 +3861,7 @@ static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
     acb->req.nb_sectors = nb_sectors;
     acb->req.qiov = qiov;
     acb->is_write = is_write;
+    acb->done = NULL;
 
     co = qemu_coroutine_create(bdrv_co_do_rw);
     qemu_coroutine_enter(co, acb);
@@ -3860,6 +3888,8 @@ BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
     BlockDriverAIOCBCoroutine *acb;
 
     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->done = NULL;
+
     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
     qemu_coroutine_enter(co, acb);
 
@@ -3888,6 +3918,7 @@ BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
     acb->req.nb_sectors = nb_sectors;
+    acb->done = NULL;
     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
     qemu_coroutine_enter(co, acb);
 
@@ -4111,7 +4142,13 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
         return -EIO;
     } else if (bs->read_only) {
         return -EROFS;
-    } else if (bs->drv->bdrv_co_discard) {
+    }
+
+    if (bs->dirty_bitmap) {
+        bdrv_reset_dirty(bs, sector_num, nb_sectors);
+    }
+
+    if (bs->drv->bdrv_co_discard) {
         return bs->drv->bdrv_co_discard(bs, sector_num, nb_sectors);
     } else if (bs->drv->bdrv_aio_discard) {
         BlockDriverAIOCB *acb;
@@ -4250,22 +4287,36 @@ void *qemu_blockalign(BlockDriverState *bs, size_t size)
     return qemu_memalign((bs && bs->buffer_alignment) ? bs->buffer_alignment : 512, size);
 }
 
-void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable)
+/*
+ * Check if all memory in this vector is sector aligned.
+ */
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
+{
+    int i;
+
+    for (i = 0; i < qiov->niov; i++) {
+        if ((uintptr_t) qiov->iov[i].iov_base % bs->buffer_alignment) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity)
 {
     int64_t bitmap_size;
 
-    bs->dirty_count = 0;
-    if (enable) {
-        if (!bs->dirty_bitmap) {
-            bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS) +
-                    BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG - 1;
-            bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG;
+    assert((granularity & (granularity - 1)) == 0);
 
-            bs->dirty_bitmap = g_new0(unsigned long, bitmap_size);
-        }
+    if (granularity) {
+        granularity >>= BDRV_SECTOR_BITS;
+        assert(!bs->dirty_bitmap);
+        bitmap_size = (bdrv_getlength(bs) >> BDRV_SECTOR_BITS);
+        bs->dirty_bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
     } else {
         if (bs->dirty_bitmap) {
-            g_free(bs->dirty_bitmap);
+            hbitmap_free(bs->dirty_bitmap);
             bs->dirty_bitmap = NULL;
         }
     }
@@ -4273,67 +4324,37 @@ void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable)
 
 int bdrv_get_dirty(BlockDriverState *bs, int64_t sector)
 {
-    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    if (bs->dirty_bitmap &&
-        (sector << BDRV_SECTOR_BITS) < bdrv_getlength(bs)) {
-        return !!(bs->dirty_bitmap[chunk / BITS_PER_LONG] &
-            (1UL << (chunk % BITS_PER_LONG)));
+    if (bs->dirty_bitmap) {
+        return hbitmap_get(bs->dirty_bitmap, sector);
     } else {
         return 0;
     }
 }
 
-int64_t bdrv_get_next_dirty(BlockDriverState *bs, int64_t sector)
+void bdrv_dirty_iter_init(BlockDriverState *bs, HBitmapIter *hbi)
 {
-    int64_t chunk;
-    int bit, elem;
-
-    /* Avoid an infinite loop.  */
-    assert(bs->dirty_count > 0);
-
-    sector = (sector | (BDRV_SECTORS_PER_DIRTY_CHUNK - 1)) + 1;
-    chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    QEMU_BUILD_BUG_ON(sizeof(bs->dirty_bitmap[0]) * 8 != BITS_PER_LONG);
-    elem = chunk / BITS_PER_LONG;
-    bit = chunk % BITS_PER_LONG;
-    for (;;) {
-        if (sector >= bs->total_sectors) {
-            sector = 0;
-            bit = elem = 0;
-        }
-        if (bit == 0 && bs->dirty_bitmap[elem] == 0) {
-            sector += BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG;
-            elem++;
-        } else {
-            if (bs->dirty_bitmap[elem] & (1UL << bit)) {
-                return sector;
-            }
-            sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
-            if (++bit == BITS_PER_LONG) {
-                bit = 0;
-                elem++;
-            }
-        }
-    }
+    hbitmap_iter_init(hbi, bs->dirty_bitmap, 0);
 }
 
 void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
                     int nr_sectors)
 {
-    set_dirty_bitmap(bs, cur_sector, nr_sectors, 1);
+    hbitmap_set(bs->dirty_bitmap, cur_sector, nr_sectors);
 }
 
 void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
                       int nr_sectors)
 {
-    set_dirty_bitmap(bs, cur_sector, nr_sectors, 0);
+    hbitmap_reset(bs->dirty_bitmap, cur_sector, nr_sectors);
 }
 
 int64_t bdrv_get_dirty_count(BlockDriverState *bs)
 {
-    return bs->dirty_count;
+    if (bs->dirty_bitmap) {
+        return hbitmap_count(bs->dirty_bitmap);
+    } else {
+        return 0;
+    }
 }
 
 void bdrv_set_in_use(BlockDriverState *bs, int in_use)
@@ -4408,9 +4429,9 @@ bdrv_acct_done(BlockDriverState *bs, BlockAcctCookie *cookie)
     bs->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
 }
 
-int bdrv_img_create(const char *filename, const char *fmt,
-                    const char *base_filename, const char *base_fmt,
-                    char *options, uint64_t img_size, int flags)
+void bdrv_img_create(const char *filename, const char *fmt,
+                     const char *base_filename, const char *base_fmt,
+                     char *options, uint64_t img_size, int flags, Error **errp)
 {
     QEMUOptionParameter *param = NULL, *create_options = NULL;
     QEMUOptionParameter *backing_fmt, *backing_file, *size;
@@ -4422,16 +4443,14 @@ int bdrv_img_create(const char *filename, const char *fmt,
     /* Find driver and parse its options */
     drv = bdrv_find_format(fmt);
     if (!drv) {
-        error_report("Unknown file format '%s'", fmt);
-        ret = -EINVAL;
-        goto out;
+        error_setg(errp, "Unknown file format '%s'", fmt);
+        return;
     }
 
     proto_drv = bdrv_find_protocol(filename);
     if (!proto_drv) {
-        error_report("Unknown protocol '%s'", filename);
-        ret = -EINVAL;
-        goto out;
+        error_setg(errp, "Unknown protocol '%s'", filename);
+        return;
     }
 
     create_options = append_option_parameters(create_options,
@@ -4448,8 +4467,7 @@ int bdrv_img_create(const char *filename, const char *fmt,
     if (options) {
         param = parse_option_parameters(options, create_options, param);
         if (param == NULL) {
-            error_report("Invalid options for file format '%s'.", fmt);
-            ret = -EINVAL;
+            error_setg(errp, "Invalid options for file format '%s'.", fmt);
             goto out;
         }
     }
@@ -4457,18 +4475,16 @@ int bdrv_img_create(const char *filename, const char *fmt,
     if (base_filename) {
         if (set_option_parameter(param, BLOCK_OPT_BACKING_FILE,
                                  base_filename)) {
-            error_report("Backing file not supported for file format '%s'",
-                         fmt);
-            ret = -EINVAL;
+            error_setg(errp, "Backing file not supported for file format '%s'",
+                       fmt);
             goto out;
         }
     }
 
     if (base_fmt) {
         if (set_option_parameter(param, BLOCK_OPT_BACKING_FMT, base_fmt)) {
-            error_report("Backing file format not supported for file "
-                         "format '%s'", fmt);
-            ret = -EINVAL;
+            error_setg(errp, "Backing file format not supported for file "
+                             "format '%s'", fmt);
             goto out;
         }
     }
@@ -4476,9 +4492,8 @@ int bdrv_img_create(const char *filename, const char *fmt,
     backing_file = get_option_parameter(param, BLOCK_OPT_BACKING_FILE);
     if (backing_file && backing_file->value.s) {
         if (!strcmp(filename, backing_file->value.s)) {
-            error_report("Error: Trying to create an image with the "
-                         "same filename as the backing file");
-            ret = -EINVAL;
+            error_setg(errp, "Error: Trying to create an image with the "
+                             "same filename as the backing file");
             goto out;
         }
     }
@@ -4487,9 +4502,8 @@ int bdrv_img_create(const char *filename, const char *fmt,
     if (backing_fmt && backing_fmt->value.s) {
         backing_drv = bdrv_find_format(backing_fmt->value.s);
         if (!backing_drv) {
-            error_report("Unknown backing file format '%s'",
-                         backing_fmt->value.s);
-            ret = -EINVAL;
+            error_setg(errp, "Unknown backing file format '%s'",
+                       backing_fmt->value.s);
             goto out;
         }
     }
@@ -4511,7 +4525,8 @@ int bdrv_img_create(const char *filename, const char *fmt,
 
             ret = bdrv_open(bs, backing_file->value.s, back_flags, backing_drv);
             if (ret < 0) {
-                error_report("Could not open '%s'", backing_file->value.s);
+                error_setg_errno(errp, -ret, "Could not open '%s'",
+                                 backing_file->value.s);
                 goto out;
             }
             bdrv_get_geometry(bs, &size);
@@ -4520,8 +4535,7 @@ int bdrv_img_create(const char *filename, const char *fmt,
             snprintf(buf, sizeof(buf), "%" PRId64, size);
             set_option_parameter(param, BLOCK_OPT_SIZE, buf);
         } else {
-            error_report("Image creation needs a size parameter");
-            ret = -EINVAL;
+            error_setg(errp, "Image creation needs a size parameter");
             goto out;
         }
     }
@@ -4531,17 +4545,16 @@ int bdrv_img_create(const char *filename, const char *fmt,
     puts("");
 
     ret = bdrv_create(drv, filename, param);
-
     if (ret < 0) {
         if (ret == -ENOTSUP) {
-            error_report("Formatting or formatting option not supported for "
-                         "file format '%s'", fmt);
+            error_setg(errp,"Formatting or formatting option not supported for "
+                            "file format '%s'", fmt);
         } else if (ret == -EFBIG) {
-            error_report("The image size is too large for file format '%s'",
-                         fmt);
+            error_setg(errp, "The image size is too large for file format '%s'",
+                       fmt);
         } else {
-            error_report("%s: error while creating %s: %s", filename, fmt,
-                         strerror(-ret));
+            error_setg(errp, "%s: error while creating %s: %s", filename, fmt,
+                       strerror(-ret));
         }
     }
 
@@ -4552,6 +4565,4 @@ out:
     if (bs) {
         bdrv_delete(bs);
     }
-
-    return ret;
 }
diff --git a/block/Makefile.objs b/block/Makefile.objs
index 7f01510..c067f38 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -18,3 +18,5 @@ endif
 common-obj-y += stream.o
 common-obj-y += commit.o
 common-obj-y += mirror.o
+
+$(obj)/curl.o: QEMU_CFLAGS+=$(CURL_CFLAGS)
diff --git a/block/blkdebug.c b/block/blkdebug.c
index d61ece8..6f74637 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -23,14 +23,17 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "qemu/config-file.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 typedef struct BDRVBlkdebugState {
     int state;
     int new_state;
+
     QLIST_HEAD(, BlkdebugRule) rules[BLKDBG_EVENT_MAX];
     QSIMPLEQ_HEAD(, BlkdebugRule) active_rules;
+    QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;
 } BDRVBlkdebugState;
 
 typedef struct BlkdebugAIOCB {
@@ -39,6 +42,12 @@ typedef struct BlkdebugAIOCB {
     int ret;
 } BlkdebugAIOCB;
 
+typedef struct BlkdebugSuspendedReq {
+    Coroutine *co;
+    char *tag;
+    QLIST_ENTRY(BlkdebugSuspendedReq) next;
+} BlkdebugSuspendedReq;
+
 static void blkdebug_aio_cancel(BlockDriverAIOCB *blockacb);
 
 static const AIOCBInfo blkdebug_aiocb_info = {
@@ -49,6 +58,7 @@ static const AIOCBInfo blkdebug_aiocb_info = {
 enum {
     ACTION_INJECT_ERROR,
     ACTION_SET_STATE,
+    ACTION_SUSPEND,
 };
 
 typedef struct BlkdebugRule {
@@ -65,6 +75,9 @@ typedef struct BlkdebugRule {
         struct {
             int new_state;
         } set_state;
+        struct {
+            char *tag;
+        } suspend;
     } options;
     QLIST_ENTRY(BlkdebugRule) next;
     QSIMPLEQ_ENTRY(BlkdebugRule) active_next;
@@ -226,6 +239,11 @@ static int add_rule(QemuOpts *opts, void *opaque)
         rule->options.set_state.new_state =
             qemu_opt_get_number(opts, "new_state", 0);
         break;
+
+    case ACTION_SUSPEND:
+        rule->options.suspend.tag =
+            g_strdup(qemu_opt_get(opts, "tag"));
+        break;
     };
 
     /* Add the rule */
@@ -234,12 +252,32 @@ static int add_rule(QemuOpts *opts, void *opaque)
     return 0;
 }
 
+static void remove_rule(BlkdebugRule *rule)
+{
+    switch (rule->action) {
+    case ACTION_INJECT_ERROR:
+    case ACTION_SET_STATE:
+        break;
+    case ACTION_SUSPEND:
+        g_free(rule->options.suspend.tag);
+        break;
+    }
+
+    QLIST_REMOVE(rule, next);
+    g_free(rule);
+}
+
 static int read_config(BDRVBlkdebugState *s, const char *filename)
 {
     FILE *f;
     int ret;
     struct add_rule_data d;
 
+    /* Allow usage without config file */
+    if (!*filename) {
+        return 0;
+    }
+
     f = fopen(filename, "r");
     if (f == NULL) {
         return -errno;
@@ -389,6 +427,7 @@ static BlockDriverAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
     return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
 }
 
+
 static void blkdebug_close(BlockDriverState *bs)
 {
     BDRVBlkdebugState *s = bs->opaque;
@@ -397,12 +436,32 @@ static void blkdebug_close(BlockDriverState *bs)
 
     for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
         QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) {
-            QLIST_REMOVE(rule, next);
-            g_free(rule);
+            remove_rule(rule);
         }
     }
 }
 
+static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugSuspendedReq r;
+
+    r = (BlkdebugSuspendedReq) {
+        .co         = qemu_coroutine_self(),
+        .tag        = g_strdup(rule->options.suspend.tag),
+    };
+
+    remove_rule(rule);
+    QLIST_INSERT_HEAD(&s->suspended_reqs, &r, next);
+
+    printf("blkdebug: Suspended request '%s'\n", r.tag);
+    qemu_coroutine_yield();
+    printf("blkdebug: Resuming request '%s'\n", r.tag);
+
+    QLIST_REMOVE(&r, next);
+    g_free(r.tag);
+}
+
 static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
     bool injected)
 {
@@ -426,6 +485,10 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
     case ACTION_SET_STATE:
         s->new_state = rule->options.set_state.new_state;
         break;
+
+    case ACTION_SUSPEND:
+        suspend_request(bs, rule);
+        break;
     }
     return injected;
 }
@@ -433,19 +496,72 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
 static void blkdebug_debug_event(BlockDriverState *bs, BlkDebugEvent event)
 {
     BDRVBlkdebugState *s = bs->opaque;
-    struct BlkdebugRule *rule;
+    struct BlkdebugRule *rule, *next;
     bool injected;
 
     assert((int)event >= 0 && event < BLKDBG_EVENT_MAX);
 
     injected = false;
     s->new_state = s->state;
-    QLIST_FOREACH(rule, &s->rules[event], next) {
+    QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {
         injected = process_rule(bs, rule, injected);
     }
     s->state = s->new_state;
 }
 
+static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,
+                                     const char *tag)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    struct BlkdebugRule *rule;
+    BlkDebugEvent blkdebug_event;
+
+    if (get_event_by_name(event, &blkdebug_event) < 0) {
+        return -ENOENT;
+    }
+
+
+    rule = g_malloc(sizeof(*rule));
+    *rule = (struct BlkdebugRule) {
+        .event  = blkdebug_event,
+        .action = ACTION_SUSPEND,
+        .state  = 0,
+        .options.suspend.tag = g_strdup(tag),
+    };
+
+    QLIST_INSERT_HEAD(&s->rules[blkdebug_event], rule, next);
+
+    return 0;
+}
+
+static int blkdebug_debug_resume(BlockDriverState *bs, const char *tag)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugSuspendedReq *r;
+
+    QLIST_FOREACH(r, &s->suspended_reqs, next) {
+        if (!strcmp(r->tag, tag)) {
+            qemu_coroutine_enter(r->co, NULL);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+
+static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+    BlkdebugSuspendedReq *r;
+
+    QLIST_FOREACH(r, &s->suspended_reqs, next) {
+        if (!strcmp(r->tag, tag)) {
+            return true;
+        }
+    }
+    return false;
+}
+
 static int64_t blkdebug_getlength(BlockDriverState *bs)
 {
     return bdrv_getlength(bs->file);
@@ -464,7 +580,10 @@ static BlockDriver bdrv_blkdebug = {
     .bdrv_aio_readv     = blkdebug_aio_readv,
     .bdrv_aio_writev    = blkdebug_aio_writev,
 
-    .bdrv_debug_event   = blkdebug_debug_event,
+    .bdrv_debug_event           = blkdebug_debug_event,
+    .bdrv_debug_breakpoint      = blkdebug_debug_breakpoint,
+    .bdrv_debug_resume          = blkdebug_debug_resume,
+    .bdrv_debug_is_suspended    = blkdebug_debug_is_suspended,
 };
 
 static void bdrv_blkdebug_init(void)
diff --git a/block/blkverify.c b/block/blkverify.c
index 4beede7..a7dd459 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -8,8 +8,8 @@
  */
 
 #include <stdarg.h>
-#include "qemu_socket.h" /* for EINPROGRESS on Windows */
-#include "block_int.h"
+#include "qemu/sockets.h" /* for EINPROGRESS on Windows */
+#include "block/block_int.h"
 
 typedef struct {
     BlockDriverState *test_file;
diff --git a/block/bochs.c b/block/bochs.c
index ab7944d..a6eb33d 100644
--- a/block/bochs.c
+++ b/block/bochs.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 /**************************************************************/
 
@@ -114,11 +114,13 @@ static int bochs_open(BlockDriverState *bs, int flags)
     int i;
     struct bochs_header bochs;
     struct bochs_header_v1 header_v1;
+    int ret;
 
     bs->read_only = 1; // no write support yet
 
-    if (bdrv_pread(bs->file, 0, &bochs, sizeof(bochs)) != sizeof(bochs)) {
-        goto fail;
+    ret = bdrv_pread(bs->file, 0, &bochs, sizeof(bochs));
+    if (ret < 0) {
+        return ret;
     }
 
     if (strcmp(bochs.magic, HEADER_MAGIC) ||
@@ -126,7 +128,7 @@ static int bochs_open(BlockDriverState *bs, int flags)
         strcmp(bochs.subtype, GROWING_TYPE) ||
 	((le32_to_cpu(bochs.version) != HEADER_VERSION) &&
 	(le32_to_cpu(bochs.version) != HEADER_V1))) {
-        goto fail;
+        return -EMEDIUMTYPE;
     }
 
     if (le32_to_cpu(bochs.version) == HEADER_V1) {
@@ -138,9 +140,13 @@ static int bochs_open(BlockDriverState *bs, int flags)
 
     s->catalog_size = le32_to_cpu(bochs.extra.redolog.catalog);
     s->catalog_bitmap = g_malloc(s->catalog_size * 4);
-    if (bdrv_pread(bs->file, le32_to_cpu(bochs.header), s->catalog_bitmap,
-                   s->catalog_size * 4) != s->catalog_size * 4)
-	goto fail;
+
+    ret = bdrv_pread(bs->file, le32_to_cpu(bochs.header), s->catalog_bitmap,
+                     s->catalog_size * 4);
+    if (ret < 0) {
+        goto fail;
+    }
+
     for (i = 0; i < s->catalog_size; i++)
 	le32_to_cpus(&s->catalog_bitmap[i]);
 
@@ -153,8 +159,10 @@ static int bochs_open(BlockDriverState *bs, int flags)
 
     qemu_co_mutex_init(&s->lock);
     return 0;
- fail:
-    return -1;
+
+fail:
+    g_free(s->catalog_bitmap);
+    return ret;
 }
 
 static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
diff --git a/block/cloop.c b/block/cloop.c
index 7570eb8..8fe13e9 100644
--- a/block/cloop.c
+++ b/block/cloop.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include <zlib.h>
 
 typedef struct BDRVCloopState {
@@ -57,27 +57,32 @@ static int cloop_open(BlockDriverState *bs, int flags)
 {
     BDRVCloopState *s = bs->opaque;
     uint32_t offsets_size, max_compressed_block_size = 1, i;
+    int ret;
 
     bs->read_only = 1;
 
     /* read header */
-    if (bdrv_pread(bs->file, 128, &s->block_size, 4) < 4) {
-        goto cloop_close;
+    ret = bdrv_pread(bs->file, 128, &s->block_size, 4);
+    if (ret < 0) {
+        return ret;
     }
     s->block_size = be32_to_cpu(s->block_size);
 
-    if (bdrv_pread(bs->file, 128 + 4, &s->n_blocks, 4) < 4) {
-        goto cloop_close;
+    ret = bdrv_pread(bs->file, 128 + 4, &s->n_blocks, 4);
+    if (ret < 0) {
+        return ret;
     }
     s->n_blocks = be32_to_cpu(s->n_blocks);
 
     /* read offsets */
     offsets_size = s->n_blocks * sizeof(uint64_t);
     s->offsets = g_malloc(offsets_size);
-    if (bdrv_pread(bs->file, 128 + 4 + 4, s->offsets, offsets_size) <
-            offsets_size) {
-        goto cloop_close;
+
+    ret = bdrv_pread(bs->file, 128 + 4 + 4, s->offsets, offsets_size);
+    if (ret < 0) {
+        goto fail;
     }
+
     for(i=0;i<s->n_blocks;i++) {
         s->offsets[i] = be64_to_cpu(s->offsets[i]);
         if (i > 0) {
@@ -92,7 +97,8 @@ static int cloop_open(BlockDriverState *bs, int flags)
     s->compressed_block = g_malloc(max_compressed_block_size + 1);
     s->uncompressed_block = g_malloc(s->block_size);
     if (inflateInit(&s->zstream) != Z_OK) {
-        goto cloop_close;
+        ret = -EINVAL;
+        goto fail;
     }
     s->current_block = s->n_blocks;
 
@@ -101,8 +107,11 @@ static int cloop_open(BlockDriverState *bs, int flags)
     qemu_co_mutex_init(&s->lock);
     return 0;
 
-cloop_close:
-    return -1;
+fail:
+    g_free(s->offsets);
+    g_free(s->compressed_block);
+    g_free(s->uncompressed_block);
+    return ret;
 }
 
 static inline int cloop_read_block(BlockDriverState *bs, int block_num)
diff --git a/block/commit.c b/block/commit.c
index fae7958..553447e 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -13,8 +13,8 @@
  */
 
 #include "trace.h"
-#include "block_int.h"
-#include "blockjob.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
 #include "qemu/ratelimit.h"
 
 enum {
@@ -65,7 +65,7 @@ static void coroutine_fn commit_run(void *opaque)
     BlockDriverState *active = s->active;
     BlockDriverState *top = s->top;
     BlockDriverState *base = s->base;
-    BlockDriverState *overlay_bs = NULL;
+    BlockDriverState *overlay_bs;
     int64_t sector_num, end;
     int ret = 0;
     int n = 0;
@@ -92,8 +92,6 @@ static void coroutine_fn commit_run(void *opaque)
         }
     }
 
-    overlay_bs = bdrv_find_overlay(active, top);
-
     end = s->common.len >> BDRV_SECTOR_BITS;
     buf = qemu_blockalign(top, COMMIT_BUFFER_SIZE);
 
@@ -103,7 +101,7 @@ static void coroutine_fn commit_run(void *opaque)
 
 wait:
         /* Note that even when no rate limit is applied we need to yield
-         * with no pending I/O here so that qemu_aio_flush() returns.
+         * with no pending I/O here so that bdrv_drain_all() returns.
          */
         block_job_sleep_ns(&s->common, rt_clock, delay_ns);
         if (block_job_is_cancelled(&s->common)) {
@@ -156,7 +154,8 @@ exit_restore_reopen:
     if (s->base_flags != bdrv_get_flags(base)) {
         bdrv_reopen(base, s->base_flags, NULL);
     }
-    if (s->orig_overlay_flags != bdrv_get_flags(overlay_bs)) {
+    overlay_bs = bdrv_find_overlay(active, top);
+    if (overlay_bs && s->orig_overlay_flags != bdrv_get_flags(overlay_bs)) {
         bdrv_reopen(overlay_bs, s->orig_overlay_flags, NULL);
     }
 
diff --git a/block/cow.c b/block/cow.c
index a5a00eb..4baf904 100644
--- a/block/cow.c
+++ b/block/cow.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 /**************************************************************/
 /* COW block driver using file system holes */
@@ -73,7 +73,7 @@ static int cow_open(BlockDriverState *bs, int flags)
     }
 
     if (be32_to_cpu(cow_header.magic) != COW_MAGIC) {
-        ret = -EINVAL;
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
 
diff --git a/block/curl.c b/block/curl.c
index 1179484..98947da 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include <curl/curl.h>
 
 // #define DEBUG
@@ -34,6 +34,10 @@
 #define DPRINTF(fmt, ...) do { } while (0)
 #endif
 
+#define PROTOCOLS (CURLPROTO_HTTP | CURLPROTO_HTTPS | \
+                   CURLPROTO_FTP | CURLPROTO_FTPS | \
+                   CURLPROTO_TFTP)
+
 #define CURL_NUM_STATES 8
 #define CURL_NUM_ACB    8
 #define SECTOR_SIZE     512
@@ -302,6 +306,17 @@ static CURLState *curl_init_state(BDRVCURLState *s)
     curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
     curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 
+    /* Restrict supported protocols to avoid security issues in the more
+     * obscure protocols.  For example, do not allow POP3/SMTP/IMAP see
+     * CVE-2013-0249.
+     *
+     * Restricting protocols is only supported from 7.19.4 upwards.
+     */
+#if LIBCURL_VERSION_NUM >= 0x071304
+    curl_easy_setopt(state->curl, CURLOPT_PROTOCOLS, PROTOCOLS);
+    curl_easy_setopt(state->curl, CURLOPT_REDIR_PROTOCOLS, PROTOCOLS);
+#endif
+
 #ifdef DEBUG_VERBOSE
     curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
 #endif
diff --git a/block/dmg.c b/block/dmg.c
index 37902a4..6d85801 100644
--- a/block/dmg.c
+++ b/block/dmg.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "bswap.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/bswap.h"
+#include "qemu/module.h"
 #include <zlib.h>
 
 typedef struct BDRVDMGState {
@@ -57,29 +57,42 @@ static int dmg_probe(const uint8_t *buf, int buf_size, const char *filename)
     return 0;
 }
 
-static off_t read_off(BlockDriverState *bs, int64_t offset)
+static int read_uint64(BlockDriverState *bs, int64_t offset, uint64_t *result)
 {
-	uint64_t buffer;
-	if (bdrv_pread(bs->file, offset, &buffer, 8) < 8)
-		return 0;
-	return be64_to_cpu(buffer);
+    uint64_t buffer;
+    int ret;
+
+    ret = bdrv_pread(bs->file, offset, &buffer, 8);
+    if (ret < 0) {
+        return ret;
+    }
+
+    *result = be64_to_cpu(buffer);
+    return 0;
 }
 
-static off_t read_uint32(BlockDriverState *bs, int64_t offset)
+static int read_uint32(BlockDriverState *bs, int64_t offset, uint32_t *result)
 {
-	uint32_t buffer;
-	if (bdrv_pread(bs->file, offset, &buffer, 4) < 4)
-		return 0;
-	return be32_to_cpu(buffer);
+    uint32_t buffer;
+    int ret;
+
+    ret = bdrv_pread(bs->file, offset, &buffer, 4);
+    if (ret < 0) {
+        return ret;
+    }
+
+    *result = be32_to_cpu(buffer);
+    return 0;
 }
 
 static int dmg_open(BlockDriverState *bs, int flags)
 {
     BDRVDMGState *s = bs->opaque;
-    off_t info_begin,info_end,last_in_offset,last_out_offset;
-    uint32_t count;
+    uint64_t info_begin,info_end,last_in_offset,last_out_offset;
+    uint32_t count, tmp;
     uint32_t max_compressed_size=1,max_sectors_per_chunk=1,i;
     int64_t offset;
+    int ret;
 
     bs->read_only = 1;
     s->n_chunks = 0;
@@ -88,21 +101,32 @@ static int dmg_open(BlockDriverState *bs, int flags)
     /* read offset of info blocks */
     offset = bdrv_getlength(bs->file);
     if (offset < 0) {
+        ret = offset;
         goto fail;
     }
     offset -= 0x1d8;
 
-    info_begin = read_off(bs, offset);
-    if (info_begin == 0) {
-	goto fail;
+    ret = read_uint64(bs, offset, &info_begin);
+    if (ret < 0) {
+        goto fail;
+    } else if (info_begin == 0) {
+        ret = -EINVAL;
+        goto fail;
     }
 
-    if (read_uint32(bs, info_begin) != 0x100) {
+    ret = read_uint32(bs, info_begin, &tmp);
+    if (ret < 0) {
+        goto fail;
+    } else if (tmp != 0x100) {
+        ret = -EINVAL;
         goto fail;
     }
 
-    count = read_uint32(bs, info_begin + 4);
-    if (count == 0) {
+    ret = read_uint32(bs, info_begin + 4, &count);
+    if (ret < 0) {
+        goto fail;
+    } else if (count == 0) {
+        ret = -EINVAL;
         goto fail;
     }
     info_end = info_begin + count;
@@ -114,12 +138,20 @@ static int dmg_open(BlockDriverState *bs, int flags)
     while (offset < info_end) {
         uint32_t type;
 
-	count = read_uint32(bs, offset);
-	if(count==0)
-	    goto fail;
+        ret = read_uint32(bs, offset, &count);
+        if (ret < 0) {
+            goto fail;
+        } else if (count == 0) {
+            ret = -EINVAL;
+            goto fail;
+        }
         offset += 4;
 
-	type = read_uint32(bs, offset);
+        ret = read_uint32(bs, offset, &type);
+        if (ret < 0) {
+            goto fail;
+        }
+
 	if (type == 0x6d697368 && count >= 244) {
 	    int new_size, chunk_count;
 
@@ -134,8 +166,11 @@ static int dmg_open(BlockDriverState *bs, int flags)
 	    s->sectors = g_realloc(s->sectors, new_size);
 	    s->sectorcounts = g_realloc(s->sectorcounts, new_size);
 
-	    for(i=s->n_chunks;i<s->n_chunks+chunk_count;i++) {
-		s->types[i] = read_uint32(bs, offset);
+            for (i = s->n_chunks; i < s->n_chunks + chunk_count; i++) {
+                ret = read_uint32(bs, offset, &s->types[i]);
+                if (ret < 0) {
+                    goto fail;
+                }
 		offset += 4;
 		if(s->types[i]!=0x80000005 && s->types[i]!=1 && s->types[i]!=2) {
 		    if(s->types[i]==0xffffffff) {
@@ -149,17 +184,31 @@ static int dmg_open(BlockDriverState *bs, int flags)
 		}
 		offset += 4;
 
-		s->sectors[i] = last_out_offset+read_off(bs, offset);
-		offset += 8;
-
-		s->sectorcounts[i] = read_off(bs, offset);
-		offset += 8;
-
-		s->offsets[i] = last_in_offset+read_off(bs, offset);
-		offset += 8;
-
-		s->lengths[i] = read_off(bs, offset);
-		offset += 8;
+                ret = read_uint64(bs, offset, &s->sectors[i]);
+                if (ret < 0) {
+                    goto fail;
+                }
+                s->sectors[i] += last_out_offset;
+                offset += 8;
+
+                ret = read_uint64(bs, offset, &s->sectorcounts[i]);
+                if (ret < 0) {
+                    goto fail;
+                }
+                offset += 8;
+
+                ret = read_uint64(bs, offset, &s->offsets[i]);
+                if (ret < 0) {
+                    goto fail;
+                }
+                s->offsets[i] += last_in_offset;
+                offset += 8;
+
+                ret = read_uint64(bs, offset, &s->lengths[i]);
+                if (ret < 0) {
+                    goto fail;
+                }
+                offset += 8;
 
 		if(s->lengths[i]>max_compressed_size)
 		    max_compressed_size = s->lengths[i];
@@ -173,15 +222,25 @@ static int dmg_open(BlockDriverState *bs, int flags)
     /* initialize zlib engine */
     s->compressed_chunk = g_malloc(max_compressed_size+1);
     s->uncompressed_chunk = g_malloc(512*max_sectors_per_chunk);
-    if(inflateInit(&s->zstream) != Z_OK)
-	goto fail;
+    if(inflateInit(&s->zstream) != Z_OK) {
+        ret = -EINVAL;
+        goto fail;
+    }
 
     s->current_chunk = s->n_chunks;
 
     qemu_co_mutex_init(&s->lock);
     return 0;
+
 fail:
-    return -1;
+    g_free(s->types);
+    g_free(s->offsets);
+    g_free(s->lengths);
+    g_free(s->sectors);
+    g_free(s->sectorcounts);
+    g_free(s->compressed_chunk);
+    g_free(s->uncompressed_chunk);
+    return ret;
 }
 
 static inline int is_sector_in_chunk(BDRVDMGState* s,
@@ -296,15 +355,15 @@ static coroutine_fn int dmg_co_read(BlockDriverState *bs, int64_t sector_num,
 static void dmg_close(BlockDriverState *bs)
 {
     BDRVDMGState *s = bs->opaque;
-    if(s->n_chunks>0) {
-	free(s->types);
-	free(s->offsets);
-	free(s->lengths);
-	free(s->sectors);
-	free(s->sectorcounts);
-    }
-    free(s->compressed_chunk);
-    free(s->uncompressed_chunk);
+
+    g_free(s->types);
+    g_free(s->offsets);
+    g_free(s->lengths);
+    g_free(s->sectors);
+    g_free(s->sectorcounts);
+    g_free(s->compressed_chunk);
+    g_free(s->uncompressed_chunk);
+
     inflateEnd(&s->zstream);
 }
 
diff --git a/block/gluster.c b/block/gluster.c
index 1c90174..ccd684d 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -16,9 +16,9 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 #include <glusterfs/api/glfs.h>
-#include "block_int.h"
-#include "qemu_socket.h"
-#include "uri.h"
+#include "block/block_int.h"
+#include "qemu/sockets.h"
+#include "qemu/uri.h"
 
 typedef struct GlusterAIOCB {
     BlockDriverAIOCB common;
@@ -217,7 +217,7 @@ static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename)
     ret = glfs_init(glfs);
     if (ret) {
         error_report("Gluster connection failed for server=%s port=%d "
-             "volume=%s image=%s transport=%s\n", gconf->server, gconf->port,
+             "volume=%s image=%s transport=%s", gconf->server, gconf->port,
              gconf->volname, gconf->image, gconf->transport);
         goto out;
     }
diff --git a/block/iscsi.c b/block/iscsi.c
index c0b70b3..deb3b68 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -27,8 +27,9 @@
 #include <poll.h>
 #include <arpa/inet.h>
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "block_int.h"
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "block/block_int.h"
 #include "trace.h"
 #include "hw/scsi-defs.h"
 
@@ -47,6 +48,7 @@ typedef struct IscsiLun {
     int block_size;
     uint64_t num_blocks;
     int events;
+    QEMUTimer *nop_timer;
 } IscsiLun;
 
 typedef struct IscsiAIOCB {
@@ -65,6 +67,9 @@ typedef struct IscsiAIOCB {
 #endif
 } IscsiAIOCB;
 
+#define NOP_INTERVAL 5000
+#define MAX_NOP_FAILURES 3
+
 static void
 iscsi_bh_cb(void *p)
 {
@@ -72,6 +77,9 @@ iscsi_bh_cb(void *p)
 
     qemu_bh_delete(acb->bh);
 
+    g_free(acb->buf);
+    acb->buf = NULL;
+
     if (acb->canceled == 0) {
         acb->common.cb(acb->common.opaque, acb->status);
     }
@@ -193,6 +201,7 @@ iscsi_aio_write16_cb(struct iscsi_context *iscsi, int status,
     trace_iscsi_aio_write16_cb(iscsi, status, acb, acb->canceled);
 
     g_free(acb->buf);
+    acb->buf = NULL;
 
     if (acb->canceled != 0) {
         return;
@@ -225,7 +234,10 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     size_t size;
     uint32_t num_sectors;
     uint64_t lba;
+#if !defined(LIBISCSI_FEATURE_IOVECTOR)
     struct iscsi_data data;
+#endif
+    int ret;
 
     acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
     trace_iscsi_aio_writev(iscsi, sector_num, nb_sectors, opaque, acb);
@@ -236,12 +248,23 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     acb->canceled   = 0;
     acb->bh         = NULL;
     acb->status     = -EINPROGRESS;
+    acb->buf        = NULL;
 
-    /* XXX we should pass the iovec to write16 to avoid the extra copy */
     /* this will allow us to get rid of 'buf' completely */
     size = nb_sectors * BDRV_SECTOR_SIZE;
-    acb->buf = g_malloc(size);
-    qemu_iovec_to_buf(acb->qiov, 0, acb->buf, size);
+
+#if !defined(LIBISCSI_FEATURE_IOVECTOR)
+    data.size = MIN(size, acb->qiov->size);
+
+    /* if the iovec only contains one buffer we can pass it directly */
+    if (acb->qiov->niov == 1) {
+        data.data = acb->qiov->iov[0].iov_base;
+    } else {
+        acb->buf = g_malloc(data.size);
+        qemu_iovec_to_buf(acb->qiov, 0, acb->buf, data.size);
+        data.data = acb->buf;
+    }
+#endif
 
     acb->task = malloc(sizeof(struct scsi_task));
     if (acb->task == NULL) {
@@ -262,19 +285,28 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     *(uint32_t *)&acb->task->cdb[10] = htonl(num_sectors);
     acb->task->expxferlen = size;
 
-    data.data = acb->buf;
-    data.size = size;
-
-    if (iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
-                                 iscsi_aio_write16_cb,
-                                 &data,
-                                 acb) != 0) {
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    ret = iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
+                                   iscsi_aio_write16_cb,
+                                   NULL,
+                                   acb);
+#else
+    ret = iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
+                                   iscsi_aio_write16_cb,
+                                   &data,
+                                   acb);
+#endif
+    if (ret != 0) {
         scsi_free_scsi_task(acb->task);
         g_free(acb->buf);
         qemu_aio_release(acb);
         return NULL;
     }
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    scsi_task_set_iov_out(acb->task, (struct scsi_iovec*) acb->qiov->iov, acb->qiov->niov);
+#endif
+
     iscsi_set_events(iscsilun);
 
     return &acb->common;
@@ -312,7 +344,10 @@ iscsi_aio_readv(BlockDriverState *bs, int64_t sector_num,
     struct iscsi_context *iscsi = iscsilun->iscsi;
     IscsiAIOCB *acb;
     size_t qemu_read_size;
+#if !defined(LIBISCSI_FEATURE_IOVECTOR)
     int i;
+#endif
+    int ret;
     uint64_t lba;
     uint32_t num_sectors;
 
@@ -374,20 +409,25 @@ iscsi_aio_readv(BlockDriverState *bs, int64_t sector_num,
         break;
     }
 
-    if (iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
-                                 iscsi_aio_read16_cb,
-                                 NULL,
-                                 acb) != 0) {
+    ret = iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
+                                   iscsi_aio_read16_cb,
+                                   NULL,
+                                   acb);
+    if (ret != 0) {
         scsi_free_scsi_task(acb->task);
         qemu_aio_release(acb);
         return NULL;
     }
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
+    scsi_task_set_iov_in(acb->task, (struct scsi_iovec*) acb->qiov->iov, acb->qiov->niov);
+#else
     for (i = 0; i < acb->qiov->niov; i++) {
         scsi_task_add_data_in_buffer(acb->task,
                 acb->qiov->iov[i].iov_len,
                 acb->qiov->iov[i].iov_base);
     }
+#endif
 
     iscsi_set_events(iscsilun);
 
@@ -429,6 +469,7 @@ iscsi_aio_flush(BlockDriverState *bs,
     acb->canceled   = 0;
     acb->bh         = NULL;
     acb->status     = -EINPROGRESS;
+    acb->buf        = NULL;
 
     acb->task = iscsi_synchronizecache10_task(iscsi, iscsilun->lun,
                                          0, 0, 0, 0,
@@ -482,6 +523,7 @@ iscsi_aio_discard(BlockDriverState *bs,
     acb->canceled   = 0;
     acb->bh         = NULL;
     acb->status     = -EINPROGRESS;
+    acb->buf        = NULL;
 
     list[0].lba = sector_qemu2lun(sector_num, iscsilun);
     list[0].num = nb_sectors * BDRV_SECTOR_SIZE / iscsilun->block_size;
@@ -761,6 +803,26 @@ static char *parse_initiator_name(const char *target)
     }
 }
 
+#if defined(LIBISCSI_FEATURE_NOP_COUNTER)
+static void iscsi_nop_timed_event(void *opaque)
+{
+    IscsiLun *iscsilun = opaque;
+
+    if (iscsi_get_nops_in_flight(iscsilun->iscsi) > MAX_NOP_FAILURES) {
+        error_report("iSCSI: NOP timeout. Reconnecting...");
+        iscsi_reconnect(iscsilun->iscsi);
+    }
+
+    if (iscsi_nop_out_async(iscsilun->iscsi, NULL, NULL, 0, NULL) != 0) {
+        error_report("iSCSI: failed to sent NOP-Out. Disabling NOP messages.");
+        return;
+    }
+
+    qemu_mod_timer(iscsilun->nop_timer, qemu_get_clock_ms(rt_clock) + NOP_INTERVAL);
+    iscsi_set_events(iscsilun);
+}
+#endif
+
 /*
  * We support iscsi url's on the form
  * iscsi://[<username>%<password>@]<host>[:<port>]/<targetname>/<lun>
@@ -921,6 +983,12 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
 
     ret = 0;
 
+#if defined(LIBISCSI_FEATURE_NOP_COUNTER)
+    /* Set up a timer for sending out iSCSI NOPs */
+    iscsilun->nop_timer = qemu_new_timer_ms(rt_clock, iscsi_nop_timed_event, iscsilun);
+    qemu_mod_timer(iscsilun->nop_timer, qemu_get_clock_ms(rt_clock) + NOP_INTERVAL);
+#endif
+
 out:
     if (initiator_name != NULL) {
         g_free(initiator_name);
@@ -946,6 +1014,10 @@ static void iscsi_close(BlockDriverState *bs)
     IscsiLun *iscsilun = bs->opaque;
     struct iscsi_context *iscsi = iscsilun->iscsi;
 
+    if (iscsilun->nop_timer) {
+        qemu_del_timer(iscsilun->nop_timer);
+        qemu_free_timer(iscsilun->nop_timer);
+    }
     qemu_aio_set_fd_handler(iscsi_get_fd(iscsi), NULL, NULL, NULL, NULL);
     iscsi_destroy_context(iscsi);
     memset(iscsilun, 0, sizeof(IscsiLun));
@@ -956,6 +1028,60 @@ static int iscsi_has_zero_init(BlockDriverState *bs)
     return 0;
 }
 
+static int iscsi_create(const char *filename, QEMUOptionParameter *options)
+{
+    int ret = 0;
+    int64_t total_size = 0;
+    BlockDriverState bs;
+    IscsiLun *iscsilun = NULL;
+
+    memset(&bs, 0, sizeof(BlockDriverState));
+
+    /* Read out options */
+    while (options && options->name) {
+        if (!strcmp(options->name, "size")) {
+            total_size = options->value.n / BDRV_SECTOR_SIZE;
+        }
+        options++;
+    }
+
+    bs.opaque = g_malloc0(sizeof(struct IscsiLun));
+    iscsilun = bs.opaque;
+
+    ret = iscsi_open(&bs, filename, 0);
+    if (ret != 0) {
+        goto out;
+    }
+    if (iscsilun->nop_timer) {
+        qemu_del_timer(iscsilun->nop_timer);
+        qemu_free_timer(iscsilun->nop_timer);
+    }
+    if (iscsilun->type != TYPE_DISK) {
+        ret = -ENODEV;
+        goto out;
+    }
+    if (bs.total_sectors < total_size) {
+        ret = -ENOSPC;
+    }
+
+    ret = 0;
+out:
+    if (iscsilun->iscsi != NULL) {
+        iscsi_destroy_context(iscsilun->iscsi);
+    }
+    g_free(bs.opaque);
+    return ret;
+}
+
+static QEMUOptionParameter iscsi_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    { NULL }
+};
+
 static BlockDriver bdrv_iscsi = {
     .format_name     = "iscsi",
     .protocol_name   = "iscsi",
@@ -963,6 +1089,8 @@ static BlockDriver bdrv_iscsi = {
     .instance_size   = sizeof(IscsiLun),
     .bdrv_file_open  = iscsi_open,
     .bdrv_close      = iscsi_close,
+    .bdrv_create     = iscsi_create,
+    .create_options  = iscsi_create_options,
 
     .bdrv_getlength  = iscsi_getlength,
 
@@ -979,9 +1107,36 @@ static BlockDriver bdrv_iscsi = {
 #endif
 };
 
+static QemuOptsList qemu_iscsi_opts = {
+    .name = "iscsi",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_iscsi_opts.head),
+    .desc = {
+        {
+            .name = "user",
+            .type = QEMU_OPT_STRING,
+            .help = "username for CHAP authentication to target",
+        },{
+            .name = "password",
+            .type = QEMU_OPT_STRING,
+            .help = "password for CHAP authentication to target",
+        },{
+            .name = "header-digest",
+            .type = QEMU_OPT_STRING,
+            .help = "HeaderDigest setting. "
+                    "{CRC32C|CRC32C-NONE|NONE-CRC32C|NONE}",
+        },{
+            .name = "initiator-name",
+            .type = QEMU_OPT_STRING,
+            .help = "Initiator iqn name to use when connecting",
+        },
+        { /* end of list */ }
+    },
+};
+
 static void iscsi_block_init(void)
 {
     bdrv_register(&bdrv_iscsi);
+    qemu_add_opts(&qemu_iscsi_opts);
 }
 
 block_init(iscsi_block_init);
diff --git a/block/linux-aio.c b/block/linux-aio.c
index 91ef863..ee0f8d1 100644
--- a/block/linux-aio.c
+++ b/block/linux-aio.c
@@ -8,10 +8,10 @@
  * See the COPYING file in the top-level directory.
  */
 #include "qemu-common.h"
-#include "qemu-aio.h"
-#include "qemu-queue.h"
+#include "block/aio.h"
+#include "qemu/queue.h"
 #include "block/raw-aio.h"
-#include "event_notifier.h"
+#include "qemu/event_notifier.h"
 
 #include <libaio.h>
 
diff --git a/block/mirror.c b/block/mirror.c
index d6618a4..a62ad86 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -12,20 +12,20 @@
  */
 
 #include "trace.h"
-#include "blockjob.h"
-#include "block_int.h"
+#include "block/blockjob.h"
+#include "block/block_int.h"
 #include "qemu/ratelimit.h"
+#include "qemu/bitmap.h"
 
-enum {
-    /*
-     * Size of data buffer for populating the image file.  This should be large
-     * enough to process multiple clusters in a single call, so that populating
-     * contiguous regions of the image is efficient.
-     */
-    BLOCK_SIZE = 512 * BDRV_SECTORS_PER_DIRTY_CHUNK, /* in bytes */
-};
+#define SLICE_TIME    100000000ULL /* ns */
+#define MAX_IN_FLIGHT 16
 
-#define SLICE_TIME 100000000ULL /* ns */
+/* The mirroring buffer is a list of granularity-sized chunks.
+ * Free chunks are organized in a list.
+ */
+typedef struct MirrorBuffer {
+    QSIMPLEQ_ENTRY(MirrorBuffer) next;
+} MirrorBuffer;
 
 typedef struct MirrorBlockJob {
     BlockJob common;
@@ -36,9 +36,26 @@ typedef struct MirrorBlockJob {
     bool synced;
     bool should_complete;
     int64_t sector_num;
+    int64_t granularity;
+    size_t buf_size;
+    unsigned long *cow_bitmap;
+    HBitmapIter hbi;
     uint8_t *buf;
+    QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
+    int buf_free_count;
+
+    unsigned long *in_flight_bitmap;
+    int in_flight;
+    int ret;
 } MirrorBlockJob;
 
+typedef struct MirrorOp {
+    MirrorBlockJob *s;
+    QEMUIOVector qiov;
+    int64_t sector_num;
+    int nb_sectors;
+} MirrorOp;
+
 static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
                                             int error)
 {
@@ -52,51 +69,234 @@ static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
     }
 }
 
-static int coroutine_fn mirror_iteration(MirrorBlockJob *s,
-                                         BlockErrorAction *p_action)
+static void mirror_iteration_done(MirrorOp *op, int ret)
 {
-    BlockDriverState *source = s->common.bs;
-    BlockDriverState *target = s->target;
-    QEMUIOVector qiov;
-    int ret, nb_sectors;
-    int64_t end;
-    struct iovec iov;
+    MirrorBlockJob *s = op->s;
+    struct iovec *iov;
+    int64_t chunk_num;
+    int i, nb_chunks, sectors_per_chunk;
+
+    trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret);
+
+    s->in_flight--;
+    iov = op->qiov.iov;
+    for (i = 0; i < op->qiov.niov; i++) {
+        MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
+        QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
+        s->buf_free_count++;
+    }
 
-    end = s->common.len >> BDRV_SECTOR_BITS;
-    s->sector_num = bdrv_get_next_dirty(source, s->sector_num);
-    nb_sectors = MIN(BDRV_SECTORS_PER_DIRTY_CHUNK, end - s->sector_num);
-    bdrv_reset_dirty(source, s->sector_num, nb_sectors);
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    chunk_num = op->sector_num / sectors_per_chunk;
+    nb_chunks = op->nb_sectors / sectors_per_chunk;
+    bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
+    if (s->cow_bitmap && ret >= 0) {
+        bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
+    }
 
-    /* Copy the dirty cluster.  */
-    iov.iov_base = s->buf;
-    iov.iov_len  = nb_sectors * 512;
-    qemu_iovec_init_external(&qiov, &iov, 1);
+    g_slice_free(MirrorOp, op);
+    qemu_coroutine_enter(s->common.co, NULL);
+}
 
-    trace_mirror_one_iteration(s, s->sector_num, nb_sectors);
-    ret = bdrv_co_readv(source, s->sector_num, nb_sectors, &qiov);
+static void mirror_write_complete(void *opaque, int ret)
+{
+    MirrorOp *op = opaque;
+    MirrorBlockJob *s = op->s;
     if (ret < 0) {
-        *p_action = mirror_error_action(s, true, -ret);
-        goto fail;
+        BlockDriverState *source = s->common.bs;
+        BlockErrorAction action;
+
+        bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
+        action = mirror_error_action(s, false, -ret);
+        if (action == BDRV_ACTION_REPORT && s->ret >= 0) {
+            s->ret = ret;
+        }
     }
-    ret = bdrv_co_writev(target, s->sector_num, nb_sectors, &qiov);
+    mirror_iteration_done(op, ret);
+}
+
+static void mirror_read_complete(void *opaque, int ret)
+{
+    MirrorOp *op = opaque;
+    MirrorBlockJob *s = op->s;
     if (ret < 0) {
-        *p_action = mirror_error_action(s, false, -ret);
-        s->synced = false;
-        goto fail;
+        BlockDriverState *source = s->common.bs;
+        BlockErrorAction action;
+
+        bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
+        action = mirror_error_action(s, true, -ret);
+        if (action == BDRV_ACTION_REPORT && s->ret >= 0) {
+            s->ret = ret;
+        }
+
+        mirror_iteration_done(op, ret);
+        return;
+    }
+    bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors,
+                    mirror_write_complete, op);
+}
+
+static void coroutine_fn mirror_iteration(MirrorBlockJob *s)
+{
+    BlockDriverState *source = s->common.bs;
+    int nb_sectors, sectors_per_chunk, nb_chunks;
+    int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector;
+    MirrorOp *op;
+
+    s->sector_num = hbitmap_iter_next(&s->hbi);
+    if (s->sector_num < 0) {
+        bdrv_dirty_iter_init(source, &s->hbi);
+        s->sector_num = hbitmap_iter_next(&s->hbi);
+        trace_mirror_restart_iter(s, bdrv_get_dirty_count(source));
+        assert(s->sector_num >= 0);
+    }
+
+    hbitmap_next_sector = s->sector_num;
+    sector_num = s->sector_num;
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    end = s->common.len >> BDRV_SECTOR_BITS;
+
+    /* Extend the QEMUIOVector to include all adjacent blocks that will
+     * be copied in this operation.
+     *
+     * We have to do this if we have no backing file yet in the destination,
+     * and the cluster size is very large.  Then we need to do COW ourselves.
+     * The first time a cluster is copied, copy it entirely.  Note that,
+     * because both the granularity and the cluster size are powers of two,
+     * the number of sectors to copy cannot exceed one cluster.
+     *
+     * We also want to extend the QEMUIOVector to include more adjacent
+     * dirty blocks if possible, to limit the number of I/O operations and
+     * run efficiently even with a small granularity.
+     */
+    nb_chunks = 0;
+    nb_sectors = 0;
+    next_sector = sector_num;
+    next_chunk = sector_num / sectors_per_chunk;
+
+    /* Wait for I/O to this cluster (from a previous iteration) to be done.  */
+    while (test_bit(next_chunk, s->in_flight_bitmap)) {
+        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
+        qemu_coroutine_yield();
+    }
+
+    do {
+        int added_sectors, added_chunks;
+
+        if (!bdrv_get_dirty(source, next_sector) ||
+            test_bit(next_chunk, s->in_flight_bitmap)) {
+            assert(nb_sectors > 0);
+            break;
+        }
+
+        added_sectors = sectors_per_chunk;
+        if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) {
+            bdrv_round_to_clusters(s->target,
+                                   next_sector, added_sectors,
+                                   &next_sector, &added_sectors);
+
+            /* On the first iteration, the rounding may make us copy
+             * sectors before the first dirty one.
+             */
+            if (next_sector < sector_num) {
+                assert(nb_sectors == 0);
+                sector_num = next_sector;
+                next_chunk = next_sector / sectors_per_chunk;
+            }
+        }
+
+        added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors));
+        added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk;
+
+        /* When doing COW, it may happen that there is not enough space for
+         * a full cluster.  Wait if that is the case.
+         */
+        while (nb_chunks == 0 && s->buf_free_count < added_chunks) {
+            trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight);
+            qemu_coroutine_yield();
+        }
+        if (s->buf_free_count < nb_chunks + added_chunks) {
+            trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight);
+            break;
+        }
+
+        /* We have enough free space to copy these sectors.  */
+        bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks);
+
+        nb_sectors += added_sectors;
+        nb_chunks += added_chunks;
+        next_sector += added_sectors;
+        next_chunk += added_chunks;
+    } while (next_sector < end);
+
+    /* Allocate a MirrorOp that is used as an AIO callback.  */
+    op = g_slice_new(MirrorOp);
+    op->s = s;
+    op->sector_num = sector_num;
+    op->nb_sectors = nb_sectors;
+
+    /* Now make a QEMUIOVector taking enough granularity-sized chunks
+     * from s->buf_free.
+     */
+    qemu_iovec_init(&op->qiov, nb_chunks);
+    next_sector = sector_num;
+    while (nb_chunks-- > 0) {
+        MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
+        QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
+        s->buf_free_count--;
+        qemu_iovec_add(&op->qiov, buf, s->granularity);
+
+        /* Advance the HBitmapIter in parallel, so that we do not examine
+         * the same sector twice.
+         */
+        if (next_sector > hbitmap_next_sector && bdrv_get_dirty(source, next_sector)) {
+            hbitmap_next_sector = hbitmap_iter_next(&s->hbi);
+        }
+
+        next_sector += sectors_per_chunk;
     }
-    return 0;
 
-fail:
-    /* Try again later.  */
-    bdrv_set_dirty(source, s->sector_num, nb_sectors);
-    return ret;
+    bdrv_reset_dirty(source, sector_num, nb_sectors);
+
+    /* Copy the dirty cluster.  */
+    s->in_flight++;
+    trace_mirror_one_iteration(s, sector_num, nb_sectors);
+    bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
+                   mirror_read_complete, op);
+}
+
+static void mirror_free_init(MirrorBlockJob *s)
+{
+    int granularity = s->granularity;
+    size_t buf_size = s->buf_size;
+    uint8_t *buf = s->buf;
+
+    assert(s->buf_free_count == 0);
+    QSIMPLEQ_INIT(&s->buf_free);
+    while (buf_size != 0) {
+        MirrorBuffer *cur = (MirrorBuffer *)buf;
+        QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
+        s->buf_free_count++;
+        buf_size -= granularity;
+        buf += granularity;
+    }
+}
+
+static void mirror_drain(MirrorBlockJob *s)
+{
+    while (s->in_flight > 0) {
+        qemu_coroutine_yield();
+    }
 }
 
 static void coroutine_fn mirror_run(void *opaque)
 {
     MirrorBlockJob *s = opaque;
     BlockDriverState *bs = s->common.bs;
-    int64_t sector_num, end;
+    int64_t sector_num, end, sectors_per_chunk, length;
+    uint64_t last_pause_ns;
+    BlockDriverInfo bdi;
+    char backing_filename[1024];
     int ret = 0;
     int n;
 
@@ -105,20 +305,39 @@ static void coroutine_fn mirror_run(void *opaque)
     }
 
     s->common.len = bdrv_getlength(bs);
-    if (s->common.len < 0) {
+    if (s->common.len <= 0) {
         block_job_completed(&s->common, s->common.len);
         return;
     }
 
+    length = (bdrv_getlength(bs) + s->granularity - 1) / s->granularity;
+    s->in_flight_bitmap = bitmap_new(length);
+
+    /* If we have no backing file yet in the destination, we cannot let
+     * the destination do COW.  Instead, we copy sectors around the
+     * dirty data if needed.  We need a bitmap to do that.
+     */
+    bdrv_get_backing_filename(s->target, backing_filename,
+                              sizeof(backing_filename));
+    if (backing_filename[0] && !s->target->backing_hd) {
+        bdrv_get_info(s->target, &bdi);
+        if (s->granularity < bdi.cluster_size) {
+            s->buf_size = MAX(s->buf_size, bdi.cluster_size);
+            s->cow_bitmap = bitmap_new(length);
+        }
+    }
+
     end = s->common.len >> BDRV_SECTOR_BITS;
-    s->buf = qemu_blockalign(bs, BLOCK_SIZE);
+    s->buf = qemu_blockalign(bs, s->buf_size);
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    mirror_free_init(s);
 
     if (s->mode != MIRROR_SYNC_MODE_NONE) {
         /* First part, loop on the sectors and initialize the dirty bitmap.  */
         BlockDriverState *base;
         base = s->mode == MIRROR_SYNC_MODE_FULL ? NULL : bs->backing_hd;
         for (sector_num = 0; sector_num < end; ) {
-            int64_t next = (sector_num | (BDRV_SECTORS_PER_DIRTY_CHUNK - 1)) + 1;
+            int64_t next = (sector_num | (sectors_per_chunk - 1)) + 1;
             ret = bdrv_co_is_allocated_above(bs, base,
                                              sector_num, next - sector_num, &n);
 
@@ -136,24 +355,40 @@ static void coroutine_fn mirror_run(void *opaque)
         }
     }
 
-    s->sector_num = -1;
+    bdrv_dirty_iter_init(bs, &s->hbi);
+    last_pause_ns = qemu_get_clock_ns(rt_clock);
     for (;;) {
         uint64_t delay_ns;
         int64_t cnt;
         bool should_complete;
 
+        if (s->ret < 0) {
+            ret = s->ret;
+            goto immediate_exit;
+        }
+
         cnt = bdrv_get_dirty_count(bs);
-        if (cnt != 0) {
-            BlockErrorAction action = BDRV_ACTION_REPORT;
-            ret = mirror_iteration(s, &action);
-            if (ret < 0 && action == BDRV_ACTION_REPORT) {
-                goto immediate_exit;
+
+        /* Note that even when no rate limit is applied we need to yield
+         * periodically with no pending I/O so that qemu_aio_flush() returns.
+         * We do so every SLICE_TIME nanoseconds, or when there is an error,
+         * or when the source is clean, whichever comes first.
+         */
+        if (qemu_get_clock_ns(rt_clock) - last_pause_ns < SLICE_TIME &&
+            s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+            if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 ||
+                (cnt == 0 && s->in_flight > 0)) {
+                trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt);
+                qemu_coroutine_yield();
+                continue;
+            } else if (cnt != 0) {
+                mirror_iteration(s);
+                continue;
             }
-            cnt = bdrv_get_dirty_count(bs);
         }
 
         should_complete = false;
-        if (cnt == 0) {
+        if (s->in_flight == 0 && cnt == 0) {
             trace_mirror_before_flush(s);
             ret = bdrv_flush(s->target);
             if (ret < 0) {
@@ -196,23 +431,20 @@ static void coroutine_fn mirror_run(void *opaque)
         trace_mirror_before_sleep(s, cnt, s->synced);
         if (!s->synced) {
             /* Publish progress */
-            s->common.offset = end * BDRV_SECTOR_SIZE - cnt * BLOCK_SIZE;
+            s->common.offset = (end - cnt) * BDRV_SECTOR_SIZE;
 
             if (s->common.speed) {
-                delay_ns = ratelimit_calculate_delay(&s->limit, BDRV_SECTORS_PER_DIRTY_CHUNK);
+                delay_ns = ratelimit_calculate_delay(&s->limit, sectors_per_chunk);
             } else {
                 delay_ns = 0;
             }
 
-            /* Note that even when no rate limit is applied we need to yield
-             * with no pending I/O here so that qemu_aio_flush() returns.
-             */
             block_job_sleep_ns(&s->common, rt_clock, delay_ns);
             if (block_job_is_cancelled(&s->common)) {
                 break;
             }
         } else if (!should_complete) {
-            delay_ns = (cnt == 0 ? SLICE_TIME : 0);
+            delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0);
             block_job_sleep_ns(&s->common, rt_clock, delay_ns);
         } else if (cnt == 0) {
             /* The two disks are in sync.  Exit and report successful
@@ -222,11 +454,24 @@ static void coroutine_fn mirror_run(void *opaque)
             s->common.cancelled = false;
             break;
         }
+        last_pause_ns = qemu_get_clock_ns(rt_clock);
     }
 
 immediate_exit:
-    g_free(s->buf);
-    bdrv_set_dirty_tracking(bs, false);
+    if (s->in_flight > 0) {
+        /* We get here only if something went wrong.  Either the job failed,
+         * or it was cancelled prematurely so that we do not guarantee that
+         * the target is a copy of the source.
+         */
+        assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common)));
+        mirror_drain(s);
+    }
+
+    assert(s->in_flight == 0);
+    qemu_vfree(s->buf);
+    g_free(s->cow_bitmap);
+    g_free(s->in_flight_bitmap);
+    bdrv_set_dirty_tracking(bs, 0);
     bdrv_iostatus_disable(s->target);
     if (s->should_complete && ret == 0) {
         if (bdrv_get_flags(s->target) != bdrv_get_flags(s->common.bs)) {
@@ -288,14 +533,28 @@ static BlockJobType mirror_job_type = {
 };
 
 void mirror_start(BlockDriverState *bs, BlockDriverState *target,
-                  int64_t speed, MirrorSyncMode mode,
-                  BlockdevOnError on_source_error,
+                  int64_t speed, int64_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockDriverCompletionFunc *cb,
                   void *opaque, Error **errp)
 {
     MirrorBlockJob *s;
 
+    if (granularity == 0) {
+        /* Choose the default granularity based on the target file's cluster
+         * size, clamped between 4k and 64k.  */
+        BlockDriverInfo bdi;
+        if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) {
+            granularity = MAX(4096, bdi.cluster_size);
+            granularity = MIN(65536, granularity);
+        } else {
+            granularity = 65536;
+        }
+    }
+
+    assert ((granularity & (granularity - 1)) == 0);
+
     if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
          on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
         !bdrv_iostatus_is_enabled(bs)) {
@@ -312,7 +571,10 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
     s->on_target_error = on_target_error;
     s->target = target;
     s->mode = mode;
-    bdrv_set_dirty_tracking(bs, true);
+    s->granularity = granularity;
+    s->buf_size = MAX(buf_size, granularity);
+
+    bdrv_set_dirty_tracking(bs, granularity);
     bdrv_set_enable_write_cache(s->target, true);
     bdrv_set_on_error(s->target, on_target_error, on_target_error);
     bdrv_iostatus_enable(s->target);
diff --git a/block/nbd.c b/block/nbd.c
index e87c248..a581294 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -27,11 +27,11 @@
  */
 
 #include "qemu-common.h"
-#include "nbd.h"
-#include "uri.h"
-#include "block_int.h"
-#include "module.h"
-#include "qemu_socket.h"
+#include "block/nbd.h"
+#include "qemu/uri.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "qemu/sockets.h"
 
 #include <sys/types.h>
 #include <unistd.h>
diff --git a/block/parallels.c b/block/parallels.c
index d30f0ec..8688f6c 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -24,8 +24,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 /**************************************************************/
 
@@ -73,14 +73,18 @@ static int parallels_open(BlockDriverState *bs, int flags)
     BDRVParallelsState *s = bs->opaque;
     int i;
     struct parallels_header ph;
+    int ret;
 
     bs->read_only = 1; // no write support yet
 
-    if (bdrv_pread(bs->file, 0, &ph, sizeof(ph)) != sizeof(ph))
+    ret = bdrv_pread(bs->file, 0, &ph, sizeof(ph));
+    if (ret < 0) {
         goto fail;
+    }
 
     if (memcmp(ph.magic, HEADER_MAGIC, 16) ||
-	(le32_to_cpu(ph.version) != HEADER_VERSION)) {
+        (le32_to_cpu(ph.version) != HEADER_VERSION)) {
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
 
@@ -90,18 +94,21 @@ static int parallels_open(BlockDriverState *bs, int flags)
 
     s->catalog_size = le32_to_cpu(ph.catalog_entries);
     s->catalog_bitmap = g_malloc(s->catalog_size * 4);
-    if (bdrv_pread(bs->file, 64, s->catalog_bitmap, s->catalog_size * 4) !=
-	s->catalog_size * 4)
-	goto fail;
+
+    ret = bdrv_pread(bs->file, 64, s->catalog_bitmap, s->catalog_size * 4);
+    if (ret < 0) {
+        goto fail;
+    }
+
     for (i = 0; i < s->catalog_size; i++)
 	le32_to_cpus(&s->catalog_bitmap[i]);
 
     qemu_co_mutex_init(&s->lock);
     return 0;
+
 fail:
-    if (s->catalog_bitmap)
-	g_free(s->catalog_bitmap);
-    return -1;
+    g_free(s->catalog_bitmap);
+    return ret;
 }
 
 static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
diff --git a/block/qcow.c b/block/qcow.c
index b239c82..a7135ee 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include <zlib.h>
-#include "aes.h"
-#include "migration.h"
+#include "block/aes.h"
+#include "migration/migration.h"
 
 /**************************************************************/
 /* QEMU COW block driver with compression and encryption support */
@@ -112,7 +112,7 @@ static int qcow_open(BlockDriverState *bs, int flags)
     be64_to_cpus(&header.l1_table_offset);
 
     if (header.magic != QCOW_MAGIC) {
-        ret = -EINVAL;
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
     if (header.version != QCOW_VERSION) {
diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c
index 2d4322a..2f3114e 100644
--- a/block/qcow2-cache.c
+++ b/block/qcow2-cache.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "block_int.h"
+#include "block/block_int.h"
 #include "qemu-common.h"
 #include "qcow2.h"
 #include "trace.h"
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index e179211..56fccf9 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -25,7 +25,7 @@
 #include <zlib.h>
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "block/qcow2.h"
 #include "trace.h"
 
@@ -615,57 +615,67 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
     return cluster_offset;
 }
 
-int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
+static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 {
     BDRVQcowState *s = bs->opaque;
-    int i, j = 0, l2_index, ret;
-    uint64_t *old_cluster, start_sect, *l2_table;
-    uint64_t cluster_offset = m->alloc_offset;
-    bool cow = false;
-
-    trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
+    int ret;
 
-    if (m->nb_clusters == 0)
+    if (r->nb_sectors == 0) {
         return 0;
+    }
 
-    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
+    qemu_co_mutex_unlock(&s->lock);
+    ret = copy_sectors(bs, m->offset / BDRV_SECTOR_SIZE, m->alloc_offset,
+                       r->offset / BDRV_SECTOR_SIZE,
+                       r->offset / BDRV_SECTOR_SIZE + r->nb_sectors);
+    qemu_co_mutex_lock(&s->lock);
 
-    /* copy content of unmodified sectors */
-    start_sect = (m->offset & ~(s->cluster_size - 1)) >> 9;
-    if (m->n_start) {
-        cow = true;
-        qemu_co_mutex_unlock(&s->lock);
-        ret = copy_sectors(bs, start_sect, cluster_offset, 0, m->n_start);
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0)
-            goto err;
-    }
-
-    if (m->nb_available & (s->cluster_sectors - 1)) {
-        cow = true;
-        qemu_co_mutex_unlock(&s->lock);
-        ret = copy_sectors(bs, start_sect, cluster_offset, m->nb_available,
-                           align_offset(m->nb_available, s->cluster_sectors));
-        qemu_co_mutex_lock(&s->lock);
-        if (ret < 0)
-            goto err;
+    if (ret < 0) {
+        return ret;
     }
 
     /*
-     * Update L2 table.
-     *
      * Before we update the L2 table to actually point to the new cluster, we
      * need to be sure that the refcounts have been increased and COW was
      * handled.
      */
-    if (cow) {
-        qcow2_cache_depends_on_flush(s->l2_table_cache);
+    qcow2_cache_depends_on_flush(s->l2_table_cache);
+
+    return 0;
+}
+
+int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
+{
+    BDRVQcowState *s = bs->opaque;
+    int i, j = 0, l2_index, ret;
+    uint64_t *old_cluster, *l2_table;
+    uint64_t cluster_offset = m->alloc_offset;
+
+    trace_qcow2_cluster_link_l2(qemu_coroutine_self(), m->nb_clusters);
+    assert(m->nb_clusters > 0);
+
+    old_cluster = g_malloc(m->nb_clusters * sizeof(uint64_t));
+
+    /* copy content of unmodified sectors */
+    ret = perform_cow(bs, m, &m->cow_start);
+    if (ret < 0) {
+        goto err;
+    }
+
+    ret = perform_cow(bs, m, &m->cow_end);
+    if (ret < 0) {
+        goto err;
     }
 
+    /* Update L2 table. */
+    if (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS) {
+        qcow2_mark_dirty(bs);
+    }
     if (qcow2_need_accurate_refcounts(s)) {
         qcow2_cache_set_dependency(bs, s->l2_table_cache,
                                    s->refcount_block_cache);
     }
+
     ret = get_cluster_table(bs, m->offset, &l2_table, &l2_index);
     if (ret < 0) {
         goto err;
@@ -743,38 +753,16 @@ out:
 }
 
 /*
- * Allocates new clusters for the given guest_offset.
- *
- * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
- * contain the number of clusters that have been allocated and are contiguous
- * in the image file.
- *
- * If *host_offset is non-zero, it specifies the offset in the image file at
- * which the new clusters must start. *nb_clusters can be 0 on return in this
- * case if the cluster at host_offset is already in use. If *host_offset is
- * zero, the clusters can be allocated anywhere in the image file.
- *
- * *host_offset is updated to contain the offset into the image file at which
- * the first allocated cluster starts.
- *
- * Return 0 on success and -errno in error cases. -EAGAIN means that the
- * function has been waiting for another request and the allocation must be
- * restarted, but the whole request should not be failed.
+ * Check if there already is an AIO write request in flight which allocates
+ * the same cluster. In this case we need to wait until the previous
+ * request has completed and updated the L2 table accordingly.
  */
-static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, unsigned int *nb_clusters)
+static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
+    unsigned int *nb_clusters)
 {
     BDRVQcowState *s = bs->opaque;
     QCowL2Meta *old_alloc;
 
-    trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
-                                         *host_offset, *nb_clusters);
-
-    /*
-     * Check if there already is an AIO write request in flight which allocates
-     * the same cluster. In this case we need to wait until the previous
-     * request has completed and updated the L2 table accordingly.
-     */
     QLIST_FOREACH(old_alloc, &s->cluster_allocs, next_in_flight) {
 
         uint64_t start = guest_offset >> s->cluster_bits;
@@ -807,6 +795,42 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
         abort();
     }
 
+    return 0;
+}
+
+/*
+ * Allocates new clusters for the given guest_offset.
+ *
+ * At most *nb_clusters are allocated, and on return *nb_clusters is updated to
+ * contain the number of clusters that have been allocated and are contiguous
+ * in the image file.
+ *
+ * If *host_offset is non-zero, it specifies the offset in the image file at
+ * which the new clusters must start. *nb_clusters can be 0 on return in this
+ * case if the cluster at host_offset is already in use. If *host_offset is
+ * zero, the clusters can be allocated anywhere in the image file.
+ *
+ * *host_offset is updated to contain the offset into the image file at which
+ * the first allocated cluster starts.
+ *
+ * Return 0 on success and -errno in error cases. -EAGAIN means that the
+ * function has been waiting for another request and the allocation must be
+ * restarted, but the whole request should not be failed.
+ */
+static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
+    uint64_t *host_offset, unsigned int *nb_clusters)
+{
+    BDRVQcowState *s = bs->opaque;
+    int ret;
+
+    trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
+                                         *host_offset, *nb_clusters);
+
+    ret = handle_dependencies(bs, guest_offset, nb_clusters);
+    if (ret < 0) {
+        return ret;
+    }
+
     /* Allocate new clusters */
     trace_qcow2_cluster_alloc_phys(qemu_coroutine_self());
     if (*host_offset == 0) {
@@ -818,7 +842,7 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
         *host_offset = cluster_offset;
         return 0;
     } else {
-        int ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
+        ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
         if (ret < 0) {
             return ret;
         }
@@ -847,7 +871,7 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
  * Return 0 on success and -errno in error cases
  */
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int n_start, int n_end, int *num, QCowL2Meta *m)
+    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m)
 {
     BDRVQcowState *s = bs->opaque;
     int l2_index, ret, sectors;
@@ -919,12 +943,6 @@ again:
     }
 
     /* If there is something left to allocate, do that now */
-    *m = (QCowL2Meta) {
-        .cluster_offset     = cluster_offset,
-        .nb_clusters        = 0,
-    };
-    qemu_co_queue_init(&m->dependent_requests);
-
     if (nb_clusters > 0) {
         uint64_t alloc_offset;
         uint64_t alloc_cluster_offset;
@@ -957,22 +975,40 @@ again:
              *
              * avail_sectors: Number of sectors from the start of the first
              * newly allocated to the end of the last newly allocated cluster.
+             *
+             * nb_sectors: The number of sectors from the start of the first
+             * newly allocated cluster to the end of the aread that the write
+             * request actually writes to (excluding COW at the end)
              */
             int requested_sectors = n_end - keep_clusters * s->cluster_sectors;
             int avail_sectors = nb_clusters
                                 << (s->cluster_bits - BDRV_SECTOR_BITS);
+            int alloc_n_start = keep_clusters == 0 ? n_start : 0;
+            int nb_sectors = MIN(requested_sectors, avail_sectors);
+
+            if (keep_clusters == 0) {
+                cluster_offset = alloc_cluster_offset;
+            }
+
+            *m = g_malloc0(sizeof(**m));
 
-            *m = (QCowL2Meta) {
-                .cluster_offset = keep_clusters == 0 ?
-                                  alloc_cluster_offset : cluster_offset,
+            **m = (QCowL2Meta) {
                 .alloc_offset   = alloc_cluster_offset,
-                .offset         = alloc_offset,
-                .n_start        = keep_clusters == 0 ? n_start : 0,
+                .offset         = alloc_offset & ~(s->cluster_size - 1),
                 .nb_clusters    = nb_clusters,
-                .nb_available   = MIN(requested_sectors, avail_sectors),
+                .nb_available   = nb_sectors,
+
+                .cow_start = {
+                    .offset     = 0,
+                    .nb_sectors = alloc_n_start,
+                },
+                .cow_end = {
+                    .offset     = nb_sectors * BDRV_SECTOR_SIZE,
+                    .nb_sectors = avail_sectors - nb_sectors,
+                },
             };
-            qemu_co_queue_init(&m->dependent_requests);
-            QLIST_INSERT_HEAD(&s->cluster_allocs, m, next_in_flight);
+            qemu_co_queue_init(&(*m)->dependent_requests);
+            QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
         }
     }
 
@@ -984,12 +1020,13 @@ again:
 
     assert(sectors > n_start);
     *num = sectors - n_start;
+    *host_offset = cluster_offset;
 
     return 0;
 
 fail:
-    if (m->nb_clusters > 0) {
-        QLIST_REMOVE(m, next_in_flight);
+    if (*m && (*m)->nb_clusters > 0) {
+        QLIST_REMOVE(*m, next_in_flight);
     }
     return ret;
 }
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 96224d1..bc1784c 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "block/qcow2.h"
 
 static int64_t alloc_clusters_noref(BlockDriverState *bs, int64_t size);
@@ -737,11 +737,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
      * l1_table_offset when it is the current s->l1_table_offset! Be careful
      * when changing this! */
     if (l1_table_offset != s->l1_table_offset) {
-        if (l1_size2 != 0) {
-            l1_table = g_malloc0(align_offset(l1_size2, 512));
-        } else {
-            l1_table = NULL;
-        }
+        l1_table = g_malloc0(align_offset(l1_size2, 512));
         l1_allocated = 1;
         if (bdrv_pread(bs->file, l1_table_offset,
                        l1_table, l1_size2) != l1_size2)
diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index 4e7c93b..eb8fcd5 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
+#include "block/block_int.h"
 #include "block/qcow2.h"
 
 typedef struct QEMU_PACKED QCowSnapshotHeader {
diff --git a/block/qcow2.c b/block/qcow2.c
index c1ff31f..7610e56 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -22,13 +22,13 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include <zlib.h>
-#include "aes.h"
+#include "block/aes.h"
 #include "block/qcow2.h"
-#include "qemu-error.h"
-#include "qerror.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/qerror.h"
 #include "trace.h"
 
 /*
@@ -222,7 +222,7 @@ static void report_unsupported_feature(BlockDriverState *bs,
  * updated successfully.  Therefore it is not required to check the return
  * value of this function.
  */
-static int qcow2_mark_dirty(BlockDriverState *bs)
+int qcow2_mark_dirty(BlockDriverState *bs)
 {
     BDRVQcowState *s = bs->opaque;
     uint64_t val;
@@ -311,7 +311,7 @@ static int qcow2_open(BlockDriverState *bs, int flags)
     be32_to_cpus(&header.nb_snapshots);
 
     if (header.magic != QCOW_MAGIC) {
-        ret = -EINVAL;
+        ret = -EMEDIUMTYPE;
         goto fail;
     }
     if (header.version < 2 || header.version > 3) {
@@ -745,21 +745,6 @@ fail:
     return ret;
 }
 
-static void run_dependent_requests(BDRVQcowState *s, QCowL2Meta *m)
-{
-    /* Take the request off the list of running requests */
-    if (m->nb_clusters != 0) {
-        QLIST_REMOVE(m, next_in_flight);
-    }
-
-    /* Restart all dependent requests */
-    if (!qemu_co_queue_empty(&m->dependent_requests)) {
-        qemu_co_mutex_unlock(&s->lock);
-        qemu_co_queue_restart_all(&m->dependent_requests);
-        qemu_co_mutex_lock(&s->lock);
-    }
-}
-
 static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                            int64_t sector_num,
                            int remaining_sectors,
@@ -774,15 +759,11 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
     QEMUIOVector hd_qiov;
     uint64_t bytes_done = 0;
     uint8_t *cluster_data = NULL;
-    QCowL2Meta l2meta = {
-        .nb_clusters = 0,
-    };
+    QCowL2Meta *l2meta = NULL;
 
     trace_qcow2_writev_start_req(qemu_coroutine_self(), sector_num,
                                  remaining_sectors);
 
-    qemu_co_queue_init(&l2meta.dependent_requests);
-
     qemu_iovec_init(&hd_qiov, qiov->niov);
 
     s->cluster_cache_offset = -1; /* disable compressed cache */
@@ -791,6 +772,8 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
 
     while (remaining_sectors != 0) {
 
+        l2meta = NULL;
+
         trace_qcow2_writev_start_part(qemu_coroutine_self());
         index_in_cluster = sector_num & (s->cluster_sectors - 1);
         n_end = index_in_cluster + remaining_sectors;
@@ -800,17 +783,11 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
         }
 
         ret = qcow2_alloc_cluster_offset(bs, sector_num << 9,
-            index_in_cluster, n_end, &cur_nr_sectors, &l2meta);
+            index_in_cluster, n_end, &cur_nr_sectors, &cluster_offset, &l2meta);
         if (ret < 0) {
             goto fail;
         }
 
-        if (l2meta.nb_clusters > 0 &&
-            (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS)) {
-            qcow2_mark_dirty(bs);
-        }
-
-        cluster_offset = l2meta.cluster_offset;
         assert((cluster_offset & 511) == 0);
 
         qemu_iovec_reset(&hd_qiov);
@@ -835,8 +812,8 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                 cur_nr_sectors * 512);
         }
 
-        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         qemu_co_mutex_unlock(&s->lock);
+        BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         trace_qcow2_writev_data(qemu_coroutine_self(),
                                 (cluster_offset >> 9) + index_in_cluster);
         ret = bdrv_co_writev(bs->file,
@@ -847,12 +824,24 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
             goto fail;
         }
 
-        ret = qcow2_alloc_cluster_link_l2(bs, &l2meta);
-        if (ret < 0) {
-            goto fail;
-        }
+        if (l2meta != NULL) {
+            ret = qcow2_alloc_cluster_link_l2(bs, l2meta);
+            if (ret < 0) {
+                goto fail;
+            }
+
+            /* Take the request off the list of running requests */
+            if (l2meta->nb_clusters != 0) {
+                QLIST_REMOVE(l2meta, next_in_flight);
+            }
+
+            qemu_co_mutex_unlock(&s->lock);
+            qemu_co_queue_restart_all(&l2meta->dependent_requests);
+            qemu_co_mutex_lock(&s->lock);
 
-        run_dependent_requests(s, &l2meta);
+            g_free(l2meta);
+            l2meta = NULL;
+        }
 
         remaining_sectors -= cur_nr_sectors;
         sector_num += cur_nr_sectors;
@@ -862,10 +851,16 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
     ret = 0;
 
 fail:
-    run_dependent_requests(s, &l2meta);
-
     qemu_co_mutex_unlock(&s->lock);
 
+    if (l2meta != NULL) {
+        if (l2meta->nb_clusters != 0) {
+            QLIST_REMOVE(l2meta, next_in_flight);
+        }
+        qemu_co_queue_restart_all(&l2meta->dependent_requests);
+        g_free(l2meta);
+    }
+
     qemu_iovec_destroy(&hd_qiov);
     qemu_vfree(cluster_data);
     trace_qcow2_writev_done_req(qemu_coroutine_self(), ret);
@@ -1128,31 +1123,33 @@ static int preallocate(BlockDriverState *bs)
 {
     uint64_t nb_sectors;
     uint64_t offset;
+    uint64_t host_offset = 0;
     int num;
     int ret;
-    QCowL2Meta meta;
+    QCowL2Meta *meta;
 
     nb_sectors = bdrv_getlength(bs) >> 9;
     offset = 0;
-    qemu_co_queue_init(&meta.dependent_requests);
-    meta.cluster_offset = 0;
 
     while (nb_sectors) {
         num = MIN(nb_sectors, INT_MAX >> 9);
-        ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num, &meta);
+        ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,
+                                         &host_offset, &meta);
         if (ret < 0) {
             return ret;
         }
 
-        ret = qcow2_alloc_cluster_link_l2(bs, &meta);
+        ret = qcow2_alloc_cluster_link_l2(bs, meta);
         if (ret < 0) {
-            qcow2_free_any_clusters(bs, meta.cluster_offset, meta.nb_clusters);
+            qcow2_free_any_clusters(bs, meta->alloc_offset, meta->nb_clusters);
             return ret;
         }
 
         /* There are no dependent requests, but we need to remove our request
          * from the list of in-flight requests */
-        run_dependent_requests(bs->opaque, &meta);
+        if (meta != NULL) {
+            QLIST_REMOVE(meta, next_in_flight);
+        }
 
         /* TODO Preallocate data if requested */
 
@@ -1165,10 +1162,10 @@ static int preallocate(BlockDriverState *bs)
      * all of the allocated clusters (otherwise we get failing reads after
      * EOF). Extend the image to the last allocated sector.
      */
-    if (meta.cluster_offset != 0) {
+    if (host_offset != 0) {
         uint8_t buf[512];
         memset(buf, 0, 512);
-        ret = bdrv_write(bs->file, (meta.cluster_offset >> 9) + num - 1, buf, 1);
+        ret = bdrv_write(bs->file, (host_offset >> 9) + num - 1, buf, 1);
         if (ret < 0) {
             return ret;
         }
diff --git a/block/qcow2.h b/block/qcow2.h
index b4eb654..718b52b 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -25,8 +25,8 @@
 #ifndef BLOCK_QCOW2_H
 #define BLOCK_QCOW2_H
 
-#include "aes.h"
-#include "qemu-coroutine.h"
+#include "block/aes.h"
+#include "block/coroutine.h"
 
 //#define DEBUG_ALLOC
 //#define DEBUG_ALLOC2
@@ -196,17 +196,56 @@ typedef struct QCowCreateState {
 
 struct QCowAIOCB;
 
-/* XXX This could be private for qcow2-cluster.c */
+typedef struct Qcow2COWRegion {
+    /**
+     * Offset of the COW region in bytes from the start of the first cluster
+     * touched by the request.
+     */
+    uint64_t    offset;
+
+    /** Number of sectors to copy */
+    int         nb_sectors;
+} Qcow2COWRegion;
+
+/**
+ * Describes an in-flight (part of a) write request that writes to clusters
+ * that are not referenced in their L2 table yet.
+ */
 typedef struct QCowL2Meta
 {
+    /** Guest offset of the first newly allocated cluster */
     uint64_t offset;
-    uint64_t cluster_offset;
+
+    /** Host offset of the first newly allocated cluster */
     uint64_t alloc_offset;
-    int n_start;
+
+    /**
+     * Number of sectors from the start of the first allocated cluster to
+     * the end of the (possibly shortened) request
+     */
     int nb_available;
+
+    /** Number of newly allocated clusters */
     int nb_clusters;
+
+    /**
+     * Requests that overlap with this allocation and wait to be restarted
+     * when the allocating request has completed.
+     */
     CoQueue dependent_requests;
 
+    /**
+     * The COW Region between the start of the first allocated cluster and the
+     * area the guest actually writes to.
+     */
+    Qcow2COWRegion cow_start;
+
+    /**
+     * The COW Region between the area the guest actually writes to and the
+     * end of the last allocated cluster.
+     */
+    Qcow2COWRegion cow_end;
+
     QLIST_ENTRY(QCowL2Meta) next_in_flight;
 } QCowL2Meta;
 
@@ -264,6 +303,8 @@ static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
 /* qcow2.c functions */
 int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
                   int64_t sector_num, int nb_sectors);
+
+int qcow2_mark_dirty(BlockDriverState *bs);
 int qcow2_update_header(BlockDriverState *bs);
 
 /* qcow2-refcount.c functions */
@@ -297,7 +338,7 @@ void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
 int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
     int *num, uint64_t *cluster_offset);
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int n_start, int n_end, int *num, QCowL2Meta *m);
+    int n_start, int n_end, int *num, uint64_t *host_offset, QCowL2Meta **m);
 uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
                                          uint64_t offset,
                                          int compressed_size);
diff --git a/block/qed-table.c b/block/qed-table.c
index de845ec..76d2dcc 100644
--- a/block/qed-table.c
+++ b/block/qed-table.c
@@ -13,7 +13,7 @@
  */
 
 #include "trace.h"
-#include "qemu_socket.h" /* for EINPROGRESS on Windows */
+#include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "qed.h"
 
 typedef struct {
diff --git a/block/qed.c b/block/qed.c
index 0b5374a..b8515e5 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -12,11 +12,11 @@
  *
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "trace.h"
 #include "qed.h"
-#include "qerror.h"
-#include "migration.h"
+#include "qapi/qmp/qerror.h"
+#include "migration/migration.h"
 
 static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
 {
@@ -390,7 +390,7 @@ static int bdrv_qed_open(BlockDriverState *bs, int flags)
     qed_header_le_to_cpu(&le_header, &s->header);
 
     if (s->header.magic != QED_MAGIC) {
-        return -EINVAL;
+        return -EMEDIUMTYPE;
     }
     if (s->header.features & ~QED_FEATURE_MASK) {
         /* image uses unsupported feature bits */
diff --git a/block/qed.h b/block/qed.h
index a063bf7..2b4dded 100644
--- a/block/qed.h
+++ b/block/qed.h
@@ -15,7 +15,7 @@
 #ifndef BLOCK_QED_H
 #define BLOCK_QED_H
 
-#include "block_int.h"
+#include "block/block_int.h"
 
 /* The layout of a QED file is as follows:
  *
diff --git a/block/raw-aio.h b/block/raw-aio.h
index e77f361..c61f159 100644
--- a/block/raw-aio.h
+++ b/block/raw-aio.h
@@ -20,11 +20,14 @@
 #define QEMU_AIO_WRITE        0x0002
 #define QEMU_AIO_IOCTL        0x0004
 #define QEMU_AIO_FLUSH        0x0008
+#define QEMU_AIO_DISCARD      0x0010
 #define QEMU_AIO_TYPE_MASK \
-	(QEMU_AIO_READ|QEMU_AIO_WRITE|QEMU_AIO_IOCTL|QEMU_AIO_FLUSH)
+        (QEMU_AIO_READ|QEMU_AIO_WRITE|QEMU_AIO_IOCTL|QEMU_AIO_FLUSH| \
+         QEMU_AIO_DISCARD)
 
 /* AIO flags */
 #define QEMU_AIO_MISALIGNED   0x1000
+#define QEMU_AIO_BLKDEV       0x2000
 
 
 /* linux-aio.c - Linux native implementation */
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 550c81f..4dfdf98 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -22,14 +22,13 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
-#include "block_int.h"
-#include "module.h"
+#include "qemu/timer.h"
+#include "qemu/log.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include "trace.h"
-#include "thread-pool.h"
-#include "iov.h"
+#include "block/thread-pool.h"
+#include "qemu/iov.h"
 #include "raw-aio.h"
 
 #if defined(__APPLE__) && (__MACH__)
@@ -60,6 +59,9 @@
 #ifdef CONFIG_FIEMAP
 #include <linux/fiemap.h>
 #endif
+#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
+#include <linux/falloc.h>
+#endif
 #if defined (__FreeBSD__) || defined(__FreeBSD_kernel__)
 #include <sys/disk.h>
 #include <sys/cdio.h>
@@ -139,6 +141,7 @@ typedef struct BDRVRawState {
 #ifdef CONFIG_XFS
     bool is_xfs : 1;
 #endif
+    bool has_discard : 1;
 } BDRVRawState;
 
 typedef struct BDRVRawReopenState {
@@ -160,7 +163,7 @@ typedef struct RawPosixAIOData {
         void *aio_ioctl_buf;
     };
     int aio_niov;
-    size_t aio_nbytes;
+    uint64_t aio_nbytes;
 #define aio_ioctl_cmd   aio_nbytes /* for QEMU_AIO_IOCTL */
     off_t aio_offset;
     int aio_type;
@@ -290,6 +293,7 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
     }
 #endif
 
+    s->has_discard = 1;
 #ifdef CONFIG_XFS
     if (platform_test_xfs_fd(s->fd)) {
         s->is_xfs = 1;
@@ -341,11 +345,20 @@ static int raw_reopen_prepare(BDRVReopenState *state,
 
     raw_s->fd = -1;
 
-    int fcntl_flags = O_APPEND | O_ASYNC | O_NONBLOCK;
+    int fcntl_flags = O_APPEND | O_NONBLOCK;
 #ifdef O_NOATIME
     fcntl_flags |= O_NOATIME;
 #endif
 
+#ifdef O_ASYNC
+    /* Not all operating systems have O_ASYNC, and those that don't
+     * will not let us track the state into raw_s->open_flags (typically
+     * you achieve the same effect with an ioctl, for example I_SETSIG
+     * on Solaris). But we do not use O_ASYNC, so that's fine.
+     */
+    assert((s->open_flags & O_ASYNC) == 0);
+#endif
+
     if ((raw_s->open_flags & ~fcntl_flags) == (s->open_flags & ~fcntl_flags)) {
         /* dup the original fd */
         /* TODO: use qemu fcntl wrapper */
@@ -431,22 +444,6 @@ static void raw_reopen_abort(BDRVReopenState *state)
 #endif
 */
 
-/*
- * Check if all memory in this vector is sector aligned.
- */
-static int qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
-{
-    int i;
-
-    for (i = 0; i < qiov->niov; i++) {
-        if ((uintptr_t) qiov->iov[i].iov_base % bs->buffer_alignment) {
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
 static ssize_t handle_aiocb_ioctl(RawPosixAIOData *aiocb)
 {
     int ret;
@@ -456,15 +453,7 @@ static ssize_t handle_aiocb_ioctl(RawPosixAIOData *aiocb)
         return -errno;
     }
 
-    /*
-     * This looks weird, but the aio code only considers a request
-     * successful if it has written the full number of bytes.
-     *
-     * Now we overload aio_nbytes as aio_ioctl_cmd for the ioctl command,
-     * so in fact we return the ioctl command here to make posix_aio_read()
-     * happy..
-     */
-    return aiocb->aio_nbytes;
+    return 0;
 }
 
 static ssize_t handle_aiocb_flush(RawPosixAIOData *aiocb)
@@ -643,6 +632,72 @@ static ssize_t handle_aiocb_rw(RawPosixAIOData *aiocb)
     return nbytes;
 }
 
+#ifdef CONFIG_XFS
+static int xfs_discard(BDRVRawState *s, int64_t offset, uint64_t bytes)
+{
+    struct xfs_flock64 fl;
+
+    memset(&fl, 0, sizeof(fl));
+    fl.l_whence = SEEK_SET;
+    fl.l_start = offset;
+    fl.l_len = bytes;
+
+    if (xfsctl(NULL, s->fd, XFS_IOC_UNRESVSP64, &fl) < 0) {
+        DEBUG_BLOCK_PRINT("cannot punch hole (%s)\n", strerror(errno));
+        return -errno;
+    }
+
+    return 0;
+}
+#endif
+
+static ssize_t handle_aiocb_discard(RawPosixAIOData *aiocb)
+{
+    int ret = -EOPNOTSUPP;
+    BDRVRawState *s = aiocb->bs->opaque;
+
+    if (s->has_discard == 0) {
+        return 0;
+    }
+
+    if (aiocb->aio_type & QEMU_AIO_BLKDEV) {
+#ifdef BLKDISCARD
+        do {
+            uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };
+            if (ioctl(aiocb->aio_fildes, BLKDISCARD, range) == 0) {
+                return 0;
+            }
+        } while (errno == EINTR);
+
+        ret = -errno;
+#endif
+    } else {
+#ifdef CONFIG_XFS
+        if (s->is_xfs) {
+            return xfs_discard(s, aiocb->aio_offset, aiocb->aio_nbytes);
+        }
+#endif
+
+#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
+        do {
+            if (fallocate(s->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                          aiocb->aio_offset, aiocb->aio_nbytes) == 0) {
+                return 0;
+            }
+        } while (errno == EINTR);
+
+        ret = -errno;
+#endif
+    }
+
+    if (ret == -ENODEV || ret == -ENOSYS || ret == -EOPNOTSUPP ||
+        ret == -ENOTTY) {
+        s->has_discard = 0;
+        ret = 0;
+    }
+    return ret;
+}
+
 static int aio_worker(void *arg)
 {
     RawPosixAIOData *aiocb = arg;
@@ -677,6 +732,9 @@ static int aio_worker(void *arg)
     case QEMU_AIO_IOCTL:
         ret = handle_aiocb_ioctl(aiocb);
         break;
+    case QEMU_AIO_DISCARD:
+        ret = handle_aiocb_discard(aiocb);
+        break;
     default:
         fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
         ret = -EINVAL;
@@ -708,22 +766,6 @@ static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
     return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
 }
 
-static BlockDriverAIOCB *paio_ioctl(BlockDriverState *bs, int fd,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
-
-    acb->bs = bs;
-    acb->aio_type = QEMU_AIO_IOCTL;
-    acb->aio_fildes = fd;
-    acb->aio_offset = 0;
-    acb->aio_ioctl_buf = buf;
-    acb->aio_ioctl_cmd = req;
-
-    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
-}
-
 static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque, int type)
@@ -739,7 +781,7 @@ static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
      * driver that it needs to copy the buffer.
      */
     if ((bs->open_flags & BDRV_O_NOCACHE)) {
-        if (!qiov_is_aligned(bs, qiov)) {
+        if (!bdrv_qiov_is_aligned(bs, qiov)) {
             type |= QEMU_AIO_MISALIGNED;
 #ifdef CONFIG_LINUX_AIO
         } else if (s->use_aio) {
@@ -1093,37 +1135,14 @@ static int coroutine_fn raw_co_is_allocated(BlockDriverState *bs,
     }
 }
 
-#ifdef CONFIG_XFS
-static int xfs_discard(BDRVRawState *s, int64_t sector_num, int nb_sectors)
-{
-    struct xfs_flock64 fl;
-
-    memset(&fl, 0, sizeof(fl));
-    fl.l_whence = SEEK_SET;
-    fl.l_start = sector_num << 9;
-    fl.l_len = (int64_t)nb_sectors << 9;
-
-    if (xfsctl(NULL, s->fd, XFS_IOC_UNRESVSP64, &fl) < 0) {
-        DEBUG_BLOCK_PRINT("cannot punch hole (%s)\n", strerror(errno));
-        return -errno;
-    }
-
-    return 0;
-}
-#endif
-
-static coroutine_fn int raw_co_discard(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors)
+static coroutine_fn BlockDriverAIOCB *raw_aio_discard(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors,
+    BlockDriverCompletionFunc *cb, void *opaque)
 {
-#ifdef CONFIG_XFS
     BDRVRawState *s = bs->opaque;
 
-    if (s->is_xfs) {
-        return xfs_discard(s, sector_num, nb_sectors);
-    }
-#endif
-
-    return 0;
+    return paio_submit(bs, s->fd, sector_num, NULL, nb_sectors,
+                       cb, opaque, QEMU_AIO_DISCARD);
 }
 
 static QEMUOptionParameter raw_create_options[] = {
@@ -1146,12 +1165,12 @@ static BlockDriver bdrv_file = {
     .bdrv_reopen_abort = raw_reopen_abort,
     .bdrv_close = raw_close,
     .bdrv_create = raw_create,
-    .bdrv_co_discard = raw_co_discard,
     .bdrv_co_is_allocated = raw_co_is_allocated,
 
     .bdrv_aio_readv = raw_aio_readv,
     .bdrv_aio_writev = raw_aio_writev,
     .bdrv_aio_flush = raw_aio_flush,
+    .bdrv_aio_discard = raw_aio_discard,
 
     .bdrv_truncate = raw_truncate,
     .bdrv_getlength = raw_getlength,
@@ -1238,9 +1257,43 @@ static int hdev_probe_device(const char *filename)
     return 0;
 }
 
+static int check_hdev_writable(BDRVRawState *s)
+{
+#if defined(BLKROGET)
+    /* Linux block devices can be configured "read-only" using blockdev(8).
+     * This is independent of device node permissions and therefore open(2)
+     * with O_RDWR succeeds.  Actual writes fail with EPERM.
+     *
+     * bdrv_open() is supposed to fail if the disk is read-only.  Explicitly
+     * check for read-only block devices so that Linux block devices behave
+     * properly.
+     */
+    struct stat st;
+    int readonly = 0;
+
+    if (fstat(s->fd, &st)) {
+        return -errno;
+    }
+
+    if (!S_ISBLK(st.st_mode)) {
+        return 0;
+    }
+
+    if (ioctl(s->fd, BLKROGET, &readonly) < 0) {
+        return -errno;
+    }
+
+    if (readonly) {
+        return -EACCES;
+    }
+#endif /* defined(BLKROGET) */
+    return 0;
+}
+
 static int hdev_open(BlockDriverState *bs, const char *filename, int flags)
 {
     BDRVRawState *s = bs->opaque;
+    int ret;
 
 #if defined(__APPLE__) && defined(__MACH__)
     if (strstart(filename, "/dev/cdrom", NULL)) {
@@ -1281,7 +1334,20 @@ static int hdev_open(BlockDriverState *bs, const char *filename, int flags)
     }
 #endif
 
-    return raw_open_common(bs, filename, flags, 0);
+    ret = raw_open_common(bs, filename, flags, 0);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (flags & BDRV_O_RDWR) {
+        ret = check_hdev_writable(s);
+        if (ret < 0) {
+            raw_close(bs);
+            return ret;
+        }
+    }
+
+    return ret;
 }
 
 #if defined(__linux__)
@@ -1346,10 +1412,19 @@ static BlockDriverAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
         BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
+    RawPosixAIOData *acb;
 
     if (fd_open(bs) < 0)
         return NULL;
-    return paio_ioctl(bs, s->fd, req, buf, cb, opaque);
+
+    acb = g_slice_new(RawPosixAIOData);
+    acb->bs = bs;
+    acb->aio_type = QEMU_AIO_IOCTL;
+    acb->aio_fildes = s->fd;
+    acb->aio_offset = 0;
+    acb->aio_ioctl_buf = buf;
+    acb->aio_ioctl_cmd = req;
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
 }
 
 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
@@ -1371,6 +1446,19 @@ static int fd_open(BlockDriverState *bs)
 
 #endif /* !linux && !FreeBSD */
 
+static coroutine_fn BlockDriverAIOCB *hdev_aio_discard(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors,
+    BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BDRVRawState *s = bs->opaque;
+
+    if (fd_open(bs) < 0) {
+        return NULL;
+    }
+    return paio_submit(bs, s->fd, sector_num, NULL, nb_sectors,
+                       cb, opaque, QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV);
+}
+
 static int hdev_create(const char *filename, QEMUOptionParameter *options)
 {
     int fd;
@@ -1423,6 +1511,7 @@ static BlockDriver bdrv_host_device = {
     .bdrv_aio_readv	= raw_aio_readv,
     .bdrv_aio_writev	= raw_aio_writev,
     .bdrv_aio_flush	= raw_aio_flush,
+    .bdrv_aio_discard   = hdev_aio_discard,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength	= raw_getlength,
@@ -1784,6 +1873,40 @@ static BlockDriver bdrv_host_cdrom = {
 };
 #endif /* __FreeBSD__ */
 
+#ifdef CONFIG_LINUX_AIO
+/**
+ * Return the file descriptor for Linux AIO
+ *
+ * This function is a layering violation and should be removed when it becomes
+ * possible to call the block layer outside the global mutex.  It allows the
+ * caller to hijack the file descriptor so I/O can be performed outside the
+ * block layer.
+ */
+int raw_get_aio_fd(BlockDriverState *bs)
+{
+    BDRVRawState *s;
+
+    if (!bs->drv) {
+        return -ENOMEDIUM;
+    }
+
+    if (bs->drv == bdrv_find_format("raw")) {
+        bs = bs->file;
+    }
+
+    /* raw-posix has several protocols so just check for raw_aio_readv */
+    if (bs->drv->bdrv_aio_readv != raw_aio_readv) {
+        return -ENOTSUP;
+    }
+
+    s = bs->opaque;
+    if (!s->use_aio) {
+        return -ENOTSUP;
+    }
+    return s->fd;
+}
+#endif /* CONFIG_LINUX_AIO */
+
 static void bdrv_file_init(void)
 {
     /*
diff --git a/block/raw-win32.c b/block/raw-win32.c
index 0c05c58..b89ac19 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -22,13 +22,13 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "block_int.h"
-#include "module.h"
+#include "qemu/timer.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include "raw-aio.h"
 #include "trace.h"
-#include "thread-pool.h"
-#include "iov.h"
+#include "block/thread-pool.h"
+#include "qemu/iov.h"
 #include <windows.h>
 #include <winioctl.h>
 
@@ -303,13 +303,24 @@ static int raw_truncate(BlockDriverState *bs, int64_t offset)
 {
     BDRVRawState *s = bs->opaque;
     LONG low, high;
+    DWORD dwPtrLow;
 
     low = offset;
     high = offset >> 32;
-    if (!SetFilePointer(s->hfile, low, &high, FILE_BEGIN))
-	return -EIO;
-    if (!SetEndOfFile(s->hfile))
+
+    /*
+     * An error has occurred if the return value is INVALID_SET_FILE_POINTER
+     * and GetLastError doesn't return NO_ERROR.
+     */
+    dwPtrLow = SetFilePointer(s->hfile, low, &high, FILE_BEGIN);
+    if (dwPtrLow == INVALID_SET_FILE_POINTER && GetLastError() != NO_ERROR) {
+        fprintf(stderr, "SetFilePointer error: %lu\n", GetLastError());
+        return -EIO;
+    }
+    if (SetEndOfFile(s->hfile) == 0) {
+        fprintf(stderr, "SetEndOfFile error: %lu\n", GetLastError());
         return -EIO;
+    }
     return 0;
 }
 
diff --git a/block/raw.c b/block/raw.c
index 253e949..75812db 100644
--- a/block/raw.c
+++ b/block/raw.c
@@ -1,7 +1,7 @@
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 
 static int raw_open(BlockDriverState *bs, int flags)
 {
diff --git a/block/rbd.c b/block/rbd.c
index f3becc7..8cd10a7 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -14,8 +14,8 @@
 #include <inttypes.h>
 
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "block_int.h"
+#include "qemu/error-report.h"
+#include "block/block_int.h"
 
 #include <rbd/librbd.h>
 
@@ -77,6 +77,7 @@ typedef struct RBDAIOCB {
     int error;
     struct BDRVRBDState *s;
     int cancelled;
+    int status;
 } RBDAIOCB;
 
 typedef struct RADOSCB {
@@ -376,12 +377,6 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
     RBDAIOCB *acb = rcb->acb;
     int64_t r;
 
-    if (acb->cancelled) {
-        qemu_vfree(acb->bounce);
-        qemu_aio_release(acb);
-        goto done;
-    }
-
     r = rcb->ret;
 
     if (acb->cmd == RBD_AIO_WRITE ||
@@ -409,7 +404,6 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
     /* Note that acb->bh can be NULL in case where the aio was cancelled */
     acb->bh = qemu_bh_new(rbd_aio_bh_cb, acb);
     qemu_bh_schedule(acb->bh);
-done:
     g_free(rcb);
 }
 
@@ -568,6 +562,12 @@ static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb)
 {
     RBDAIOCB *acb = (RBDAIOCB *) blockacb;
     acb->cancelled = 1;
+
+    while (acb->status == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+
+    qemu_aio_release(acb);
 }
 
 static const AIOCBInfo rbd_aiocb_info = {
@@ -639,8 +639,11 @@ static void rbd_aio_bh_cb(void *opaque)
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
     qemu_bh_delete(acb->bh);
     acb->bh = NULL;
+    acb->status = 0;
 
-    qemu_aio_release(acb);
+    if (!acb->cancelled) {
+        qemu_aio_release(acb);
+    }
 }
 
 static int rbd_aio_discard_wrapper(rbd_image_t image,
@@ -685,6 +688,7 @@ static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->s = s;
     acb->cancelled = 0;
     acb->bh = NULL;
+    acb->status = -EINPROGRESS;
 
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
diff --git a/block/sheepdog.c b/block/sheepdog.c
index a48f58c..d466b23 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -13,10 +13,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu_socket.h"
-#include "block_int.h"
-#include "bitops.h"
+#include "qemu/error-report.h"
+#include "qemu/sockets.h"
+#include "block/block_int.h"
+#include "qemu/bitops.h"
 
 #define SD_PROTO_VER 0x01
 
@@ -36,7 +36,8 @@
 
 #define SD_FLAG_CMD_WRITE    0x01
 #define SD_FLAG_CMD_COW      0x02
-#define SD_FLAG_CMD_CACHE    0x04
+#define SD_FLAG_CMD_CACHE    0x04 /* Writeback mode for cache */
+#define SD_FLAG_CMD_DIRECT   0x08 /* Don't use cache */
 
 #define SD_RES_SUCCESS       0x00 /* Success */
 #define SD_RES_UNKNOWN       0x01 /* Unknown error */
@@ -144,7 +145,7 @@ typedef struct SheepdogVdiReq {
     uint32_t id;
     uint32_t data_length;
     uint64_t vdi_size;
-    uint32_t base_vdi_id;
+    uint32_t vdi_id;
     uint32_t copies;
     uint32_t snapid;
     uint32_t pad[3];
@@ -265,6 +266,7 @@ typedef struct AIOReq {
 enum AIOCBState {
     AIOCB_WRITE_UDATA,
     AIOCB_READ_UDATA,
+    AIOCB_FLUSH_CACHE,
 };
 
 struct SheepdogAIOCB {
@@ -293,12 +295,11 @@ typedef struct BDRVSheepdogState {
 
     char name[SD_MAX_VDI_LEN];
     bool is_snapshot;
-    bool cache_enabled;
+    uint32_t cache_flags;
 
     char *addr;
     char *port;
     int fd;
-    int flush_fd;
 
     CoMutex lock;
     Coroutine *co_send;
@@ -426,12 +427,11 @@ static const AIOCBInfo sd_aiocb_info = {
 };
 
 static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
-                                   int64_t sector_num, int nb_sectors,
-                                   BlockDriverCompletionFunc *cb, void *opaque)
+                                   int64_t sector_num, int nb_sectors)
 {
     SheepdogAIOCB *acb;
 
-    acb = qemu_aio_get(&sd_aiocb_info, bs, cb, opaque);
+    acb = qemu_aio_get(&sd_aiocb_info, bs, NULL, NULL);
 
     acb->qiov = qiov;
 
@@ -714,16 +714,17 @@ static void coroutine_fn aio_read_response(void *opaque)
              * and max_dirty_data_idx are changed to include updated
              * index between them.
              */
-            s->inode.data_vdi_id[idx] = s->inode.vdi_id;
-            s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
-            s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
-
+            if (rsp.result == SD_RES_SUCCESS) {
+                s->inode.data_vdi_id[idx] = s->inode.vdi_id;
+                s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
+                s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
+            }
             /*
              * Some requests may be blocked because simultaneous
              * create requests are not allowed, so we search the
              * pending requests here.
              */
-            send_pending_req(s, vid_to_data_oid(s->inode.vdi_id, idx));
+            send_pending_req(s, aio_req->oid);
         }
         break;
     case AIOCB_READ_UDATA:
@@ -734,6 +735,13 @@ static void coroutine_fn aio_read_response(void *opaque)
             goto out;
         }
         break;
+    case AIOCB_FLUSH_CACHE:
+        if (rsp.result == SD_RES_INVALID_PARMS) {
+            dprintf("disable cache since the server doesn't support it\n");
+            s->cache_flags = SD_FLAG_CMD_DIRECT;
+            rsp.result = SD_RES_SUCCESS;
+        }
+        break;
     }
 
     if (rsp.result != SD_RES_SUCCESS) {
@@ -948,7 +956,7 @@ static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 {
     int nr_copies = s->inode.nr_copies;
     SheepdogObjReq hdr;
-    unsigned int wlen;
+    unsigned int wlen = 0;
     int ret;
     uint64_t oid = aio_req->oid;
     unsigned int datalen = aio_req->data_len;
@@ -962,22 +970,27 @@ static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
     memset(&hdr, 0, sizeof(hdr));
 
-    if (aiocb_type == AIOCB_READ_UDATA) {
-        wlen = 0;
+    switch (aiocb_type) {
+    case AIOCB_FLUSH_CACHE:
+        hdr.opcode = SD_OP_FLUSH_VDI;
+        break;
+    case AIOCB_READ_UDATA:
         hdr.opcode = SD_OP_READ_OBJ;
         hdr.flags = flags;
-    } else if (create) {
-        wlen = datalen;
-        hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
-        hdr.flags = SD_FLAG_CMD_WRITE | flags;
-    } else {
+        break;
+    case AIOCB_WRITE_UDATA:
+        if (create) {
+            hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
+        } else {
+            hdr.opcode = SD_OP_WRITE_OBJ;
+        }
         wlen = datalen;
-        hdr.opcode = SD_OP_WRITE_OBJ;
         hdr.flags = SD_FLAG_CMD_WRITE | flags;
+        break;
     }
 
-    if (s->cache_enabled) {
-        hdr.flags |= SD_FLAG_CMD_CACHE;
+    if (s->cache_flags) {
+        hdr.flags |= s->cache_flags;
     }
 
     hdr.oid = oid;
@@ -1022,7 +1035,7 @@ static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
 static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
                              unsigned int datalen, uint64_t offset,
-                             bool write, bool create, bool cache)
+                             bool write, bool create, uint32_t cache_flags)
 {
     SheepdogObjReq hdr;
     SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
@@ -1046,9 +1059,7 @@ static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
         hdr.opcode = SD_OP_READ_OBJ;
     }
 
-    if (cache) {
-        hdr.flags |= SD_FLAG_CMD_CACHE;
-    }
+    hdr.flags |= cache_flags;
 
     hdr.oid = oid;
     hdr.data_length = datalen;
@@ -1071,18 +1082,19 @@ static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
 }
 
 static int read_object(int fd, char *buf, uint64_t oid, int copies,
-                       unsigned int datalen, uint64_t offset, bool cache)
+                       unsigned int datalen, uint64_t offset,
+                       uint32_t cache_flags)
 {
     return read_write_object(fd, buf, oid, copies, datalen, offset, false,
-                             false, cache);
+                             false, cache_flags);
 }
 
 static int write_object(int fd, char *buf, uint64_t oid, int copies,
                         unsigned int datalen, uint64_t offset, bool create,
-                        bool cache)
+                        uint32_t cache_flags)
 {
     return read_write_object(fd, buf, oid, copies, datalen, offset, true,
-                             create, cache);
+                             create, cache_flags);
 }
 
 static int sd_open(BlockDriverState *bs, const char *filename, int flags)
@@ -1117,12 +1129,13 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
         goto out;
     }
 
-    s->cache_enabled = true;
-    s->flush_fd = connect_to_sdog(s->addr, s->port);
-    if (s->flush_fd < 0) {
-        error_report("failed to connect");
-        ret = s->flush_fd;
-        goto out;
+    /*
+     * QEMU block layer emulates writethrough cache as 'writeback + flush', so
+     * we always set SD_FLAG_CMD_CACHE (writeback cache) as default.
+     */
+    s->cache_flags = SD_FLAG_CMD_CACHE;
+    if (flags & BDRV_O_NOCACHE) {
+        s->cache_flags = SD_FLAG_CMD_DIRECT;
     }
 
     if (snapid || tag[0] != '\0') {
@@ -1139,7 +1152,7 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
 
     buf = g_malloc(SD_INODE_SIZE);
     ret = read_object(fd, buf, vid_to_vdi_oid(vid), 0, SD_INODE_SIZE, 0,
-                      s->cache_enabled);
+                      s->cache_flags);
 
     closesocket(fd);
 
@@ -1188,7 +1201,7 @@ static int do_sd_create(char *filename, int64_t vdi_size,
 
     memset(&hdr, 0, sizeof(hdr));
     hdr.opcode = SD_OP_NEW_VDI;
-    hdr.base_vdi_id = base_vid;
+    hdr.vdi_id = base_vid;
 
     wlen = SD_MAX_VDI_LEN;
 
@@ -1371,6 +1384,7 @@ static void sd_close(BlockDriverState *bs)
     memset(&hdr, 0, sizeof(hdr));
 
     hdr.opcode = SD_OP_RELEASE_VDI;
+    hdr.vdi_id = s->inode.vdi_id;
     wlen = strlen(s->name) + 1;
     hdr.data_length = wlen;
     hdr.flags = SD_FLAG_CMD_WRITE;
@@ -1386,9 +1400,6 @@ static void sd_close(BlockDriverState *bs)
 
     qemu_aio_set_fd_handler(s->fd, NULL, NULL, NULL, NULL);
     closesocket(s->fd);
-    if (s->cache_enabled) {
-        closesocket(s->flush_fd);
-    }
     g_free(s->addr);
 }
 
@@ -1422,7 +1433,7 @@ static int sd_truncate(BlockDriverState *bs, int64_t offset)
     datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
     s->inode.vdi_size = offset;
     ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
-                       s->inode.nr_copies, datalen, 0, false, s->cache_enabled);
+                       s->inode.nr_copies, datalen, 0, false, s->cache_flags);
     close(fd);
 
     if (ret < 0) {
@@ -1505,7 +1516,7 @@ static int sd_create_branch(BDRVSheepdogState *s)
     }
 
     ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
-                      SD_INODE_SIZE, 0, s->cache_enabled);
+                      SD_INODE_SIZE, 0, s->cache_flags);
 
     closesocket(fd);
 
@@ -1661,7 +1672,7 @@ static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
         bs->total_sectors = sector_num + nb_sectors;
     }
 
-    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
+    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors);
     acb->aio_done_func = sd_write_done;
     acb->aiocb_type = AIOCB_WRITE_UDATA;
 
@@ -1682,7 +1693,7 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
     SheepdogAIOCB *acb;
     int ret;
 
-    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
+    acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors);
     acb->aiocb_type = AIOCB_READ_UDATA;
     acb->aio_done_func = sd_finish_aiocb;
 
@@ -1700,39 +1711,31 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
 static int coroutine_fn sd_co_flush_to_disk(BlockDriverState *bs)
 {
     BDRVSheepdogState *s = bs->opaque;
-    SheepdogObjReq hdr = { 0 };
-    SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
-    SheepdogInode *inode = &s->inode;
+    SheepdogAIOCB *acb;
+    AIOReq *aio_req;
     int ret;
-    unsigned int wlen = 0, rlen = 0;
 
-    if (!s->cache_enabled) {
+    if (s->cache_flags != SD_FLAG_CMD_CACHE) {
         return 0;
     }
 
-    hdr.opcode = SD_OP_FLUSH_VDI;
-    hdr.oid = vid_to_vdi_oid(inode->vdi_id);
+    acb = sd_aio_setup(bs, NULL, 0, 0);
+    acb->aiocb_type = AIOCB_FLUSH_CACHE;
+    acb->aio_done_func = sd_finish_aiocb;
 
-    ret = do_req(s->flush_fd, (SheepdogReq *)&hdr, NULL, &wlen, &rlen);
-    if (ret) {
-        error_report("failed to send a request to the sheep");
+    aio_req = alloc_aio_req(s, acb, vid_to_vdi_oid(s->inode.vdi_id),
+                            0, 0, 0, 0, 0);
+    QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
+    ret = add_aio_request(s, aio_req, NULL, 0, false, acb->aiocb_type);
+    if (ret < 0) {
+        error_report("add_aio_request is failed");
+        free_aio_req(s, aio_req);
+        qemu_aio_release(acb);
         return ret;
     }
 
-    if (rsp->result == SD_RES_INVALID_PARMS) {
-        dprintf("disable write cache since the server doesn't support it\n");
-
-        s->cache_enabled = false;
-        closesocket(s->flush_fd);
-        return 0;
-    }
-
-    if (rsp->result != SD_RES_SUCCESS) {
-        error_report("%s", sd_strerror(rsp->result));
-        return -EIO;
-    }
-
-    return 0;
+    qemu_coroutine_yield();
+    return acb->ret;
 }
 
 static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
@@ -1773,7 +1776,7 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     }
 
     ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
-                       s->inode.nr_copies, datalen, 0, false, s->cache_enabled);
+                       s->inode.nr_copies, datalen, 0, false, s->cache_flags);
     if (ret < 0) {
         error_report("failed to write snapshot's inode.");
         goto cleanup;
@@ -1790,7 +1793,7 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     inode = (SheepdogInode *)g_malloc(datalen);
 
     ret = read_object(fd, (char *)inode, vid_to_vdi_oid(new_vid),
-                      s->inode.nr_copies, datalen, 0, s->cache_enabled);
+                      s->inode.nr_copies, datalen, 0, s->cache_flags);
 
     if (ret < 0) {
         error_report("failed to read new inode info. %s", strerror(errno));
@@ -1844,7 +1847,7 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 
     buf = g_malloc(SD_INODE_SIZE);
     ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
-                      SD_INODE_SIZE, 0, s->cache_enabled);
+                      SD_INODE_SIZE, 0, s->cache_flags);
 
     closesocket(fd);
 
@@ -1941,7 +1944,7 @@ static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
         /* we don't need to read entire object */
         ret = read_object(fd, (char *)&inode, vid_to_vdi_oid(vid),
                           0, SD_INODE_SIZE - sizeof(inode.data_vdi_id), 0,
-                          s->cache_enabled);
+                          s->cache_flags);
 
         if (ret) {
             continue;
@@ -2002,11 +2005,11 @@ static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
         if (load) {
             ret = read_object(fd, (char *)data, vmstate_oid,
                               s->inode.nr_copies, data_len, offset,
-                              s->cache_enabled);
+                              s->cache_flags);
         } else {
             ret = write_object(fd, (char *)data, vmstate_oid,
                                s->inode.nr_copies, data_len, offset, create,
-                               s->cache_enabled);
+                               s->cache_flags);
         }
 
         if (ret < 0) {
diff --git a/block/stream.c b/block/stream.c
index 0c0fc7a..d6df06f 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -12,8 +12,8 @@
  */
 
 #include "trace.h"
-#include "block_int.h"
-#include "blockjob.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
 #include "qemu/ratelimit.h"
 
 enum {
@@ -108,7 +108,7 @@ static void coroutine_fn stream_run(void *opaque)
 
 wait:
         /* Note that even when no rate limit is applied we need to yield
-         * with no pending I/O here so that qemu_aio_flush() returns.
+         * with no pending I/O here so that bdrv_drain_all() returns.
          */
         block_job_sleep_ns(&s->common, rt_clock, delay_ns);
         if (block_job_is_cancelled(&s->common)) {
diff --git a/block/vdi.c b/block/vdi.c
index c8330b7..87c691b 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -50,15 +50,15 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
 
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
 #else
 /* TODO: move uuid emulation to some central place in QEMU. */
-#include "sysemu.h"     /* UUID_FMT */
+#include "sysemu/sysemu.h"     /* UUID_FMT */
 typedef unsigned char uuid_t[16];
 #endif
 
@@ -246,7 +246,7 @@ static void vdi_header_print(VdiHeader *header)
 {
     char uuid[37];
     logout("text        %s", header->text);
-    logout("signature   0x%04x\n", header->signature);
+    logout("signature   0x%08x\n", header->signature);
     logout("header size 0x%04x\n", header->header_size);
     logout("image type  0x%04x\n", header->image_type);
     logout("image flags 0x%04x\n", header->image_flags);
@@ -369,10 +369,12 @@ static int vdi_open(BlockDriverState *bs, int flags)
     BDRVVdiState *s = bs->opaque;
     VdiHeader header;
     size_t bmap_size;
+    int ret;
 
     logout("\n");
 
-    if (bdrv_read(bs->file, 0, (uint8_t *)&header, 1) < 0) {
+    ret = bdrv_read(bs->file, 0, (uint8_t *)&header, 1);
+    if (ret < 0) {
         goto fail;
     }
 
@@ -390,33 +392,45 @@ static int vdi_open(BlockDriverState *bs, int flags)
         header.disk_size &= ~(SECTOR_SIZE - 1);
     }
 
-    if (header.version != VDI_VERSION_1_1) {
+    if (header.signature != VDI_SIGNATURE) {
+        logout("bad vdi signature %08x\n", header.signature);
+        ret = -EMEDIUMTYPE;
+        goto fail;
+    } else if (header.version != VDI_VERSION_1_1) {
         logout("unsupported version %u.%u\n",
                header.version >> 16, header.version & 0xffff);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.offset_bmap % SECTOR_SIZE != 0) {
         /* We only support block maps which start on a sector boundary. */
         logout("unsupported block map offset 0x%x B\n", header.offset_bmap);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.offset_data % SECTOR_SIZE != 0) {
         /* We only support data blocks which start on a sector boundary. */
         logout("unsupported data offset 0x%x B\n", header.offset_data);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.sector_size != SECTOR_SIZE) {
         logout("unsupported sector size %u B\n", header.sector_size);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.block_size != 1 * MiB) {
         logout("unsupported block size %u B\n", header.block_size);
+        ret = -ENOTSUP;
         goto fail;
     } else if (header.disk_size >
                (uint64_t)header.blocks_in_image * header.block_size) {
         logout("unsupported disk size %" PRIu64 " B\n", header.disk_size);
+        ret = -ENOTSUP;
         goto fail;
     } else if (!uuid_is_null(header.uuid_link)) {
         logout("link uuid != 0, unsupported\n");
+        ret = -ENOTSUP;
         goto fail;
     } else if (!uuid_is_null(header.uuid_parent)) {
         logout("parent uuid != 0, unsupported\n");
+        ret = -ENOTSUP;
         goto fail;
     }
 
@@ -429,10 +443,9 @@ static int vdi_open(BlockDriverState *bs, int flags)
 
     bmap_size = header.blocks_in_image * sizeof(uint32_t);
     bmap_size = (bmap_size + SECTOR_SIZE - 1) / SECTOR_SIZE;
-    if (bmap_size > 0) {
-        s->bmap = g_malloc(bmap_size * SECTOR_SIZE);
-    }
-    if (bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size) < 0) {
+    s->bmap = g_malloc(bmap_size * SECTOR_SIZE);
+    ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size);
+    if (ret < 0) {
         goto fail_free_bmap;
     }
 
@@ -448,7 +461,7 @@ static int vdi_open(BlockDriverState *bs, int flags)
     g_free(s->bmap);
 
  fail:
-    return -1;
+    return ret;
 }
 
 static int vdi_reopen_prepare(BDRVReopenState *state,
diff --git a/block/vmdk.c b/block/vmdk.c
index 51398c0..aef1abc 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -24,9 +24,9 @@
  */
 
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
 #include <zlib.h>
 
 #define VMDK3_MAGIC (('C' << 24) | ('O' << 16) | ('W' << 8) | 'D')
@@ -616,7 +616,7 @@ static int vmdk_open_sparse(BlockDriverState *bs,
             return vmdk_open_vmdk4(bs, file, flags);
             break;
         default:
-            return -EINVAL;
+            return -EMEDIUMTYPE;
             break;
     }
 }
@@ -641,7 +641,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
          * RW [size in sectors] SPARSE "file-name.vmdk"
          */
         flat_offset = -1;
-        ret = sscanf(p, "%10s %" SCNd64 " %10s %511s %" SCNd64,
+        ret = sscanf(p, "%10s %" SCNd64 " %10s \"%511[^\n\r\"]\" %" SCNd64,
                 access, &sectors, type, fname, &flat_offset);
         if (ret < 4 || strcmp(access, "RW")) {
             goto next_line;
@@ -653,14 +653,6 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             return -EINVAL;
         }
 
-        /* trim the quotation marks around */
-        if (fname[0] == '"') {
-            memmove(fname, fname + 1, strlen(fname));
-            if (strlen(fname) <= 1 || fname[strlen(fname) - 1] != '"') {
-                return -EINVAL;
-            }
-            fname[strlen(fname) - 1] = '\0';
-        }
         if (sectors <= 0 ||
             (strcmp(type, "FLAT") && strcmp(type, "SPARSE")) ||
             (strcmp(access, "RW"))) {
@@ -718,7 +710,7 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags,
     }
     buf[2047] = '\0';
     if (vmdk_parse_description(buf, "createType", ct, sizeof(ct))) {
-        return -EINVAL;
+        return -EMEDIUMTYPE;
     }
     if (strcmp(ct, "monolithicFlat") &&
         strcmp(ct, "twoGbMaxExtentSparse") &&
@@ -1442,6 +1434,7 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
     int fd, idx = 0;
     char desc[BUF_SIZE];
     int64_t total_size = 0, filesize;
+    const char *adapter_type = NULL;
     const char *backing_file = NULL;
     const char *fmt = NULL;
     int flags = 0;
@@ -1453,6 +1446,7 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
     const char *desc_extent_line;
     char parent_desc_line[BUF_SIZE] = "";
     uint32_t parent_cid = 0xffffffff;
+    uint32_t number_heads = 16;
     const char desc_template[] =
         "# Disk DescriptorFile\n"
         "version=1\n"
@@ -1469,9 +1463,9 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
         "\n"
         "ddb.virtualHWVersion = \"%d\"\n"
         "ddb.geometry.cylinders = \"%" PRId64 "\"\n"
-        "ddb.geometry.heads = \"16\"\n"
+        "ddb.geometry.heads = \"%d\"\n"
         "ddb.geometry.sectors = \"63\"\n"
-        "ddb.adapterType = \"ide\"\n";
+        "ddb.adapterType = \"%s\"\n";
 
     if (filename_decompose(filename, path, prefix, postfix, PATH_MAX)) {
         return -EINVAL;
@@ -1480,6 +1474,8 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
     while (options && options->name) {
         if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
             total_size = options->value.n;
+        } else if (!strcmp(options->name, BLOCK_OPT_ADAPTER_TYPE)) {
+            adapter_type = options->value.s;
         } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
             backing_file = options->value.s;
         } else if (!strcmp(options->name, BLOCK_OPT_COMPAT6)) {
@@ -1489,6 +1485,20 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
         }
         options++;
     }
+    if (!adapter_type) {
+        adapter_type = "ide";
+    } else if (strcmp(adapter_type, "ide") &&
+               strcmp(adapter_type, "buslogic") &&
+               strcmp(adapter_type, "lsilogic") &&
+               strcmp(adapter_type, "legacyESX")) {
+        fprintf(stderr, "VMDK: Unknown adapter type: '%s'.\n", adapter_type);
+        return -EINVAL;
+    }
+    if (strcmp(adapter_type, "ide") != 0) {
+        /* that's the number of heads with which vmware operates when
+           creating, exporting, etc. vmdk files with a non-ide adapter type */
+        number_heads = 255;
+    }
     if (!fmt) {
         /* Default format to monolithicSparse */
         fmt = "monolithicSparse";
@@ -1576,7 +1586,8 @@ static int vmdk_create(const char *filename, QEMUOptionParameter *options)
             parent_desc_line,
             ext_desc_lines,
             (flags & BLOCK_FLAG_COMPAT6 ? 6 : 4),
-            total_size / (int64_t)(63 * 16 * 512));
+            total_size / (int64_t)(63 * number_heads * 512), number_heads,
+                adapter_type);
     if (split || flat) {
         fd = qemu_open(filename,
                        O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_LARGEFILE,
@@ -1661,6 +1672,12 @@ static QEMUOptionParameter vmdk_create_options[] = {
         .help = "Virtual disk size"
     },
     {
+        .name = BLOCK_OPT_ADAPTER_TYPE,
+        .type = OPT_STRING,
+        .help = "Virtual adapter type, can be one of "
+                "ide (default), lsilogic, buslogic or legacyESX"
+    },
+    {
         .name = BLOCK_OPT_BACKING_FILE,
         .type = OPT_STRING,
         .help = "File name of a base image"
diff --git a/block/vpc.c b/block/vpc.c
index b6bf52f..82229ef 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -23,9 +23,12 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
+#if defined(CONFIG_UUID)
+#include <uuid/uuid.h>
+#endif
 
 /**************************************************************/
 
@@ -160,24 +163,33 @@ static int vpc_open(BlockDriverState *bs, int flags)
     struct vhd_dyndisk_header* dyndisk_header;
     uint8_t buf[HEADER_SIZE];
     uint32_t checksum;
-    int err = -1;
     int disk_type = VHD_DYNAMIC;
+    int ret;
 
-    if (bdrv_pread(bs->file, 0, s->footer_buf, HEADER_SIZE) != HEADER_SIZE)
+    ret = bdrv_pread(bs->file, 0, s->footer_buf, HEADER_SIZE);
+    if (ret < 0) {
         goto fail;
+    }
 
     footer = (struct vhd_footer*) s->footer_buf;
     if (strncmp(footer->creator, "conectix", 8)) {
         int64_t offset = bdrv_getlength(bs->file);
-        if (offset < HEADER_SIZE) {
+        if (offset < 0) {
+            ret = offset;
+            goto fail;
+        } else if (offset < HEADER_SIZE) {
+            ret = -EINVAL;
             goto fail;
         }
+
         /* If a fixed disk, the footer is found only at the end of the file */
-        if (bdrv_pread(bs->file, offset-HEADER_SIZE, s->footer_buf, HEADER_SIZE)
-                != HEADER_SIZE) {
+        ret = bdrv_pread(bs->file, offset-HEADER_SIZE, s->footer_buf,
+                         HEADER_SIZE);
+        if (ret < 0) {
             goto fail;
         }
         if (strncmp(footer->creator, "conectix", 8)) {
+            ret = -EMEDIUMTYPE;
             goto fail;
         }
         disk_type = VHD_FIXED;
@@ -198,20 +210,23 @@ static int vpc_open(BlockDriverState *bs, int flags)
     bs->total_sectors = (int64_t)
         be16_to_cpu(footer->cyls) * footer->heads * footer->secs_per_cyl;
 
-    if (bs->total_sectors >= 65535 * 16 * 255) {
-        err = -EFBIG;
+    /* Allow a maximum disk size of approximately 2 TB */
+    if (bs->total_sectors >= 65535LL * 255 * 255) {
+        ret = -EFBIG;
         goto fail;
     }
 
     if (disk_type == VHD_DYNAMIC) {
-        if (bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf,
-                HEADER_SIZE) != HEADER_SIZE) {
+        ret = bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf,
+                         HEADER_SIZE);
+        if (ret < 0) {
             goto fail;
         }
 
         dyndisk_header = (struct vhd_dyndisk_header *) buf;
 
         if (strncmp(dyndisk_header->magic, "cxsparse", 8)) {
+            ret = -EINVAL;
             goto fail;
         }
 
@@ -222,8 +237,10 @@ static int vpc_open(BlockDriverState *bs, int flags)
         s->pagetable = g_malloc(s->max_table_entries * 4);
 
         s->bat_offset = be64_to_cpu(dyndisk_header->table_offset);
-        if (bdrv_pread(bs->file, s->bat_offset, s->pagetable,
-                s->max_table_entries * 4) != s->max_table_entries * 4) {
+
+        ret = bdrv_pread(bs->file, s->bat_offset, s->pagetable,
+                         s->max_table_entries * 4);
+        if (ret < 0) {
             goto fail;
         }
 
@@ -261,8 +278,13 @@ static int vpc_open(BlockDriverState *bs, int flags)
     migrate_add_blocker(s->migration_blocker);
 
     return 0;
- fail:
-    return err;
+
+fail:
+    g_free(s->pagetable);
+#ifdef CACHE
+    g_free(s->pageentry_u8);
+#endif
+    return ret;
 }
 
 static int vpc_reopen_prepare(BDRVReopenState *state,
@@ -524,19 +546,27 @@ static coroutine_fn int vpc_co_write(BlockDriverState *bs, int64_t sector_num,
  * Note that the geometry doesn't always exactly match total_sectors but
  * may round it down.
  *
- * Returns 0 on success, -EFBIG if the size is larger than 127 GB
+ * Returns 0 on success, -EFBIG if the size is larger than ~2 TB. Override
+ * the hardware EIDE and ATA-2 limit of 16 heads (max disk size of 127 GB)
+ * and instead allow up to 255 heads.
  */
 static int calculate_geometry(int64_t total_sectors, uint16_t* cyls,
     uint8_t* heads, uint8_t* secs_per_cyl)
 {
     uint32_t cyls_times_heads;
 
-    if (total_sectors > 65535 * 16 * 255)
+    /* Allow a maximum disk size of approximately 2 TB */
+    if (total_sectors > 65535LL * 255 * 255) {
         return -EFBIG;
+    }
 
     if (total_sectors > 65535 * 16 * 63) {
         *secs_per_cyl = 255;
-        *heads = 16;
+        if (total_sectors > 65535 * 16 * 255) {
+            *heads = 255;
+        } else {
+            *heads = 16;
+        }
         cyls_times_heads = total_sectors / *secs_per_cyl;
     } else {
         *secs_per_cyl = 17;
@@ -739,7 +769,9 @@ static int vpc_create(const char *filename, QEMUOptionParameter *options)
 
     footer->type = be32_to_cpu(disk_type);
 
-    /* TODO uuid is missing */
+#if defined(CONFIG_UUID)
+    uuid_generate(footer->uuid);
+#endif
 
     footer->checksum = be32_to_cpu(vpc_checksum(buf, HEADER_SIZE));
 
diff --git a/block/vvfat.c b/block/vvfat.c
index 59d3c5b..06e6654 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -25,9 +25,9 @@
 #include <sys/stat.h>
 #include <dirent.h>
 #include "qemu-common.h"
-#include "block_int.h"
-#include "module.h"
-#include "migration.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
+#include "migration/migration.h"
 
 #ifndef S_IWGRP
 #define S_IWGRP 0
@@ -529,13 +529,9 @@ static inline uint8_t fat_chksum(const direntry_t* entry)
 /* if return_time==0, this returns the fat_date, else the fat_time */
 static uint16_t fat_datetime(time_t time,int return_time) {
     struct tm* t;
-#ifdef _WIN32
-    t=localtime(&time); /* this is not thread safe */
-#else
     struct tm t1;
     t = &t1;
     localtime_r(&time,t);
-#endif
     if(return_time)
 	return cpu_to_le16((t->tm_sec/2)|(t->tm_min<<5)|(t->tm_hour<<11));
     return cpu_to_le16((t->tm_mday)|((t->tm_mon+1)<<5)|((t->tm_year-80)<<9));
diff --git a/block/win32-aio.c b/block/win32-aio.c
index 4704ee0..5d0fbbf 100644
--- a/block/win32-aio.c
+++ b/block/win32-aio.c
@@ -22,13 +22,14 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "block_int.h"
-#include "module.h"
+#include "qemu/timer.h"
+#include "block/block_int.h"
+#include "qemu/module.h"
 #include "qemu-common.h"
-#include "qemu-aio.h"
+#include "block/aio.h"
 #include "raw-aio.h"
-#include "event_notifier.h"
+#include "qemu/event_notifier.h"
+#include "qemu/iov.h"
 #include <windows.h>
 #include <winioctl.h>
 
@@ -80,15 +81,9 @@ static void win32_aio_process_completion(QEMUWin32AIOState *s,
     if (!waiocb->is_linear) {
         if (ret == 0 && waiocb->is_read) {
             QEMUIOVector *qiov = waiocb->qiov;
-            char *p = waiocb->buf;
-            int i;
-
-            for (i = 0; i < qiov->niov; ++i) {
-                memcpy(p, qiov->iov[i].iov_base, qiov->iov[i].iov_len);
-                p += qiov->iov[i].iov_len;
-            }
-            g_free(waiocb->buf);
+            iov_from_buf(qiov->iov, qiov->niov, 0, waiocb->buf, qiov->size);
         }
+        qemu_vfree(waiocb->buf);
     }
 
 
@@ -153,13 +148,7 @@ BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
     if (qiov->niov > 1) {
         waiocb->buf = qemu_blockalign(bs, qiov->size);
         if (type & QEMU_AIO_WRITE) {
-            char *p = waiocb->buf;
-            int i;
-
-            for (i = 0; i < qiov->niov; ++i) {
-                memcpy(p, qiov->iov[i].iov_base, qiov->iov[i].iov_len);
-                p += qiov->iov[i].iov_len;
-            }
+            iov_to_buf(qiov->iov, qiov->niov, 0, waiocb->buf, qiov->size);
         }
         waiocb->is_linear = false;
     } else {
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
index 6b26bbf..dc4e9a2 100644
--- a/blockdev-nbd.c
+++ b/blockdev-nbd.c
@@ -9,15 +9,15 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "monitor.h"
-#include "qerror.h"
-#include "sysemu.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/sysemu.h"
 #include "qmp-commands.h"
 #include "trace.h"
-#include "nbd.h"
-#include "qemu_socket.h"
+#include "block/nbd.h"
+#include "qemu/sockets.h"
 
 static int server_fd = -1;
 
diff --git a/blockdev.c b/blockdev.c
index e73fd6e..63e6f1e 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -7,19 +7,19 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "blockjob.h"
-#include "monitor.h"
-#include "qerror.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-objects.h"
-#include "sysemu.h"
-#include "block_int.h"
+#include "block/blockjob.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "qapi/qmp/types.h"
+#include "sysemu/sysemu.h"
+#include "block/block_int.h"
 #include "qmp-commands.h"
 #include "trace.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
 
 static QTAILQ_HEAD(drivelist, DriveInfo) drives = QTAILQ_HEAD_INITIALIZER(drives);
 
@@ -275,7 +275,7 @@ static bool do_check_io_limits(BlockIOLimit *io_limits)
     return true;
 }
 
-DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
+DriveInfo *drive_init(QemuOpts *opts, BlockInterfaceType block_default_type)
 {
     const char *buf;
     const char *file = NULL;
@@ -325,7 +325,7 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
             return NULL;
 	}
     } else {
-        type = default_to_scsi ? IF_SCSI : IF_IDE;
+        type = block_default_type;
     }
 
     max_devs = if_max_devs[type];
@@ -568,7 +568,7 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
         break;
     case IF_VIRTIO:
         /* add virtio block device */
-        opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0, NULL);
+        opts = qemu_opts_create_nofail(qemu_find_opts("device"));
         if (arch_type == QEMU_ARCH_S390X) {
             qemu_opt_set(opts, "driver", "virtio-blk-s390");
         } else {
@@ -617,8 +617,13 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
 
     ret = bdrv_open(dinfo->bdrv, file, bdrv_flags, drv);
     if (ret < 0) {
-        error_report("could not open disk image %s: %s",
-                     file, strerror(-ret));
+        if (ret == -EMEDIUMTYPE) {
+            error_report("could not open disk image %s: not in %s format",
+                         file, drv->format_name);
+        } else {
+            error_report("could not open disk image %s: %s",
+                         file, strerror(-ret));
+        }
         goto err;
     }
 
@@ -642,21 +647,17 @@ void do_commit(Monitor *mon, const QDict *qdict)
 
     if (!strcmp(device, "all")) {
         ret = bdrv_commit_all();
-        if (ret == -EBUSY) {
-            qerror_report(QERR_DEVICE_IN_USE, device);
-            return;
-        }
     } else {
         bs = bdrv_find(device);
         if (!bs) {
-            qerror_report(QERR_DEVICE_NOT_FOUND, device);
+            monitor_printf(mon, "Device '%s' not found\n", device);
             return;
         }
         ret = bdrv_commit(bs);
-        if (ret == -EBUSY) {
-            qerror_report(QERR_DEVICE_IN_USE, device);
-            return;
-        }
+    }
+    if (ret < 0) {
+        monitor_printf(mon, "'commit' error for '%s': %s\n", device,
+                       strerror(-ret));
     }
 }
 
@@ -707,6 +708,7 @@ void qmp_transaction(BlockdevActionList *dev_list, Error **errp)
     int ret = 0;
     BlockdevActionList *dev_entry = dev_list;
     BlkTransactionStates *states, *next;
+    Error *local_err = NULL;
 
     QSIMPLEQ_HEAD(snap_bdrv_states, BlkTransactionStates) snap_bdrv_states;
     QSIMPLEQ_INIT(&snap_bdrv_states);
@@ -786,12 +788,12 @@ void qmp_transaction(BlockdevActionList *dev_list, Error **errp)
 
         /* create new image w/backing file */
         if (mode != NEW_IMAGE_MODE_EXISTING) {
-            ret = bdrv_img_create(new_image_file, format,
-                                  states->old_bs->filename,
-                                  states->old_bs->drv->format_name,
-                                  NULL, -1, flags);
-            if (ret) {
-                error_set(errp, QERR_OPEN_FILE_FAILED, new_image_file);
+            bdrv_img_create(new_image_file, format,
+                            states->old_bs->filename,
+                            states->old_bs->drv->format_name,
+                            NULL, -1, flags, &local_err);
+            if (error_is_set(&local_err)) {
+                error_propagate(errp, local_err);
                 goto delete_and_fail;
             }
         }
@@ -1187,16 +1189,19 @@ void qmp_block_commit(const char *device,
     drive_get_ref(drive_get_by_blockdev(bs));
 }
 
+#define DEFAULT_MIRROR_BUF_SIZE   (10 << 20)
+
 void qmp_drive_mirror(const char *device, const char *target,
                       bool has_format, const char *format,
                       enum MirrorSyncMode sync,
                       bool has_mode, enum NewImageMode mode,
                       bool has_speed, int64_t speed,
+                      bool has_granularity, uint32_t granularity,
+                      bool has_buf_size, int64_t buf_size,
                       bool has_on_source_error, BlockdevOnError on_source_error,
                       bool has_on_target_error, BlockdevOnError on_target_error,
                       Error **errp)
 {
-    BlockDriverInfo bdi;
     BlockDriverState *bs;
     BlockDriverState *source, *target_bs;
     BlockDriver *proto_drv;
@@ -1218,6 +1223,21 @@ void qmp_drive_mirror(const char *device, const char *target,
     if (!has_mode) {
         mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     }
+    if (!has_granularity) {
+        granularity = 0;
+    }
+    if (!has_buf_size) {
+        buf_size = DEFAULT_MIRROR_BUF_SIZE;
+    }
+
+    if (granularity != 0 && (granularity < 512 || granularity > 1048576 * 64)) {
+        error_set(errp, QERR_INVALID_PARAMETER, device);
+        return;
+    }
+    if (granularity & (granularity - 1)) {
+        error_set(errp, QERR_INVALID_PARAMETER, device);
+        return;
+    }
 
     bs = bdrv_find(device);
     if (!bs) {
@@ -1258,13 +1278,13 @@ void qmp_drive_mirror(const char *device, const char *target,
         return;
     }
 
+    bdrv_get_geometry(bs, &size);
+    size *= 512;
     if (sync == MIRROR_SYNC_MODE_FULL && mode != NEW_IMAGE_MODE_EXISTING) {
         /* create new image w/o backing file */
         assert(format && drv);
-        bdrv_get_geometry(bs, &size);
-        size *= 512;
-        ret = bdrv_img_create(target, format,
-                              NULL, NULL, NULL, size, flags);
+        bdrv_img_create(target, format,
+                        NULL, NULL, NULL, size, flags, &local_err);
     } else {
         switch (mode) {
         case NEW_IMAGE_MODE_EXISTING:
@@ -1272,21 +1292,24 @@ void qmp_drive_mirror(const char *device, const char *target,
             break;
         case NEW_IMAGE_MODE_ABSOLUTE_PATHS:
             /* create new image with backing file */
-            ret = bdrv_img_create(target, format,
-                                  source->filename,
-                                  source->drv->format_name,
-                                  NULL, -1, flags);
+            bdrv_img_create(target, format,
+                            source->filename,
+                            source->drv->format_name,
+                            NULL, size, flags, &local_err);
             break;
         default:
             abort();
         }
     }
 
-    if (ret) {
-        error_set(errp, QERR_OPEN_FILE_FAILED, target);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
         return;
     }
 
+    /* Mirroring takes care of copy-on-write using the source's backing
+     * file.
+     */
     target_bs = bdrv_new("");
     ret = bdrv_open(target_bs, target, flags | BDRV_O_NO_BACKING, drv);
 
@@ -1296,18 +1319,8 @@ void qmp_drive_mirror(const char *device, const char *target,
         return;
     }
 
-    /* We need a backing file if we will copy parts of a cluster.  */
-    if (bdrv_get_info(target_bs, &bdi) >= 0 && bdi.cluster_size != 0 &&
-        bdi.cluster_size >= BDRV_SECTORS_PER_DIRTY_CHUNK * 512) {
-        ret = bdrv_open_backing_file(target_bs);
-        if (ret < 0) {
-            bdrv_delete(target_bs);
-            error_set(errp, QERR_OPEN_FILE_FAILED, target);
-            return;
-        }
-    }
-
-    mirror_start(bs, target_bs, speed, sync, on_source_error, on_target_error,
+    mirror_start(bs, target_bs, speed, granularity, buf_size, sync,
+                 on_source_error, on_target_error,
                  block_job_cb, bs, &local_err);
     if (local_err != NULL) {
         bdrv_delete(target_bs);
@@ -1426,3 +1439,121 @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
     bdrv_iterate(do_qmp_query_block_jobs_one, &prev);
     return dummy.next;
 }
+
+QemuOptsList qemu_drive_opts = {
+    .name = "drive",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_drive_opts.head),
+    .desc = {
+        {
+            .name = "bus",
+            .type = QEMU_OPT_NUMBER,
+            .help = "bus number",
+        },{
+            .name = "unit",
+            .type = QEMU_OPT_NUMBER,
+            .help = "unit number (i.e. lun for scsi)",
+        },{
+            .name = "if",
+            .type = QEMU_OPT_STRING,
+            .help = "interface (ide, scsi, sd, mtd, floppy, pflash, virtio)",
+        },{
+            .name = "index",
+            .type = QEMU_OPT_NUMBER,
+            .help = "index number",
+        },{
+            .name = "cyls",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of cylinders (ide disk geometry)",
+        },{
+            .name = "heads",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of heads (ide disk geometry)",
+        },{
+            .name = "secs",
+            .type = QEMU_OPT_NUMBER,
+            .help = "number of sectors (ide disk geometry)",
+        },{
+            .name = "trans",
+            .type = QEMU_OPT_STRING,
+            .help = "chs translation (auto, lba. none)",
+        },{
+            .name = "media",
+            .type = QEMU_OPT_STRING,
+            .help = "media type (disk, cdrom)",
+        },{
+            .name = "snapshot",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable snapshot mode",
+        },{
+            .name = "file",
+            .type = QEMU_OPT_STRING,
+            .help = "disk image",
+        },{
+            .name = "cache",
+            .type = QEMU_OPT_STRING,
+            .help = "host cache usage (none, writeback, writethrough, "
+                    "directsync, unsafe)",
+        },{
+            .name = "aio",
+            .type = QEMU_OPT_STRING,
+            .help = "host AIO implementation (threads, native)",
+        },{
+            .name = "format",
+            .type = QEMU_OPT_STRING,
+            .help = "disk format (raw, qcow2, ...)",
+        },{
+            .name = "serial",
+            .type = QEMU_OPT_STRING,
+            .help = "disk serial number",
+        },{
+            .name = "rerror",
+            .type = QEMU_OPT_STRING,
+            .help = "read error action",
+        },{
+            .name = "werror",
+            .type = QEMU_OPT_STRING,
+            .help = "write error action",
+        },{
+            .name = "addr",
+            .type = QEMU_OPT_STRING,
+            .help = "pci address (virtio only)",
+        },{
+            .name = "readonly",
+            .type = QEMU_OPT_BOOL,
+            .help = "open drive file as read-only",
+        },{
+            .name = "iops",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit total I/O operations per second",
+        },{
+            .name = "iops_rd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit read operations per second",
+        },{
+            .name = "iops_wr",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit write operations per second",
+        },{
+            .name = "bps",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit total bytes per second",
+        },{
+            .name = "bps_rd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit read bytes per second",
+        },{
+            .name = "bps_wr",
+            .type = QEMU_OPT_NUMBER,
+            .help = "limit write bytes per second",
+        },{
+            .name = "copy-on-read",
+            .type = QEMU_OPT_BOOL,
+            .help = "copy read data from backing file into image file",
+        },{
+            .name = "boot",
+            .type = QEMU_OPT_BOOL,
+            .help = "(deprecated, ignored)",
+        },
+        { /* end of list */ }
+    },
+};
diff --git a/blockjob.c b/blockjob.c
index cda12c6..ca80df1 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -26,14 +26,14 @@
 #include "config-host.h"
 #include "qemu-common.h"
 #include "trace.h"
-#include "monitor.h"
-#include "block.h"
-#include "blockjob.h"
-#include "block_int.h"
-#include "qjson.h"
-#include "qemu-coroutine.h"
+#include "monitor/monitor.h"
+#include "block/block.h"
+#include "block/blockjob.h"
+#include "block/block_int.h"
+#include "qapi/qmp/qjson.h"
+#include "block/coroutine.h"
 #include "qmp-commands.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
                        int64_t speed, BlockDriverCompletionFunc *cb,
diff --git a/bsd-user/elfload.c b/bsd-user/elfload.c
index 55b2136..a6cd3ab 100644
--- a/bsd-user/elfload.c
+++ b/bsd-user/elfload.c
@@ -10,7 +10,7 @@
 #include <string.h>
 
 #include "qemu.h"
-#include "disas.h"
+#include "disas/disas.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
diff --git a/bsd-user/main.c b/bsd-user/main.c
index 095ae8e..ae24723 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -31,8 +31,8 @@
 /* For tb_lock */
 #include "cpu.h"
 #include "tcg.h"
-#include "qemu-timer.h"
-#include "envlist.h"
+#include "qemu/timer.h"
+#include "qemu/envlist.h"
 
 #define DEBUG_LOGFILE "/tmp/qemu.log"
 
@@ -917,7 +917,7 @@ int main(int argc, char **argv)
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-#if defined(TARGET_I386) || defined(TARGET_SPARC) || defined(TARGET_PPC)
+#if defined(TARGET_SPARC) || defined(TARGET_PPC)
     cpu_reset(ENV_GET_CPU(env));
 #endif
     thread_env = env;
diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
index 5d6cffc..aae8ea1 100644
--- a/bsd-user/mmap.c
+++ b/bsd-user/mmap.c
@@ -74,7 +74,7 @@ void mmap_unlock(void)
 }
 #endif
 
-void *qemu_vmalloc(size_t size)
+static void *bsd_vmalloc(size_t size)
 {
     void *p;
     mmap_lock();
@@ -98,7 +98,7 @@ void *g_malloc(size_t size)
 {
     char * p;
     size += 16;
-    p = qemu_vmalloc(size);
+    p = bsd_vmalloc(size);
     *(size_t *)p = size;
     return p + 16;
 }
diff --git a/bsd-user/qemu-types.h b/bsd-user/qemu-types.h
deleted file mode 100644
index 1adda9f..0000000
--- a/bsd-user/qemu-types.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#ifndef QEMU_TYPES_H
-#define QEMU_TYPES_H
-#include "cpu.h"
-
-#ifdef TARGET_ABI32
-typedef uint32_t abi_ulong;
-typedef int32_t abi_long;
-#define TARGET_ABI_FMT_lx "%08x"
-#define TARGET_ABI_FMT_ld "%d"
-#define TARGET_ABI_FMT_lu "%u"
-#define TARGET_ABI_BITS 32
-#else
-typedef target_ulong abi_ulong;
-typedef target_long abi_long;
-#define TARGET_ABI_FMT_lx TARGET_FMT_lx
-#define TARGET_ABI_FMT_ld TARGET_FMT_ld
-#define TARGET_ABI_FMT_lu TARGET_FMT_lu
-#define TARGET_ABI_BITS TARGET_LONG_BITS
-/* for consistency, define ABI32 too */
-#if TARGET_ABI_BITS == 32
-#define TARGET_ABI32 1
-#endif
-#endif
-#endif
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index 8a5ee3d..a826086 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -11,7 +11,7 @@
 #include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
-#include "qemu-types.h"
+#include "exec/user/abitypes.h"
 
 enum BSDType {
     target_freebsd,
@@ -23,7 +23,7 @@ extern enum BSDType bsd_type;
 #include "syscall_defs.h"
 #include "syscall.h"
 #include "target_signal.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 
 #if defined(CONFIG_USE_NPTL)
 #define THREAD __thread
@@ -146,7 +146,7 @@ int get_osversion(void);
 void fork_start(void);
 void fork_end(int child);
 
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 /* strace.c */
 void
diff --git a/bswap.h b/bswap.h
deleted file mode 100644
index cc7f84d..0000000
--- a/bswap.h
+++ /dev/null
@@ -1,713 +0,0 @@
-#ifndef BSWAP_H
-#define BSWAP_H
-
-#include "config-host.h"
-
-#include <inttypes.h>
-#include "softfloat.h"
-
-#ifdef CONFIG_MACHINE_BSWAP_H
-#include <sys/endian.h>
-#include <sys/types.h>
-#include <machine/bswap.h>
-#else
-
-#ifdef CONFIG_BYTESWAP_H
-#include <byteswap.h>
-#else
-
-#define bswap_16(x) \
-({ \
-	uint16_t __x = (x); \
-	((uint16_t)( \
-		(((uint16_t)(__x) & (uint16_t)0x00ffU) << 8) | \
-		(((uint16_t)(__x) & (uint16_t)0xff00U) >> 8) )); \
-})
-
-#define bswap_32(x) \
-({ \
-	uint32_t __x = (x); \
-	((uint32_t)( \
-		(((uint32_t)(__x) & (uint32_t)0x000000ffUL) << 24) | \
-		(((uint32_t)(__x) & (uint32_t)0x0000ff00UL) <<  8) | \
-		(((uint32_t)(__x) & (uint32_t)0x00ff0000UL) >>  8) | \
-		(((uint32_t)(__x) & (uint32_t)0xff000000UL) >> 24) )); \
-})
-
-#define bswap_64(x) \
-({ \
-	uint64_t __x = (x); \
-	((uint64_t)( \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x00000000000000ffULL) << 56) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x000000000000ff00ULL) << 40) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x00000000ff000000ULL) <<  8) | \
-	        (uint64_t)(((uint64_t)(__x) & (uint64_t)0x000000ff00000000ULL) >>  8) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \
-		(uint64_t)(((uint64_t)(__x) & (uint64_t)0xff00000000000000ULL) >> 56) )); \
-})
-
-#endif /* !CONFIG_BYTESWAP_H */
-
-static inline uint16_t bswap16(uint16_t x)
-{
-    return bswap_16(x);
-}
-
-static inline uint32_t bswap32(uint32_t x)
-{
-    return bswap_32(x);
-}
-
-static inline uint64_t bswap64(uint64_t x)
-{
-    return bswap_64(x);
-}
-
-#endif /* ! CONFIG_MACHINE_BSWAP_H */
-
-static inline void bswap16s(uint16_t *s)
-{
-    *s = bswap16(*s);
-}
-
-static inline void bswap32s(uint32_t *s)
-{
-    *s = bswap32(*s);
-}
-
-static inline void bswap64s(uint64_t *s)
-{
-    *s = bswap64(*s);
-}
-
-#if defined(HOST_WORDS_BIGENDIAN)
-#define be_bswap(v, size) (v)
-#define le_bswap(v, size) bswap ## size(v)
-#define be_bswaps(v, size)
-#define le_bswaps(p, size) *p = bswap ## size(*p);
-#else
-#define le_bswap(v, size) (v)
-#define be_bswap(v, size) bswap ## size(v)
-#define le_bswaps(v, size)
-#define be_bswaps(p, size) *p = bswap ## size(*p);
-#endif
-
-#define CPU_CONVERT(endian, size, type)\
-static inline type endian ## size ## _to_cpu(type v)\
-{\
-    return endian ## _bswap(v, size);\
-}\
-\
-static inline type cpu_to_ ## endian ## size(type v)\
-{\
-    return endian ## _bswap(v, size);\
-}\
-\
-static inline void endian ## size ## _to_cpus(type *p)\
-{\
-    endian ## _bswaps(p, size)\
-}\
-\
-static inline void cpu_to_ ## endian ## size ## s(type *p)\
-{\
-    endian ## _bswaps(p, size)\
-}\
-\
-static inline type endian ## size ## _to_cpup(const type *p)\
-{\
-    return endian ## size ## _to_cpu(*p);\
-}\
-\
-static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\
-{\
-     *p = cpu_to_ ## endian ## size(v);\
-}
-
-CPU_CONVERT(be, 16, uint16_t)
-CPU_CONVERT(be, 32, uint32_t)
-CPU_CONVERT(be, 64, uint64_t)
-
-CPU_CONVERT(le, 16, uint16_t)
-CPU_CONVERT(le, 32, uint32_t)
-CPU_CONVERT(le, 64, uint64_t)
-
-/* unaligned versions (optimized for frequent unaligned accesses)*/
-
-#if defined(__i386__) || defined(_ARCH_PPC)
-
-#define cpu_to_le16wu(p, v) cpu_to_le16w(p, v)
-#define cpu_to_le32wu(p, v) cpu_to_le32w(p, v)
-#define le16_to_cpupu(p) le16_to_cpup(p)
-#define le32_to_cpupu(p) le32_to_cpup(p)
-#define be32_to_cpupu(p) be32_to_cpup(p)
-
-#define cpu_to_be16wu(p, v) cpu_to_be16w(p, v)
-#define cpu_to_be32wu(p, v) cpu_to_be32w(p, v)
-#define cpu_to_be64wu(p, v) cpu_to_be64w(p, v)
-
-#else
-
-static inline void cpu_to_le16wu(uint16_t *p, uint16_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v & 0xff;
-    p1[1] = v >> 8;
-}
-
-static inline void cpu_to_le32wu(uint32_t *p, uint32_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v & 0xff;
-    p1[1] = v >> 8;
-    p1[2] = v >> 16;
-    p1[3] = v >> 24;
-}
-
-static inline uint16_t le16_to_cpupu(const uint16_t *p)
-{
-    const uint8_t *p1 = (const uint8_t *)p;
-    return p1[0] | (p1[1] << 8);
-}
-
-static inline uint32_t le32_to_cpupu(const uint32_t *p)
-{
-    const uint8_t *p1 = (const uint8_t *)p;
-    return p1[0] | (p1[1] << 8) | (p1[2] << 16) | (p1[3] << 24);
-}
-
-static inline uint32_t be32_to_cpupu(const uint32_t *p)
-{
-    const uint8_t *p1 = (const uint8_t *)p;
-    return p1[3] | (p1[2] << 8) | (p1[1] << 16) | (p1[0] << 24);
-}
-
-static inline void cpu_to_be16wu(uint16_t *p, uint16_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v >> 8;
-    p1[1] = v & 0xff;
-}
-
-static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v >> 24;
-    p1[1] = v >> 16;
-    p1[2] = v >> 8;
-    p1[3] = v & 0xff;
-}
-
-static inline void cpu_to_be64wu(uint64_t *p, uint64_t v)
-{
-    uint8_t *p1 = (uint8_t *)p;
-
-    p1[0] = v >> 56;
-    p1[1] = v >> 48;
-    p1[2] = v >> 40;
-    p1[3] = v >> 32;
-    p1[4] = v >> 24;
-    p1[5] = v >> 16;
-    p1[6] = v >> 8;
-    p1[7] = v & 0xff;
-}
-
-#endif
-
-#ifdef HOST_WORDS_BIGENDIAN
-#define cpu_to_32wu cpu_to_be32wu
-#define leul_to_cpu(v) glue(glue(le,HOST_LONG_BITS),_to_cpu)(v)
-#else
-#define cpu_to_32wu cpu_to_le32wu
-#define leul_to_cpu(v) (v)
-#endif
-
-#undef le_bswap
-#undef be_bswap
-#undef le_bswaps
-#undef be_bswaps
-
-/* len must be one of 1, 2, 4 */
-static inline uint32_t qemu_bswap_len(uint32_t value, int len)
-{
-    return bswap32(value) >> (32 - 8 * len);
-}
-
-typedef union {
-    float32 f;
-    uint32_t l;
-} CPU_FloatU;
-
-typedef union {
-    float64 d;
-#if defined(HOST_WORDS_BIGENDIAN)
-    struct {
-        uint32_t upper;
-        uint32_t lower;
-    } l;
-#else
-    struct {
-        uint32_t lower;
-        uint32_t upper;
-    } l;
-#endif
-    uint64_t ll;
-} CPU_DoubleU;
-
-typedef union {
-     floatx80 d;
-     struct {
-         uint64_t lower;
-         uint16_t upper;
-     } l;
-} CPU_LDoubleU;
-
-typedef union {
-    float128 q;
-#if defined(HOST_WORDS_BIGENDIAN)
-    struct {
-        uint32_t upmost;
-        uint32_t upper;
-        uint32_t lower;
-        uint32_t lowest;
-    } l;
-    struct {
-        uint64_t upper;
-        uint64_t lower;
-    } ll;
-#else
-    struct {
-        uint32_t lowest;
-        uint32_t lower;
-        uint32_t upper;
-        uint32_t upmost;
-    } l;
-    struct {
-        uint64_t lower;
-        uint64_t upper;
-    } ll;
-#endif
-} CPU_QuadU;
-
-/* unaligned/endian-independent pointer access */
-
-/*
- * the generic syntax is:
- *
- * load: ld{type}{sign}{size}{endian}_p(ptr)
- *
- * store: st{type}{size}{endian}_p(ptr, val)
- *
- * Note there are small differences with the softmmu access API!
- *
- * type is:
- * (empty): integer access
- *   f    : float access
- *
- * sign is:
- * (empty): for floats or 32 bit size
- *   u    : unsigned
- *   s    : signed
- *
- * size is:
- *   b: 8 bits
- *   w: 16 bits
- *   l: 32 bits
- *   q: 64 bits
- *
- * endian is:
- * (empty): 8 bit access
- *   be   : big endian
- *   le   : little endian
- */
-static inline int ldub_p(const void *ptr)
-{
-    return *(uint8_t *)ptr;
-}
-
-static inline int ldsb_p(const void *ptr)
-{
-    return *(int8_t *)ptr;
-}
-
-static inline void stb_p(void *ptr, int v)
-{
-    *(uint8_t *)ptr = v;
-}
-
-/* NOTE: on arm, putting 2 in /proc/sys/debug/alignment so that the
-   kernel handles unaligned load/stores may give better results, but
-   it is a system wide setting : bad */
-#if defined(HOST_WORDS_BIGENDIAN) || defined(WORDS_ALIGNED)
-
-/* conservative code for little endian unaligned accesses */
-static inline int lduw_le_p(const void *ptr)
-{
-#ifdef _ARCH_PPC
-    int val;
-    __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (ptr));
-    return val;
-#else
-    const uint8_t *p = ptr;
-    return p[0] | (p[1] << 8);
-#endif
-}
-
-static inline int ldsw_le_p(const void *ptr)
-{
-#ifdef _ARCH_PPC
-    int val;
-    __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (ptr));
-    return (int16_t)val;
-#else
-    const uint8_t *p = ptr;
-    return (int16_t)(p[0] | (p[1] << 8));
-#endif
-}
-
-static inline int ldl_le_p(const void *ptr)
-{
-#ifdef _ARCH_PPC
-    int val;
-    __asm__ __volatile__ ("lwbrx %0,0,%1" : "=r" (val) : "r" (ptr));
-    return val;
-#else
-    const uint8_t *p = ptr;
-    return p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
-#endif
-}
-
-static inline uint64_t ldq_le_p(const void *ptr)
-{
-    const uint8_t *p = ptr;
-    uint32_t v1, v2;
-    v1 = ldl_le_p(p);
-    v2 = ldl_le_p(p + 4);
-    return v1 | ((uint64_t)v2 << 32);
-}
-
-static inline void stw_le_p(void *ptr, int v)
-{
-#ifdef _ARCH_PPC
-    __asm__ __volatile__ ("sthbrx %1,0,%2" : "=m" (*(uint16_t *)ptr) : "r" (v), "r" (ptr));
-#else
-    uint8_t *p = ptr;
-    p[0] = v;
-    p[1] = v >> 8;
-#endif
-}
-
-static inline void stl_le_p(void *ptr, int v)
-{
-#ifdef _ARCH_PPC
-    __asm__ __volatile__ ("stwbrx %1,0,%2" : "=m" (*(uint32_t *)ptr) : "r" (v), "r" (ptr));
-#else
-    uint8_t *p = ptr;
-    p[0] = v;
-    p[1] = v >> 8;
-    p[2] = v >> 16;
-    p[3] = v >> 24;
-#endif
-}
-
-static inline void stq_le_p(void *ptr, uint64_t v)
-{
-    uint8_t *p = ptr;
-    stl_le_p(p, (uint32_t)v);
-    stl_le_p(p + 4, v >> 32);
-}
-
-/* float access */
-
-static inline float32 ldfl_le_p(const void *ptr)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.i = ldl_le_p(ptr);
-    return u.f;
-}
-
-static inline void stfl_le_p(void *ptr, float32 v)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.f = v;
-    stl_le_p(ptr, u.i);
-}
-
-static inline float64 ldfq_le_p(const void *ptr)
-{
-    CPU_DoubleU u;
-    u.l.lower = ldl_le_p(ptr);
-    u.l.upper = ldl_le_p(ptr + 4);
-    return u.d;
-}
-
-static inline void stfq_le_p(void *ptr, float64 v)
-{
-    CPU_DoubleU u;
-    u.d = v;
-    stl_le_p(ptr, u.l.lower);
-    stl_le_p(ptr + 4, u.l.upper);
-}
-
-#else
-
-static inline int lduw_le_p(const void *ptr)
-{
-    return *(uint16_t *)ptr;
-}
-
-static inline int ldsw_le_p(const void *ptr)
-{
-    return *(int16_t *)ptr;
-}
-
-static inline int ldl_le_p(const void *ptr)
-{
-    return *(uint32_t *)ptr;
-}
-
-static inline uint64_t ldq_le_p(const void *ptr)
-{
-    return *(uint64_t *)ptr;
-}
-
-static inline void stw_le_p(void *ptr, int v)
-{
-    *(uint16_t *)ptr = v;
-}
-
-static inline void stl_le_p(void *ptr, int v)
-{
-    *(uint32_t *)ptr = v;
-}
-
-static inline void stq_le_p(void *ptr, uint64_t v)
-{
-    *(uint64_t *)ptr = v;
-}
-
-/* float access */
-
-static inline float32 ldfl_le_p(const void *ptr)
-{
-    return *(float32 *)ptr;
-}
-
-static inline float64 ldfq_le_p(const void *ptr)
-{
-    return *(float64 *)ptr;
-}
-
-static inline void stfl_le_p(void *ptr, float32 v)
-{
-    *(float32 *)ptr = v;
-}
-
-static inline void stfq_le_p(void *ptr, float64 v)
-{
-    *(float64 *)ptr = v;
-}
-#endif
-
-#if !defined(HOST_WORDS_BIGENDIAN) || defined(WORDS_ALIGNED)
-
-static inline int lduw_be_p(const void *ptr)
-{
-#if defined(__i386__)
-    int val;
-    asm volatile ("movzwl %1, %0\n"
-                  "xchgb %b0, %h0\n"
-                  : "=q" (val)
-                  : "m" (*(uint16_t *)ptr));
-    return val;
-#else
-    const uint8_t *b = ptr;
-    return ((b[0] << 8) | b[1]);
-#endif
-}
-
-static inline int ldsw_be_p(const void *ptr)
-{
-#if defined(__i386__)
-    int val;
-    asm volatile ("movzwl %1, %0\n"
-                  "xchgb %b0, %h0\n"
-                  : "=q" (val)
-                  : "m" (*(uint16_t *)ptr));
-    return (int16_t)val;
-#else
-    const uint8_t *b = ptr;
-    return (int16_t)((b[0] << 8) | b[1]);
-#endif
-}
-
-static inline int ldl_be_p(const void *ptr)
-{
-#if defined(__i386__) || defined(__x86_64__)
-    int val;
-    asm volatile ("movl %1, %0\n"
-                  "bswap %0\n"
-                  : "=r" (val)
-                  : "m" (*(uint32_t *)ptr));
-    return val;
-#else
-    const uint8_t *b = ptr;
-    return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3];
-#endif
-}
-
-static inline uint64_t ldq_be_p(const void *ptr)
-{
-    uint32_t a,b;
-    a = ldl_be_p(ptr);
-    b = ldl_be_p((uint8_t *)ptr + 4);
-    return (((uint64_t)a<<32)|b);
-}
-
-static inline void stw_be_p(void *ptr, int v)
-{
-#if defined(__i386__)
-    asm volatile ("xchgb %b0, %h0\n"
-                  "movw %w0, %1\n"
-                  : "=q" (v)
-                  : "m" (*(uint16_t *)ptr), "0" (v));
-#else
-    uint8_t *d = (uint8_t *) ptr;
-    d[0] = v >> 8;
-    d[1] = v;
-#endif
-}
-
-static inline void stl_be_p(void *ptr, int v)
-{
-#if defined(__i386__) || defined(__x86_64__)
-    asm volatile ("bswap %0\n"
-                  "movl %0, %1\n"
-                  : "=r" (v)
-                  : "m" (*(uint32_t *)ptr), "0" (v));
-#else
-    uint8_t *d = (uint8_t *) ptr;
-    d[0] = v >> 24;
-    d[1] = v >> 16;
-    d[2] = v >> 8;
-    d[3] = v;
-#endif
-}
-
-static inline void stq_be_p(void *ptr, uint64_t v)
-{
-    stl_be_p(ptr, v >> 32);
-    stl_be_p((uint8_t *)ptr + 4, v);
-}
-
-/* float access */
-
-static inline float32 ldfl_be_p(const void *ptr)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.i = ldl_be_p(ptr);
-    return u.f;
-}
-
-static inline void stfl_be_p(void *ptr, float32 v)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.f = v;
-    stl_be_p(ptr, u.i);
-}
-
-static inline float64 ldfq_be_p(const void *ptr)
-{
-    CPU_DoubleU u;
-    u.l.upper = ldl_be_p(ptr);
-    u.l.lower = ldl_be_p((uint8_t *)ptr + 4);
-    return u.d;
-}
-
-static inline void stfq_be_p(void *ptr, float64 v)
-{
-    CPU_DoubleU u;
-    u.d = v;
-    stl_be_p(ptr, u.l.upper);
-    stl_be_p((uint8_t *)ptr + 4, u.l.lower);
-}
-
-#else
-
-static inline int lduw_be_p(const void *ptr)
-{
-    return *(uint16_t *)ptr;
-}
-
-static inline int ldsw_be_p(const void *ptr)
-{
-    return *(int16_t *)ptr;
-}
-
-static inline int ldl_be_p(const void *ptr)
-{
-    return *(uint32_t *)ptr;
-}
-
-static inline uint64_t ldq_be_p(const void *ptr)
-{
-    return *(uint64_t *)ptr;
-}
-
-static inline void stw_be_p(void *ptr, int v)
-{
-    *(uint16_t *)ptr = v;
-}
-
-static inline void stl_be_p(void *ptr, int v)
-{
-    *(uint32_t *)ptr = v;
-}
-
-static inline void stq_be_p(void *ptr, uint64_t v)
-{
-    *(uint64_t *)ptr = v;
-}
-
-/* float access */
-
-static inline float32 ldfl_be_p(const void *ptr)
-{
-    return *(float32 *)ptr;
-}
-
-static inline float64 ldfq_be_p(const void *ptr)
-{
-    return *(float64 *)ptr;
-}
-
-static inline void stfl_be_p(void *ptr, float32 v)
-{
-    *(float32 *)ptr = v;
-}
-
-static inline void stfq_be_p(void *ptr, float64 v)
-{
-    *(float64 *)ptr = v;
-}
-
-#endif
-
-#endif /* BSWAP_H */
diff --git a/bt-host.c b/bt-host.c
index 0d3ad28..2092754 100644
--- a/bt-host.c
+++ b/bt-host.c
@@ -18,9 +18,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "net.h"
-#include "bt-host.h"
+#include "bt/bt.h"
+#include "qemu/main-loop.h"
 
 #ifndef _WIN32
 # include <errno.h>
diff --git a/bt-host.h b/bt-host.h
deleted file mode 100644
index f1eff65..0000000
--- a/bt-host.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#ifndef BT_HOST_H
-#define BT_HOST_H
-
-struct HCIInfo;
-
-/* bt-host.c */
-struct HCIInfo *bt_host_hci(const char *id);
-
-#endif
diff --git a/bt-vhci.c b/bt-vhci.c
index bbc1029..a6a7ab0 100644
--- a/bt-vhci.c
+++ b/bt-vhci.c
@@ -18,9 +18,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "hw/bt.h"
+#include "qemu/main-loop.h"
 
 #define VHCI_DEV	"/dev/vhci"
 #define VHCI_UDEV	"/dev/hci_vhci"
diff --git a/buffered_file.c b/buffered_file.c
deleted file mode 100644
index bd0f61d..0000000
--- a/buffered_file.c
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * QEMU buffered QEMUFile
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "hw/hw.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-
-//#define DEBUG_BUFFERED_FILE
-
-typedef struct QEMUFileBuffered
-{
-    MigrationState *migration_state;
-    QEMUFile *file;
-    int freeze_output;
-    size_t bytes_xfer;
-    size_t xfer_limit;
-    uint8_t *buffer;
-    size_t buffer_size;
-    size_t buffer_capacity;
-    QEMUTimer *timer;
-} QEMUFileBuffered;
-
-#ifdef DEBUG_BUFFERED_FILE
-#define DPRINTF(fmt, ...) \
-    do { printf("buffered-file: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-static void buffered_append(QEMUFileBuffered *s,
-                            const uint8_t *buf, size_t size)
-{
-    if (size > (s->buffer_capacity - s->buffer_size)) {
-        DPRINTF("increasing buffer capacity from %zu by %zu\n",
-                s->buffer_capacity, size + 1024);
-
-        s->buffer_capacity += size + 1024;
-
-        s->buffer = g_realloc(s->buffer, s->buffer_capacity);
-    }
-
-    memcpy(s->buffer + s->buffer_size, buf, size);
-    s->buffer_size += size;
-}
-
-static ssize_t buffered_flush(QEMUFileBuffered *s)
-{
-    size_t offset = 0;
-    ssize_t ret = 0;
-
-    DPRINTF("flushing %zu byte(s) of data\n", s->buffer_size);
-
-    while (s->bytes_xfer < s->xfer_limit && offset < s->buffer_size) {
-
-        ret = migrate_fd_put_buffer(s->migration_state, s->buffer + offset,
-                                    s->buffer_size - offset);
-        if (ret == -EAGAIN) {
-            DPRINTF("backend not ready, freezing\n");
-            ret = 0;
-            s->freeze_output = 1;
-            break;
-        }
-
-        if (ret <= 0) {
-            DPRINTF("error flushing data, %zd\n", ret);
-            break;
-        } else {
-            DPRINTF("flushed %zd byte(s)\n", ret);
-            offset += ret;
-            s->bytes_xfer += ret;
-        }
-    }
-
-    DPRINTF("flushed %zu of %zu byte(s)\n", offset, s->buffer_size);
-    memmove(s->buffer, s->buffer + offset, s->buffer_size - offset);
-    s->buffer_size -= offset;
-
-    if (ret < 0) {
-        return ret;
-    }
-    return offset;
-}
-
-static int buffered_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileBuffered *s = opaque;
-    ssize_t error;
-
-    DPRINTF("putting %d bytes at %" PRId64 "\n", size, pos);
-
-    error = qemu_file_get_error(s->file);
-    if (error) {
-        DPRINTF("flush when error, bailing: %s\n", strerror(-error));
-        return error;
-    }
-
-    DPRINTF("unfreezing output\n");
-    s->freeze_output = 0;
-
-    if (size > 0) {
-        DPRINTF("buffering %d bytes\n", size - offset);
-        buffered_append(s, buf, size);
-    }
-
-    error = buffered_flush(s);
-    if (error < 0) {
-        DPRINTF("buffered flush error. bailing: %s\n", strerror(-error));
-        return error;
-    }
-
-    if (pos == 0 && size == 0) {
-        DPRINTF("file is ready\n");
-        if (!s->freeze_output && s->bytes_xfer < s->xfer_limit) {
-            DPRINTF("notifying client\n");
-            migrate_fd_put_ready(s->migration_state);
-        }
-    }
-
-    return size;
-}
-
-static int buffered_close(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-    ssize_t ret = 0;
-    int ret2;
-
-    DPRINTF("closing\n");
-
-    s->xfer_limit = INT_MAX;
-    while (!qemu_file_get_error(s->file) && s->buffer_size) {
-        ret = buffered_flush(s);
-        if (ret < 0) {
-            break;
-        }
-        if (s->freeze_output) {
-            ret = migrate_fd_wait_for_unfreeze(s->migration_state);
-            if (ret < 0) {
-                break;
-            }
-        }
-    }
-
-    ret2 = migrate_fd_close(s->migration_state);
-    if (ret >= 0) {
-        ret = ret2;
-    }
-    qemu_del_timer(s->timer);
-    qemu_free_timer(s->timer);
-    g_free(s->buffer);
-    g_free(s);
-
-    return ret;
-}
-
-/*
- * The meaning of the return values is:
- *   0: We can continue sending
- *   1: Time to stop
- *   negative: There has been an error
- */
-static int buffered_get_fd(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-
-    return qemu_get_fd(s->file);
-}
-
-static int buffered_rate_limit(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-    int ret;
-
-    ret = qemu_file_get_error(s->file);
-    if (ret) {
-        return ret;
-    }
-    if (s->freeze_output)
-        return 1;
-
-    if (s->bytes_xfer > s->xfer_limit)
-        return 1;
-
-    return 0;
-}
-
-static int64_t buffered_set_rate_limit(void *opaque, int64_t new_rate)
-{
-    QEMUFileBuffered *s = opaque;
-    if (qemu_file_get_error(s->file)) {
-        goto out;
-    }
-    if (new_rate > SIZE_MAX) {
-        new_rate = SIZE_MAX;
-    }
-
-    s->xfer_limit = new_rate / 10;
-    
-out:
-    return s->xfer_limit;
-}
-
-static int64_t buffered_get_rate_limit(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-  
-    return s->xfer_limit;
-}
-
-static void buffered_rate_tick(void *opaque)
-{
-    QEMUFileBuffered *s = opaque;
-
-    if (qemu_file_get_error(s->file)) {
-        buffered_close(s);
-        return;
-    }
-
-    qemu_mod_timer(s->timer, qemu_get_clock_ms(rt_clock) + 100);
-
-    if (s->freeze_output)
-        return;
-
-    s->bytes_xfer = 0;
-
-    buffered_put_buffer(s, NULL, 0, 0);
-}
-
-static const QEMUFileOps buffered_file_ops = {
-    .get_fd =         buffered_get_fd,
-    .put_buffer =     buffered_put_buffer,
-    .close =          buffered_close,
-    .rate_limit =     buffered_rate_limit,
-    .get_rate_limit = buffered_get_rate_limit,
-    .set_rate_limit = buffered_set_rate_limit,
-};
-
-QEMUFile *qemu_fopen_ops_buffered(MigrationState *migration_state)
-{
-    QEMUFileBuffered *s;
-
-    s = g_malloc0(sizeof(*s));
-
-    s->migration_state = migration_state;
-    s->xfer_limit = migration_state->bandwidth_limit / 10;
-
-    s->file = qemu_fopen_ops(s, &buffered_file_ops);
-
-    s->timer = qemu_new_timer_ms(rt_clock, buffered_rate_tick, s);
-
-    qemu_mod_timer(s->timer, qemu_get_clock_ms(rt_clock) + 100);
-
-    return s->file;
-}
diff --git a/buffered_file.h b/buffered_file.h
deleted file mode 100644
index ef010fe..0000000
--- a/buffered_file.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * QEMU buffered QEMUFile
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef QEMU_BUFFERED_FILE_H
-#define QEMU_BUFFERED_FILE_H
-
-#include "hw/hw.h"
-#include "migration.h"
-
-QEMUFile *qemu_fopen_ops_buffered(MigrationState *migration_state);
-
-#endif
diff --git a/cmd.c b/cmd.c
index f40f09b..10a8688 100644
--- a/cmd.c
+++ b/cmd.c
@@ -24,8 +24,8 @@
 #include <getopt.h>
 
 #include "cmd.h"
-#include "qemu-aio.h"
-#include "main-loop.h"
+#include "block/aio.h"
+#include "qemu/main-loop.h"
 
 #define _(x)	x	/* not gettext support yet */
 
diff --git a/configure b/configure
index 88960fc..6058fd8 100755
--- a/configure
+++ b/configure
@@ -116,7 +116,7 @@ audio_drv_list=""
 audio_card_list="ac97 es1370 sb16 hda"
 audio_possible_cards="ac97 es1370 sb16 cs4231a adlib gus hda"
 block_drv_whitelist=""
-host_cc="gcc"
+host_cc="cc"
 libs_softmmu=""
 libs_tools=""
 audio_pt_int=""
@@ -158,6 +158,7 @@ vnc_tls=""
 vnc_sasl=""
 vnc_jpeg=""
 vnc_png=""
+vnc_ws=""
 xen=""
 xen_ctrl_version=""
 xen_pci_passthrough=""
@@ -176,6 +177,8 @@ strip_opt="yes"
 tcg_interpreter="no"
 bigendian="no"
 mingw32="no"
+gcov="no"
+gcov_tool="gcov"
 EXESUF=""
 prefix="/usr/local"
 mandir="\${prefix}/share/man"
@@ -212,7 +215,6 @@ trace_backend="nop"
 trace_file="trace"
 spice=""
 rbd=""
-smartcard=""
 smartcard_nss=""
 usb_redir=""
 opengl=""
@@ -223,6 +225,7 @@ libiscsi=""
 coroutine=""
 seccomp=""
 glusterfs=""
+virtio_blk_data_plane=""
 
 # If this is a Linaro QEMU tarball then default the pkgversion
 # string to say so, so that we clearly distinguish ourselves
@@ -244,8 +247,10 @@ for opt do
   --cpu=*) cpu="$optarg"
   ;;
   --extra-cflags=*) QEMU_CFLAGS="$optarg $QEMU_CFLAGS"
+                    EXTRA_CFLAGS="$optarg"
   ;;
   --extra-ldflags=*) LDFLAGS="$optarg $LDFLAGS"
+                     EXTRA_LDFLAGS="$optarg"
   ;;
   --enable-debug-info) debug_info="yes"
   ;;
@@ -257,8 +262,19 @@ done
 # Using uname is really, really broken.  Once we have the right set of checks
 # we can eliminate its usage altogether.
 
-cc="${CC-${cross_prefix}gcc}"
+# Preferred compiler:
+#  ${CC} (if set)
+#  ${cross_prefix}gcc (if cross-prefix specified)
+#  system compiler
+if test -z "${CC}${cross_prefix}"; then
+  cc="$host_cc"
+else
+  cc="${CC-${cross_prefix}gcc}"
+fi
+
 ar="${AR-${cross_prefix}ar}"
+as="${AS-${cross_prefix}as}"
+cpp="${CPP-$cc -E}"
 objcopy="${OBJCOPY-${cross_prefix}objcopy}"
 ld="${LD-${cross_prefix}ld}"
 libtool="${LIBTOOL-${cross_prefix}libtool}"
@@ -276,7 +292,7 @@ QEMU_CFLAGS="-fno-strict-aliasing $QEMU_CFLAGS"
 QEMU_CFLAGS="-Wall -Wundef -Wwrite-strings -Wmissing-prototypes $QEMU_CFLAGS"
 QEMU_CFLAGS="-Wstrict-prototypes -Wredundant-decls $QEMU_CFLAGS"
 QEMU_CFLAGS="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE $QEMU_CFLAGS"
-QEMU_INCLUDES="-I. -I\$(SRC_PATH) -I\$(SRC_PATH)/fpu"
+QEMU_INCLUDES="-I. -I\$(SRC_PATH) -I\$(SRC_PATH)/include"
 if test "$debug_info" = "yes"; then
     CFLAGS="-g $CFLAGS"
     LDFLAGS="-g $LDFLAGS"
@@ -597,6 +613,8 @@ for opt do
   ;;
   --python=*) python="$optarg"
   ;;
+  --gcov=*) gcov_tool="$optarg"
+  ;;
   --smbd=*) smbd="$optarg"
   ;;
   --extra-cflags=*)
@@ -617,6 +635,8 @@ for opt do
   ;;
   --enable-gprof) gprof="yes"
   ;;
+  --enable-gcov) gcov="yes"
+  ;;
   --static)
     static="yes"
     LDFLAGS="-static $LDFLAGS"
@@ -654,6 +674,8 @@ for opt do
   ;;
   --without-system-pixman) pixman="internal"
   ;;
+  --without-pixman) pixman="none"
+  ;;
   --disable-sdl) sdl="no"
   ;;
   --enable-sdl) sdl="yes"
@@ -710,6 +732,10 @@ for opt do
   ;;
   --enable-vnc-png) vnc_png="yes"
   ;;
+  --disable-vnc-ws) vnc_ws="no"
+  ;;
+  --enable-vnc-ws) vnc_ws="yes"
+  ;;
   --disable-slirp) slirp="no"
   ;;
   --disable-uuid) uuid="no"
@@ -848,10 +874,6 @@ for opt do
   ;;
   --enable-xfsctl) xfs="yes"
   ;;
-  --disable-smartcard) smartcard="no"
-  ;;
-  --enable-smartcard) smartcard="yes"
-  ;;
   --disable-smartcard-nss) smartcard_nss="no"
   ;;
   --enable-smartcard-nss) smartcard_nss="yes"
@@ -878,6 +900,10 @@ for opt do
   ;;
   --enable-glusterfs) glusterfs="yes"
   ;;
+  --disable-virtio-blk-data-plane) virtio_blk_data_plane="no"
+  ;;
+  --enable-virtio-blk-data-plane) virtio_blk_data_plane="yes"
+  ;;
   *) echo "ERROR: unknown option $opt"; show_help="yes"
   ;;
   esac
@@ -1055,6 +1081,8 @@ echo "  --disable-vnc-jpeg       disable JPEG lossy compression for VNC server"
 echo "  --enable-vnc-jpeg        enable JPEG lossy compression for VNC server"
 echo "  --disable-vnc-png        disable PNG compression for VNC server (default)"
 echo "  --enable-vnc-png         enable PNG compression for VNC server"
+echo "  --disable-vnc-ws         disable Websockets support for VNC server"
+echo "  --enable-vnc-ws          enable Websockets support for VNC server"
 echo "  --disable-curses         disable curses output"
 echo "  --enable-curses          enable curses output"
 echo "  --disable-curl           disable curl connectivity"
@@ -1087,7 +1115,6 @@ echo "  --fmod-inc               path to FMOD includes"
 echo "  --oss-lib                path to OSS library"
 echo "  --enable-uname-release=R Return R for uname -r in usermode emulation"
 echo "  --cpu=CPU                Build for host CPU [$cpu]"
-echo "  --sparc_cpu=V            Build qemu for Sparc architecture v7, v8, v8plus, v8plusa, v9"
 echo "  --disable-uuid           disable uuid support"
 echo "  --enable-uuid            enable uuid support"
 echo "  --disable-vde            disable support for vde network"
@@ -1112,8 +1139,6 @@ echo "  --enable-spice           enable spice"
 echo "  --enable-rbd             enable building the rados block device (rbd)"
 echo "  --disable-libiscsi       disable iscsi support"
 echo "  --enable-libiscsi        enable iscsi support"
-echo "  --disable-smartcard      disable smartcard support"
-echo "  --enable-smartcard       enable smartcard support"
 echo "  --disable-smartcard-nss  disable smartcard nss support"
 echo "  --enable-smartcard-nss   enable smartcard nss support"
 echo "  --disable-usb-redir      disable usb network redirection support"
@@ -1126,6 +1151,8 @@ echo "  --with-coroutine=BACKEND coroutine backend. Supported options:"
 echo "                           gthread, ucontext, sigaltstack, windows"
 echo "  --enable-glusterfs       enable GlusterFS backend"
 echo "  --disable-glusterfs      disable GlusterFS backend"
+echo "  --enable-gcov            enable test coverage analysis with gcov"
+echo "  --gcov=GCOV              use specified gcov [$gcov_tool]"
 echo ""
 echo "NOTE: The object files are built at the place where configure is launched"
 exit 1
@@ -1160,7 +1187,7 @@ fi
 z_version=`cut -f3 -d. $source_path/VERSION`
 
 if test -z "$werror" ; then
-    if test "$z_version" = "50" -a \
+    if test -d "$source_path/.git" -a \
         "$linux" = "yes" ; then
         werror="yes"
     else
@@ -1358,6 +1385,14 @@ esac
 fi
 
 ##########################################
+# pkg-config probe
+
+if ! has "$pkg_config_exe"; then
+  echo "Error: pkg-config binary '$pkg_config_exe' not found"
+  exit 1
+fi
+
+##########################################
 # NPTL probe
 
 if test "$nptl" != "no" ; then
@@ -1406,7 +1441,7 @@ fi
 
 if test "$seccomp" != "no" ; then
     if $pkg_config --atleast-version=1.0.0 libseccomp --modversion >/dev/null 2>&1; then
-        LIBS=`$pkg_config --libs libseccomp`
+        libs_softmmu="$libs_softmmu `$pkg_config --libs libseccomp`"
 	seccomp="yes"
     else
 	if test "$seccomp" = "yes"; then
@@ -1588,14 +1623,6 @@ if test "$xen_pci_passthrough" != "no"; then
 fi
 
 ##########################################
-# pkg-config probe
-
-if ! has "$pkg_config_exe"; then
-  echo "Error: pkg-config binary '$pkg_config_exe' not found"
-  exit 1
-fi
-
-##########################################
 # libtool probe
 
 if ! has $libtool; then
@@ -1699,8 +1726,8 @@ EOF
 fi
 
 ##########################################
-# VNC TLS detection
-if test "$vnc" = "yes" -a "$vnc_tls" != "no" ; then
+# VNC TLS/WS detection
+if test "$vnc" = "yes" -a \( "$vnc_tls" != "no" -o "$vnc_ws" != "no" \) ; then
   cat > $TMPC <<EOF
 #include <gnutls/gnutls.h>
 int main(void) { gnutls_session_t s; gnutls_init(&s, GNUTLS_SERVER); return 0; }
@@ -1708,13 +1735,23 @@ EOF
   vnc_tls_cflags=`$pkg_config --cflags gnutls 2> /dev/null`
   vnc_tls_libs=`$pkg_config --libs gnutls 2> /dev/null`
   if compile_prog "$vnc_tls_cflags" "$vnc_tls_libs" ; then
-    vnc_tls=yes
+    if test "$vnc_tls" != "no" ; then
+      vnc_tls=yes
+    fi
+    if test "$vnc_ws" != "no" ; then
+      vnc_ws=yes
+    fi
     libs_softmmu="$vnc_tls_libs $libs_softmmu"
+    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_tls_cflags"
   else
     if test "$vnc_tls" = "yes" ; then
       feature_not_found "vnc-tls"
     fi
+    if test "$vnc_ws" = "yes" ; then
+      feature_not_found "vnc-ws"
+    fi
     vnc_tls=no
+    vnc_ws=no
   fi
 fi
 
@@ -1732,6 +1769,7 @@ EOF
   if compile_prog "$vnc_sasl_cflags" "$vnc_sasl_libs" ; then
     vnc_sasl=yes
     libs_softmmu="$vnc_sasl_libs $libs_softmmu"
+    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_sasl_cflags"
   else
     if test "$vnc_sasl" = "yes" ; then
       feature_not_found "vnc-sasl"
@@ -1753,6 +1791,7 @@ EOF
   if compile_prog "$vnc_jpeg_cflags" "$vnc_jpeg_libs" ; then
     vnc_jpeg=yes
     libs_softmmu="$vnc_jpeg_libs $libs_softmmu"
+    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_jpeg_cflags"
   else
     if test "$vnc_jpeg" = "yes" ; then
       feature_not_found "vnc-jpeg"
@@ -2023,7 +2062,7 @@ fi
 if test "$mingw32" = "yes" ; then
     curses_list="-lpdcurses"
 else
-    curses_list="-lncurses -lcurses"
+    curses_list="-lncurses:-lcurses:$($pkg_config --libs ncurses 2>/dev/null)"
 fi
 
 if test "$curses" != "no" ; then
@@ -2036,13 +2075,16 @@ int main(void) {
   return s != 0;
 }
 EOF
+  IFS=:
   for curses_lib in $curses_list; do
+    unset IFS
     if compile_prog "" "$curses_lib" ; then
       curses_found=yes
       libs_softmmu="$curses_lib $libs_softmmu"
       break
     fi
   done
+  unset IFS
   if test "$curses_found" = "yes" ; then
     curses=yes
   else
@@ -2125,19 +2167,31 @@ fi
 # pixman support probe
 
 if test "$pixman" = ""; then
-  if $pkg_config pixman-1 > /dev/null 2>&1; then
+  if test "$want_tools" = "no" -a "$softmmu" = "no"; then
+    pixman="none"
+  elif $pkg_config pixman-1 > /dev/null 2>&1; then
     pixman="system"
   else
     pixman="internal"
   fi
 fi
-if test "$pixman" = "system"; then
+if test "$pixman" = "none"; then
+  if test "$want_tools" != "no" -o "$softmmu" != "no"; then
+    echo "ERROR: pixman disabled but system emulation or tools build"
+    echo "       enabled.  You can turn off pixman only if you also"
+    echo "       disable all system emulation targets and the tools"
+    echo "       build with '--disable-tools --disable-system'."
+    exit 1
+  fi
+  pixman_cflags=
+  pixman_libs=
+elif test "$pixman" = "system"; then
   pixman_cflags=`$pkg_config --cflags pixman-1 2>/dev/null`
   pixman_libs=`$pkg_config --libs pixman-1 2>/dev/null`
 else
   if test ! -d ${source_path}/pixman/pixman; then
     echo "ERROR: pixman not present. Your options:"
-    echo "  (1) Prefered: Install the pixman devel package (any recent"
+    echo "  (1) Preferred: Install the pixman devel package (any recent"
     echo "      distro should have packages as Xorg needs pixman too)."
     echo "  (2) Fetch the pixman submodule, using:"
     echo "      git submodule update --init pixman"
@@ -2255,6 +2309,17 @@ EOF
 fi
 
 ##########################################
+# adjust virtio-blk-data-plane based on linux-aio
+
+if test "$virtio_blk_data_plane" = "yes" -a \
+	"$linux_aio" != "yes" ; then
+  echo "Error: virtio-blk-data-plane requires Linux AIO, please try --enable-linux-aio"
+  exit 1
+elif test -z "$virtio_blk_data_plane" ; then
+  virtio_blk_data_plane=$linux_aio
+fi
+
+##########################################
 # attr probe
 
 if test "$attr" != "no" ; then
@@ -2330,7 +2395,7 @@ fi
 ##########################################
 # opengl probe, used by milkymist-tmu2
 if test "$opengl" != "no" ; then
-  opengl_libs="-lGL"
+  opengl_libs="-lGL -lX11"
   cat > $TMPC << EOF
 #include <X11/Xlib.h>
 #include <GL/gl.h>
@@ -2542,6 +2607,22 @@ if compile_prog "" "" ; then
   fallocate=yes
 fi
 
+# check for fallocate hole punching
+fallocate_punch_hole=no
+cat > $TMPC << EOF
+#include <fcntl.h>
+#include <linux/falloc.h>
+
+int main(void)
+{
+    fallocate(0, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, 0);
+    return 0;
+}
+EOF
+if compile_prog "" "" ; then
+  fallocate_punch_hole=yes
+fi
+
 # check for sync_file_range
 sync_file_range=no
 cat > $TMPC << EOF
@@ -2663,7 +2744,7 @@ if compile_prog "" "" ; then
   byteswap_h=yes
 fi
 
-# Search for bswap_32 function
+# Search for bswap32 function
 bswap_h=no
 cat > $TMPC << EOF
 #include <sys/endian.h>
@@ -2769,47 +2850,42 @@ EOF
 fi
 
 # check for libcacard for smartcard support
-if test "$smartcard" != "no" ; then
-    smartcard="yes"
-    smartcard_cflags=""
-    # TODO - what's the minimal nss version we support?
-    if test "$smartcard_nss" != "no"; then
-      cat > $TMPC << EOF
+smartcard_cflags=""
+# TODO - what's the minimal nss version we support?
+if test "$smartcard_nss" != "no"; then
+  cat > $TMPC << EOF
 #include <pk11pub.h>
 int main(void) { PK11_FreeSlot(0); return 0; }
 EOF
-        smartcard_includes="-I\$(SRC_PATH)/libcacard"
-        libcacard_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
-        libcacard_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
-        test_cflags="$libcacard_cflags"
-        # The header files in nss < 3.13.3 have a bug which causes them to
-        # emit a warning. If we're going to compile QEMU with -Werror, then
-        # test that the headers don't have this bug. Otherwise we would pass
-        # the configure test but fail to compile QEMU later.
-        if test "$werror" = "yes"; then
-            test_cflags="-Werror $test_cflags"
-        fi
-        if $pkg_config --atleast-version=3.12.8 nss >/dev/null 2>&1 && \
-          compile_prog "$test_cflags" "$libcacard_libs"; then
-            smartcard_nss="yes"
-            QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
-            QEMU_INCLUDES="$QEMU_INCLUDES $smartcard_includes"
-            libs_softmmu="$libcacard_libs $libs_softmmu"
-        else
-            if test "$smartcard_nss" = "yes"; then
-                feature_not_found "nss"
-            fi
-            smartcard_nss="no"
+    smartcard_includes="-I\$(SRC_PATH)/libcacard"
+    libcacard_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
+    libcacard_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
+    test_cflags="$libcacard_cflags"
+    # The header files in nss < 3.13.3 have a bug which causes them to
+    # emit a warning. If we're going to compile QEMU with -Werror, then
+    # test that the headers don't have this bug. Otherwise we would pass
+    # the configure test but fail to compile QEMU later.
+    if test "$werror" = "yes"; then
+        test_cflags="-Werror $test_cflags"
+    fi
+    if test -n "$libtool" &&
+            $pkg_config --atleast-version=3.12.8 nss >/dev/null 2>&1 && \
+      compile_prog "$test_cflags" "$libcacard_libs"; then
+        smartcard_nss="yes"
+        QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
+        QEMU_INCLUDES="$QEMU_INCLUDES $smartcard_includes"
+        libs_softmmu="$libcacard_libs $libs_softmmu"
+    else
+        if test "$smartcard_nss" = "yes"; then
+            feature_not_found "nss"
         fi
+        smartcard_nss="no"
     fi
 fi
-if test "$smartcard" = "no" ; then
-    smartcard_nss="no"
-fi
 
 # check for usbredirparser for usb network redirection support
 if test "$usb_redir" != "no" ; then
-    if $pkg_config --atleast-version=0.5.3 libusbredirparser-0.5 >/dev/null 2>&1 ; then
+    if $pkg_config --atleast-version=0.6 libusbredirparser-0.5 >/dev/null 2>&1 ; then
         usb_redir="yes"
         usb_redir_cflags=$($pkg_config --cflags libusbredirparser-0.5 2>/dev/null)
         usb_redir_libs=$($pkg_config --libs libusbredirparser-0.5 2>/dev/null)
@@ -2976,8 +3052,6 @@ EOF
     else
 	coroutine_backend=gthread
     fi
-  else
-    echo "Silently falling back into gthread backend under darwin"
   fi
 elif test "$coroutine" = "gthread" ; then
   coroutine_backend=gthread
@@ -3036,7 +3110,7 @@ int main(void) {
 }
 EOF
 if compile_prog "-Werror" "" ; then
-    pragma_disable_unused_but_set=yes
+    pragma_diagnostic_available=yes
 fi
 
 ########################################
@@ -3069,14 +3143,33 @@ if compile_prog "" "" ; then
     has_environ=yes
 fi
 
+########################################
+# check if cpuid.h is usable.
+
+cpuid_h=no
+cat > $TMPC << EOF
+#include <cpuid.h>
+int main(void) {
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    cpuid_h=yes
+fi
+
+
 ##########################################
 # End of CC checks
 # After here, no more $cc or $ld runs
 
-if test "$debug" = "no" ; then
+if test "$gcov" = "yes" ; then
+  CFLAGS="-fprofile-arcs -ftest-coverage -g $CFLAGS"
+  LDFLAGS="-fprofile-arcs -ftest-coverage $LDFLAGS"
+elif test "$debug" = "no" ; then
   CFLAGS="-O2 -D_FORTIFY_SOURCE=2 $CFLAGS"
 fi
 
+
 # Disable zero malloc errors for official releases unless explicitly told to
 # enable/disable
 if test -z "$zero_malloc" ; then
@@ -3142,9 +3235,6 @@ if test "$softmmu" = yes ; then
       tools="qemu-ga\$(EXESUF) $tools"
     fi
   fi
-  if test "$smartcard_nss" = "yes" ; then
-    tools="vscclient\$(EXESUF) $tools"
-  fi
 fi
 
 # Mac OS X ships with a broken assembler
@@ -3216,6 +3306,7 @@ if test "$vnc" = "yes" ; then
     echo "VNC SASL support  $vnc_sasl"
     echo "VNC JPEG support  $vnc_jpeg"
     echo "VNC PNG support   $vnc_png"
+    echo "VNC WS support    $vnc_ws"
 fi
 if test -n "$sparc_cpu"; then
     echo "Target Sparc Arch $sparc_cpu"
@@ -3257,6 +3348,9 @@ echo "build guest agent $guest_agent"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine_backend"
 echo "GlusterFS support $glusterfs"
+echo "virtio-blk-data-plane $virtio_blk_data_plane"
+echo "gcov              $gcov_tool"
+echo "gcov enabled      $gcov"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -3265,6 +3359,8 @@ fi
 config_host_mak="config-host.mak"
 config_host_ld="config-host.ld"
 
+echo "# Automatically generated by configure - do not modify" >config-all-disas.mak
+
 echo "# Automatically generated by configure - do not modify" > $config_host_mak
 printf "# Configured with:" >> $config_host_mak
 printf " '%s'" "$0" "$@" >> $config_host_mak
@@ -3283,6 +3379,8 @@ echo "qemu_datadir=$qemu_datadir" >> $config_host_mak
 echo "qemu_docdir=$qemu_docdir" >> $config_host_mak
 echo "qemu_localstatedir=$local_statedir" >> $config_host_mak
 echo "qemu_helperdir=$libexecdir" >> $config_host_mak
+echo "extra_cflags=$EXTRA_CFLAGS" >> $config_host_mak
+echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak
 if test "$debug_tcg" = "yes" ; then
@@ -3343,7 +3441,6 @@ fi
 if test "$slirp" = "yes" ; then
   echo "CONFIG_SLIRP=y" >> $config_host_mak
   echo "CONFIG_SMBD_COMMAND=\"$smbd\"" >> $config_host_mak
-  QEMU_INCLUDES="-I\$(SRC_PATH)/slirp $QEMU_INCLUDES"
 fi
 if test "$vde" = "yes" ; then
   echo "CONFIG_VDE=y" >> $config_host_mak
@@ -3378,19 +3475,19 @@ if test "$vnc" = "yes" ; then
 fi
 if test "$vnc_tls" = "yes" ; then
   echo "CONFIG_VNC_TLS=y" >> $config_host_mak
-  echo "VNC_TLS_CFLAGS=$vnc_tls_cflags" >> $config_host_mak
 fi
 if test "$vnc_sasl" = "yes" ; then
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
-  echo "VNC_SASL_CFLAGS=$vnc_sasl_cflags" >> $config_host_mak
 fi
 if test "$vnc_jpeg" = "yes" ; then
   echo "CONFIG_VNC_JPEG=y" >> $config_host_mak
-  echo "VNC_JPEG_CFLAGS=$vnc_jpeg_cflags" >> $config_host_mak
 fi
 if test "$vnc_png" = "yes" ; then
   echo "CONFIG_VNC_PNG=y" >> $config_host_mak
-  echo "VNC_PNG_CFLAGS=$vnc_png_cflags" >> $config_host_mak
+fi
+if test "$vnc_ws" = "yes" ; then
+  echo "CONFIG_VNC_WS=y" >> $config_host_mak
+  echo "VNC_WS_CFLAGS=$vnc_ws_cflags" >> $config_host_mak
 fi
 if test "$fnmatch" = "yes" ; then
   echo "CONFIG_FNMATCH=y" >> $config_host_mak
@@ -3440,6 +3537,9 @@ fi
 if test "$fallocate" = "yes" ; then
   echo "CONFIG_FALLOCATE=y" >> $config_host_mak
 fi
+if test "$fallocate_punch_hole" = "yes" ; then
+  echo "CONFIG_FALLOCATE_PUNCH_HOLE=y" >> $config_host_mak
+fi
 if test "$sync_file_range" = "yes" ; then
   echo "CONFIG_SYNC_FILE_RANGE=y" >> $config_host_mak
 fi
@@ -3533,10 +3633,6 @@ if test "$spice" = "yes" ; then
   echo "CONFIG_SPICE=y" >> $config_host_mak
 fi
 
-if test "$smartcard" = "yes" ; then
-  echo "CONFIG_SMARTCARD=y" >> $config_host_mak
-fi
-
 if test "$smartcard_nss" = "yes" ; then
   echo "CONFIG_SMARTCARD_NSS=y" >> $config_host_mak
   echo "libcacard_libs=$libcacard_libs" >> $config_host_mak
@@ -3587,8 +3683,8 @@ if test "$linux_magic_h" = "yes" ; then
   echo "CONFIG_LINUX_MAGIC_H=y" >> $config_host_mak
 fi
 
-if test "$pragma_disable_unused_but_set" = "yes" ; then
-  echo "CONFIG_PRAGMA_DISABLE_UNUSED_BUT_SET=y" >> $config_host_mak
+if test "$pragma_diagnostic_available" = "yes" ; then
+  echo "CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE=y" >> $config_host_mak
 fi
 
 if test "$valgrind_h" = "yes" ; then
@@ -3599,10 +3695,18 @@ if test "$has_environ" = "yes" ; then
   echo "CONFIG_HAS_ENVIRON=y" >> $config_host_mak
 fi
 
+if test "$cpuid_h" = "yes" ; then
+  echo "CONFIG_CPUID_H=y" >> $config_host_mak
+fi
+
 if test "$glusterfs" = "yes" ; then
   echo "CONFIG_GLUSTERFS=y" >> $config_host_mak
 fi
 
+if test "$virtio_blk_data_plane" = "yes" ; then
+  echo "CONFIG_VIRTIO_BLK_DATA_PLANE=y" >> $config_host_mak
+fi
+
 # USB host support
 case "$usb" in
 linux)
@@ -3652,13 +3756,21 @@ echo "MAKE=$make" >> $config_host_mak
 echo "INSTALL=$install" >> $config_host_mak
 echo "INSTALL_DIR=$install -d -m 0755" >> $config_host_mak
 echo "INSTALL_DATA=$install -c -m 0644" >> $config_host_mak
-echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
+if test -n "$libtool"; then
+  echo "INSTALL_PROG=\$(LIBTOOL) --mode=install $install -c -m 0755" >> $config_host_mak
+  echo "INSTALL_LIB=\$(LIBTOOL) --mode=install $install -c -m 0644" >> $config_host_mak
+else
+  echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
+  echo "INSTALL_LIB=$install -c -m 0644" >> $config_host_mak
+fi
 echo "PYTHON=$python" >> $config_host_mak
 echo "CC=$cc" >> $config_host_mak
 echo "CC_I386=$cc_i386" >> $config_host_mak
 echo "HOST_CC=$host_cc" >> $config_host_mak
 echo "OBJCC=$objcc" >> $config_host_mak
 echo "AR=$ar" >> $config_host_mak
+echo "AS=$as" >> $config_host_mak
+echo "CPP=$cpp" >> $config_host_mak
 echo "OBJCOPY=$objcopy" >> $config_host_mak
 echo "LD=$ld" >> $config_host_mak
 echo "WINDRES=$windres" >> $config_host_mak
@@ -3685,6 +3797,10 @@ echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
 echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
+if test "$gcov" = "yes" ; then
+  echo "CONFIG_GCOV=y" >> $config_host_mak
+  echo "GCOV=$gcov_tool" >> $config_host_mak
+fi
 
 # generate list of library paths for linker script
 
@@ -3698,11 +3814,6 @@ if test -f ${config_host_ld}~ ; then
   fi
 fi
 
-for d in libdis libdis-user; do
-    symlink "$source_path/Makefile.dis" "$d/Makefile"
-    echo > $d/config.mak
-done
-
 # use included Linux headers
 if test "$linux" = "yes" ; then
   mkdir -p linux-headers
@@ -3979,9 +4090,6 @@ fi
 if test "$target_softmmu" = "yes" ; then
   echo "CONFIG_SOFTMMU=y" >> $config_target_mak
   echo "LIBS+=$libs_softmmu $target_libs_softmmu" >> $config_target_mak
-  if test "$smartcard_nss" = "yes" ; then
-    echo "subdir-$target: subdir-libcacard" >> $config_host_mak
-  fi
   case "$target_arch2" in
     i386|x86_64)
       echo "CONFIG_HAVE_CORE_DUMP=y" >> $config_target_mak
@@ -4044,83 +4152,77 @@ if test "$linux" = "yes" ; then
   includes="-I\$(SRC_PATH)/linux-headers $includes"
 fi
 
-if test "$target_user_only" = "yes" ; then
-    libdis_config_mak=libdis-user/config.mak
-else
-    libdis_config_mak=libdis/config.mak
-fi
-
 for i in $ARCH $TARGET_BASE_ARCH ; do
   case "$i" in
   alpha)
     echo "CONFIG_ALPHA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ALPHA_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_ALPHA_DIS=y"  >> config-all-disas.mak
   ;;
   arm)
     echo "CONFIG_ARM_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ARM_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_ARM_DIS=y"  >> config-all-disas.mak
   ;;
   cris)
     echo "CONFIG_CRIS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_CRIS_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_CRIS_DIS=y"  >> config-all-disas.mak
   ;;
   hppa)
     echo "CONFIG_HPPA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_HPPA_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_HPPA_DIS=y"  >> config-all-disas.mak
   ;;
   i386|x86_64)
     echo "CONFIG_I386_DIS=y"  >> $config_target_mak
-    echo "CONFIG_I386_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_I386_DIS=y"  >> config-all-disas.mak
   ;;
   ia64*)
     echo "CONFIG_IA64_DIS=y"  >> $config_target_mak
-    echo "CONFIG_IA64_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_IA64_DIS=y"  >> config-all-disas.mak
   ;;
   lm32)
     echo "CONFIG_LM32_DIS=y"  >> $config_target_mak
-    echo "CONFIG_LM32_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_LM32_DIS=y"  >> config-all-disas.mak
   ;;
   m68k)
     echo "CONFIG_M68K_DIS=y"  >> $config_target_mak
-    echo "CONFIG_M68K_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_M68K_DIS=y"  >> config-all-disas.mak
   ;;
   microblaze*)
     echo "CONFIG_MICROBLAZE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_MICROBLAZE_DIS=y"  >> config-all-disas.mak
   ;;
   mips*)
     echo "CONFIG_MIPS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MIPS_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_MIPS_DIS=y"  >> config-all-disas.mak
   ;;
   or32)
     echo "CONFIG_OPENRISC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_OPENRISC_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_OPENRISC_DIS=y"  >> config-all-disas.mak
   ;;
   ppc*)
     echo "CONFIG_PPC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_PPC_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_PPC_DIS=y"  >> config-all-disas.mak
   ;;
   s390*)
     echo "CONFIG_S390_DIS=y"  >> $config_target_mak
-    echo "CONFIG_S390_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_S390_DIS=y"  >> config-all-disas.mak
   ;;
   sh4)
     echo "CONFIG_SH4_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SH4_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_SH4_DIS=y"  >> config-all-disas.mak
   ;;
   sparc*)
     echo "CONFIG_SPARC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SPARC_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_SPARC_DIS=y"  >> config-all-disas.mak
   ;;
   xtensa*)
     echo "CONFIG_XTENSA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_XTENSA_DIS=y"  >> $libdis_config_mak
+    echo "CONFIG_XTENSA_DIS=y"  >> config-all-disas.mak
   ;;
   esac
 done
 if test "$tcg_interpreter" = "yes" ; then
   echo "CONFIG_TCI_DIS=y"  >> $config_target_mak
-  echo "CONFIG_TCI_DIS=y"  >> $libdis_config_mak
+  echo "CONFIG_TCI_DIS=y"  >> config-all-disas.mak
 fi
 
 case "$ARCH" in
@@ -4159,7 +4261,7 @@ fi
 if test "$ARCH" = "tci"; then
   linker_script=""
 else
-  linker_script="-Wl,-T../config-host.ld -Wl,-T,\$(SRC_PATH)/\$(ARCH).ld"
+  linker_script="-Wl,-T../config-host.ld -Wl,-T,\$(SRC_PATH)/ldscripts/\$(ARCH).ld"
 fi
 
 if test "$target_linux_user" = "yes" -o "$target_bsd_user" = "yes" ; then
@@ -4188,15 +4290,15 @@ DIRS="tests tests/tcg tests/tcg/cris tests/tcg/lm32"
 DIRS="$DIRS pc-bios/optionrom pc-bios/spapr-rtas"
 DIRS="$DIRS roms/seabios roms/vgabios"
 DIRS="$DIRS qapi-generated"
-DIRS="$DIRS libcacard libcacard/libcacard libcacard/trace"
 FILES="Makefile tests/tcg/Makefile qdict-test-data.txt"
 FILES="$FILES tests/tcg/cris/Makefile tests/tcg/cris/.gdbinit"
-FILES="$FILES tests/tcg/lm32/Makefile libcacard/Makefile"
+FILES="$FILES tests/tcg/lm32/Makefile"
 FILES="$FILES pc-bios/optionrom/Makefile pc-bios/keymaps"
 FILES="$FILES pc-bios/spapr-rtas/Makefile"
 FILES="$FILES roms/seabios/Makefile roms/vgabios/Makefile"
 for bios_file in \
     $source_path/pc-bios/*.bin \
+    $source_path/pc-bios/*.aml \
     $source_path/pc-bios/*.rom \
     $source_path/pc-bios/*.dtb \
     $source_path/pc-bios/openbios-* \
@@ -4217,17 +4319,15 @@ for rom in seabios vgabios ; do
     config_mak=roms/$rom/config.mak
     echo "# Automatically generated by configure - do not modify" > $config_mak
     echo "SRC_PATH=$source_path/roms/$rom" >> $config_mak
+    echo "AS=$as" >> $config_mak
     echo "CC=$cc" >> $config_mak
     echo "BCC=bcc" >> $config_mak
-    echo "CPP=${cross_prefix}cpp" >> $config_mak
+    echo "CPP=$cpp" >> $config_mak
     echo "OBJCOPY=objcopy" >> $config_mak
     echo "IASL=iasl" >> $config_mak
     echo "LD=$ld" >> $config_mak
 done
 
-d=libuser
-symlink "$source_path/Makefile.user" "$d/Makefile"
-
 if test "$docs" = "yes" ; then
   mkdir -p QMP
 fi
diff --git a/coroutine-gthread.c b/coroutine-gthread.c
index 30c24c9..d3e5b99 100644
--- a/coroutine-gthread.c
+++ b/coroutine-gthread.c
@@ -20,7 +20,7 @@
 
 #include <glib.h>
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 typedef struct {
     Coroutine base;
diff --git a/coroutine-sigaltstack.c b/coroutine-sigaltstack.c
index 39dbaa5..e37ebac 100644
--- a/coroutine-sigaltstack.c
+++ b/coroutine-sigaltstack.c
@@ -31,7 +31,7 @@
 #include <pthread.h>
 #include <signal.h>
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 enum {
     /* Maximum free pool size prevents holding too many freed coroutines */
diff --git a/coroutine-ucontext.c b/coroutine-ucontext.c
index 784081a..a9c30e9 100644
--- a/coroutine-ucontext.c
+++ b/coroutine-ucontext.c
@@ -28,7 +28,7 @@
 #include <pthread.h>
 #include <ucontext.h>
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 #ifdef CONFIG_VALGRIND_H
 #include <valgrind/valgrind.h>
@@ -200,7 +200,7 @@ Coroutine *qemu_coroutine_new(void)
 }
 
 #ifdef CONFIG_VALGRIND_H
-#ifdef CONFIG_PRAGMA_DISABLE_UNUSED_BUT_SET
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
 /* Work around an unused variable in the valgrind.h macro... */
 #pragma GCC diagnostic ignored "-Wunused-but-set-variable"
 #endif
@@ -208,7 +208,7 @@ static inline void valgrind_stack_deregister(CoroutineUContext *co)
 {
     VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
 }
-#ifdef CONFIG_PRAGMA_DISABLE_UNUSED_BUT_SET
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
 #pragma GCC diagnostic error "-Wunused-but-set-variable"
 #endif
 #endif
diff --git a/coroutine-win32.c b/coroutine-win32.c
index 4179609..edc1f72 100644
--- a/coroutine-win32.c
+++ b/coroutine-win32.c
@@ -23,7 +23,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine_int.h"
 
 typedef struct
 {
diff --git a/cpu-exec.c b/cpu-exec.c
index 904ee73..19ebb4a 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -18,10 +18,10 @@
  */
 #include "config.h"
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg.h"
-#include "qemu-barrier.h"
-#include "qtest.h"
+#include "qemu/atomic.h"
+#include "sysemu/qtest.h"
 
 int tb_invalidated_flag;
 
diff --git a/cpus.c b/cpus.c
index d9c332f..41779eb 100644
--- a/cpus.c
+++ b/cpus.c
@@ -25,21 +25,21 @@
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
 
-#include "monitor.h"
-#include "sysemu.h"
-#include "gdbstub.h"
-#include "dma.h"
-#include "kvm.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "exec/gdbstub.h"
+#include "sysemu/dma.h"
+#include "sysemu/kvm.h"
 #include "qmp-commands.h"
 
-#include "qemu-thread.h"
-#include "cpus.h"
-#include "qtest.h"
-#include "main-loop.h"
-#include "bitmap.h"
+#include "qemu/thread.h"
+#include "sysemu/cpus.h"
+#include "sysemu/qtest.h"
+#include "qemu/main-loop.h"
+#include "qemu/bitmap.h"
 
 #ifndef _WIN32
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 #endif
 
 #ifdef CONFIG_LINUX
@@ -390,13 +390,15 @@ void hw_error(const char *fmt, ...)
 {
     va_list ap;
     CPUArchState *env;
+    CPUState *cpu;
 
     va_start(ap, fmt);
     fprintf(stderr, "qemu: hardware error: ");
     vfprintf(stderr, fmt, ap);
     fprintf(stderr, "\n");
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        fprintf(stderr, "CPU #%d:\n", env->cpu_index);
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
+        fprintf(stderr, "CPU #%d:\n", cpu->cpu_index);
         cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU);
     }
     va_end(ap);
@@ -515,7 +517,7 @@ static void qemu_init_sigbus(void)
     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 }
 
-static void qemu_kvm_eat_signals(CPUArchState *env)
+static void qemu_kvm_eat_signals(CPUState *cpu)
 {
     struct timespec ts = { 0, 0 };
     siginfo_t siginfo;
@@ -536,7 +538,7 @@ static void qemu_kvm_eat_signals(CPUArchState *env)
 
         switch (r) {
         case SIGBUS:
-            if (kvm_on_sigbus_vcpu(env, siginfo.si_code, siginfo.si_addr)) {
+            if (kvm_on_sigbus_vcpu(cpu, siginfo.si_code, siginfo.si_addr)) {
                 sigbus_reraise();
             }
             break;
@@ -558,7 +560,7 @@ static void qemu_init_sigbus(void)
 {
 }
 
-static void qemu_kvm_eat_signals(CPUArchState *env)
+static void qemu_kvm_eat_signals(CPUState *cpu)
 {
 }
 #endif /* !CONFIG_LINUX */
@@ -725,7 +727,7 @@ static void qemu_kvm_wait_io_event(CPUArchState *env)
         qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
-    qemu_kvm_eat_signals(env);
+    qemu_kvm_eat_signals(cpu);
     qemu_wait_io_event_common(cpu);
 }
 
@@ -740,7 +742,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
     cpu->thread_id = qemu_get_thread_id();
     cpu_single_env = env;
 
-    r = kvm_init_vcpu(env);
+    r = kvm_init_vcpu(cpu);
     if (r < 0) {
         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
         exit(1);
@@ -1041,8 +1043,8 @@ void qemu_init_vcpu(void *_env)
     CPUArchState *env = _env;
     CPUState *cpu = ENV_GET_CPU(env);
 
-    env->nr_cores = smp_cores;
-    env->nr_threads = smp_threads;
+    cpu->nr_cores = smp_cores;
+    cpu->nr_threads = smp_threads;
     cpu->stopped = true;
     if (kvm_enabled()) {
         qemu_kvm_start_vcpu(env);
@@ -1160,12 +1162,14 @@ static void tcg_exec_all(void)
 void set_numa_modes(void)
 {
     CPUArchState *env;
+    CPUState *cpu;
     int i;
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
         for (i = 0; i < nb_numa_nodes; i++) {
-            if (test_bit(env->cpu_index, node_cpumask[i])) {
-                env->numa_node = i;
+            if (test_bit(cpu->cpu_index, node_cpumask[i])) {
+                cpu->numa_node = i;
             }
         }
     }
@@ -1213,7 +1217,7 @@ CpuInfoList *qmp_query_cpus(Error **errp)
 
         info = g_malloc0(sizeof(*info));
         info->value = g_malloc0(sizeof(*info->value));
-        info->value->CPU = env->cpu_index;
+        info->value->CPU = cpu->cpu_index;
         info->value->current = (env == first_cpu);
         info->value->halted = env->halted;
         info->value->thread_id = cpu->thread_id;
@@ -1251,6 +1255,7 @@ void qmp_memsave(int64_t addr, int64_t size, const char *filename,
     FILE *f;
     uint32_t l;
     CPUArchState *env;
+    CPUState *cpu;
     uint8_t buf[1024];
 
     if (!has_cpu) {
@@ -1258,7 +1263,8 @@ void qmp_memsave(int64_t addr, int64_t size, const char *filename,
     }
 
     for (env = first_cpu; env; env = env->next_cpu) {
-        if (cpu_index == env->cpu_index) {
+        cpu = ENV_GET_CPU(env);
+        if (cpu_index == cpu->cpu_index) {
             break;
         }
     }
diff --git a/cputlb.c b/cputlb.c
index d6d0372..88239c4 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -19,13 +19,13 @@
 
 #include "config.h"
 #include "cpu.h"
-#include "exec-all.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/exec-all.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
-#include "cputlb.h"
+#include "exec/cputlb.h"
 
-#include "memory-internal.h"
+#include "exec/memory-internal.h"
 
 //#define DEBUG_TLB
 //#define DEBUG_TLB_CHECK
@@ -347,15 +347,15 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 #define SOFTMMU_CODE_ACCESS
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #undef env
diff --git a/default-configs/pci.mak b/default-configs/pci.mak
index ae9d1eb..ee2d18d 100644
--- a/default-configs/pci.mak
+++ b/default-configs/pci.mak
@@ -21,3 +21,4 @@ CONFIG_ESP=y
 CONFIG_ESP_PCI=y
 CONFIG_SERIAL=y
 CONFIG_SERIAL_PCI=y
+CONFIG_IPACK=y
diff --git a/default-configs/ppc-softmmu.mak b/default-configs/ppc-softmmu.mak
index d0fde7b..1f4a1cf 100644
--- a/default-configs/ppc-softmmu.mak
+++ b/default-configs/ppc-softmmu.mak
@@ -8,6 +8,7 @@ CONFIG_M48T59=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
+CONFIG_PARALLEL=y
 CONFIG_I8254=y
 CONFIG_PCKBD=y
 CONFIG_FDC=y
@@ -16,6 +17,7 @@ CONFIG_I82374=y
 CONFIG_OPENPIC=y
 CONFIG_PREP_PCI=y
 CONFIG_I82378=y
+CONFIG_PC87312=y
 CONFIG_MACIO=y
 CONFIG_PCSPK=y
 CONFIG_CUDA=y
diff --git a/default-configs/ppc64-softmmu.mak b/default-configs/ppc64-softmmu.mak
index e4265b4..5ff406c 100644
--- a/default-configs/ppc64-softmmu.mak
+++ b/default-configs/ppc64-softmmu.mak
@@ -8,13 +8,18 @@ CONFIG_M48T59=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
+CONFIG_PARALLEL=y
 CONFIG_I8254=y
 CONFIG_PCKBD=y
 CONFIG_FDC=y
 CONFIG_DMA=y
+CONFIG_I82374=y
 CONFIG_OPENPIC=y
 CONFIG_PREP_PCI=y
+CONFIG_I82378=y
+CONFIG_PC87312=y
 CONFIG_MACIO=y
+CONFIG_PCSPK=y
 CONFIG_CUDA=y
 CONFIG_ADB=y
 CONFIG_MAC_NVRAM=y
diff --git a/device_tree.c b/device_tree.c
index a923613..56af24b 100644
--- a/device_tree.c
+++ b/device_tree.c
@@ -20,10 +20,10 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "hw/loader.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 
 #include <libfdt.h>
 
diff --git a/disas.c b/disas.c
index 6da1dd0..a46faee 100644
--- a/disas.c
+++ b/disas.c
@@ -1,11 +1,11 @@
 /* General "disassemble this chunk" code.  Used for debugging. */
 #include "config.h"
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #include "elf.h"
 #include <errno.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 
 typedef struct CPUDebug {
     struct disassemble_info info;
@@ -374,7 +374,7 @@ const char *lookup_symbol(target_ulong orig_addr)
 
 #if !defined(CONFIG_USER_ONLY)
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 static int monitor_disas_is_physical;
 
diff --git a/disas/Makefile.objs b/disas/Makefile.objs
new file mode 100644
index 0000000..ed75f9a
--- /dev/null
+++ b/disas/Makefile.objs
@@ -0,0 +1,18 @@
+common-obj-$(CONFIG_ALPHA_DIS) += alpha.o
+common-obj-$(CONFIG_ARM_DIS) += arm.o
+common-obj-$(CONFIG_CRIS_DIS) += cris.o
+common-obj-$(CONFIG_HPPA_DIS) += hppa.o
+common-obj-$(CONFIG_I386_DIS) += i386.o
+common-obj-$(CONFIG_IA64_DIS) += ia64.o
+common-obj-$(CONFIG_M68K_DIS) += m68k.o
+common-obj-$(CONFIG_MICROBLAZE_DIS) += microblaze.o
+common-obj-$(CONFIG_MIPS_DIS) += mips.o
+common-obj-$(CONFIG_PPC_DIS) += ppc.o
+common-obj-$(CONFIG_S390_DIS) += s390.o
+common-obj-$(CONFIG_SH4_DIS) += sh4.o
+common-obj-$(CONFIG_SPARC_DIS) += sparc.o
+common-obj-$(CONFIG_LM32_DIS) += lm32.o
+
+# TODO: As long as the TCG interpreter and its generated code depend
+# on the QEMU target, we cannot compile the disassembler here.
+#common-obj-$(CONFIG_TCI_DIS) += tci.o
diff --git a/alpha-dis.c b/disas/alpha.c
index ae331b3..a950b9c 100644
--- a/alpha-dis.c
+++ b/disas/alpha.c
@@ -20,7 +20,7 @@ along with this file; see the file COPYING.  If not, see
 <http://www.gnu.org/licenses/>. */
 
 #include <stdio.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* MAX is redefined below, so remove any previous definition. */
 #undef MAX
diff --git a/arm-dis.c b/disas/arm.c
index 6bc4d71..4927d8a 100644
--- a/arm-dis.c
+++ b/disas/arm.c
@@ -22,7 +22,7 @@
 /* Start of qemu specific additions.  Mostly this is stub definitions
    for things we don't care about.  */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #define ATTRIBUTE_UNUSED __attribute__((unused))
 #define ISSPACE(x) ((x) == ' ' || (x) == '\t' || (x) == '\n')
 
diff --git a/cris-dis.c b/disas/cris.c
index 1d174ba..9dfb4e3 100644
--- a/cris-dis.c
+++ b/disas/cris.c
@@ -19,7 +19,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
 #include "qemu-common.h"
-#include "dis-asm.h"
+#include "disas/bfd.h"
 //#include "sysdep.h"
 #include "target-cris/opcode-cris.h"
 //#include "libiberty.h"
diff --git a/hppa-dis.c b/disas/hppa.c
index 420a7d2..c7c8be6 100644
--- a/hppa-dis.c
+++ b/disas/hppa.c
@@ -18,7 +18,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* HP PA-RISC SOM object file format:  definitions internal to BFD.
    Copyright 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000,
diff --git a/i386-dis.c b/disas/i386.c
index c4a81c9..3b006b1 100644
--- a/i386-dis.c
+++ b/disas/i386.c
@@ -32,7 +32,7 @@
    the Intel manual for details.  */
 
 #include <stdlib.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 /* include/opcode/i386.h r1.78 */
 
 /* opcode/i386.h -- Intel 80386 opcode macros
diff --git a/ia64-dis.c b/disas/ia64.c
index 2a103e6..a8fe26c 100644
--- a/ia64-dis.c
+++ b/disas/ia64.c
@@ -21,7 +21,7 @@
 #include <assert.h>
 #include <string.h>
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* ia64.h -- Header file for ia64 opcode table
    Copyright (C) 1998, 1999, 2000, 2002, 2005, 2006
diff --git a/lm32-dis.c b/disas/lm32.c
index 709ed32..a8eefe0 100644
--- a/lm32-dis.c
+++ b/disas/lm32.c
@@ -19,7 +19,7 @@
  */
 
 #include <stdio.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 typedef enum {
     LM32_OP_SRUI = 0, LM32_OP_NORI, LM32_OP_MULI, LM32_OP_SH, LM32_OP_LB,
diff --git a/m68k-dis.c b/disas/m68k.c
index 2b155de..c950241 100644
--- a/m68k-dis.c
+++ b/disas/m68k.c
@@ -5,7 +5,7 @@
 #include <math.h>
 #include <stdio.h>
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* **** floatformat.h from sourceware.org CVS 2005-08-14.  */
 /* IEEE floating point support declarations, for GDB, the GNU Debugger.
diff --git a/microblaze-dis.c b/disas/microblaze.c
index 16c312f..ec91af3 100644
--- a/microblaze-dis.c
+++ b/disas/microblaze.c
@@ -582,7 +582,7 @@ char pvr_register_prefix[] = "rpvr";
 
 #endif /* MICROBLAZE_OPC */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #include <strings.h>
 
 #define get_field_rd(instr) get_field(instr, RD_MASK, RD_LOW)
diff --git a/mips-dis.c b/disas/mips.c
index e3a6e0b..2106b57 100644
--- a/mips-dis.c
+++ b/disas/mips.c
@@ -19,7 +19,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* mips.h.  Mips opcode list for GDB, the GNU debugger.
    Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003
diff --git a/ppc-dis.c b/disas/ppc.c
index bc98cbe..c149506 100644
--- a/ppc-dis.c
+++ b/disas/ppc.c
@@ -18,7 +18,7 @@ the GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this file; see the file COPYING.  If not,
 see <http://www.gnu.org/licenses/>.  */
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #define BFD_DEFAULT_TARGET_SIZE 64
 
 /* ppc.h -- Header file for PowerPC opcode table
diff --git a/s390-dis.c b/disas/s390.c
index 8abcdf0..25499ba 100644
--- a/s390-dis.c
+++ b/disas/s390.c
@@ -21,7 +21,7 @@
    02110-1301, USA.  */
 
 #include "qemu-common.h"
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* include/opcode/s390.h revision 1.9 */
 /* s390.h -- Header file for S390 opcode table
@@ -589,6 +589,16 @@ static const struct s390_operand s390_operands[] =
   { 4, 32, S390_OPERAND_CCODE },
 #define I8_32 46                  /* 8 bit signed value starting at 32 */
   { 8, 32, S390_OPERAND_SIGNED },
+#define U8_24 47                  /* 8 bit unsigned value starting at 24 */
+  { 8, 24, 0 },
+#define U8_32 48                  /* 8 bit unsigned value starting at 32 */
+  { 8, 32, 0 },
+#define I16_32 49
+  { 16, 32, S390_OPERAND_SIGNED },
+#define M4_16 50                  /* 4-bit condition-code starting at 12 */
+  { 4, 16, S390_OPERAND_CCODE },
+#define I8_16 51
+  { 8, 16, S390_OPERAND_SIGNED },
 /* QEMU-END */
 };
 
@@ -663,7 +673,9 @@ static const struct s390_operand s390_operands[] =
    This is just a workaround for existing code e.g. glibc.  */
 #define INSTR_RRE_RR_OPT 4, { R_24,RO_28,0,0,0,0 }             /* efpc, sfpc */
 #define INSTR_RRF_F0FF   4, { F_16,F_24,F_28,0,0,0 }           /* e.g. madbr */
-#define INSTR_RRF_F0FF2  4, { F_24,F_16,F_28,0,0,0 }           /* e.g. cpsdr */
+/* QEMU-MOD */
+#define INSTR_RRF_F0FF2  4, { F_24,F_28,F_16,0,0,0 }           /* e.g. cpsdr */
+/* QEMU-END */
 #define INSTR_RRF_F0FR   4, { F_24,F_16,R_28,0,0,0 }           /* e.g. iedtr */
 #define INSTR_RRF_FUFF   4, { F_24,F_16,F_28,U4_20,0,0 }       /* e.g. didbr */
 #define INSTR_RRF_RURR   4, { R_24,R_28,R_16,U4_20,0,0 }       /* e.g. .insn */
@@ -801,11 +813,35 @@ static const struct s390_operand s390_operands[] =
 #define MASK_SSF_RRDRD   { 0xff, 0x0f, 0x00, 0x00, 0x00, 0x00 }
 
 /* QEMU-ADD: */
-#define INSTR_RIE_MRRP   6, { M4_32,R_8,R_12,J16_16,0,0 }	/* e.g. crj */
+#define INSTR_RIE_MRRP   6, { M4_32, R_8, R_12, J16_16, 0, 0 } /* e.g. crj */
 #define MASK_RIE_MRRP    { 0xff, 0x00, 0x00, 0x00, 0x0f, 0xff }
 
-#define INSTR_RIE_MRIP   6, { M4_12,R_8,I8_32,J16_16,0,0 }      /* e.g. cij */
+#define INSTR_RIE_MRIP   6, { M4_12, R_8, I8_32, J16_16, 0, 0 } /* e.g. cij */
 #define MASK_RIE_MRIP    { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_RIE_RRIII  6, { R_8, R_12, U8_16, U8_24, U8_32, 0 } /* risbg */
+#define MASK_RIE_RRIII   { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+#define INSTR_RIE_MRI    6, { M4_32, R_8, I16_16, 0, 0, 0 }    /* e.g. cit */
+#define MASK_RIE_MRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+#define INSTR_RIE_MRU    6, { M4_32, R_8, U16_16, 0, 0, 0 }    /* e.g. clfit */
+#define MASK_RIE_MRU     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+#define INSTR_RIE_RRI    6, { R_8, R_12, I16_16, 0, 0, 0 }
+#define MASK_RIE_RRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_RXY_URRD   6, { U8_8, D20_20, X_12, B_16, 0, 0 }
+#define MASK_RXY_URRD    { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_SIL_DRI    6, { D_20, B_16, I16_32, 0, 0, 0 }
+#define MASK_SIL_DRI     { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }
+
+#define INSTR_RSY_MRRD   6, { M4_12, R_8, D20_20, B_16, 0, 0 }
+#define MASK_SRY_MRRD    { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
+
+#define INSTR_RRF_MRR    6, { M4_16, R_24, R_28, 0, 0, 0 }
+#define MASK_RRF_MRR     { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }
+
+#define INSTR_SIY_DRI    6, { D20_20, B_16, I8_16, 0, 0, 0 }
+#define MASK_SIY_DRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
 /* QEMU-END */
 
 /* The opcode formats table (blueprints for .insn pseudo mnemonic).  */
@@ -926,6 +962,30 @@ static const struct s390_opcode s390_opcodes[] =
   { "ldeb", OP48(0xed0000000004LL), MASK_RXE_FRRD, INSTR_RXE_FRRD, 3, 0},
   { "brxlg", OP48(0xec0000000045LL), MASK_RIE_RRP, INSTR_RIE_RRP, 2, 2},
   { "brxhg", OP48(0xec0000000044LL), MASK_RIE_RRP, INSTR_RIE_RRP, 2, 2},
+/* QEMU-ADD: */
+  { "crj",   OP48(0xec0000000076LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "cgrj",  OP48(0xec0000000064LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "clrj",  OP48(0xec0000000077LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "clgrj", OP48(0xec0000000065LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
+  { "cij",   OP48(0xec000000007eLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "cgij",  OP48(0xec000000007cLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "clij",  OP48(0xec000000007fLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "clgij", OP48(0xec000000007dLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
+  { "risbg", OP48(0xec0000000055LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "risbhg", OP48(0xec000000005dLL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "risblg", OP48(0xec0000000051LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "rnsbg", OP48(0xec0000000054LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "rosbg", OP48(0xec0000000056LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "rxsbg", OP48(0xec0000000057LL), MASK_RIE_RRIII, INSTR_RIE_RRIII, 3, 6},
+  { "cit", OP48(0xec0000000072LL), MASK_RIE_MRI, INSTR_RIE_MRI, 3, 6},
+  { "cgit", OP48(0xec0000000070LL), MASK_RIE_MRI, INSTR_RIE_MRI, 3, 6},
+  { "clfit", OP48(0xec0000000073LL), MASK_RIE_MRU, INSTR_RIE_MRU, 3, 6},
+  { "clgit", OP48(0xec0000000071LL), MASK_RIE_MRU, INSTR_RIE_MRU, 3, 6},
+  { "ahik", OP48(0xec00000000d8LL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+  { "aghik", OP48(0xec00000000d9LL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+  { "alhsik", OP48(0xec00000000daLL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+  { "alghsik", OP48(0xec00000000dbLL), MASK_RIE_RRI, INSTR_RIE_RRI, 3, 6},
+/* QEMU-END */
   { "tp", OP48(0xeb00000000c0LL), MASK_RSL_R0RD, INSTR_RSL_R0RD, 3, 0},
   { "stamy", OP48(0xeb000000009bLL), MASK_RSY_AARD, INSTR_RSY_AARD, 2, 3},
   { "lamy", OP48(0xeb000000009aLL), MASK_RSY_AARD, INSTR_RSY_AARD, 2, 3},
@@ -985,6 +1045,20 @@ static const struct s390_opcode s390_opcodes[] =
   { "srag", OP48(0xeb000000000aLL), MASK_RSE_RRRD, INSTR_RSE_RRRD, 2, 2},
   { "lmg", OP48(0xeb0000000004LL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 2, 3},
   { "lmg", OP48(0xeb0000000004LL), MASK_RSE_RRRD, INSTR_RSE_RRRD, 2, 2},
+/* QEMU-ADD: */
+  { "loc", OP48(0xeb00000000f2LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "locg", OP48(0xeb00000000e2LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "stoc", OP48(0xeb00000000f3LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "stocg", OP48(0xeb00000000e3LL), MASK_SRY_MRRD, INSTR_RSY_MRRD, 3, 6},
+  { "srak", OP48(0xeb00000000dcLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "slak", OP48(0xeb00000000ddLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "srlk", OP48(0xeb00000000deLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "sllk", OP48(0xeb00000000dfLL), MASK_RSY_RRRD, INSTR_RSY_RRRD, 3, 6},
+  { "asi", OP48(0xeb000000006aLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+  { "alsi", OP48(0xeb000000006eLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+  { "agsi", OP48(0xeb000000007aLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+  { "algsi", OP48(0xeb000000007eLL), MASK_SIY_DRI, INSTR_SIY_DRI, 3, 6},
+/* QEMU-END */
   { "unpka", OP8(0xeaLL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
   { "pka", OP8(0xe9LL), MASK_SS_L2RDRD, INSTR_SS_L2RDRD, 3, 0},
   { "mvcin", OP8(0xe8LL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
@@ -993,6 +1067,17 @@ static const struct s390_opcode s390_opcodes[] =
   { "tprot", OP16(0xe501LL), MASK_SSE_RDRD, INSTR_SSE_RDRD, 3, 0},
   { "strag", OP48(0xe50000000002LL), MASK_SSE_RDRD, INSTR_SSE_RDRD, 2, 2},
   { "lasp", OP16(0xe500LL), MASK_SSE_RDRD, INSTR_SSE_RDRD, 3, 0},
+/* QEMU-ADD: */
+  { "mvhhi", OP16(0xe544LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "mvghi", OP16(0xe548LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "mvhi", OP16(0xe54cLL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "chhsi", OP16(0xe554LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "clhhsi", OP16(0xe555LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "cghsi", OP16(0xe558LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "clghsi", OP16(0xe559LL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "chsi", OP16(0xe55cLL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+  { "clfhsi", OP16(0xe55dLL), MASK_SIL_DRI, INSTR_SIL_DRI, 3, 6},
+/* QEMU-END */
   { "slb", OP48(0xe30000000099LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 3, 3},
   { "slb", OP48(0xe30000000099LL), MASK_RXE_RRRD, INSTR_RXE_RRRD, 3, 2},
   { "alc", OP48(0xe30000000098LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 3, 3},
@@ -1116,6 +1201,9 @@ static const struct s390_opcode s390_opcodes[] =
   { "lrag", OP48(0xe30000000003LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 2, 3},
   { "lrag", OP48(0xe30000000003LL), MASK_RXE_RRRD, INSTR_RXE_RRRD, 2, 2},
   { "ltg", OP48(0xe30000000002LL), MASK_RXY_RRRD, INSTR_RXY_RRRD, 2, 4},
+/* QEMU-ADD: */
+  { "pfd", OP48(0xe30000000036LL), MASK_RXY_URRD, INSTR_RXY_URRD, 3, 6},
+/* QEMU-END */
   { "unpku", OP8(0xe2LL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
   { "pku", OP8(0xe1LL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
   { "edmk", OP8(0xdfLL), MASK_SS_L0RDRD, INSTR_SS_L0RDRD, 3, 0},
@@ -1135,6 +1223,32 @@ static const struct s390_opcode s390_opcodes[] =
   { "csst", OP16(0xc802LL), MASK_SSF_RRDRD, INSTR_SSF_RRDRD, 2, 5},
   { "ectg", OP16(0xc801LL), MASK_SSF_RRDRD, INSTR_SSF_RRDRD, 2, 5},
   { "mvcos", OP16(0xc800LL), MASK_SSF_RRDRD, INSTR_SSF_RRDRD, 2, 4},
+/* QEMU-ADD: */
+  { "exrl", OP16(0xc600ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "pfdrl", OP16(0xc602ll), MASK_RIL_UP, INSTR_RIL_UP, 3, 6},
+  { "cghrl", OP16(0xc604ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "chrl", OP16(0xc605ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clghrl", OP16(0xc606ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clhrl", OP16(0xc607ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "cgrl", OP16(0xc608ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clgrl", OP16(0xc60all), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "cgfrl", OP16(0xc60cll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "crl", OP16(0xc60dll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clgfrl", OP16(0xc60ell), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "clrl", OP16(0xc60fll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+
+  { "llhrl", OP16(0xc400ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lghrl", OP16(0xc404ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lhrl", OP16(0xc405ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "llghrl", OP16(0xc406ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "sthrl", OP16(0xc407ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lgrl", OP16(0xc408ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "stgrl", OP16(0xc40bll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lgfrl", OP16(0xc40cll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "lrl", OP16(0xc40dll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "llgfrl", OP16(0xc40ell), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+  { "strl", OP16(0xc40fll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
+/* QEMU-END */
   { "clfi", OP16(0xc20fLL), MASK_RIL_RU, INSTR_RIL_RU, 2, 4},
   { "clgfi", OP16(0xc20eLL), MASK_RIL_RU, INSTR_RIL_RU, 2, 4},
   { "cfi", OP16(0xc20dLL), MASK_RIL_RI, INSTR_RIL_RI, 2, 4},
@@ -1265,6 +1379,29 @@ static const struct s390_opcode s390_opcodes[] =
   { "ltgr", OP16(0xb902LL), MASK_RRE_RR, INSTR_RRE_RR, 2, 2},
   { "lngr", OP16(0xb901LL), MASK_RRE_RR, INSTR_RRE_RR, 2, 2},
   { "lpgr", OP16(0xb900LL), MASK_RRE_RR, INSTR_RRE_RR, 2, 2},
+/* QEMU-ADD: */
+  { "crt", OP16(0xb972LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "cgrt", OP16(0xb960LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "clrt", OP16(0xb973LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "clgrt", OP16(0xb961LL), MASK_RRF_M0RR, INSTR_RRF_M0RR, 3, 6},
+  { "locr", OP16(0xb9f2LL), MASK_RRF_MRR, INSTR_RRF_MRR, 3, 6},
+  { "locgr", OP16(0xb9e2LL), MASK_RRF_MRR, INSTR_RRF_MRR, 3, 6},
+  { "popcnt", OP16(0xb9e1LL), MASK_RRE_RR, INSTR_RRE_RR, 3, 6},
+  { "ngrk", OP16(0xb9e4LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "ogrk", OP16(0xb9e6LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "xgrk", OP16(0xb9e7LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "agrk", OP16(0xb9e8LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "sgrk", OP16(0xb9e9LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "algrk", OP16(0xb9eaLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "slgrk", OP16(0xb9ebLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "nrk", OP16(0xb9f4LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "ork", OP16(0xb9f6LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "xrk", OP16(0xb9f7LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "ark", OP16(0xb9f8LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "srk", OP16(0xb9f9LL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "alrk", OP16(0xb9faLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+  { "slrk", OP16(0xb9fbLL), MASK_RRF_R0RR, INSTR_RRF_R0RR, 3, 6},
+/* QEMU-END */
   { "lctl", OP8(0xb7LL), MASK_RS_CCRD, INSTR_RS_CCRD, 3, 0},
   { "stctl", OP8(0xb6LL), MASK_RS_CCRD, INSTR_RS_CCRD, 3, 0},
   { "rrxtr", OP16(0xb3ffLL), MASK_RRF_FFFU, INSTR_RRF_FFFU, 2, 5},
@@ -1426,6 +1563,20 @@ static const struct s390_opcode s390_opcodes[] =
   { "ltebr", OP16(0xb302LL), MASK_RRE_FF, INSTR_RRE_FF, 3, 0},
   { "lnebr", OP16(0xb301LL), MASK_RRE_FF, INSTR_RRE_FF, 3, 0},
   { "lpebr", OP16(0xb300LL), MASK_RRE_FF, INSTR_RRE_FF, 3, 0},
+/* QEMU-ADD: */
+  { "clfebr", OP16(0xb39cLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clfdbr", OP16(0xb39dLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clfxbr", OP16(0xb39eLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clgebr", OP16(0xb3acLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clgdbr", OP16(0xb3adLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "clgxbr", OP16(0xb3aeLL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "celfbr", OP16(0xb390LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cdlfbr", OP16(0xb391LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cxlfbr", OP16(0xb392LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "celgbr", OP16(0xb3a0LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cdlgbr", OP16(0xb3a1LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+  { "cxlgbr", OP16(0xb3a2LL), MASK_RRF_UUFF, INSTR_RRF_UUFF, 3, 6},
+/* QEMU-END */
   { "trap4", OP16(0xb2ffLL), MASK_S_RD, INSTR_S_RD, 3, 0},
   { "lfas", OP16(0xb2bdLL), MASK_S_RD, INSTR_S_RD, 2, 5},
   { "srnmt", OP16(0xb2b9LL), MASK_S_RD, INSTR_S_RD, 2, 5},
@@ -1774,22 +1925,6 @@ static const struct s390_opcode s390_opcodes[] =
   { "sckpf", OP16(0x0107LL), MASK_E, INSTR_E, 3, 0},
   { "upt", OP16(0x0102LL), MASK_E, INSTR_E, 3, 0},
   { "pr", OP16(0x0101LL), MASK_E, INSTR_E, 3, 0},
-
-/* QEMU-ADD: */
-  { "crj",   OP48(0xec0000000076LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-  { "cgrj",  OP48(0xec0000000064LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-  { "clrj",  OP48(0xec0000000077LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-  { "clgrj", OP48(0xec0000000065LL), MASK_RIE_MRRP, INSTR_RIE_MRRP, 3, 6},
-
-  { "cij",   OP48(0xec000000007eLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-  { "cgij",  OP48(0xec000000007cLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-  { "clij",  OP48(0xec000000007fLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-  { "clgij", OP48(0xec000000007dLL), MASK_RIE_MRIP, INSTR_RIE_MRIP, 3, 6},
-
-  { "lrl",   OP16(0xc40dll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
-  { "lgrl",  OP16(0xc408ll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
-  { "lgfrl", OP16(0xc40cll), MASK_RIL_RP, INSTR_RIL_RP, 3, 6},
-/* QEMU-END */
 };
 
 static const int s390_num_opcodes =
diff --git a/sh4-dis.c b/disas/sh4.c
index 673bc78..f6cadd5 100644
--- a/sh4-dis.c
+++ b/disas/sh4.c
@@ -16,7 +16,7 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
 #include <stdio.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 #define DEFINE_TABLE
 
diff --git a/sparc-dis.c b/disas/sparc.c
index 1d017fa..8eb22e6 100644
--- a/sparc-dis.c
+++ b/disas/sparc.c
@@ -27,7 +27,7 @@
    see <http://www.gnu.org/licenses/>.  */
 
 #include <stdlib.h>
-#include "dis-asm.h"
+#include "disas/bfd.h"
 
 /* The SPARC opcode table (and other related data) is defined in
    the opcodes library in sparc-opc.c.  If you change anything here, make
diff --git a/tci-dis.c b/disas/tci.c
index 10c411b..a606b63 100644
--- a/tci-dis.c
+++ b/disas/tci.c
@@ -17,7 +17,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dis-asm.h"
+#include "disas/bfd.h"
 #include "tcg/tcg.h"
 
 /* Disassemble TCI bytecode. */
diff --git a/dma-helpers.c b/dma-helpers.c
index 4f5fb64..272632f 100644
--- a/dma-helpers.c
+++ b/dma-helpers.c
@@ -7,10 +7,10 @@
  * (GNU GPL), version 2 or later.
  */
 
-#include "dma.h"
+#include "sysemu/dma.h"
 #include "trace.h"
-#include "range.h"
-#include "qemu-thread.h"
+#include "qemu/range.h"
+#include "qemu/thread.h"
 
 /* #define DEBUG_IOMMU */
 
diff --git a/docs/q35-chipset.cfg b/docs/q35-chipset.cfg
new file mode 100644
index 0000000..1b6efc0
--- /dev/null
+++ b/docs/q35-chipset.cfg
@@ -0,0 +1,129 @@
+################################################################
+#
+# qemu -M q35 creates a bare machine with just the very essential
+# chipset devices being present:
+#
+#     00.0 - Host bridge
+#     1f.0 - ISA bridge / LPC
+#     1f.2 - SATA (AHCI) controller
+#     1f.3 - SMBus controller
+#
+# This config file documents the other devices and how they are
+# created.  You can simply use "-readconfig $thisfile" to create
+# them all.  Here is a overview:
+#
+#     19.0 - Ethernet controller (not created, our e1000 emulation
+#                                 doesn't emulate the ich9 device).
+#     1a.* - USB Controller #2 (ehci + uhci companions)
+#     1b.0 - HD Audio Controller
+#     1c.* - PCI Express Ports
+#     1d.* - USB Controller #1 (ehci + uhci companions,
+#                               "qemu -M q35 -usb" creates these too)
+#     1e.0 - PCI Bridge
+#
+
+[device "ich9-ehci-2"]
+  driver = "ich9-usb-ehci2"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.7"
+
+[device "ich9-uhci-4"]
+  driver = "ich9-usb-uhci4"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.0"
+  masterbus = "ich9-ehci-2.0"
+  firstport = "0"
+
+[device "ich9-uhci-5"]
+  driver = "ich9-usb-uhci5"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.1"
+  masterbus = "ich9-ehci-2.0"
+  firstport = "2"
+
+[device "ich9-uhci-6"]
+  driver = "ich9-usb-uhci6"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1a.2"
+  masterbus = "ich9-ehci-2.0"
+  firstport = "4"
+
+
+[device "ich9-hda-audio"]
+  driver = "ich9-intel-hda"
+  bus = "pcie.0"
+  addr = "1b.0"
+
+
+[device "ich9-pcie-port-1"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.0"
+  port = "1"
+  chassis = "1"
+
+[device "ich9-pcie-port-2"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.1"
+  port = "2"
+  chassis = "2"
+
+[device "ich9-pcie-port-3"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.2"
+  port = "3"
+  chassis = "3"
+
+[device "ich9-pcie-port-4"]
+  driver = "ioh3420"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1c.3"
+  port = "4"
+  chassis = "4"
+
+
+[device "ich9-ehci-1"]
+  driver = "ich9-usb-ehci1"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.7"
+
+[device "ich9-uhci-1"]
+  driver = "ich9-usb-uhci1"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.0"
+  masterbus = "ich9-ehci-1.0"
+  firstport = "0"
+
+[device "ich9-uhci-2"]
+  driver = "ich9-usb-uhci2"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.1"
+  masterbus = "ich9-ehci-1.0"
+  firstport = "2"
+
+[device "ich9-uhci-3"]
+  driver = "ich9-usb-uhci3"
+  multifunction = "on"
+  bus = "pcie.0"
+  addr = "1d.2"
+  masterbus = "ich9-ehci-1.0"
+  firstport = "4"
+
+
+[device "ich9-pci-bridge"]
+  driver = "i82801b11-bridge"
+  bus = "pcie.0"
+  addr = "1e.0"
diff --git a/docs/specs/pci-ids.txt b/docs/specs/pci-ids.txt
new file mode 100644
index 0000000..3c65e1a
--- /dev/null
+++ b/docs/specs/pci-ids.txt
@@ -0,0 +1,50 @@
+
+PCI IDs for qemu
+================
+
+Red Hat, Inc. donates a part of its device ID range to qemu, to be used for
+virtual devices.  The vendor IDs are 1af4 (formerly Qumranet ID) and 1b36.
+
+Contact Gerd Hoffmann <kraxel@redhat.com> to get a device ID assigned
+for your devices.
+
+1af4 vendor ID
+--------------
+
+The 1000 -> 10ff device ID range is used as follows for virtio-pci devices.
+Note that this allocation separate from the virtio device IDs, which are
+maintained as part of the virtio specification.
+
+1af4:1000  network device
+1af4:1001  block device
+1af4:1002  balloon device
+1af4:1003  console device
+1af4:1004  SCSI host bus adapter device
+1af4:1005  entropy generator device
+1af4:1009  9p filesystem device
+
+1af4:10f0  Available for experimental usage without registration.  Must get
+   to      official ID when the code leaves the test lab (i.e. when seeking
+1af4:10ff  upstream merge or shipping a distro/product) to avoid conflicts.
+
+1af4:1100  Used as PCI Subsystem ID for existing hardware devices emulated
+           by qemu.
+
+1af4:1110  ivshmem device (shared memory, docs/specs/ivshmem_device_spec.txt)
+
+All other device IDs are reserved.
+
+1b36 vendor ID
+--------------
+
+The 0000 -> 00ff device ID range is used as follows for QEMU-specific
+PCI devices (other than virtio):
+
+1b36:0001  PCI-PCI bridge
+1b36:0002  PCI serial port (16550A) adapter (docs/specs/pci-serial.txt)
+1b36:0003  PCI Dual-port 16550A adapter (docs/specs/pci-serial.txt)
+1b36:0004  PCI Quad-port 16550A adapter (docs/specs/pci-serial.txt)
+
+All these devices are documented in docs/specs.
+
+The 0100 device ID is used for the QXL video card device.
diff --git a/docs/spice-port-fqdn.txt b/docs/spice-port-fqdn.txt
new file mode 100644
index 0000000..5077895
--- /dev/null
+++ b/docs/spice-port-fqdn.txt
@@ -0,0 +1,19 @@
+A Spice port channel is an arbitrary communication between the Spice
+server host side and the client side.
+
+Thanks to the associated reverse fully qualified domain name (fqdn),
+a Spice client can handle the various ports appropriately.
+
+The following fqdn names are reserved by the QEMU project:
+
+org.qemu.monitor.hmp.0
+  QEMU human monitor
+
+org.qemu.monitor.qmp.0:
+  QEMU control monitor
+
+org.qemu.console.serial.0
+  QEMU virtual serial port
+
+org.qemu.console.debug.0
+  QEMU debug console
diff --git a/docs/tracing.txt b/docs/tracing.txt
index 453cc4a..14db3bf 100644
--- a/docs/tracing.txt
+++ b/docs/tracing.txt
@@ -23,7 +23,7 @@ for debugging, profiling, and observing execution.
 
 4. Pretty-print the binary trace file:
 
-    ./simpletrace.py trace-events trace-*
+    ./scripts/simpletrace.py trace-events trace-*
 
 == Trace events ==
 
@@ -198,7 +198,7 @@ The "simple" backend produces binary trace files that can be formatted with the
 simpletrace.py script.  The script takes the "trace-events" file and the binary
 trace:
 
-    ./simpletrace.py trace-events trace-12345
+    ./scripts/simpletrace.py trace-events trace-12345
 
 You must ensure that the same "trace-events" file was used to build QEMU,
 otherwise trace event declarations may have changed and output will not be
diff --git a/docs/usb-storage.txt b/docs/usb-storage.txt
index e58e849..fa93111 100644
--- a/docs/usb-storage.txt
+++ b/docs/usb-storage.txt
@@ -2,7 +2,7 @@
 qemu usb storage emulation
 --------------------------
 
-QEMU has two emulations for usb storage devices.
+QEMU has three devices for usb storage emulation.
 
 Number one emulates the classic bulk-only transport protocol which is
 used by 99% of the usb sticks on the marked today and is called
@@ -31,6 +31,15 @@ with tree logical units:
        -device scsi-cd,bus=uas.0,scsi-id=0,lun=5,drive=uas-cdrom
 
 
+Number three emulates the classic bulk-only transport protocol too.
+It's called "usb-bot".  It shares most code with "usb-storage", and
+the guest will not be able to see the difference.  The qemu command
+line interface is simliar to usb-uas though, i.e. no automatic scsi
+disk creation.  It also features support for up to 16 LUNs.  The LUN
+numbers must be continous, i.e. for three devices you must use 0+1+2.
+The 0+1+5 numbering from the "usb-uas" example isn't going to work
+with "usb-bot".
+
 enjoy,
   Gerd
 
diff --git a/docs/virtio-balloon-stats.txt b/docs/virtio-balloon-stats.txt
new file mode 100644
index 0000000..f74612f
--- /dev/null
+++ b/docs/virtio-balloon-stats.txt
@@ -0,0 +1,104 @@
+virtio balloon memory statistics
+================================
+
+The virtio balloon driver supports guest memory statistics reporting. These
+statistics are available to QEMU users as QOM (QEMU Object Model) device
+properties via a polling mechanism.
+
+Before querying the available stats, clients first have to enable polling.
+This is done by writing a time interval value (in seconds) to the
+guest-stats-polling-interval property. This value can be:
+
+  > 0  enables polling in the specified interval. If polling is already
+       enabled, the polling time interval is changed to the new value
+
+  0    disables polling. Previous polled statistics are still valid and
+       can be queried.
+
+Once polling is enabled, the virtio-balloon device in QEMU will start
+polling the guest's balloon driver for new stats in the specified time
+interval.
+
+To retrieve those stats, clients have to query the guest-stats property,
+which will return a dictionary containing:
+
+  o A key named 'stats', containing all available stats. If the guest
+    doesn't support a particular stat, or if it couldn't be retrieved,
+    its value will be -1. Currently, the following stats are supported:
+
+      - stat-swap-in
+      - stat-swap-out
+      - stat-major-faults
+      - stat-minor-faults
+      - stat-free-memory
+      - stat-total-memory
+
+  o A key named last-update, which contains the last stats update
+    timestamp in seconds. Since this timestamp is generated by the host,
+    a buggy guest can't influence its value
+
+It's also important to note the following:
+
+ - Previously polled statistics remain available even if the polling is
+   later disabled
+
+ - As noted above, if a guest doesn't support a particular stat its value
+   will always be -1. However, it's also possible that a guest temporarily
+   couldn't update one or even all stats. If this happens, just wait for
+   the next update
+
+ - Polling can be enabled even if the guest doesn't have stats support
+   or the balloon driver wasn't loaded in the guest. If this is the case
+   and stats are queried, an error will be returned
+
+ - The polling timer is only re-armed when the guest responds to the
+   statistics request. This means that if a (buggy) guest doesn't ever
+   respond to the request the timer will never be re-armed, which has
+   the same effect as disabling polling
+
+Here are a few examples. QEMU is started with '-balloon virtio', which
+generates '/machine/peripheral-anon/device[1]' as the QOM path for the
+balloon device.
+
+Enable polling with 2 seconds interval:
+
+{ "execute": "qom-set",
+             "arguments": { "path": "/machine/peripheral-anon/device[1]",
+			 "property": "guest-stats-polling-interval", "value": 2 } }
+
+{ "return": {} }
+
+Change polling to 10 seconds:
+
+{ "execute": "qom-set",
+             "arguments": { "path": "/machine/peripheral-anon/device[1]",
+			 "property": "guest-stats-polling-interval", "value": 10 } }
+
+{ "return": {} }
+
+Get stats:
+
+{ "execute": "qom-get",
+  "arguments": { "path": "/machine/peripheral-anon/device[1]",
+  "property": "guest-stats" } }
+{
+    "return": {
+        "stats": {
+            "stat-swap-out": 0,
+            "stat-free-memory": 844943360,
+            "stat-minor-faults": 219028,
+            "stat-major-faults": 235,
+            "stat-total-memory": 1044406272,
+            "stat-swap-in": 0
+        },
+        "last-update": 1358529861
+    }
+}
+
+Disable polling:
+
+{ "execute": "qom-set",
+             "arguments": { "path": "/machine/peripheral-anon/device[1]",
+			 "property": "stats-polling-interval", "value": 0 } }
+
+{ "return": {} }
diff --git a/dump-stub.c b/dump-stub.c
index 56d4564..a9d0b3c 100644
--- a/dump-stub.c
+++ b/dump-stub.c
@@ -12,8 +12,8 @@
  */
 
 #include "qemu-common.h"
-#include "dump.h"
-#include "qerror.h"
+#include "sysemu/dump.h"
+#include "qapi/qmp/qerror.h"
 #include "qmp-commands.h"
 
 /* we need this function in hmp.c */
diff --git a/dump.c b/dump.c
index 5640c2c..4ed1fa8 100644
--- a/dump.c
+++ b/dump.c
@@ -14,16 +14,16 @@
 #include "qemu-common.h"
 #include "elf.h"
 #include "cpu.h"
-#include "cpu-all.h"
-#include "hwaddr.h"
-#include "monitor.h"
-#include "kvm.h"
-#include "dump.h"
-#include "sysemu.h"
-#include "memory_mapping.h"
-#include "error.h"
+#include "exec/cpu-all.h"
+#include "exec/hwaddr.h"
+#include "monitor/monitor.h"
+#include "sysemu/kvm.h"
+#include "sysemu/dump.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/memory_mapping.h"
+#include "qapi/error.h"
 #include "qmp-commands.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 
 static uint16_t cpu_convert_to_target16(uint16_t val, int endian)
 {
@@ -427,7 +427,7 @@ static hwaddr get_offset(hwaddr phys_addr,
         }
     }
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (s->has_filter) {
             if (block->offset >= s->begin + s->length ||
                 block->offset + block->length <= s->begin) {
@@ -594,7 +594,7 @@ static int dump_completed(DumpState *s)
 static int get_next_block(DumpState *s, RAMBlock *block)
 {
     while (1) {
-        block = QLIST_NEXT(block, next);
+        block = QTAILQ_NEXT(block, next);
         if (!block) {
             /* no more block */
             return 1;
@@ -670,11 +670,11 @@ static ram_addr_t get_start_block(DumpState *s)
     RAMBlock *block;
 
     if (!s->has_filter) {
-        s->block = QLIST_FIRST(&ram_list.blocks);
+        s->block = QTAILQ_FIRST(&ram_list.blocks);
         return 0;
     }
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (block->offset >= s->begin + s->length ||
             block->offset + block->length <= s->begin) {
             /* This block is out of the range */
diff --git a/exec.c b/exec.c
index 8435de0..b85508b 100644
--- a/exec.c
+++ b/exec.c
@@ -1,5 +1,5 @@
 /*
- *  virtual page mapping and translated block handling
+ *  Virtual page mapping
  *
  *  Copyright (c) 2003 Fabrice Bellard
  *
@@ -29,75 +29,35 @@
 #include "tcg.h"
 #include "hw/hw.h"
 #include "hw/qdev.h"
-#include "osdep.h"
-#include "kvm.h"
+#include "qemu/osdep.h"
+#include "sysemu/kvm.h"
 #include "hw/xen.h"
-#include "qemu-timer.h"
-#include "memory.h"
-#include "dma.h"
-#include "exec-memory.h"
+#include "qemu/timer.h"
+#include "qemu/config-file.h"
+#include "exec/memory.h"
+#include "sysemu/dma.h"
+#include "exec/address-spaces.h"
 #if defined(CONFIG_USER_ONLY)
 #include <qemu.h>
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
-#include <sys/param.h>
-#if __FreeBSD_version >= 700104
-#define HAVE_KINFO_GETVMMAP
-#define sigqueue sigqueue_freebsd  /* avoid redefinition */
-#include <sys/time.h>
-#include <sys/proc.h>
-#include <machine/profile.h>
-#define _KERNEL
-#include <sys/user.h>
-#undef _KERNEL
-#undef sigqueue
-#include <libutil.h>
-#endif
-#endif
 #else /* !CONFIG_USER_ONLY */
-#include "xen-mapcache.h"
+#include "sysemu/xen-mapcache.h"
 #include "trace.h"
 #endif
+#include "exec/cpu-all.h"
 
-#include "cputlb.h"
+#include "exec/cputlb.h"
+#include "translate-all.h"
 
-#include "memory-internal.h"
+#include "exec/memory-internal.h"
 
-//#define DEBUG_TB_INVALIDATE
-//#define DEBUG_FLUSH
 //#define DEBUG_UNASSIGNED
-
-/* make various TB consistency checks */
-//#define DEBUG_TB_CHECK
-
-//#define DEBUG_IOPORT
 //#define DEBUG_SUBPAGE
 
 #if !defined(CONFIG_USER_ONLY)
-/* TB consistency checks only implemented for usermode emulation.  */
-#undef DEBUG_TB_CHECK
-#endif
-
-#define SMC_BITMAP_USE_THRESHOLD 10
-
-static TranslationBlock *tbs;
-static int code_gen_max_blocks;
-TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
-static int nb_tbs;
-/* any access to the tbs or the page table must use this lock */
-spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
-
-uint8_t *code_gen_prologue;
-static uint8_t *code_gen_buffer;
-static size_t code_gen_buffer_size;
-/* threshold to flush the translated code buffer */
-static size_t code_gen_buffer_max_size;
-static uint8_t *code_gen_ptr;
-
-#if !defined(CONFIG_USER_ONLY)
 int phys_ram_fd;
 static int in_migration;
 
-RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
+RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
 
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;
@@ -118,60 +78,7 @@ DEFINE_TLS(CPUArchState *,cpu_single_env);
 /* 0 = Do not count executed instructions.
    1 = Precise instruction counting.
    2 = Adaptive rate instruction counting.  */
-int use_icount = 0;
-
-typedef struct PageDesc {
-    /* list of TBs intersecting this ram page */
-    TranslationBlock *first_tb;
-    /* in order to optimize self modifying code, we count the number
-       of lookups we do to a given page to use a bitmap */
-    unsigned int code_write_count;
-    uint8_t *code_bitmap;
-#if defined(CONFIG_USER_ONLY)
-    unsigned long flags;
-#endif
-} PageDesc;
-
-/* In system mode we want L1_MAP to be based on ram offsets,
-   while in user mode we want it to be based on virtual addresses.  */
-#if !defined(CONFIG_USER_ONLY)
-#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
-# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
-#else
-# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
-#endif
-#else
-# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
-#endif
-
-/* Size of the L2 (and L3, etc) page tables.  */
-#define L2_BITS 10
-#define L2_SIZE (1 << L2_BITS)
-
-#define P_L2_LEVELS \
-    (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
-
-/* The bits remaining after N lower levels of page tables.  */
-#define V_L1_BITS_REM \
-    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
-
-#if V_L1_BITS_REM < 4
-#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
-#else
-#define V_L1_BITS  V_L1_BITS_REM
-#endif
-
-#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
-
-#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
-
-uintptr_t qemu_real_host_page_size;
-uintptr_t qemu_host_page_size;
-uintptr_t qemu_host_page_mask;
-
-/* This is a multi-level map on the virtual address space.
-   The bottom level has pointers to PageDesc.  */
-static void *l1_map[V_L1_SIZE];
+int use_icount;
 
 #if !defined(CONFIG_USER_ONLY)
 
@@ -194,179 +101,6 @@ static void *qemu_safe_ram_ptr(ram_addr_t addr);
 
 static MemoryRegion io_mem_watch;
 #endif
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
-                         tb_page_addr_t phys_page2);
-
-/* statistics */
-static int tb_flush_count;
-static int tb_phys_invalidate_count;
-
-#ifdef _WIN32
-static inline void map_exec(void *addr, long size)
-{
-    DWORD old_protect;
-    VirtualProtect(addr, size,
-                   PAGE_EXECUTE_READWRITE, &old_protect);
-    
-}
-#else
-static inline void map_exec(void *addr, long size)
-{
-    unsigned long start, end, page_size;
-    
-    page_size = getpagesize();
-    start = (unsigned long)addr;
-    start &= ~(page_size - 1);
-    
-    end = (unsigned long)addr + size;
-    end += page_size - 1;
-    end &= ~(page_size - 1);
-    
-    mprotect((void *)start, end - start,
-             PROT_READ | PROT_WRITE | PROT_EXEC);
-}
-#endif
-
-static void page_init(void)
-{
-    /* NOTE: we can always suppose that qemu_host_page_size >=
-       TARGET_PAGE_SIZE */
-#ifdef _WIN32
-    {
-        SYSTEM_INFO system_info;
-
-        GetSystemInfo(&system_info);
-        qemu_real_host_page_size = system_info.dwPageSize;
-    }
-#else
-    qemu_real_host_page_size = getpagesize();
-#endif
-    if (qemu_host_page_size == 0)
-        qemu_host_page_size = qemu_real_host_page_size;
-    if (qemu_host_page_size < TARGET_PAGE_SIZE)
-        qemu_host_page_size = TARGET_PAGE_SIZE;
-    qemu_host_page_mask = ~(qemu_host_page_size - 1);
-
-#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
-    {
-#ifdef HAVE_KINFO_GETVMMAP
-        struct kinfo_vmentry *freep;
-        int i, cnt;
-
-        freep = kinfo_getvmmap(getpid(), &cnt);
-        if (freep) {
-            mmap_lock();
-            for (i = 0; i < cnt; i++) {
-                unsigned long startaddr, endaddr;
-
-                startaddr = freep[i].kve_start;
-                endaddr = freep[i].kve_end;
-                if (h2g_valid(startaddr)) {
-                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
-
-                    if (h2g_valid(endaddr)) {
-                        endaddr = h2g(endaddr);
-                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
-                    } else {
-#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
-                        endaddr = ~0ul;
-                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
-#endif
-                    }
-                }
-            }
-            free(freep);
-            mmap_unlock();
-        }
-#else
-        FILE *f;
-
-        last_brk = (unsigned long)sbrk(0);
-
-        f = fopen("/compat/linux/proc/self/maps", "r");
-        if (f) {
-            mmap_lock();
-
-            do {
-                unsigned long startaddr, endaddr;
-                int n;
-
-                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
-
-                if (n == 2 && h2g_valid(startaddr)) {
-                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
-
-                    if (h2g_valid(endaddr)) {
-                        endaddr = h2g(endaddr);
-                    } else {
-                        endaddr = ~0ul;
-                    }
-                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
-                }
-            } while (!feof(f));
-
-            fclose(f);
-            mmap_unlock();
-        }
-#endif
-    }
-#endif
-}
-
-static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
-{
-    PageDesc *pd;
-    void **lp;
-    int i;
-
-#if defined(CONFIG_USER_ONLY)
-    /* We can't use g_malloc because it may recurse into a locked mutex. */
-# define ALLOC(P, SIZE)                                 \
-    do {                                                \
-        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
-                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
-    } while (0)
-#else
-# define ALLOC(P, SIZE) \
-    do { P = g_malloc0(SIZE); } while (0)
-#endif
-
-    /* Level 1.  Always allocated.  */
-    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
-
-    /* Level 2..N-1.  */
-    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
-        void **p = *lp;
-
-        if (p == NULL) {
-            if (!alloc) {
-                return NULL;
-            }
-            ALLOC(p, sizeof(void *) * L2_SIZE);
-            *lp = p;
-        }
-
-        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
-    }
-
-    pd = *lp;
-    if (pd == NULL) {
-        if (!alloc) {
-            return NULL;
-        }
-        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
-        *lp = pd;
-    }
-
-#undef ALLOC
-
-    return pd + (index & (L2_SIZE - 1));
-}
-
-static inline PageDesc *page_find(tb_page_addr_t index)
-{
-    return page_find_alloc(index, 0);
-}
 
 #if !defined(CONFIG_USER_ONLY)
 
@@ -474,178 +208,12 @@ bool memory_region_is_unassigned(MemoryRegion *mr)
         && mr != &io_mem_notdirty && !mr->rom_device
         && mr != &io_mem_watch;
 }
-
-#define mmap_lock() do { } while(0)
-#define mmap_unlock() do { } while(0)
 #endif
 
-#if defined(CONFIG_USER_ONLY)
-/* Currently it is not recommended to allocate big chunks of data in
-   user mode. It will change when a dedicated libc will be used.  */
-/* ??? 64-bit hosts ought to have no problem mmaping data outside the
-   region in which the guest needs to run.  Revisit this.  */
-#define USE_STATIC_CODE_GEN_BUFFER
-#endif
-
-/* ??? Should configure for this, not list operating systems here.  */
-#if (defined(__linux__) \
-    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
-    || defined(__DragonFly__) || defined(__OpenBSD__) \
-    || defined(__NetBSD__))
-# define USE_MMAP
-#endif
-
-/* Minimum size of the code gen buffer.  This number is randomly chosen,
-   but not so small that we can't have a fair number of TB's live.  */
-#define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
-
-/* Maximum size of the code gen buffer we'd like to use.  Unless otherwise
-   indicated, this is constrained by the range of direct branches on the
-   host cpu, as used by the TCG implementation of goto_tb.  */
-#if defined(__x86_64__)
-# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
-#elif defined(__sparc__)
-# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
-#elif defined(__arm__)
-# define MAX_CODE_GEN_BUFFER_SIZE  (16u * 1024 * 1024)
-#elif defined(__s390x__)
-  /* We have a +- 4GB range on the branches; leave some slop.  */
-# define MAX_CODE_GEN_BUFFER_SIZE  (3ul * 1024 * 1024 * 1024)
-#else
-# define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
-#endif
-
-#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
-
-#define DEFAULT_CODE_GEN_BUFFER_SIZE \
-  (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
-   ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
-
-static inline size_t size_code_gen_buffer(size_t tb_size)
-{
-    /* Size the buffer.  */
-    if (tb_size == 0) {
-#ifdef USE_STATIC_CODE_GEN_BUFFER
-        tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
-#else
-        /* ??? Needs adjustments.  */
-        /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
-           static buffer, we could size this on RESERVED_VA, on the text
-           segment size of the executable, or continue to use the default.  */
-        tb_size = (unsigned long)(ram_size / 4);
-#endif
-    }
-    if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
-        tb_size = MIN_CODE_GEN_BUFFER_SIZE;
-    }
-    if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
-        tb_size = MAX_CODE_GEN_BUFFER_SIZE;
-    }
-    code_gen_buffer_size = tb_size;
-    return tb_size;
-}
-
-#ifdef USE_STATIC_CODE_GEN_BUFFER
-static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
-    __attribute__((aligned(CODE_GEN_ALIGN)));
-
-static inline void *alloc_code_gen_buffer(void)
-{
-    map_exec(static_code_gen_buffer, code_gen_buffer_size);
-    return static_code_gen_buffer;
-}
-#elif defined(USE_MMAP)
-static inline void *alloc_code_gen_buffer(void)
-{
-    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
-    uintptr_t start = 0;
-    void *buf;
-
-    /* Constrain the position of the buffer based on the host cpu.
-       Note that these addresses are chosen in concert with the
-       addresses assigned in the relevant linker script file.  */
-# if defined(__PIE__) || defined(__PIC__)
-    /* Don't bother setting a preferred location if we're building
-       a position-independent executable.  We're more likely to get
-       an address near the main executable if we let the kernel
-       choose the address.  */
-# elif defined(__x86_64__) && defined(MAP_32BIT)
-    /* Force the memory down into low memory with the executable.
-       Leave the choice of exact location with the kernel.  */
-    flags |= MAP_32BIT;
-    /* Cannot expect to map more than 800MB in low memory.  */
-    if (code_gen_buffer_size > 800u * 1024 * 1024) {
-        code_gen_buffer_size = 800u * 1024 * 1024;
-    }
-# elif defined(__sparc__)
-    start = 0x40000000ul;
-# elif defined(__s390x__)
-    start = 0x90000000ul;
-# endif
-
-    buf = mmap((void *)start, code_gen_buffer_size,
-               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
-    return buf == MAP_FAILED ? NULL : buf;
-}
-#else
-static inline void *alloc_code_gen_buffer(void)
-{
-    void *buf = g_malloc(code_gen_buffer_size);
-    if (buf) {
-        map_exec(buf, code_gen_buffer_size);
-    }
-    return buf;
-}
-#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
-
-static inline void code_gen_alloc(size_t tb_size)
-{
-    code_gen_buffer_size = size_code_gen_buffer(tb_size);
-    code_gen_buffer = alloc_code_gen_buffer();
-    if (code_gen_buffer == NULL) {
-        fprintf(stderr, "Could not allocate dynamic translator buffer\n");
-        exit(1);
-    }
-
-    /* Steal room for the prologue at the end of the buffer.  This ensures
-       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
-       from TB's to the prologue are going to be in range.  It also means
-       that we don't need to mark (additional) portions of the data segment
-       as executable.  */
-    code_gen_prologue = code_gen_buffer + code_gen_buffer_size - 1024;
-    code_gen_buffer_size -= 1024;
-
-    code_gen_buffer_max_size = code_gen_buffer_size -
-        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
-    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
-    tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
-}
-
-/* Must be called before using the QEMU cpus. 'tb_size' is the size
-   (in bytes) allocated to the translation buffer. Zero means default
-   size. */
-void tcg_exec_init(unsigned long tb_size)
-{
-    cpu_gen_init();
-    code_gen_alloc(tb_size);
-    code_gen_ptr = code_gen_buffer;
-    tcg_register_jit(code_gen_buffer, code_gen_buffer_size);
-    page_init();
-#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
-    /* There's no guest base to take into account, so go ahead and
-       initialize the prologue now.  */
-    tcg_prologue_init(&tcg_ctx);
-#endif
-}
-
-bool tcg_enabled(void)
-{
-    return code_gen_buffer != NULL;
-}
-
 void cpu_exec_init_all(void)
 {
 #if !defined(CONFIG_USER_ONLY)
+    qemu_mutex_init(&ram_list.mutex);
     memory_map_init();
     io_mem_init();
 #endif
@@ -679,24 +247,25 @@ static const VMStateDescription vmstate_cpu_common = {
 };
 #endif
 
-CPUArchState *qemu_get_cpu(int cpu)
+CPUState *qemu_get_cpu(int index)
 {
     CPUArchState *env = first_cpu;
+    CPUState *cpu = NULL;
 
     while (env) {
-        if (env->cpu_index == cpu)
+        cpu = ENV_GET_CPU(env);
+        if (cpu->cpu_index == index) {
             break;
+        }
         env = env->next_cpu;
     }
 
-    return env;
+    return cpu;
 }
 
 void cpu_exec_init(CPUArchState *env)
 {
-#ifndef CONFIG_USER_ONLY
     CPUState *cpu = ENV_GET_CPU(env);
-#endif
     CPUArchState **penv;
     int cpu_index;
 
@@ -710,8 +279,8 @@ void cpu_exec_init(CPUArchState *env)
         penv = &(*penv)->next_cpu;
         cpu_index++;
     }
-    env->cpu_index = cpu_index;
-    env->numa_node = 0;
+    cpu->cpu_index = cpu_index;
+    cpu->numa_node = 0;
     QTAILQ_INIT(&env->breakpoints);
     QTAILQ_INIT(&env->watchpoints);
 #ifndef CONFIG_USER_ONLY
@@ -728,763 +297,6 @@ void cpu_exec_init(CPUArchState *env)
 #endif
 }
 
-/* Allocate a new translation block. Flush the translation buffer if
-   too many translation blocks or too much generated code. */
-static TranslationBlock *tb_alloc(target_ulong pc)
-{
-    TranslationBlock *tb;
-
-    if (nb_tbs >= code_gen_max_blocks ||
-        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
-        return NULL;
-    tb = &tbs[nb_tbs++];
-    tb->pc = pc;
-    tb->cflags = 0;
-    return tb;
-}
-
-void tb_free(TranslationBlock *tb)
-{
-    /* In practice this is mostly used for single use temporary TB
-       Ignore the hard cases and just back up if this TB happens to
-       be the last one generated.  */
-    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
-        code_gen_ptr = tb->tc_ptr;
-        nb_tbs--;
-    }
-}
-
-static inline void invalidate_page_bitmap(PageDesc *p)
-{
-    if (p->code_bitmap) {
-        g_free(p->code_bitmap);
-        p->code_bitmap = NULL;
-    }
-    p->code_write_count = 0;
-}
-
-/* Set to NULL all the 'first_tb' fields in all PageDescs. */
-
-static void page_flush_tb_1 (int level, void **lp)
-{
-    int i;
-
-    if (*lp == NULL) {
-        return;
-    }
-    if (level == 0) {
-        PageDesc *pd = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            pd[i].first_tb = NULL;
-            invalidate_page_bitmap(pd + i);
-        }
-    } else {
-        void **pp = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            page_flush_tb_1 (level - 1, pp + i);
-        }
-    }
-}
-
-static void page_flush_tb(void)
-{
-    int i;
-    for (i = 0; i < V_L1_SIZE; i++) {
-        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
-    }
-}
-
-/* flush all the translation blocks */
-/* XXX: tb_flush is currently not thread safe */
-void tb_flush(CPUArchState *env1)
-{
-    CPUArchState *env;
-#if defined(DEBUG_FLUSH)
-    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
-           (unsigned long)(code_gen_ptr - code_gen_buffer),
-           nb_tbs, nb_tbs > 0 ?
-           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
-#endif
-    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
-        cpu_abort(env1, "Internal error: code buffer overflow\n");
-
-    nb_tbs = 0;
-
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
-    }
-
-    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
-    page_flush_tb();
-
-    code_gen_ptr = code_gen_buffer;
-    /* XXX: flush processor icache at this point if cache flush is
-       expensive */
-    tb_flush_count++;
-}
-
-#ifdef DEBUG_TB_CHECK
-
-static void tb_invalidate_check(target_ulong address)
-{
-    TranslationBlock *tb;
-    int i;
-    address &= TARGET_PAGE_MASK;
-    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
-            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
-                  address >= tb->pc + tb->size)) {
-                printf("ERROR invalidate: address=" TARGET_FMT_lx
-                       " PC=%08lx size=%04x\n",
-                       address, (long)tb->pc, tb->size);
-            }
-        }
-    }
-}
-
-/* verify that all the pages have correct rights for code */
-static void tb_page_check(void)
-{
-    TranslationBlock *tb;
-    int i, flags1, flags2;
-
-    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
-            flags1 = page_get_flags(tb->pc);
-            flags2 = page_get_flags(tb->pc + tb->size - 1);
-            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
-                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
-                       (long)tb->pc, tb->size, flags1, flags2);
-            }
-        }
-    }
-}
-
-#endif
-
-/* invalidate one TB */
-static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
-                             int next_offset)
-{
-    TranslationBlock *tb1;
-    for(;;) {
-        tb1 = *ptb;
-        if (tb1 == tb) {
-            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
-            break;
-        }
-        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
-    }
-}
-
-static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
-{
-    TranslationBlock *tb1;
-    unsigned int n1;
-
-    for(;;) {
-        tb1 = *ptb;
-        n1 = (uintptr_t)tb1 & 3;
-        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-        if (tb1 == tb) {
-            *ptb = tb1->page_next[n1];
-            break;
-        }
-        ptb = &tb1->page_next[n1];
-    }
-}
-
-static inline void tb_jmp_remove(TranslationBlock *tb, int n)
-{
-    TranslationBlock *tb1, **ptb;
-    unsigned int n1;
-
-    ptb = &tb->jmp_next[n];
-    tb1 = *ptb;
-    if (tb1) {
-        /* find tb(n) in circular list */
-        for(;;) {
-            tb1 = *ptb;
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-            if (n1 == n && tb1 == tb)
-                break;
-            if (n1 == 2) {
-                ptb = &tb1->jmp_first;
-            } else {
-                ptb = &tb1->jmp_next[n1];
-            }
-        }
-        /* now we can suppress tb(n) from the list */
-        *ptb = tb->jmp_next[n];
-
-        tb->jmp_next[n] = NULL;
-    }
-}
-
-/* reset the jump entry 'n' of a TB so that it is not chained to
-   another TB */
-static inline void tb_reset_jump(TranslationBlock *tb, int n)
-{
-    tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
-}
-
-void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
-{
-    CPUArchState *env;
-    PageDesc *p;
-    unsigned int h, n1;
-    tb_page_addr_t phys_pc;
-    TranslationBlock *tb1, *tb2;
-
-    /* remove the TB from the hash list */
-    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
-    h = tb_phys_hash_func(phys_pc);
-    tb_remove(&tb_phys_hash[h], tb,
-              offsetof(TranslationBlock, phys_hash_next));
-
-    /* remove the TB from the page list */
-    if (tb->page_addr[0] != page_addr) {
-        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
-        tb_page_remove(&p->first_tb, tb);
-        invalidate_page_bitmap(p);
-    }
-    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
-        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
-        tb_page_remove(&p->first_tb, tb);
-        invalidate_page_bitmap(p);
-    }
-
-    tb_invalidated_flag = 1;
-
-    /* remove the TB from the hash list */
-    h = tb_jmp_cache_hash_func(tb->pc);
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->tb_jmp_cache[h] == tb)
-            env->tb_jmp_cache[h] = NULL;
-    }
-
-    /* suppress this TB from the two jump lists */
-    tb_jmp_remove(tb, 0);
-    tb_jmp_remove(tb, 1);
-
-    /* suppress any remaining jumps to this TB */
-    tb1 = tb->jmp_first;
-    for(;;) {
-        n1 = (uintptr_t)tb1 & 3;
-        if (n1 == 2)
-            break;
-        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-        tb2 = tb1->jmp_next[n1];
-        tb_reset_jump(tb1, n1);
-        tb1->jmp_next[n1] = NULL;
-        tb1 = tb2;
-    }
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
-
-    tb_phys_invalidate_count++;
-}
-
-static inline void set_bits(uint8_t *tab, int start, int len)
-{
-    int end, mask, end1;
-
-    end = start + len;
-    tab += start >> 3;
-    mask = 0xff << (start & 7);
-    if ((start & ~7) == (end & ~7)) {
-        if (start < end) {
-            mask &= ~(0xff << (end & 7));
-            *tab |= mask;
-        }
-    } else {
-        *tab++ |= mask;
-        start = (start + 8) & ~7;
-        end1 = end & ~7;
-        while (start < end1) {
-            *tab++ = 0xff;
-            start += 8;
-        }
-        if (start < end) {
-            mask = ~(0xff << (end & 7));
-            *tab |= mask;
-        }
-    }
-}
-
-static void build_page_bitmap(PageDesc *p)
-{
-    int n, tb_start, tb_end;
-    TranslationBlock *tb;
-
-    p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
-
-    tb = p->first_tb;
-    while (tb != NULL) {
-        n = (uintptr_t)tb & 3;
-        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
-        /* NOTE: this is subtle as a TB may span two physical pages */
-        if (n == 0) {
-            /* NOTE: tb_end may be after the end of the page, but
-               it is not a problem */
-            tb_start = tb->pc & ~TARGET_PAGE_MASK;
-            tb_end = tb_start + tb->size;
-            if (tb_end > TARGET_PAGE_SIZE)
-                tb_end = TARGET_PAGE_SIZE;
-        } else {
-            tb_start = 0;
-            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
-        }
-        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
-        tb = tb->page_next[n];
-    }
-}
-
-TranslationBlock *tb_gen_code(CPUArchState *env,
-                              target_ulong pc, target_ulong cs_base,
-                              int flags, int cflags)
-{
-    TranslationBlock *tb;
-    uint8_t *tc_ptr;
-    tb_page_addr_t phys_pc, phys_page2;
-    target_ulong virt_page2;
-    int code_gen_size;
-
-    phys_pc = get_page_addr_code(env, pc);
-    tb = tb_alloc(pc);
-    if (!tb) {
-        /* flush must be done */
-        tb_flush(env);
-        /* cannot fail at this point */
-        tb = tb_alloc(pc);
-        /* Don't forget to invalidate previous TB info.  */
-        tb_invalidated_flag = 1;
-    }
-    tc_ptr = code_gen_ptr;
-    tb->tc_ptr = tc_ptr;
-    tb->cs_base = cs_base;
-    tb->flags = flags;
-    tb->cflags = cflags;
-    cpu_gen_code(env, tb, &code_gen_size);
-    code_gen_ptr = (void *)(((uintptr_t)code_gen_ptr + code_gen_size +
-                             CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
-
-    /* check next page if needed */
-    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
-    phys_page2 = -1;
-    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
-        phys_page2 = get_page_addr_code(env, virt_page2);
-    }
-    tb_link_page(tb, phys_pc, phys_page2);
-    return tb;
-}
-
-/*
- * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end may refer to *different* physical pages.
- * 'is_cpu_write_access' should be true if called from a real cpu write
- * access: the virtual CPU will exit the current TB if code is modified inside
- * this TB.
- */
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
-                              int is_cpu_write_access)
-{
-    while (start < end) {
-        tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
-        start &= TARGET_PAGE_MASK;
-        start += TARGET_PAGE_SIZE;
-    }
-}
-
-/*
- * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end must refer to the *same* physical page.
- * 'is_cpu_write_access' should be true if called from a real cpu write
- * access: the virtual CPU will exit the current TB if code is modified inside
- * this TB.
- */
-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
-                                   int is_cpu_write_access)
-{
-    TranslationBlock *tb, *tb_next, *saved_tb;
-    CPUArchState *env = cpu_single_env;
-    tb_page_addr_t tb_start, tb_end;
-    PageDesc *p;
-    int n;
-#ifdef TARGET_HAS_PRECISE_SMC
-    int current_tb_not_found = is_cpu_write_access;
-    TranslationBlock *current_tb = NULL;
-    int current_tb_modified = 0;
-    target_ulong current_pc = 0;
-    target_ulong current_cs_base = 0;
-    int current_flags = 0;
-#endif /* TARGET_HAS_PRECISE_SMC */
-
-    p = page_find(start >> TARGET_PAGE_BITS);
-    if (!p)
-        return;
-    if (!p->code_bitmap &&
-        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
-        is_cpu_write_access) {
-        /* build code bitmap */
-        build_page_bitmap(p);
-    }
-
-    /* we remove all the TBs in the range [start, end[ */
-    /* XXX: see if in some cases it could be faster to invalidate all the code */
-    tb = p->first_tb;
-    while (tb != NULL) {
-        n = (uintptr_t)tb & 3;
-        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
-        tb_next = tb->page_next[n];
-        /* NOTE: this is subtle as a TB may span two physical pages */
-        if (n == 0) {
-            /* NOTE: tb_end may be after the end of the page, but
-               it is not a problem */
-            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
-            tb_end = tb_start + tb->size;
-        } else {
-            tb_start = tb->page_addr[1];
-            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
-        }
-        if (!(tb_end <= start || tb_start >= end)) {
-#ifdef TARGET_HAS_PRECISE_SMC
-            if (current_tb_not_found) {
-                current_tb_not_found = 0;
-                current_tb = NULL;
-                if (env->mem_io_pc) {
-                    /* now we have a real cpu fault */
-                    current_tb = tb_find_pc(env->mem_io_pc);
-                }
-            }
-            if (current_tb == tb &&
-                (current_tb->cflags & CF_COUNT_MASK) != 1) {
-                /* If we are modifying the current TB, we must stop
-                its execution. We could be more precise by checking
-                that the modification is after the current PC, but it
-                would require a specialized function to partially
-                restore the CPU state */
-
-                current_tb_modified = 1;
-                cpu_restore_state(current_tb, env, env->mem_io_pc);
-                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
-                                     &current_flags);
-            }
-#endif /* TARGET_HAS_PRECISE_SMC */
-            /* we need to do that to handle the case where a signal
-               occurs while doing tb_phys_invalidate() */
-            saved_tb = NULL;
-            if (env) {
-                saved_tb = env->current_tb;
-                env->current_tb = NULL;
-            }
-            tb_phys_invalidate(tb, -1);
-            if (env) {
-                env->current_tb = saved_tb;
-                if (env->interrupt_request && env->current_tb)
-                    cpu_interrupt(env, env->interrupt_request);
-            }
-        }
-        tb = tb_next;
-    }
-#if !defined(CONFIG_USER_ONLY)
-    /* if no code remaining, no need to continue to use slow writes */
-    if (!p->first_tb) {
-        invalidate_page_bitmap(p);
-        if (is_cpu_write_access) {
-            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
-        }
-    }
-#endif
-#ifdef TARGET_HAS_PRECISE_SMC
-    if (current_tb_modified) {
-        /* we generate a block containing just the instruction
-           modifying the memory. It will ensure that it cannot modify
-           itself */
-        env->current_tb = NULL;
-        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
-        cpu_resume_from_signal(env, NULL);
-    }
-#endif
-}
-
-/* len must be <= 8 and start must be a multiple of len */
-static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
-{
-    PageDesc *p;
-    int offset, b;
-#if 0
-    if (1) {
-        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
-                  cpu_single_env->mem_io_vaddr, len,
-                  cpu_single_env->eip,
-                  cpu_single_env->eip +
-                  (intptr_t)cpu_single_env->segs[R_CS].base);
-    }
-#endif
-    p = page_find(start >> TARGET_PAGE_BITS);
-    if (!p)
-        return;
-    if (p->code_bitmap) {
-        offset = start & ~TARGET_PAGE_MASK;
-        b = p->code_bitmap[offset >> 3] >> (offset & 7);
-        if (b & ((1 << len) - 1))
-            goto do_invalidate;
-    } else {
-    do_invalidate:
-        tb_invalidate_phys_page_range(start, start + len, 1);
-    }
-}
-
-#if !defined(CONFIG_SOFTMMU)
-static void tb_invalidate_phys_page(tb_page_addr_t addr,
-                                    uintptr_t pc, void *puc)
-{
-    TranslationBlock *tb;
-    PageDesc *p;
-    int n;
-#ifdef TARGET_HAS_PRECISE_SMC
-    TranslationBlock *current_tb = NULL;
-    CPUArchState *env = cpu_single_env;
-    int current_tb_modified = 0;
-    target_ulong current_pc = 0;
-    target_ulong current_cs_base = 0;
-    int current_flags = 0;
-#endif
-
-    addr &= TARGET_PAGE_MASK;
-    p = page_find(addr >> TARGET_PAGE_BITS);
-    if (!p)
-        return;
-    tb = p->first_tb;
-#ifdef TARGET_HAS_PRECISE_SMC
-    if (tb && pc != 0) {
-        current_tb = tb_find_pc(pc);
-    }
-#endif
-    while (tb != NULL) {
-        n = (uintptr_t)tb & 3;
-        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
-#ifdef TARGET_HAS_PRECISE_SMC
-        if (current_tb == tb &&
-            (current_tb->cflags & CF_COUNT_MASK) != 1) {
-                /* If we are modifying the current TB, we must stop
-                   its execution. We could be more precise by checking
-                   that the modification is after the current PC, but it
-                   would require a specialized function to partially
-                   restore the CPU state */
-
-            current_tb_modified = 1;
-            cpu_restore_state(current_tb, env, pc);
-            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
-                                 &current_flags);
-        }
-#endif /* TARGET_HAS_PRECISE_SMC */
-        tb_phys_invalidate(tb, addr);
-        tb = tb->page_next[n];
-    }
-    p->first_tb = NULL;
-#ifdef TARGET_HAS_PRECISE_SMC
-    if (current_tb_modified) {
-        /* we generate a block containing just the instruction
-           modifying the memory. It will ensure that it cannot modify
-           itself */
-        env->current_tb = NULL;
-        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
-        cpu_resume_from_signal(env, puc);
-    }
-#endif
-}
-#endif
-
-/* add the tb in the target page and protect it if necessary */
-static inline void tb_alloc_page(TranslationBlock *tb,
-                                 unsigned int n, tb_page_addr_t page_addr)
-{
-    PageDesc *p;
-#ifndef CONFIG_USER_ONLY
-    bool page_already_protected;
-#endif
-
-    tb->page_addr[n] = page_addr;
-    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
-    tb->page_next[n] = p->first_tb;
-#ifndef CONFIG_USER_ONLY
-    page_already_protected = p->first_tb != NULL;
-#endif
-    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
-    invalidate_page_bitmap(p);
-
-#if defined(TARGET_HAS_SMC) || 1
-
-#if defined(CONFIG_USER_ONLY)
-    if (p->flags & PAGE_WRITE) {
-        target_ulong addr;
-        PageDesc *p2;
-        int prot;
-
-        /* force the host page as non writable (writes will have a
-           page fault + mprotect overhead) */
-        page_addr &= qemu_host_page_mask;
-        prot = 0;
-        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
-            addr += TARGET_PAGE_SIZE) {
-
-            p2 = page_find (addr >> TARGET_PAGE_BITS);
-            if (!p2)
-                continue;
-            prot |= p2->flags;
-            p2->flags &= ~PAGE_WRITE;
-          }
-        mprotect(g2h(page_addr), qemu_host_page_size,
-                 (prot & PAGE_BITS) & ~PAGE_WRITE);
-#ifdef DEBUG_TB_INVALIDATE
-        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
-               page_addr);
-#endif
-    }
-#else
-    /* if some code is already present, then the pages are already
-       protected. So we handle the case where only the first TB is
-       allocated in a physical page */
-    if (!page_already_protected) {
-        tlb_protect_code(page_addr);
-    }
-#endif
-
-#endif /* TARGET_HAS_SMC */
-}
-
-/* add a new TB and link it to the physical page tables. phys_page2 is
-   (-1) to indicate that only one page contains the TB. */
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
-                         tb_page_addr_t phys_page2)
-{
-    unsigned int h;
-    TranslationBlock **ptb;
-
-    /* Grab the mmap lock to stop another thread invalidating this TB
-       before we are done.  */
-    mmap_lock();
-    /* add in the physical hash table */
-    h = tb_phys_hash_func(phys_pc);
-    ptb = &tb_phys_hash[h];
-    tb->phys_hash_next = *ptb;
-    *ptb = tb;
-
-    /* add in the page list */
-    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
-    if (phys_page2 != -1)
-        tb_alloc_page(tb, 1, phys_page2);
-    else
-        tb->page_addr[1] = -1;
-
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
-    tb->jmp_next[0] = NULL;
-    tb->jmp_next[1] = NULL;
-
-    /* init original jump addresses */
-    if (tb->tb_next_offset[0] != 0xffff)
-        tb_reset_jump(tb, 0);
-    if (tb->tb_next_offset[1] != 0xffff)
-        tb_reset_jump(tb, 1);
-
-#ifdef DEBUG_TB_CHECK
-    tb_page_check();
-#endif
-    mmap_unlock();
-}
-
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-/* check whether the given addr is in TCG generated code buffer or not */
-bool is_tcg_gen_code(uintptr_t tc_ptr)
-{
-    /* This can be called during code generation, code_gen_buffer_max_size
-       is used instead of code_gen_ptr for upper boundary checking */
-    return (tc_ptr >= (uintptr_t)code_gen_buffer &&
-            tc_ptr < (uintptr_t)(code_gen_buffer + code_gen_buffer_max_size));
-}
-#endif
-
-/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
-   tb[1].tc_ptr. Return NULL if not found */
-TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
-{
-    int m_min, m_max, m;
-    uintptr_t v;
-    TranslationBlock *tb;
-
-    if (nb_tbs <= 0)
-        return NULL;
-    if (tc_ptr < (uintptr_t)code_gen_buffer ||
-        tc_ptr >= (uintptr_t)code_gen_ptr) {
-        return NULL;
-    }
-    /* binary search (cf Knuth) */
-    m_min = 0;
-    m_max = nb_tbs - 1;
-    while (m_min <= m_max) {
-        m = (m_min + m_max) >> 1;
-        tb = &tbs[m];
-        v = (uintptr_t)tb->tc_ptr;
-        if (v == tc_ptr)
-            return tb;
-        else if (tc_ptr < v) {
-            m_max = m - 1;
-        } else {
-            m_min = m + 1;
-        }
-    }
-    return &tbs[m_max];
-}
-
-static void tb_reset_jump_recursive(TranslationBlock *tb);
-
-static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
-{
-    TranslationBlock *tb1, *tb_next, **ptb;
-    unsigned int n1;
-
-    tb1 = tb->jmp_next[n];
-    if (tb1 != NULL) {
-        /* find head of list */
-        for(;;) {
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-            if (n1 == 2)
-                break;
-            tb1 = tb1->jmp_next[n1];
-        }
-        /* we are now sure now that tb jumps to tb1 */
-        tb_next = tb1;
-
-        /* remove tb from the jmp_first list */
-        ptb = &tb_next->jmp_first;
-        for(;;) {
-            tb1 = *ptb;
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-            if (n1 == n && tb1 == tb)
-                break;
-            ptb = &tb1->jmp_next[n1];
-        }
-        *ptb = tb->jmp_next[n];
-        tb->jmp_next[n] = NULL;
-
-        /* suppress the jump to next tb in generated code */
-        tb_reset_jump(tb, n);
-
-        /* suppress jumps in the tb on which we could have jumped */
-        tb_reset_jump_recursive(tb_next);
-    }
-}
-
-static void tb_reset_jump_recursive(TranslationBlock *tb)
-{
-    tb_reset_jump_recursive2(tb, 0);
-    tb_reset_jump_recursive2(tb, 1);
-}
-
 #if defined(TARGET_HAS_ICE)
 #if defined(CONFIG_USER_ONLY)
 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
@@ -1492,21 +304,6 @@ static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 }
 #else
-void tb_invalidate_phys_addr(hwaddr addr)
-{
-    ram_addr_t ram_addr;
-    MemoryRegionSection *section;
-
-    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
-    if (!(memory_region_is_ram(section->mr)
-          || (section->mr->rom_device && section->mr->readable))) {
-        return;
-    }
-    ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
-        + memory_region_section_addr(section, addr);
-    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
-}
-
 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 {
     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
@@ -1688,67 +485,6 @@ void cpu_single_step(CPUArchState *env, int enabled)
 #endif
 }
 
-static void cpu_unlink_tb(CPUArchState *env)
-{
-    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
-       problem and hope the cpu will stop of its own accord.  For userspace
-       emulation this often isn't actually as bad as it sounds.  Often
-       signals are used primarily to interrupt blocking syscalls.  */
-    TranslationBlock *tb;
-    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
-
-    spin_lock(&interrupt_lock);
-    tb = env->current_tb;
-    /* if the cpu is currently executing code, we must unlink it and
-       all the potentially executing TB */
-    if (tb) {
-        env->current_tb = NULL;
-        tb_reset_jump_recursive(tb);
-    }
-    spin_unlock(&interrupt_lock);
-}
-
-#ifndef CONFIG_USER_ONLY
-/* mask must never be zero, except for A20 change call */
-static void tcg_handle_interrupt(CPUArchState *env, int mask)
-{
-    CPUState *cpu = ENV_GET_CPU(env);
-    int old_mask;
-
-    old_mask = env->interrupt_request;
-    env->interrupt_request |= mask;
-
-    /*
-     * If called from iothread context, wake the target cpu in
-     * case its halted.
-     */
-    if (!qemu_cpu_is_self(cpu)) {
-        qemu_cpu_kick(cpu);
-        return;
-    }
-
-    if (use_icount) {
-        env->icount_decr.u16.high = 0xffff;
-        if (!can_do_io(env)
-            && (mask & ~old_mask) != 0) {
-            cpu_abort(env, "Raised interrupt while not in I/O function");
-        }
-    } else {
-        cpu_unlink_tb(env);
-    }
-}
-
-CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
-
-#else /* CONFIG_USER_ONLY */
-
-void cpu_interrupt(CPUArchState *env, int mask)
-{
-    env->interrupt_request |= mask;
-    cpu_unlink_tb(env);
-}
-#endif /* CONFIG_USER_ONLY */
-
 void cpu_reset_interrupt(CPUArchState *env, int mask)
 {
     env->interrupt_request &= ~mask;
@@ -1796,7 +532,6 @@ CPUArchState *cpu_copy(CPUArchState *env)
 {
     CPUArchState *new_env = cpu_init(env->cpu_model_str);
     CPUArchState *next_cpu = new_env->next_cpu;
-    int cpu_index = new_env->cpu_index;
 #if defined(TARGET_HAS_ICE)
     CPUBreakpoint *bp;
     CPUWatchpoint *wp;
@@ -1804,9 +539,8 @@ CPUArchState *cpu_copy(CPUArchState *env)
 
     memcpy(new_env, env, sizeof(CPUArchState));
 
-    /* Preserve chaining and index. */
+    /* Preserve chaining. */
     new_env->next_cpu = next_cpu;
-    new_env->cpu_index = cpu_index;
 
     /* Clone all break/watchpoints.
        Note: Once we support ptrace with hw-debug register access, make sure
@@ -1827,21 +561,6 @@ CPUArchState *cpu_copy(CPUArchState *env)
 }
 
 #if !defined(CONFIG_USER_ONLY)
-void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
-{
-    unsigned int i;
-
-    /* Discard jump cache entries for any tb which might potentially
-       overlap the flushed page.  */
-    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
-    memset (&env->tb_jmp_cache[i], 0, 
-            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
-
-    i = tb_jmp_cache_hash_page(addr);
-    memset (&env->tb_jmp_cache[i], 0, 
-            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
-}
-
 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
                                       uintptr_t length)
 {
@@ -1931,264 +650,6 @@ hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 
     return iotlb;
 }
-
-#else
-/*
- * Walks guest process memory "regions" one by one
- * and calls callback function 'fn' for each region.
- */
-
-struct walk_memory_regions_data
-{
-    walk_memory_regions_fn fn;
-    void *priv;
-    uintptr_t start;
-    int prot;
-};
-
-static int walk_memory_regions_end(struct walk_memory_regions_data *data,
-                                   abi_ulong end, int new_prot)
-{
-    if (data->start != -1ul) {
-        int rc = data->fn(data->priv, data->start, end, data->prot);
-        if (rc != 0) {
-            return rc;
-        }
-    }
-
-    data->start = (new_prot ? end : -1ul);
-    data->prot = new_prot;
-
-    return 0;
-}
-
-static int walk_memory_regions_1(struct walk_memory_regions_data *data,
-                                 abi_ulong base, int level, void **lp)
-{
-    abi_ulong pa;
-    int i, rc;
-
-    if (*lp == NULL) {
-        return walk_memory_regions_end(data, base, 0);
-    }
-
-    if (level == 0) {
-        PageDesc *pd = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            int prot = pd[i].flags;
-
-            pa = base | (i << TARGET_PAGE_BITS);
-            if (prot != data->prot) {
-                rc = walk_memory_regions_end(data, pa, prot);
-                if (rc != 0) {
-                    return rc;
-                }
-            }
-        }
-    } else {
-        void **pp = *lp;
-        for (i = 0; i < L2_SIZE; ++i) {
-            pa = base | ((abi_ulong)i <<
-                (TARGET_PAGE_BITS + L2_BITS * level));
-            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
-            if (rc != 0) {
-                return rc;
-            }
-        }
-    }
-
-    return 0;
-}
-
-int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
-{
-    struct walk_memory_regions_data data;
-    uintptr_t i;
-
-    data.fn = fn;
-    data.priv = priv;
-    data.start = -1ul;
-    data.prot = 0;
-
-    for (i = 0; i < V_L1_SIZE; i++) {
-        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
-                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
-        if (rc != 0) {
-            return rc;
-        }
-    }
-
-    return walk_memory_regions_end(&data, 0, 0);
-}
-
-static int dump_region(void *priv, abi_ulong start,
-    abi_ulong end, unsigned long prot)
-{
-    FILE *f = (FILE *)priv;
-
-    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
-        " "TARGET_ABI_FMT_lx" %c%c%c\n",
-        start, end, end - start,
-        ((prot & PAGE_READ) ? 'r' : '-'),
-        ((prot & PAGE_WRITE) ? 'w' : '-'),
-        ((prot & PAGE_EXEC) ? 'x' : '-'));
-
-    return (0);
-}
-
-/* dump memory mappings */
-void page_dump(FILE *f)
-{
-    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
-            "start", "end", "size", "prot");
-    walk_memory_regions(f, dump_region);
-}
-
-int page_get_flags(target_ulong address)
-{
-    PageDesc *p;
-
-    p = page_find(address >> TARGET_PAGE_BITS);
-    if (!p)
-        return 0;
-    return p->flags;
-}
-
-/* Modify the flags of a page and invalidate the code if necessary.
-   The flag PAGE_WRITE_ORG is positioned automatically depending
-   on PAGE_WRITE.  The mmap_lock should already be held.  */
-void page_set_flags(target_ulong start, target_ulong end, int flags)
-{
-    target_ulong addr, len;
-
-    /* This function should never be called with addresses outside the
-       guest address space.  If this assert fires, it probably indicates
-       a missing call to h2g_valid.  */
-#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
-    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
-#endif
-    assert(start < end);
-
-    start = start & TARGET_PAGE_MASK;
-    end = TARGET_PAGE_ALIGN(end);
-
-    if (flags & PAGE_WRITE) {
-        flags |= PAGE_WRITE_ORG;
-    }
-
-    for (addr = start, len = end - start;
-         len != 0;
-         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
-        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
-
-        /* If the write protection bit is set, then we invalidate
-           the code inside.  */
-        if (!(p->flags & PAGE_WRITE) &&
-            (flags & PAGE_WRITE) &&
-            p->first_tb) {
-            tb_invalidate_phys_page(addr, 0, NULL);
-        }
-        p->flags = flags;
-    }
-}
-
-int page_check_range(target_ulong start, target_ulong len, int flags)
-{
-    PageDesc *p;
-    target_ulong end;
-    target_ulong addr;
-
-    /* This function should never be called with addresses outside the
-       guest address space.  If this assert fires, it probably indicates
-       a missing call to h2g_valid.  */
-#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
-    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
-#endif
-
-    if (len == 0) {
-        return 0;
-    }
-    if (start + len - 1 < start) {
-        /* We've wrapped around.  */
-        return -1;
-    }
-
-    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
-    start = start & TARGET_PAGE_MASK;
-
-    for (addr = start, len = end - start;
-         len != 0;
-         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
-        p = page_find(addr >> TARGET_PAGE_BITS);
-        if( !p )
-            return -1;
-        if( !(p->flags & PAGE_VALID) )
-            return -1;
-
-        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
-            return -1;
-        if (flags & PAGE_WRITE) {
-            if (!(p->flags & PAGE_WRITE_ORG))
-                return -1;
-            /* unprotect the page if it was put read-only because it
-               contains translated code */
-            if (!(p->flags & PAGE_WRITE)) {
-                if (!page_unprotect(addr, 0, NULL))
-                    return -1;
-            }
-            return 0;
-        }
-    }
-    return 0;
-}
-
-/* called from signal handler: invalidate the code and unprotect the
-   page. Return TRUE if the fault was successfully handled. */
-int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
-{
-    unsigned int prot;
-    PageDesc *p;
-    target_ulong host_start, host_end, addr;
-
-    /* Technically this isn't safe inside a signal handler.  However we
-       know this only ever happens in a synchronous SEGV handler, so in
-       practice it seems to be ok.  */
-    mmap_lock();
-
-    p = page_find(address >> TARGET_PAGE_BITS);
-    if (!p) {
-        mmap_unlock();
-        return 0;
-    }
-
-    /* if the page was really writable, then we change its
-       protection back to writable */
-    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
-        host_start = address & qemu_host_page_mask;
-        host_end = host_start + qemu_host_page_size;
-
-        prot = 0;
-        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
-            p = page_find(addr >> TARGET_PAGE_BITS);
-            p->flags |= PAGE_WRITE;
-            prot |= p->flags;
-
-            /* and since the content will be modified, we must invalidate
-               the corresponding translated code. */
-            tb_invalidate_phys_page(addr, pc, puc);
-#ifdef DEBUG_TB_CHECK
-            tb_invalidate_check(addr);
-#endif
-        }
-        mprotect((void *)g2h(host_start), qemu_host_page_size,
-                 prot & PAGE_BITS);
-
-        mmap_unlock();
-        return 1;
-    }
-    mmap_unlock();
-    return 0;
-}
 #endif /* defined(CONFIG_USER_ONLY) */
 
 #if !defined(CONFIG_USER_ONLY)
@@ -2340,6 +801,16 @@ void qemu_flush_coalesced_mmio_buffer(void)
         kvm_flush_coalesced_mmio_buffer();
 }
 
+void qemu_mutex_lock_ramlist(void)
+{
+    qemu_mutex_lock(&ram_list.mutex);
+}
+
+void qemu_mutex_unlock_ramlist(void)
+{
+    qemu_mutex_unlock(&ram_list.mutex);
+}
+
 #if defined(__linux__) && !defined(TARGET_S390X)
 
 #include <sys/vfs.h>
@@ -2392,18 +863,16 @@ static void *file_ram_alloc(RAMBlock *block,
         return NULL;
     }
 
-    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
-        return NULL;
-    }
+    filename = g_strdup_printf("%s/qemu_back_mem.XXXXXX", path);
 
     fd = mkstemp(filename);
     if (fd < 0) {
         perror("unable to create backing store for hugepages");
-        free(filename);
+        g_free(filename);
         return NULL;
     }
     unlink(filename);
-    free(filename);
+    g_free(filename);
 
     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 
@@ -2441,15 +910,15 @@ static ram_addr_t find_ram_offset(ram_addr_t size)
     RAMBlock *block, *next_block;
     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 
-    if (QLIST_EMPTY(&ram_list.blocks))
+    if (QTAILQ_EMPTY(&ram_list.blocks))
         return 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         ram_addr_t end, next = RAM_ADDR_MAX;
 
         end = block->offset + block->length;
 
-        QLIST_FOREACH(next_block, &ram_list.blocks, next) {
+        QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
             if (next_block->offset >= end) {
                 next = MIN(next, next_block->offset);
             }
@@ -2474,7 +943,7 @@ ram_addr_t last_ram_offset(void)
     RAMBlock *block;
     ram_addr_t last = 0;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next)
+    QTAILQ_FOREACH(block, &ram_list.blocks, next)
         last = MAX(last, block->offset + block->length);
 
     return last;
@@ -2503,7 +972,7 @@ void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
     RAMBlock *new_block, *block;
 
     new_block = NULL;
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (block->offset == addr) {
             new_block = block;
             break;
@@ -2521,13 +990,16 @@ void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
     }
     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                     new_block->idstr);
             abort();
         }
     }
+    qemu_mutex_unlock_ramlist();
 }
 
 static int memory_try_enable_merging(void *addr, size_t len)
@@ -2546,11 +1018,13 @@ static int memory_try_enable_merging(void *addr, size_t len)
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                    MemoryRegion *mr)
 {
-    RAMBlock *new_block;
+    RAMBlock *block, *new_block;
 
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
 
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
     new_block->mr = mr;
     new_block->offset = find_ram_offset(size);
     if (host) {
@@ -2582,7 +1056,21 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
     }
     new_block->length = size;
 
-    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
+    /* Keep the list sorted from biggest to smallest block.  */
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+        if (block->length < new_block->length) {
+            break;
+        }
+    }
+    if (block) {
+        QTAILQ_INSERT_BEFORE(block, new_block, next);
+    } else {
+        QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
+    }
+    ram_list.mru_block = NULL;
+
+    ram_list.version++;
+    qemu_mutex_unlock_ramlist();
 
     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
                                        last_ram_offset() >> TARGET_PAGE_BITS);
@@ -2608,22 +1096,31 @@ void qemu_ram_free_from_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr == block->offset) {
-            QLIST_REMOVE(block, next);
+            QTAILQ_REMOVE(&ram_list.blocks, block, next);
+            ram_list.mru_block = NULL;
+            ram_list.version++;
             g_free(block);
-            return;
+            break;
         }
     }
+    qemu_mutex_unlock_ramlist();
 }
 
 void qemu_ram_free(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* This assumes the iothread lock is taken here too.  */
+    qemu_mutex_lock_ramlist();
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr == block->offset) {
-            QLIST_REMOVE(block, next);
+            QTAILQ_REMOVE(&ram_list.blocks, block, next);
+            ram_list.mru_block = NULL;
+            ram_list.version++;
             if (block->flags & RAM_PREALLOC_MASK) {
                 ;
             } else if (mem_path) {
@@ -2649,9 +1146,10 @@ void qemu_ram_free(ram_addr_t addr)
 #endif
             }
             g_free(block);
-            return;
+            break;
         }
     }
+    qemu_mutex_unlock_ramlist();
 
 }
 
@@ -2663,7 +1161,7 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
     int flags;
     void *area, *vaddr;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         offset = addr - block->offset;
         if (offset < block->length) {
             vaddr = block->host + offset;
@@ -2729,43 +1227,48 @@ void *qemu_get_ram_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* The list is protected by the iothread lock here.  */
+    block = ram_list.mru_block;
+    if (block && addr - block->offset < block->length) {
+        goto found;
+    }
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr - block->offset < block->length) {
-            /* Move this entry to to start of the list.  */
-            if (block != QLIST_FIRST(&ram_list.blocks)) {
-                QLIST_REMOVE(block, next);
-                QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
-            }
-            if (xen_enabled()) {
-                /* We need to check if the requested address is in the RAM
-                 * because we don't want to map the entire memory in QEMU.
-                 * In that case just map until the end of the page.
-                 */
-                if (block->offset == 0) {
-                    return xen_map_cache(addr, 0, 0);
-                } else if (block->host == NULL) {
-                    block->host =
-                        xen_map_cache(block->offset, block->length, 1);
-                }
-            }
-            return block->host + (addr - block->offset);
+            goto found;
         }
     }
 
     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
     abort();
 
-    return NULL;
+found:
+    ram_list.mru_block = block;
+    if (xen_enabled()) {
+        /* We need to check if the requested address is in the RAM
+         * because we don't want to map the entire memory in QEMU.
+         * In that case just map until the end of the page.
+         */
+        if (block->offset == 0) {
+            return xen_map_cache(addr, 0, 0);
+        } else if (block->host == NULL) {
+            block->host =
+                xen_map_cache(block->offset, block->length, 1);
+        }
+    }
+    return block->host + (addr - block->offset);
 }
 
-/* Return a host pointer to ram allocated with qemu_ram_alloc.
- * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
+/* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
+ * qemu_get_ram_ptr but do not touch ram_list.mru_block.
+ *
+ * ??? Is this still necessary?
  */
 static void *qemu_safe_ram_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    /* The list is protected by the iothread lock here.  */
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         if (addr - block->offset < block->length) {
             if (xen_enabled()) {
                 /* We need to check if the requested address is in the RAM
@@ -2801,7 +1304,7 @@ static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
     } else {
         RAMBlock *block;
 
-        QLIST_FOREACH(block, &ram_list.blocks, next) {
+        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
             if (addr - block->offset < block->length) {
                 if (addr - block->offset + *size > block->length)
                     *size = block->length - addr + block->offset;
@@ -2829,7 +1332,7 @@ int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
         return 0;
     }
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         /* This case append when the block is not mapped. */
         if (block->host == NULL) {
             continue;
@@ -2952,7 +1455,6 @@ static void check_watchpoint(int offset, int len_mask, int flags)
 {
     CPUArchState *env = cpu_single_env;
     target_ulong pc, cs_base;
-    TranslationBlock *tb;
     target_ulong vaddr;
     CPUWatchpoint *wp;
     int cpu_flags;
@@ -2971,13 +1473,7 @@ static void check_watchpoint(int offset, int len_mask, int flags)
             wp->flags |= BP_WATCHPOINT_HIT;
             if (!env->watchpoint_hit) {
                 env->watchpoint_hit = wp;
-                tb = tb_find_pc(env->mem_io_pc);
-                if (!tb) {
-                    cpu_abort(env, "check_watchpoint: could not find TB for "
-                              "pc=%p", (void *)env->mem_io_pc);
-                }
-                cpu_restore_state(tb, env, env->mem_io_pc);
-                tb_phys_invalidate(tb, -1);
+                tb_check_watchpoint(env);
                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                     env->exception_index = EXCP_DEBUG;
                     cpu_loop_exit(env);
@@ -4088,119 +2584,8 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
 }
 #endif
 
-/* in deterministic execution mode, instructions doing device I/Os
-   must be at the end of the TB */
-void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-    uint32_t n, cflags;
-    target_ulong pc, cs_base;
-    uint64_t flags;
-
-    tb = tb_find_pc(retaddr);
-    if (!tb) {
-        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
-                  (void *)retaddr);
-    }
-    n = env->icount_decr.u16.low + tb->icount;
-    cpu_restore_state(tb, env, retaddr);
-    /* Calculate how many instructions had been executed before the fault
-       occurred.  */
-    n = n - env->icount_decr.u16.low;
-    /* Generate a new TB ending on the I/O insn.  */
-    n++;
-    /* On MIPS and SH, delay slot instructions can only be restarted if
-       they were already the first instruction in the TB.  If this is not
-       the first instruction in a TB then re-execute the preceding
-       branch.  */
-#if defined(TARGET_MIPS)
-    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
-        env->active_tc.PC -= 4;
-        env->icount_decr.u16.low++;
-        env->hflags &= ~MIPS_HFLAG_BMASK;
-    }
-#elif defined(TARGET_SH4)
-    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
-            && n > 1) {
-        env->pc -= 2;
-        env->icount_decr.u16.low++;
-        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
-    }
-#endif
-    /* This should never happen.  */
-    if (n > CF_COUNT_MASK)
-        cpu_abort(env, "TB too big during recompile");
-
-    cflags = n | CF_LAST_IO;
-    pc = tb->pc;
-    cs_base = tb->cs_base;
-    flags = tb->flags;
-    tb_phys_invalidate(tb, -1);
-    /* FIXME: In theory this could raise an exception.  In practice
-       we have already translated the block once so it's probably ok.  */
-    tb_gen_code(env, pc, cs_base, flags, cflags);
-    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
-       the first in the TB) then we end up generating a whole new TB and
-       repeating the fault, which is horribly inefficient.
-       Better would be to execute just this insn uncached, or generate a
-       second new TB.  */
-    cpu_resume_from_signal(env, NULL);
-}
-
 #if !defined(CONFIG_USER_ONLY)
 
-void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
-{
-    int i, target_code_size, max_target_code_size;
-    int direct_jmp_count, direct_jmp2_count, cross_page;
-    TranslationBlock *tb;
-
-    target_code_size = 0;
-    max_target_code_size = 0;
-    cross_page = 0;
-    direct_jmp_count = 0;
-    direct_jmp2_count = 0;
-    for(i = 0; i < nb_tbs; i++) {
-        tb = &tbs[i];
-        target_code_size += tb->size;
-        if (tb->size > max_target_code_size)
-            max_target_code_size = tb->size;
-        if (tb->page_addr[1] != -1)
-            cross_page++;
-        if (tb->tb_next_offset[0] != 0xffff) {
-            direct_jmp_count++;
-            if (tb->tb_next_offset[1] != 0xffff) {
-                direct_jmp2_count++;
-            }
-        }
-    }
-    /* XXX: avoid using doubles ? */
-    cpu_fprintf(f, "Translation buffer state:\n");
-    cpu_fprintf(f, "gen code size       %td/%zd\n",
-                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
-    cpu_fprintf(f, "TB count            %d/%d\n", 
-                nb_tbs, code_gen_max_blocks);
-    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
-                nb_tbs ? target_code_size / nb_tbs : 0,
-                max_target_code_size);
-    cpu_fprintf(f, "TB avg host size    %td bytes (expansion ratio: %0.1f)\n",
-                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
-                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
-    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
-            cross_page,
-            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
-    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
-                direct_jmp_count,
-                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
-                direct_jmp2_count,
-                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
-    cpu_fprintf(f, "\nStatistics:\n");
-    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
-    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
-    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
-    tcg_dump_info(f, cpu_fprintf);
-}
-
 /*
  * A helper function for the _utterly broken_ virtio device model to find out if
  * it's running on a big endian machine. Don't do this at home kids!
diff --git a/fpu/softfloat.c b/fpu/softfloat.c
index 8413146..83ccc4b 100644
--- a/fpu/softfloat.c
+++ b/fpu/softfloat.c
@@ -40,7 +40,7 @@ these four paragraphs for those parts of this code that are retained.
  */
 #include "config.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 /*----------------------------------------------------------------------------
 | Primitive arithmetic functions, including multi-word arithmetic, and
@@ -1271,11 +1271,18 @@ float64 int64_to_float64( int64 a STATUS_PARAM )
 
 }
 
-float64 uint64_to_float64( uint64 a STATUS_PARAM )
+float64 uint64_to_float64(uint64 a STATUS_PARAM)
 {
-    if ( a == 0 ) return float64_zero;
-    return normalizeRoundAndPackFloat64( 0, 0x43C, a STATUS_VAR );
+    int exp =  0x43C;
 
+    if (a == 0) {
+        return float64_zero;
+    }
+    if ((int64_t)a < 0) {
+        shift64RightJamming(a, 1, &a);
+        exp += 1;
+    }
+    return normalizeRoundAndPackFloat64(0, exp, a STATUS_VAR);
 }
 
 /*----------------------------------------------------------------------------
@@ -1332,6 +1339,14 @@ float128 int64_to_float128( int64 a STATUS_PARAM )
 
 }
 
+float128 uint64_to_float128(uint64 a STATUS_PARAM)
+{
+    if (a == 0) {
+        return float128_zero;
+    }
+    return normalizeRoundAndPackFloat128(0, 0x406E, a, 0 STATUS_VAR);
+}
+
 /*----------------------------------------------------------------------------
 | Returns the result of converting the single-precision floating-point value
 | `a' to the 32-bit two's complement integer format.  The conversion is
@@ -2219,7 +2234,7 @@ float32 float32_muladd(float32 a, float32 b, float32 c, int flags STATUS_PARAM)
             }
         }
         /* Zero plus something non-zero : just return the something */
-        return make_float32(float32_val(c) ^ (signflip << 31));
+        return packFloat32(cSign ^ signflip, cExp, cSig);
     }
 
     if (aExp == 0) {
@@ -3772,7 +3787,7 @@ float64 float64_muladd(float64 a, float64 b, float64 c, int flags STATUS_PARAM)
             }
         }
         /* Zero plus something non-zero : just return the something */
-        return make_float64(float64_val(c) ^ ((uint64_t)signflip << 63));
+        return packFloat64(cSign ^ signflip, cExp, cSig);
     }
 
     if (aExp == 0) {
diff --git a/fsdev/Makefile.objs b/fsdev/Makefile.objs
index cb1e250..206289c 100644
--- a/fsdev/Makefile.objs
+++ b/fsdev/Makefile.objs
@@ -1,9 +1,10 @@
 ifeq ($(CONFIG_REALLY_VIRTFS),y)
 common-obj-y = qemu-fsdev.o virtio-9p-marshal.o
-
-# Toplevel always builds this; targets without virtio will put it in
-# common-obj-y
-extra-obj-y = qemu-fsdev-dummy.o
 else
 common-obj-y = qemu-fsdev-dummy.o
 endif
+common-obj-y += qemu-fsdev-opts.o
+
+# Toplevel always builds this; targets without virtio will put it in
+# common-obj-y
+common-obj-$(CONFIG_ALL) += qemu-fsdev-dummy.o
diff --git a/fsdev/qemu-fsdev-dummy.c b/fsdev/qemu-fsdev-dummy.c
index 300f275..7dc2630 100644
--- a/fsdev/qemu-fsdev-dummy.c
+++ b/fsdev/qemu-fsdev-dummy.c
@@ -13,17 +13,10 @@
 #include <stdio.h>
 #include <string.h>
 #include "qemu-fsdev.h"
-#include "qemu-config.h"
-#include "module.h"
+#include "qemu/config-file.h"
+#include "qemu/module.h"
 
 int qemu_fsdev_add(QemuOpts *opts)
 {
     return 0;
 }
-
-static void fsdev_register_config(void)
-{
-    qemu_add_opts(&qemu_fsdev_opts);
-    qemu_add_opts(&qemu_virtfs_opts);
-}
-machine_init(fsdev_register_config);
diff --git a/fsdev/qemu-fsdev-opts.c b/fsdev/qemu-fsdev-opts.c
new file mode 100644
index 0000000..6311c7a
--- /dev/null
+++ b/fsdev/qemu-fsdev-opts.c
@@ -0,0 +1,85 @@
+/*
+ * Virtio 9p
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/config-file.h"
+#include "qemu/option.h"
+#include "qemu/module.h"
+
+static QemuOptsList qemu_fsdev_opts = {
+    .name = "fsdev",
+    .implied_opt_name = "fsdriver",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_fsdev_opts.head),
+    .desc = {
+        {
+            .name = "fsdriver",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "security_model",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "writeout",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "readonly",
+            .type = QEMU_OPT_BOOL,
+
+        }, {
+            .name = "socket",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "sock_fd",
+            .type = QEMU_OPT_NUMBER,
+        },
+
+        { /*End of list */ }
+    },
+};
+
+static QemuOptsList qemu_virtfs_opts = {
+    .name = "virtfs",
+    .implied_opt_name = "fsdriver",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_virtfs_opts.head),
+    .desc = {
+        {
+            .name = "fsdriver",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "mount_tag",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "security_model",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "writeout",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "readonly",
+            .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "socket",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "sock_fd",
+            .type = QEMU_OPT_NUMBER,
+        },
+
+        { /*End of list */ }
+    },
+};
+
+static void fsdev_register_config(void)
+{
+    qemu_add_opts(&qemu_fsdev_opts);
+    qemu_add_opts(&qemu_virtfs_opts);
+}
+machine_init(fsdev_register_config);
diff --git a/fsdev/qemu-fsdev.c b/fsdev/qemu-fsdev.c
index e20202a..6eaf36d 100644
--- a/fsdev/qemu-fsdev.c
+++ b/fsdev/qemu-fsdev.c
@@ -13,10 +13,10 @@
 #include <stdio.h>
 #include <string.h>
 #include "qemu-fsdev.h"
-#include "qemu-queue.h"
-#include "osdep.h"
+#include "qemu/queue.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "qemu-config.h"
+#include "qemu/config-file.h"
 
 static QTAILQ_HEAD(FsDriverEntry_head, FsDriverListEntry) fsdriver_entries =
     QTAILQ_HEAD_INITIALIZER(fsdriver_entries);
@@ -97,11 +97,3 @@ FsDriverEntry *get_fsdev_fsentry(char *id)
     }
     return NULL;
 }
-
-static void fsdev_register_config(void)
-{
-    qemu_add_opts(&qemu_fsdev_opts);
-    qemu_add_opts(&qemu_virtfs_opts);
-}
-machine_init(fsdev_register_config);
-
diff --git a/fsdev/qemu-fsdev.h b/fsdev/qemu-fsdev.h
index 1af1f54..9fa45bf 100644
--- a/fsdev/qemu-fsdev.h
+++ b/fsdev/qemu-fsdev.h
@@ -12,7 +12,7 @@
  */
 #ifndef QEMU_FSDEV_H
 #define QEMU_FSDEV_H
-#include "qemu-option.h"
+#include "qemu/option.h"
 #include "file-op-9p.h"
 
 
diff --git a/fsdev/virtfs-proxy-helper.c b/fsdev/virtfs-proxy-helper.c
index f9a8270..36f6616 100644
--- a/fsdev/virtfs-proxy-helper.c
+++ b/fsdev/virtfs-proxy-helper.c
@@ -21,8 +21,8 @@
 #include <linux/magic.h>
 #endif
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "qemu-xattr.h"
+#include "qemu/sockets.h"
+#include "qemu/xattr.h"
 #include "virtio-9p-marshal.h"
 #include "hw/9pfs/virtio-9p-proxy.h"
 #include "fsdev/virtio-9p-marshal.h"
@@ -272,31 +272,76 @@ static int send_status(int sockfd, struct iovec *iovec, int status)
 /*
  * from man 7 capabilities, section
  * Effect of User ID Changes on Capabilities:
- * 4. If the file system user ID is changed from 0 to nonzero (see setfsuid(2))
- * then the following capabilities are cleared from the effective set:
- * CAP_CHOWN, CAP_DAC_OVERRIDE, CAP_DAC_READ_SEARCH,  CAP_FOWNER, CAP_FSETID,
- * CAP_LINUX_IMMUTABLE  (since  Linux 2.2.30), CAP_MAC_OVERRIDE, and CAP_MKNOD
- * (since Linux 2.2.30). If the file system UID is changed from nonzero to 0,
- * then any of these capabilities that are enabled in the permitted set
- * are enabled in the effective set.
+ * If the effective user ID is changed from nonzero to 0, then the permitted
+ * set is copied to the effective set.  If the effective user ID is changed
+ * from 0 to nonzero, then all capabilities are are cleared from the effective
+ * set.
+ *
+ * The setfsuid/setfsgid man pages warn that changing the effective user ID may
+ * expose the program to unwanted signals, but this is not true anymore: for an
+ * unprivileged (without CAP_KILL) program to send a signal, the real or
+ * effective user ID of the sending process must equal the real or saved user
+ * ID of the target process.  Even when dropping privileges, it is enough to
+ * keep the saved UID to a "privileged" value and virtfs-proxy-helper won't
+ * be exposed to signals.  So just use setresuid/setresgid.
  */
-static int setfsugid(int uid, int gid)
+static int setugid(int uid, int gid, int *suid, int *sgid)
 {
+    int retval;
+
     /*
-     * We still need DAC_OVERRIDE because  we don't change
+     * We still need DAC_OVERRIDE because we don't change
      * supplementary group ids, and hence may be subjected DAC rules
      */
     cap_value_t cap_list[] = {
         CAP_DAC_OVERRIDE,
     };
 
-    setfsgid(gid);
-    setfsuid(uid);
+    *suid = geteuid();
+    *sgid = getegid();
+
+    if (setresgid(-1, gid, *sgid) == -1) {
+        retval = -errno;
+        goto err_out;
+    }
+
+    if (setresuid(-1, uid, *suid) == -1) {
+        retval = -errno;
+        goto err_sgid;
+    }
 
     if (uid != 0 || gid != 0) {
-        return do_cap_set(cap_list, ARRAY_SIZE(cap_list), 0);
+        if (do_cap_set(cap_list, ARRAY_SIZE(cap_list), 0) < 0) {
+            retval = -errno;
+            goto err_suid;
+        }
     }
     return 0;
+
+err_suid:
+    if (setresuid(-1, *suid, *suid) == -1) {
+        abort();
+    }
+err_sgid:
+    if (setresgid(-1, *sgid, *sgid) == -1) {
+        abort();
+    }
+err_out:
+    return retval;
+}
+
+/*
+ * This is used to reset the ugid back with the saved values
+ * There is nothing much we can do checking error values here.
+ */
+static void resetugid(int suid, int sgid)
+{
+    if (setresgid(-1, sgid, sgid) == -1) {
+        abort();
+    }
+    if (setresuid(-1, suid, suid) == -1) {
+        abort();
+    }
 }
 
 /*
@@ -578,18 +623,15 @@ static int do_create_others(int type, struct iovec *iovec)
 
     v9fs_string_init(&path);
     v9fs_string_init(&oldpath);
-    cur_uid = geteuid();
-    cur_gid = getegid();
 
     retval = proxy_unmarshal(iovec, offset, "dd", &uid, &gid);
     if (retval < 0) {
         return retval;
     }
     offset += retval;
-    retval = setfsugid(uid, gid);
+    retval = setugid(uid, gid, &cur_uid, &cur_gid);
     if (retval < 0) {
-        retval = -errno;
-        goto err_out;
+        goto unmarshal_err_out;
     }
     switch (type) {
     case T_MKNOD:
@@ -619,9 +661,10 @@ static int do_create_others(int type, struct iovec *iovec)
     }
 
 err_out:
+    resetugid(cur_uid, cur_gid);
+unmarshal_err_out:
     v9fs_string_free(&path);
     v9fs_string_free(&oldpath);
-    setfsugid(cur_uid, cur_gid);
     return retval;
 }
 
@@ -641,24 +684,16 @@ static int do_create(struct iovec *iovec)
     if (ret < 0) {
         goto unmarshal_err_out;
     }
-    cur_uid = geteuid();
-    cur_gid = getegid();
-    ret = setfsugid(uid, gid);
+    ret = setugid(uid, gid, &cur_uid, &cur_gid);
     if (ret < 0) {
-        /*
-         * On failure reset back to the
-         * old uid/gid
-         */
-        ret = -errno;
-        goto err_out;
+        goto unmarshal_err_out;
     }
     ret = open(path.data, flags, mode);
     if (ret < 0) {
         ret = -errno;
     }
 
-err_out:
-    setfsugid(cur_uid, cur_gid);
+    resetugid(cur_uid, cur_gid);
 unmarshal_err_out:
     v9fs_string_free(&path);
     return ret;
@@ -1004,7 +1039,7 @@ int main(int argc, char **argv)
         }
         switch (c) {
         case 'p':
-            rpath = strdup(optarg);
+            rpath = g_strdup(optarg);
             break;
         case 'n':
             is_daemon = false;
@@ -1013,7 +1048,7 @@ int main(int argc, char **argv)
             sock = atoi(optarg);
             break;
         case 's':
-            sock_name = strdup(optarg);
+            sock_name = g_strdup(optarg);
             break;
         case 'u':
             own_u = atoi(optarg);
diff --git a/fsdev/virtio-9p-marshal.c b/fsdev/virtio-9p-marshal.c
index bf980bf..20f308b 100644
--- a/fsdev/virtio-9p-marshal.c
+++ b/fsdev/virtio-9p-marshal.c
@@ -22,9 +22,9 @@
 #include <stdint.h>
 #include <errno.h>
 
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include "virtio-9p-marshal.h"
-#include "bswap.h"
+#include "qemu/bswap.h"
 
 void v9fs_string_free(V9fsString *str)
 {
diff --git a/gdbstub.c b/gdbstub.c
index d02ec75..6cd26f1 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -29,17 +29,18 @@
 
 #include "qemu.h"
 #else
-#include "monitor.h"
-#include "qemu-char.h"
-#include "sysemu.h"
-#include "gdbstub.h"
+#include "monitor/monitor.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
+#include "exec/gdbstub.h"
 #endif
 
 #define MAX_PACKET_LENGTH 4096
 
 #include "cpu.h"
-#include "qemu_socket.h"
-#include "kvm.h"
+#include "qemu/sockets.h"
+#include "sysemu/kvm.h"
+#include "qemu/bitops.h"
 
 #ifndef TARGET_CPU_MEMORY_RW_DEBUG
 static inline int target_memory_rw_debug(CPUArchState *env, target_ulong addr,
@@ -1535,27 +1536,34 @@ static int cpu_gdb_write_register(CPUAlphaState *env, uint8_t *mem_buf, int n)
 }
 #elif defined (TARGET_S390X)
 
-#define NUM_CORE_REGS S390_NUM_TOTAL_REGS
+#define NUM_CORE_REGS  S390_NUM_REGS
 
 static int cpu_gdb_read_register(CPUS390XState *env, uint8_t *mem_buf, int n)
 {
+    uint64_t val;
+    int cc_op;
+
     switch (n) {
-        case S390_PSWM_REGNUM: GET_REGL(env->psw.mask); break;
-        case S390_PSWA_REGNUM: GET_REGL(env->psw.addr); break;
-        case S390_R0_REGNUM ... S390_R15_REGNUM:
-            GET_REGL(env->regs[n-S390_R0_REGNUM]); break;
-        case S390_A0_REGNUM ... S390_A15_REGNUM:
-            GET_REG32(env->aregs[n-S390_A0_REGNUM]); break;
-        case S390_FPC_REGNUM: GET_REG32(env->fpc); break;
-        case S390_F0_REGNUM ... S390_F15_REGNUM:
-            /* XXX */
-            break;
-        case S390_PC_REGNUM: GET_REGL(env->psw.addr); break;
-        case S390_CC_REGNUM:
-            env->cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst,
-                                 env->cc_vr);
-            GET_REG32(env->cc_op);
-            break;
+    case S390_PSWM_REGNUM:
+        cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst, env->cc_vr);
+        val = deposit64(env->psw.mask, 44, 2, cc_op);
+        GET_REGL(val);
+        break;
+    case S390_PSWA_REGNUM:
+        GET_REGL(env->psw.addr);
+        break;
+    case S390_R0_REGNUM ... S390_R15_REGNUM:
+        GET_REGL(env->regs[n-S390_R0_REGNUM]);
+        break;
+    case S390_A0_REGNUM ... S390_A15_REGNUM:
+        GET_REG32(env->aregs[n-S390_A0_REGNUM]);
+        break;
+    case S390_FPC_REGNUM:
+        GET_REG32(env->fpc);
+        break;
+    case S390_F0_REGNUM ... S390_F15_REGNUM:
+        GET_REG64(env->fregs[n-S390_F0_REGNUM].ll);
+        break;
     }
 
     return 0;
@@ -1570,20 +1578,30 @@ static int cpu_gdb_write_register(CPUS390XState *env, uint8_t *mem_buf, int n)
     tmp32 = ldl_p(mem_buf);
 
     switch (n) {
-        case S390_PSWM_REGNUM: env->psw.mask = tmpl; break;
-        case S390_PSWA_REGNUM: env->psw.addr = tmpl; break;
-        case S390_R0_REGNUM ... S390_R15_REGNUM:
-            env->regs[n-S390_R0_REGNUM] = tmpl; break;
-        case S390_A0_REGNUM ... S390_A15_REGNUM:
-            env->aregs[n-S390_A0_REGNUM] = tmp32; r=4; break;
-        case S390_FPC_REGNUM: env->fpc = tmp32; r=4; break;
-        case S390_F0_REGNUM ... S390_F15_REGNUM:
-            /* XXX */
-            break;
-        case S390_PC_REGNUM: env->psw.addr = tmpl; break;
-        case S390_CC_REGNUM: env->cc_op = tmp32; r=4; break;
+    case S390_PSWM_REGNUM:
+        env->psw.mask = tmpl;
+        env->cc_op = extract64(tmpl, 44, 2);
+        break;
+    case S390_PSWA_REGNUM:
+        env->psw.addr = tmpl;
+        break;
+    case S390_R0_REGNUM ... S390_R15_REGNUM:
+        env->regs[n-S390_R0_REGNUM] = tmpl;
+        break;
+    case S390_A0_REGNUM ... S390_A15_REGNUM:
+        env->aregs[n-S390_A0_REGNUM] = tmp32;
+        r = 4;
+        break;
+    case S390_FPC_REGNUM:
+        env->fpc = tmp32;
+        r = 4;
+        break;
+    case S390_F0_REGNUM ... S390_F15_REGNUM:
+        env->fregs[n-S390_F0_REGNUM].ll = tmpl;
+        break;
+    default:
+        return 0;
     }
-
     return r;
 }
 #elif defined (TARGET_LM32)
@@ -2383,9 +2401,10 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
             thread = strtoull(p+16, (char **)&p, 16);
             env = find_cpu(thread);
             if (env != NULL) {
+                CPUState *cpu = ENV_GET_CPU(env);
                 cpu_synchronize_state(env);
                 len = snprintf((char *)mem_buf, sizeof(mem_buf),
-                               "CPU#%d [%s]", env->cpu_index,
+                               "CPU#%d [%s]", cpu->cpu_index,
                                env->halted ? "halted " : "running");
                 memtohex(buf, mem_buf, len);
                 put_packet(s, buf);
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 010b8c9..64008a9 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -837,6 +837,44 @@ STEXI
 @item nmi @var{cpu}
 @findex nmi
 Inject an NMI on the given CPU (x86 only).
+
+ETEXI
+
+    {
+        .name       = "ringbuf_write",
+        .args_type  = "device:s,data:s",
+        .params     = "device data",
+        .help       = "Write to a ring buffer character device",
+        .mhandler.cmd = hmp_ringbuf_write,
+    },
+
+STEXI
+@item ringbuf_write @var{device} @var{data}
+@findex ringbuf_write
+Write @var{data} to ring buffer character device @var{device}.
+@var{data} must be a UTF-8 string.
+
+ETEXI
+
+    {
+        .name       = "ringbuf_read",
+        .args_type  = "device:s,size:i",
+        .params     = "device size",
+        .help       = "Read from a ring buffer character device",
+        .mhandler.cmd = hmp_ringbuf_read,
+    },
+
+STEXI
+@item ringbuf_read @var{device}
+@findex ringbuf_read
+Read and print up to @var{size} bytes from ring buffer character
+device @var{device}.
+Certain non-printable characters are printed \uXXXX, where XXXX is the
+character code in hexadecimal.  Character \ is printed \\.
+Bug: can screw up when the buffer contains invalid UTF-8 sequences,
+NUL characters, after the ring buffer lost data, and when reading
+stops because the size limit is reached.
+
 ETEXI
 
     {
@@ -1484,12 +1522,46 @@ passed since 1970, i.e. unix epoch.
 @end table
 ETEXI
 
+HXCOMM Disabled for now, because it isn't built on top of QMP's chardev-add
+HXCOMM     {
+HXCOMM         .name       = "chardev-add",
+HXCOMM         .args_type  = "args:s",
+HXCOMM         .params     = "args",
+HXCOMM         .help       = "add chardev",
+HXCOMM         .mhandler.cmd = hmp_chardev_add,
+HXCOMM     },
+HXCOMM
+HXCOMM STEXI
+HXCOMM @item chardev_add args
+HXCOMM @findex chardev_add
+HXCOMM
+HXCOMM chardev_add accepts the same parameters as the -chardev command line switch.
+HXCOMM
+HXCOMM ETEXI
+HXCOMM
+HXCOMM     {
+HXCOMM         .name       = "chardev-remove",
+HXCOMM         .args_type  = "id:s",
+HXCOMM         .params     = "id",
+HXCOMM         .help       = "remove chardev",
+HXCOMM         .mhandler.cmd = hmp_chardev_remove,
+HXCOMM     },
+HXCOMM
+HXCOMM STEXI
+HXCOMM @item chardev_remove id
+HXCOMM @findex chardev_remove
+HXCOMM
+HXCOMM Removes the chardev @var{id}.
+HXCOMM
+HXCOMM ETEXI
+
     {
         .name       = "info",
         .args_type  = "item:s?",
         .params     = "[subcommand]",
         .help       = "show various information about the system state",
-        .mhandler.cmd = do_info,
+        .mhandler.cmd = do_info_help,
+        .sub_table = info_cmds,
     },
 
 STEXI
diff --git a/hmp.c b/hmp.c
index 180ba2b..2f47a8a 100644
--- a/hmp.c
+++ b/hmp.c
@@ -14,13 +14,14 @@
  */
 
 #include "hmp.h"
-#include "net.h"
-#include "qemu-option.h"
-#include "qemu-timer.h"
+#include "net/net.h"
+#include "char/char.h"
+#include "qemu/option.h"
+#include "qemu/timer.h"
 #include "qmp-commands.h"
-#include "qemu_socket.h"
-#include "monitor.h"
-#include "console.h"
+#include "qemu/sockets.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
 
 static void hmp_handle_error(Monitor *mon, Error **errp)
 {
@@ -30,7 +31,7 @@ static void hmp_handle_error(Monitor *mon, Error **errp)
     }
 }
 
-void hmp_info_name(Monitor *mon)
+void hmp_info_name(Monitor *mon, const QDict *qdict)
 {
     NameInfo *info;
 
@@ -41,7 +42,7 @@ void hmp_info_name(Monitor *mon)
     qapi_free_NameInfo(info);
 }
 
-void hmp_info_version(Monitor *mon)
+void hmp_info_version(Monitor *mon, const QDict *qdict)
 {
     VersionInfo *info;
 
@@ -54,7 +55,7 @@ void hmp_info_version(Monitor *mon)
     qapi_free_VersionInfo(info);
 }
 
-void hmp_info_kvm(Monitor *mon)
+void hmp_info_kvm(Monitor *mon, const QDict *qdict)
 {
     KvmInfo *info;
 
@@ -69,7 +70,7 @@ void hmp_info_kvm(Monitor *mon)
     qapi_free_KvmInfo(info);
 }
 
-void hmp_info_status(Monitor *mon)
+void hmp_info_status(Monitor *mon, const QDict *qdict)
 {
     StatusInfo *info;
 
@@ -88,7 +89,7 @@ void hmp_info_status(Monitor *mon)
     qapi_free_StatusInfo(info);
 }
 
-void hmp_info_uuid(Monitor *mon)
+void hmp_info_uuid(Monitor *mon, const QDict *qdict)
 {
     UuidInfo *info;
 
@@ -97,7 +98,7 @@ void hmp_info_uuid(Monitor *mon)
     qapi_free_UuidInfo(info);
 }
 
-void hmp_info_chardev(Monitor *mon)
+void hmp_info_chardev(Monitor *mon, const QDict *qdict)
 {
     ChardevInfoList *char_info, *info;
 
@@ -110,7 +111,7 @@ void hmp_info_chardev(Monitor *mon)
     qapi_free_ChardevInfoList(char_info);
 }
 
-void hmp_info_mice(Monitor *mon)
+void hmp_info_mice(Monitor *mon, const QDict *qdict)
 {
     MouseInfoList *mice_list, *mouse;
 
@@ -130,7 +131,7 @@ void hmp_info_mice(Monitor *mon)
     qapi_free_MouseInfoList(mice_list);
 }
 
-void hmp_info_migrate(Monitor *mon)
+void hmp_info_migrate(Monitor *mon, const QDict *qdict)
 {
     MigrationInfo *info;
     MigrationCapabilityStatusList *caps, *cap;
@@ -208,7 +209,7 @@ void hmp_info_migrate(Monitor *mon)
     qapi_free_MigrationCapabilityStatusList(caps);
 }
 
-void hmp_info_migrate_capabilities(Monitor *mon)
+void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict)
 {
     MigrationCapabilityStatusList *caps, *cap;
 
@@ -227,13 +228,13 @@ void hmp_info_migrate_capabilities(Monitor *mon)
     qapi_free_MigrationCapabilityStatusList(caps);
 }
 
-void hmp_info_migrate_cache_size(Monitor *mon)
+void hmp_info_migrate_cache_size(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "xbzrel cache size: %" PRId64 " kbytes\n",
                    qmp_query_migrate_cache_size(NULL) >> 10);
 }
 
-void hmp_info_cpus(Monitor *mon)
+void hmp_info_cpus(Monitor *mon, const QDict *qdict)
 {
     CpuInfoList *cpu_list, *cpu;
 
@@ -271,7 +272,7 @@ void hmp_info_cpus(Monitor *mon)
     qapi_free_CpuInfoList(cpu_list);
 }
 
-void hmp_info_block(Monitor *mon)
+void hmp_info_block(Monitor *mon, const QDict *qdict)
 {
     BlockInfoList *block_list, *info;
 
@@ -325,7 +326,7 @@ void hmp_info_block(Monitor *mon)
     qapi_free_BlockInfoList(block_list);
 }
 
-void hmp_info_blockstats(Monitor *mon)
+void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
 {
     BlockStatsList *stats_list, *stats;
 
@@ -359,7 +360,7 @@ void hmp_info_blockstats(Monitor *mon)
     qapi_free_BlockStatsList(stats_list);
 }
 
-void hmp_info_vnc(Monitor *mon)
+void hmp_info_vnc(Monitor *mon, const QDict *qdict)
 {
     VncInfo *info;
     Error *err = NULL;
@@ -405,7 +406,7 @@ out:
     qapi_free_VncInfo(info);
 }
 
-void hmp_info_spice(Monitor *mon)
+void hmp_info_spice(Monitor *mon, const QDict *qdict)
 {
     SpiceChannelList *chan;
     SpiceInfo *info;
@@ -452,7 +453,7 @@ out:
     qapi_free_SpiceInfo(info);
 }
 
-void hmp_info_balloon(Monitor *mon)
+void hmp_info_balloon(Monitor *mon, const QDict *qdict)
 {
     BalloonInfo *info;
     Error *err = NULL;
@@ -464,29 +465,7 @@ void hmp_info_balloon(Monitor *mon)
         return;
     }
 
-    monitor_printf(mon, "balloon: actual=%" PRId64, info->actual >> 20);
-    if (info->has_mem_swapped_in) {
-        monitor_printf(mon, " mem_swapped_in=%" PRId64, info->mem_swapped_in);
-    }
-    if (info->has_mem_swapped_out) {
-        monitor_printf(mon, " mem_swapped_out=%" PRId64, info->mem_swapped_out);
-    }
-    if (info->has_major_page_faults) {
-        monitor_printf(mon, " major_page_faults=%" PRId64,
-                       info->major_page_faults);
-    }
-    if (info->has_minor_page_faults) {
-        monitor_printf(mon, " minor_page_faults=%" PRId64,
-                       info->minor_page_faults);
-    }
-    if (info->has_free_mem) {
-        monitor_printf(mon, " free_mem=%" PRId64, info->free_mem);
-    }
-    if (info->has_total_mem) {
-        monitor_printf(mon, " total_mem=%" PRId64, info->total_mem);
-    }
-
-    monitor_printf(mon, "\n");
+    monitor_printf(mon, "balloon: actual=%" PRId64 "\n", info->actual >> 20);
 
     qapi_free_BalloonInfo(info);
 }
@@ -569,7 +548,7 @@ static void hmp_info_pci_device(Monitor *mon, const PciDeviceInfo *dev)
     }
 }
 
-void hmp_info_pci(Monitor *mon)
+void hmp_info_pci(Monitor *mon, const QDict *qdict)
 {
     PciInfoList *info_list, *info;
     Error *err = NULL;
@@ -592,7 +571,7 @@ void hmp_info_pci(Monitor *mon)
     qapi_free_PciInfoList(info_list);
 }
 
-void hmp_info_block_jobs(Monitor *mon)
+void hmp_info_block_jobs(Monitor *mon, const QDict *qdict)
 {
     BlockJobInfoList *list;
     Error *err = NULL;
@@ -683,6 +662,48 @@ void hmp_pmemsave(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &errp);
 }
 
+void hmp_ringbuf_write(Monitor *mon, const QDict *qdict)
+{
+    const char *chardev = qdict_get_str(qdict, "device");
+    const char *data = qdict_get_str(qdict, "data");
+    Error *errp = NULL;
+
+    qmp_ringbuf_write(chardev, data, false, 0, &errp);
+
+    hmp_handle_error(mon, &errp);
+}
+
+void hmp_ringbuf_read(Monitor *mon, const QDict *qdict)
+{
+    uint32_t size = qdict_get_int(qdict, "size");
+    const char *chardev = qdict_get_str(qdict, "device");
+    char *data;
+    Error *errp = NULL;
+    int i;
+
+    data = qmp_ringbuf_read(chardev, size, false, 0, &errp);
+    if (errp) {
+        monitor_printf(mon, "%s\n", error_get_pretty(errp));
+        error_free(errp);
+        return;
+    }
+
+    for (i = 0; data[i]; i++) {
+        unsigned char ch = data[i];
+
+        if (ch == '\\') {
+            monitor_printf(mon, "\\\\");
+        } else if ((ch < 0x20 && ch != '\n' && ch != '\t') || ch == 0x7F) {
+            monitor_printf(mon, "\\u%04X", ch);
+        } else {
+            monitor_printf(mon, "%c", ch);
+        }
+
+    }
+    monitor_printf(mon, "\n");
+    g_free(data);
+}
+
 static void hmp_cont_cb(void *opaque, int err)
 {
     if (!err) {
@@ -795,7 +816,7 @@ void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
 
     qmp_drive_mirror(device, filename, !!format, format,
                      full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
-                     true, mode, false, 0,
+                     true, mode, false, 0, false, 0, false, 0,
                      false, 0, false, 0, &errp);
     hmp_handle_error(mon, &errp);
 }
@@ -879,7 +900,7 @@ void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict)
     qapi_free_MigrationCapabilityStatusList(caps);
 
     if (err) {
-        monitor_printf(mon, "migrate_set_parameter: %s\n",
+        monitor_printf(mon, "migrate_set_capability: %s\n",
                        error_get_pretty(err));
         error_free(err);
     }
@@ -1335,3 +1356,26 @@ void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict)
     qmp_nbd_server_stop(&errp);
     hmp_handle_error(mon, &errp);
 }
+
+void hmp_chardev_add(Monitor *mon, const QDict *qdict)
+{
+    const char *args = qdict_get_str(qdict, "args");
+    Error *err = NULL;
+    QemuOpts *opts;
+
+    opts = qemu_opts_parse(qemu_find_opts("chardev"), args, 1);
+    if (opts == NULL) {
+        error_setg(&err, "Parsing chardev args failed");
+    } else {
+        qemu_chr_new_from_opts(opts, NULL, &err);
+    }
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_chardev_remove(Monitor *mon, const QDict *qdict)
+{
+    Error *local_err = NULL;
+
+    qmp_chardev_remove(qdict_get_str(qdict, "id"), &local_err);
+    hmp_handle_error(mon, &local_err);
+}
diff --git a/hmp.h b/hmp.h
index 0ab03be..30b3c20 100644
--- a/hmp.h
+++ b/hmp.h
@@ -16,26 +16,26 @@
 
 #include "qemu-common.h"
 #include "qapi-types.h"
-#include "qdict.h"
+#include "qapi/qmp/qdict.h"
 
-void hmp_info_name(Monitor *mon);
-void hmp_info_version(Monitor *mon);
-void hmp_info_kvm(Monitor *mon);
-void hmp_info_status(Monitor *mon);
-void hmp_info_uuid(Monitor *mon);
-void hmp_info_chardev(Monitor *mon);
-void hmp_info_mice(Monitor *mon);
-void hmp_info_migrate(Monitor *mon);
-void hmp_info_migrate_capabilities(Monitor *mon);
-void hmp_info_migrate_cache_size(Monitor *mon);
-void hmp_info_cpus(Monitor *mon);
-void hmp_info_block(Monitor *mon);
-void hmp_info_blockstats(Monitor *mon);
-void hmp_info_vnc(Monitor *mon);
-void hmp_info_spice(Monitor *mon);
-void hmp_info_balloon(Monitor *mon);
-void hmp_info_pci(Monitor *mon);
-void hmp_info_block_jobs(Monitor *mon);
+void hmp_info_name(Monitor *mon, const QDict *qdict);
+void hmp_info_version(Monitor *mon, const QDict *qdict);
+void hmp_info_kvm(Monitor *mon, const QDict *qdict);
+void hmp_info_status(Monitor *mon, const QDict *qdict);
+void hmp_info_uuid(Monitor *mon, const QDict *qdict);
+void hmp_info_chardev(Monitor *mon, const QDict *qdict);
+void hmp_info_mice(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate_cache_size(Monitor *mon, const QDict *qdict);
+void hmp_info_cpus(Monitor *mon, const QDict *qdict);
+void hmp_info_block(Monitor *mon, const QDict *qdict);
+void hmp_info_blockstats(Monitor *mon, const QDict *qdict);
+void hmp_info_vnc(Monitor *mon, const QDict *qdict);
+void hmp_info_spice(Monitor *mon, const QDict *qdict);
+void hmp_info_balloon(Monitor *mon, const QDict *qdict);
+void hmp_info_pci(Monitor *mon, const QDict *qdict);
+void hmp_info_block_jobs(Monitor *mon, const QDict *qdict);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
@@ -43,6 +43,8 @@ void hmp_system_powerdown(Monitor *mon, const QDict *qdict);
 void hmp_cpu(Monitor *mon, const QDict *qdict);
 void hmp_memsave(Monitor *mon, const QDict *qdict);
 void hmp_pmemsave(Monitor *mon, const QDict *qdict);
+void hmp_ringbuf_write(Monitor *mon, const QDict *qdict);
+void hmp_ringbuf_read(Monitor *mon, const QDict *qdict);
 void hmp_cont(Monitor *mon, const QDict *qdict);
 void hmp_system_wakeup(Monitor *mon, const QDict *qdict);
 void hmp_inject_nmi(Monitor *mon, const QDict *qdict);
@@ -80,5 +82,7 @@ void hmp_screen_dump(Monitor *mon, const QDict *qdict);
 void hmp_nbd_server_start(Monitor *mon, const QDict *qdict);
 void hmp_nbd_server_add(Monitor *mon, const QDict *qdict);
 void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict);
+void hmp_chardev_add(Monitor *mon, const QDict *qdict);
+void hmp_chardev_remove(Monitor *mon, const QDict *qdict);
 
 #endif
diff --git a/hw/9pfs/codir.c b/hw/9pfs/codir.c
index 3d18828..65ad329 100644
--- a/hw/9pfs/codir.c
+++ b/hw/9pfs/codir.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_readdir_r(V9fsPDU *pdu, V9fsFidState *fidp, struct dirent *dent,
diff --git a/hw/9pfs/cofile.c b/hw/9pfs/cofile.c
index 9345aae..2efebf3 100644
--- a/hw/9pfs/cofile.c
+++ b/hw/9pfs/cofile.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t st_mode,
diff --git a/hw/9pfs/cofs.c b/hw/9pfs/cofs.c
index 83f125b..3891050 100644
--- a/hw/9pfs/cofs.c
+++ b/hw/9pfs/cofs.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_readlink(V9fsPDU *pdu, V9fsPath *path, V9fsString *buf)
diff --git a/hw/9pfs/coxattr.c b/hw/9pfs/coxattr.c
index 8a48228..18ee08d 100644
--- a/hw/9pfs/coxattr.c
+++ b/hw/9pfs/coxattr.c
@@ -13,8 +13,8 @@
  */
 
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 int v9fs_co_llistxattr(V9fsPDU *pdu, V9fsPath *path, void *value, size_t size)
diff --git a/hw/9pfs/virtio-9p-coth.c b/hw/9pfs/virtio-9p-coth.c
index 25556cc..ae6cde8 100644
--- a/hw/9pfs/virtio-9p-coth.c
+++ b/hw/9pfs/virtio-9p-coth.c
@@ -12,10 +12,9 @@
  *
  */
 
-#include "qemu-char.h"
 #include "fsdev/qemu-fsdev.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p-coth.h"
 
 /* v9fs glib thread pool */
diff --git a/hw/9pfs/virtio-9p-coth.h b/hw/9pfs/virtio-9p-coth.h
index c31c965..86d5ed4 100644
--- a/hw/9pfs/virtio-9p-coth.h
+++ b/hw/9pfs/virtio-9p-coth.h
@@ -15,8 +15,8 @@
 #ifndef _QEMU_VIRTIO_9P_COTH_H
 #define _QEMU_VIRTIO_9P_COTH_H
 
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 #include "virtio-9p.h"
 #include <glib.h>
 
diff --git a/hw/9pfs/virtio-9p-device.c b/hw/9pfs/virtio-9p-device.c
index b8220ab..74155fb 100644
--- a/hw/9pfs/virtio-9p-device.c
+++ b/hw/9pfs/virtio-9p-device.c
@@ -13,7 +13,7 @@
 
 #include "hw/virtio.h"
 #include "hw/pc.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "hw/virtio-pci.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
@@ -85,11 +85,7 @@ VirtIODevice *virtio_9p_init(DeviceState *dev, V9fsConf *conf)
     }
 
     s->ctx.export_flags = fse->export_flags;
-    if (fse->path) {
-        s->ctx.fs_root = g_strdup(fse->path);
-    } else {
-        s->ctx.fs_root = NULL;
-    }
+    s->ctx.fs_root = g_strdup(fse->path);
     s->ctx.exops.get_st_gen = NULL;
     len = strlen(conf->tag);
     if (len > MAX_TAG_LEN - 1) {
@@ -98,7 +94,7 @@ VirtIODevice *virtio_9p_init(DeviceState *dev, V9fsConf *conf)
         exit(1);
     }
 
-    s->tag = strdup(conf->tag);
+    s->tag = g_strdup(conf->tag);
     s->ctx.uid = -1;
 
     s->ops = fse->ops;
@@ -170,14 +166,14 @@ static void virtio_9p_class_init(ObjectClass *klass, void *data)
 
     k->init = virtio_9p_init_pci;
     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
-    k->device_id = 0x1009;
+    k->device_id = PCI_DEVICE_ID_VIRTIO_9P;
     k->revision = VIRTIO_PCI_ABI_VERSION;
     k->class_id = 0x2;
     dc->props = virtio_9p_properties;
     dc->reset = virtio_pci_reset;
 }
 
-static TypeInfo virtio_9p_info = {
+static const TypeInfo virtio_9p_info = {
     .name          = "virtio-9p-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
diff --git a/hw/9pfs/virtio-9p-handle.c b/hw/9pfs/virtio-9p-handle.c
index f96d17a..e30fdb6 100644
--- a/hw/9pfs/virtio-9p-handle.c
+++ b/hw/9pfs/virtio-9p-handle.c
@@ -19,7 +19,7 @@
 #include <grp.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #include <unistd.h>
 #include <linux/fs.h>
 #ifdef CONFIG_LINUX_MAGIC_H
diff --git a/hw/9pfs/virtio-9p-local.c b/hw/9pfs/virtio-9p-local.c
index 33a41d2..f1b1c83 100644
--- a/hw/9pfs/virtio-9p-local.c
+++ b/hw/9pfs/virtio-9p-local.c
@@ -19,7 +19,7 @@
 #include <grp.h>
 #include <sys/socket.h>
 #include <sys/un.h>
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #include <libgen.h>
 #include <linux/fs.h>
 #ifdef CONFIG_LINUX_MAGIC_H
@@ -46,7 +46,7 @@ static const char *local_mapped_attr_path(FsContext *ctx,
                                           const char *path, char *buffer)
 {
     char *dir_name;
-    char *tmp_path = strdup(path);
+    char *tmp_path = g_strdup(path);
     char *base_name = basename(tmp_path);
 
     /* NULL terminate the directory */
@@ -55,7 +55,7 @@ static const char *local_mapped_attr_path(FsContext *ctx,
 
     snprintf(buffer, PATH_MAX, "%s/%s/%s/%s",
              ctx->fs_root, dir_name, VIRTFS_META_DIR, base_name);
-    free(tmp_path);
+    g_free(tmp_path);
     return buffer;
 }
 
@@ -130,7 +130,7 @@ static int local_create_mapped_attr_dir(FsContext *ctx, const char *path)
 {
     int err;
     char attr_dir[PATH_MAX];
-    char *tmp_path = strdup(path);
+    char *tmp_path = g_strdup(path);
 
     snprintf(attr_dir, PATH_MAX, "%s/%s/%s",
              ctx->fs_root, dirname(tmp_path), VIRTFS_META_DIR);
@@ -139,7 +139,7 @@ static int local_create_mapped_attr_dir(FsContext *ctx, const char *path)
     if (err < 0 && errno == EEXIST) {
         err = 0;
     }
-    free(tmp_path);
+    g_free(tmp_path);
     return err;
 }
 
diff --git a/hw/9pfs/virtio-9p-posix-acl.c b/hw/9pfs/virtio-9p-posix-acl.c
index c064017..08bb0e8 100644
--- a/hw/9pfs/virtio-9p-posix-acl.c
+++ b/hw/9pfs/virtio-9p-posix-acl.c
@@ -12,7 +12,7 @@
  */
 
 #include <sys/types.h>
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #include "hw/virtio.h"
 #include "virtio-9p.h"
 #include "fsdev/file-op-9p.h"
diff --git a/hw/9pfs/virtio-9p-proxy.c b/hw/9pfs/virtio-9p-proxy.c
index d5ad208..54e9875 100644
--- a/hw/9pfs/virtio-9p-proxy.c
+++ b/hw/9pfs/virtio-9p-proxy.c
@@ -521,7 +521,7 @@ static int v9fs_request(V9fsProxy *proxy, int type,
         }
         break;
     default:
-        error_report("Invalid type %d\n", type);
+        error_report("Invalid type %d", type);
         retval = -EINVAL;
         break;
     }
diff --git a/hw/9pfs/virtio-9p-synth.h b/hw/9pfs/virtio-9p-synth.h
index e03f434..ab05a8e 100644
--- a/hw/9pfs/virtio-9p-synth.h
+++ b/hw/9pfs/virtio-9p-synth.h
@@ -10,6 +10,8 @@
  * the COPYING file in the top-level directory.
  *
  */
+#ifndef HW_9PFS_VIRTIO9P_SYNTH_H
+#define HW_9PFS_VIRTIO9P_SYNTH_H 1
 
 #include <unistd.h>
 #include <sys/types.h>
@@ -48,3 +50,5 @@ extern int qemu_v9fs_synth_mkdir(V9fsSynthNode *parent, int mode,
 extern int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
                                     const char *name, v9fs_synth_read read,
                                     v9fs_synth_write write, void *arg);
+
+#endif
diff --git a/hw/9pfs/virtio-9p-xattr.h b/hw/9pfs/virtio-9p-xattr.h
index 9437280..41cc6cb 100644
--- a/hw/9pfs/virtio-9p-xattr.h
+++ b/hw/9pfs/virtio-9p-xattr.h
@@ -13,7 +13,7 @@
 #ifndef _QEMU_VIRTIO_9P_XATTR_H
 #define _QEMU_VIRTIO_9P_XATTR_H
 
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 
 typedef struct xattr_operations
 {
diff --git a/hw/9pfs/virtio-9p.c b/hw/9pfs/virtio-9p.c
index 8b9cdc9..d3ea820 100644
--- a/hw/9pfs/virtio-9p.c
+++ b/hw/9pfs/virtio-9p.c
@@ -13,14 +13,14 @@
 
 #include "hw/virtio.h"
 #include "hw/pc.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "hw/virtio-pci.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
 #include "virtio-9p-xattr.h"
 #include "virtio-9p-coth.h"
 #include "trace.h"
-#include "migration.h"
+#include "migration/migration.h"
 
 int open_fd_hw;
 int total_open_fd;
@@ -327,7 +327,7 @@ static int free_fid(V9fsPDU *pdu, V9fsFidState *fidp)
     return retval;
 }
 
-static void put_fid(V9fsPDU *pdu, V9fsFidState *fidp)
+static int put_fid(V9fsPDU *pdu, V9fsFidState *fidp)
 {
     BUG_ON(!fidp->ref);
     fidp->ref--;
@@ -348,8 +348,9 @@ static void put_fid(V9fsPDU *pdu, V9fsFidState *fidp)
                 pdu->s->migration_blocker = NULL;
             }
         }
-        free_fid(pdu, fidp);
+        return free_fid(pdu, fidp);
     }
+    return 0;
 }
 
 static V9fsFidState *clunk_fid(V9fsState *s, int32_t fid)
@@ -1537,9 +1538,10 @@ static void v9fs_clunk(void *opaque)
      * free the fid.
      */
     fidp->ref++;
-    err = offset;
-
-    put_fid(pdu, fidp);
+    err = put_fid(pdu, fidp);
+    if (!err) {
+        err = offset;
+    }
 out_nofid:
     complete_pdu(s, pdu, err);
 }
@@ -3101,11 +3103,7 @@ static void v9fs_xattrcreate(void *opaque)
     xattr_fidp->fs.xattr.flags = flags;
     v9fs_string_init(&xattr_fidp->fs.xattr.name);
     v9fs_string_copy(&xattr_fidp->fs.xattr.name, &name);
-    if (size) {
-        xattr_fidp->fs.xattr.value = g_malloc(size);
-    } else {
-        xattr_fidp->fs.xattr.value = NULL;
-    }
+    xattr_fidp->fs.xattr.value = g_malloc(size);
     err = offset;
     put_fid(pdu, file_fidp);
 out_nofid:
diff --git a/hw/9pfs/virtio-9p.h b/hw/9pfs/virtio-9p.h
index 5797944..406fe52 100644
--- a/hw/9pfs/virtio-9p.h
+++ b/hw/9pfs/virtio-9p.h
@@ -9,8 +9,8 @@
 #include "hw/virtio.h"
 #include "fsdev/file-op-9p.h"
 #include "fsdev/virtio-9p-marshal.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
 
 
 /* The feature bitmap for virtio 9P */
diff --git a/hw/Makefile.objs b/hw/Makefile.objs
index 1d50d8b..54521f2 100644
--- a/hw/Makefile.objs
+++ b/hw/Makefile.objs
@@ -1,14 +1,17 @@
-common-obj-y = usb/ ide/
+# core qdev-related obj files, also used by *-user:
+common-obj-y += qdev.o qdev-properties.o
+# irq.o needed for qdev GPIO handling:
+common-obj-y += irq.o
+
+ifeq ($(CONFIG_SOFTMMU),y)
+common-obj-y += usb/ ide/ pci/
 common-obj-y += loader.o
 common-obj-$(CONFIG_VIRTIO) += virtio-console.o
 common-obj-$(CONFIG_VIRTIO) += virtio-rng.o
 common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
+common-obj-$(CONFIG_VIRTIO) += virtio-bus.o
 common-obj-y += fw_cfg.o
-common-obj-$(CONFIG_PCI) += pci.o pci_bridge.o pci_bridge_dev.o
-common-obj-$(CONFIG_PCI) += msix.o msi.o
-common-obj-$(CONFIG_PCI) += shpc.o
-common-obj-$(CONFIG_PCI) += slotid_cap.o
-common-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
+common-obj-$(CONFIG_PCI) += pci_bridge_dev.o
 common-obj-$(CONFIG_PCI) += ioh3420.o xio3130_upstream.o xio3130_downstream.o
 common-obj-$(CONFIG_PCI) += i82801b11.o
 common-obj-y += watchdog.o
@@ -35,7 +38,7 @@ common-obj-$(CONFIG_DMA) += dma.o
 common-obj-$(CONFIG_I82374) += i82374.o
 common-obj-$(CONFIG_HPET) += hpet.o
 common-obj-$(CONFIG_APPLESMC) += applesmc.o
-common-obj-$(CONFIG_SMARTCARD) += ccid-card-passthru.o
+common-obj-y += ccid-card-passthru.o
 common-obj-$(CONFIG_SMARTCARD_NSS) += ccid-card-emulated.o
 common-obj-$(CONFIG_I8259) += i8259_common.o i8259.o
 common-obj-y += fifo.o
@@ -44,6 +47,7 @@ common-obj-y += pam.o
 # PPC devices
 common-obj-$(CONFIG_PREP_PCI) += prep_pci.o
 common-obj-$(CONFIG_I82378) += i82378.o
+common-obj-$(CONFIG_PC87312) += pc87312.o
 # Mac shared devices
 common-obj-$(CONFIG_MACIO) += macio.o
 common-obj-$(CONFIG_CUDA) += cuda.o
@@ -102,7 +106,8 @@ common-obj-$(CONFIG_XGMAC) += xgmac.o
 # PCI watchdog devices
 common-obj-$(CONFIG_PCI) += wdt_i6300esb.o
 
-common-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
+# IndustryPack
+common-obj-$(CONFIG_IPACK) += tpci200.o ipoctal232.o ipack.o
 
 # PCI network cards
 common-obj-$(CONFIG_NE2000_PCI) += ne2000.o
@@ -158,7 +163,6 @@ common-obj-$(CONFIG_SOUND) += $(sound-obj-y)
 common-obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
 
 common-obj-y += usb/
-common-obj-y += irq.o
 common-obj-$(CONFIG_PTIMER) += ptimer.o
 common-obj-$(CONFIG_MAX7310) += max7310.o
 common-obj-$(CONFIG_WM8750) += wm8750.o
@@ -185,7 +189,8 @@ common-obj-$(CONFIG_SD) += sd.o
 common-obj-y += bt.o bt-l2cap.o bt-sdp.o bt-hci.o bt-hid.o
 common-obj-y += bt-hci-csr.o
 common-obj-y += msmouse.o ps2.o
-common-obj-y += qdev.o qdev-properties.o qdev-monitor.o
+common-obj-y += qdev-monitor.o
+common-obj-y += qdev-properties-system.o
 common-obj-$(CONFIG_BRLAPI) += baum.o
 
 # xen backend driver support
@@ -195,12 +200,12 @@ common-obj-$(CONFIG_XEN_BACKEND) += xen_console.o xenfb.o xen_disk.o xen_nic.o
 # Per-target files
 # virtio has to be here due to weird dependency between PCI and virtio-net.
 # need to fix this properly
+obj-$(CONFIG_VIRTIO) += dataplane/
 obj-$(CONFIG_VIRTIO) += virtio.o virtio-blk.o virtio-balloon.o virtio-net.o
 obj-$(CONFIG_VIRTIO) += virtio-serial-bus.o virtio-scsi.o
 obj-$(CONFIG_SOFTMMU) += vhost_net.o
 obj-$(CONFIG_VHOST_NET) += vhost.o
 obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
-obj-$(CONFIG_NO_PCI) += pci-stub.o
 obj-$(CONFIG_VGA) += vga.o
 obj-$(CONFIG_SOFTMMU) += device-hotplug.o
 obj-$(CONFIG_XEN) += xen_domainbuild.o xen_machine_pv.o
@@ -210,3 +215,6 @@ ifeq ($(CONFIG_PCI), y)
 obj-$(CONFIG_KVM) += ivshmem.o
 obj-$(CONFIG_LINUX) += vfio_pci.o
 endif
+
+$(obj)/baum.o: QEMU_CFLAGS += $(SDL_CFLAGS) 
+endif
diff --git a/hw/a15mpcore.c b/hw/a15mpcore.c
index 31158f9..97abe41 100644
--- a/hw/a15mpcore.c
+++ b/hw/a15mpcore.c
@@ -19,7 +19,7 @@
  */
 
 #include "sysbus.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 /* A15MP private memory region.  */
 
@@ -41,7 +41,7 @@ static int a15mp_priv_init(SysBusDevice *dev)
 {
     A15MPPrivState *s = FROM_SYSBUS(A15MPPrivState, dev);
     SysBusDevice *busdev;
-    const char *gictype = "arm-gic";
+    const char *gictype = "arm_gic";
 
     if (kvm_irqchip_in_kernel()) {
         gictype = "kvm-arm-gic";
@@ -52,7 +52,7 @@ static int a15mp_priv_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-irq", s->num_irq);
     qdev_prop_set_uint32(s->gic, "revision", 2);
     qdev_init_nofail(s->gic);
-    busdev = sysbus_from_qdev(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, busdev);
@@ -99,7 +99,7 @@ static void a15mp_priv_class_init(ObjectClass *klass, void *data)
     /* We currently have no savable state */
 }
 
-static TypeInfo a15mp_priv_info = {
+static const TypeInfo a15mp_priv_info = {
     .name  = "a15mpcore_priv",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size  = sizeof(A15MPPrivState),
diff --git a/hw/a9mpcore.c b/hw/a9mpcore.c
index 824ff0a..673bbd8 100644
--- a/hw/a9mpcore.c
+++ b/hw/a9mpcore.c
@@ -19,7 +19,6 @@ typedef struct a9mp_priv_state {
     uint32_t old_timer_status[8];
     uint32_t num_cpu;
     MemoryRegion scu_iomem;
-    MemoryRegion ptimer_iomem;
     MemoryRegion container;
     DeviceState *mptimer;
     DeviceState *gic;
@@ -113,7 +112,7 @@ static const MemoryRegionOps a9_scu_ops = {
 
 static void a9mp_priv_reset(DeviceState *dev)
 {
-    a9mp_priv_state *s = FROM_SYSBUS(a9mp_priv_state, sysbus_from_qdev(dev));
+    a9mp_priv_state *s = FROM_SYSBUS(a9mp_priv_state, SYS_BUS_DEVICE(dev));
     int i;
     s->scu_control = 0;
     for (i = 0; i < ARRAY_SIZE(s->old_timer_status); i++) {
@@ -137,7 +136,7 @@ static int a9mp_priv_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(s->gic, "num-irq", s->num_irq);
     qdev_init_nofail(s->gic);
-    gicbusdev = sysbus_from_qdev(s->gic);
+    gicbusdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, gicbusdev);
@@ -148,7 +147,7 @@ static int a9mp_priv_init(SysBusDevice *dev)
     s->mptimer = qdev_create(NULL, "arm_mptimer");
     qdev_prop_set_uint32(s->mptimer, "num-cpu", s->num_cpu);
     qdev_init_nofail(s->mptimer);
-    busdev = sysbus_from_qdev(s->mptimer);
+    busdev = SYS_BUS_DEVICE(s->mptimer);
 
     /* Memory map (addresses are offsets from PERIPHBASE):
      *  0x0000-0x00ff -- Snoop Control Unit
@@ -227,7 +226,7 @@ static void a9mp_priv_class_init(ObjectClass *klass, void *data)
     dc->reset = a9mp_priv_reset;
 }
 
-static TypeInfo a9mp_priv_info = {
+static const TypeInfo a9mp_priv_info = {
     .name          = "a9mpcore_priv",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(a9mp_priv_state),
diff --git a/hw/ac97.c b/hw/ac97.c
index ce6a1dc..6c565e7 100644
--- a/hw/ac97.c
+++ b/hw/ac97.c
@@ -20,8 +20,8 @@
 #include "hw.h"
 #include "audiodev.h"
 #include "audio/audio.h"
-#include "pci.h"
-#include "dma.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
 
 enum {
     AC97_Reset                     = 0x00,
@@ -1423,7 +1423,7 @@ static void ac97_class_init (ObjectClass *klass, void *data)
     dc->props = ac97_properties;
 }
 
-static TypeInfo ac97_info = {
+static const TypeInfo ac97_info = {
     .name          = "AC97",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof (AC97LinkState),
diff --git a/hw/acpi.c b/hw/acpi.c
index f4aca49..8c9dcc5 100644
--- a/hw/acpi.c
+++ b/hw/acpi.c
@@ -18,11 +18,11 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "hw.h"
 #include "pc.h"
 #include "acpi.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 struct acpi_table_header {
     uint16_t _length;         /* our length, not actual part of the hdr */
@@ -104,7 +104,7 @@ int acpi_table_add(const char *t)
     /* now read in the data files, reallocating buffer as needed */
 
     for (f = strtok(buf, ":"); f; f = strtok(NULL, ":")) {
-        int fd = open(f, O_RDONLY);
+        int fd = open(f, O_RDONLY | O_BINARY);
 
         if (fd < 0) {
             fprintf(stderr, "can't open file %s: %s\n", f, strerror(errno));
@@ -275,7 +275,7 @@ uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar)
     return ar->pm1.evt.sts;
 }
 
-void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val)
+static void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val)
 {
     uint16_t pm1_sts = acpi_pm1_evt_get_sts(ar);
     if (pm1_sts & val & ACPI_BITMASK_TIMER_STATUS) {
@@ -285,7 +285,7 @@ void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val)
     ar->pm1.evt.sts &= ~val;
 }
 
-void acpi_pm1_evt_write_en(ACPIREGS *ar, uint16_t val)
+static void acpi_pm1_evt_write_en(ACPIREGS *ar, uint16_t val)
 {
     ar->pm1.evt.en = val;
     qemu_system_wakeup_enable(QEMU_WAKEUP_REASON_RTC,
@@ -310,6 +310,51 @@ void acpi_pm1_evt_reset(ACPIREGS *ar)
     qemu_system_wakeup_enable(QEMU_WAKEUP_REASON_PMTIMER, 0);
 }
 
+static uint64_t acpi_pm_evt_read(void *opaque, hwaddr addr, unsigned width)
+{
+    ACPIREGS *ar = opaque;
+    switch (addr) {
+    case 0:
+        return acpi_pm1_evt_get_sts(ar);
+    case 2:
+        return ar->pm1.evt.en;
+    default:
+        return 0;
+    }
+}
+
+static void acpi_pm_evt_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned width)
+{
+    ACPIREGS *ar = opaque;
+    switch (addr) {
+    case 0:
+        acpi_pm1_evt_write_sts(ar, val);
+        ar->pm1.evt.update_sci(ar);
+        break;
+    case 2:
+        acpi_pm1_evt_write_en(ar, val);
+        ar->pm1.evt.update_sci(ar);
+        break;
+    }
+}
+
+static const MemoryRegionOps acpi_pm_evt_ops = {
+    .read = acpi_pm_evt_read,
+    .write = acpi_pm_evt_write,
+    .valid.min_access_size = 2,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm1_evt_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                       MemoryRegion *parent)
+{
+    ar->pm1.evt.update_sci = update_sci;
+    memory_region_init_io(&ar->pm1.evt.io, &acpi_pm_evt_ops, ar, "acpi-evt", 4);
+    memory_region_add_subregion(parent, 0, &ar->pm1.evt.io);
+}
+
 /* ACPI PM_TMR */
 void acpi_pm_tmr_update(ACPIREGS *ar, bool enable)
 {
@@ -331,7 +376,7 @@ void acpi_pm_tmr_calc_overflow_time(ACPIREGS *ar)
     ar->tmr.overflow_time = (d + 0x800000LL) & ~0x7fffffLL;
 }
 
-uint32_t acpi_pm_tmr_get(ACPIREGS *ar)
+static uint32_t acpi_pm_tmr_get(ACPIREGS *ar)
 {
     uint32_t d = acpi_pm_tmr_get_clock();
     return d & 0xffffff;
@@ -344,10 +389,25 @@ static void acpi_pm_tmr_timer(void *opaque)
     ar->tmr.update_sci(ar);
 }
 
-void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci)
+static uint64_t acpi_pm_tmr_read(void *opaque, hwaddr addr, unsigned width)
+{
+    return acpi_pm_tmr_get(opaque);
+}
+
+static const MemoryRegionOps acpi_pm_tmr_ops = {
+    .read = acpi_pm_tmr_read,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                      MemoryRegion *parent)
 {
     ar->tmr.update_sci = update_sci;
     ar->tmr.timer = qemu_new_timer_ns(vm_clock, acpi_pm_tmr_timer, ar);
+    memory_region_init_io(&ar->tmr.io, &acpi_pm_tmr_ops, ar, "acpi-tmr", 4);
+    memory_region_add_subregion(parent, 8, &ar->tmr.io);
 }
 
 void acpi_pm_tmr_reset(ACPIREGS *ar)
@@ -357,13 +417,7 @@ void acpi_pm_tmr_reset(ACPIREGS *ar)
 }
 
 /* ACPI PM1aCNT */
-void acpi_pm1_cnt_init(ACPIREGS *ar)
-{
-    ar->wakeup.notify = acpi_notify_wakeup;
-    qemu_register_wakeup_notifier(&ar->wakeup);
-}
-
-void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val, char s4)
+static void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val)
 {
     ar->pm1.cnt.cnt = val & ~(ACPI_BITMASK_SLEEP_ENABLE);
 
@@ -378,7 +432,7 @@ void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val, char s4)
             qemu_system_suspend_request();
             break;
         default:
-            if (sus_typ == s4) { /* S4 request */
+            if (sus_typ == ar->pm1.cnt.s4_val) { /* S4 request */
                 monitor_protocol_event(QEVENT_SUSPEND_DISK, NULL);
                 qemu_system_shutdown_request();
             }
@@ -398,6 +452,34 @@ void acpi_pm1_cnt_update(ACPIREGS *ar,
     }
 }
 
+static uint64_t acpi_pm_cnt_read(void *opaque, hwaddr addr, unsigned width)
+{
+    ACPIREGS *ar = opaque;
+    return ar->pm1.cnt.cnt;
+}
+
+static void acpi_pm_cnt_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned width)
+{
+    acpi_pm1_cnt_write(opaque, val);
+}
+
+static const MemoryRegionOps acpi_pm_cnt_ops = {
+    .read = acpi_pm_cnt_read,
+    .write = acpi_pm_cnt_write,
+    .valid.min_access_size = 2,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent)
+{
+    ar->wakeup.notify = acpi_notify_wakeup;
+    qemu_register_wakeup_notifier(&ar->wakeup);
+    memory_region_init_io(&ar->pm1.cnt.io, &acpi_pm_cnt_ops, ar, "acpi-cnt", 2);
+    memory_region_add_subregion(parent, 4, &ar->pm1.cnt.io);
+}
+
 void acpi_pm1_cnt_reset(ACPIREGS *ar)
 {
     ar->pm1.cnt.cnt = 0;
@@ -411,11 +493,6 @@ void acpi_gpe_init(ACPIREGS *ar, uint8_t len)
     ar->gpe.en = g_malloc0(len / 2);
 }
 
-void acpi_gpe_blk(ACPIREGS *ar, uint32_t blk)
-{
-    ar->gpe.blk = blk;
-}
-
 void acpi_gpe_reset(ACPIREGS *ar)
 {
     memset(ar->gpe.sts, 0, ar->gpe.len / 2);
@@ -441,7 +518,6 @@ void acpi_gpe_ioport_writeb(ACPIREGS *ar, uint32_t addr, uint32_t val)
 {
     uint8_t *cur;
 
-    addr -= ar->gpe.blk;
     cur = acpi_gpe_ioport_get_ptr(ar, addr);
     if (addr < ar->gpe.len / 2) {
         /* GPE_STS */
@@ -459,7 +535,6 @@ uint32_t acpi_gpe_ioport_readb(ACPIREGS *ar, uint32_t addr)
     uint8_t *cur;
     uint32_t val;
 
-    addr -= ar->gpe.blk;
     cur = acpi_gpe_ioport_get_ptr(ar, addr);
     val = 0;
     if (cur != NULL) {
diff --git a/hw/acpi.h b/hw/acpi.h
index 7337f41..c3628d0 100644
--- a/hw/acpi.h
+++ b/hw/acpi.h
@@ -84,22 +84,26 @@ typedef void (*acpi_update_sci_fn)(ACPIREGS *ar);
 
 struct ACPIPMTimer {
     QEMUTimer *timer;
+    MemoryRegion io;
     int64_t overflow_time;
 
     acpi_update_sci_fn update_sci;
 };
 
 struct ACPIPM1EVT {
+    MemoryRegion io;
     uint16_t sts;
     uint16_t en;
+    acpi_update_sci_fn update_sci;
 };
 
 struct ACPIPM1CNT {
+    MemoryRegion io;
     uint16_t cnt;
+    uint8_t s4_val;
 };
 
 struct ACPIGPE {
-    uint32_t blk;
     uint8_t len;
 
     uint8_t *sts;
@@ -119,11 +123,11 @@ struct ACPIREGS {
 /* PM_TMR */
 void acpi_pm_tmr_update(ACPIREGS *ar, bool enable);
 void acpi_pm_tmr_calc_overflow_time(ACPIREGS *ar);
-uint32_t acpi_pm_tmr_get(ACPIREGS *ar);
-void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci);
+void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                      MemoryRegion *parent);
 void acpi_pm_tmr_reset(ACPIREGS *ar);
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 static inline int64_t acpi_pm_tmr_get_clock(void)
 {
     return muldiv64(qemu_get_clock_ns(vm_clock), PM_TIMER_FREQUENCY,
@@ -132,21 +136,19 @@ static inline int64_t acpi_pm_tmr_get_clock(void)
 
 /* PM1a_EVT: piix and ich9 don't implement PM1b. */
 uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar);
-void acpi_pm1_evt_write_sts(ACPIREGS *ar, uint16_t val);
-void acpi_pm1_evt_write_en(ACPIREGS *ar, uint16_t val);
 void acpi_pm1_evt_power_down(ACPIREGS *ar);
 void acpi_pm1_evt_reset(ACPIREGS *ar);
+void acpi_pm1_evt_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
+                       MemoryRegion *parent);
 
 /* PM1a_CNT: piix and ich9 don't implement PM1b CNT. */
-void acpi_pm1_cnt_init(ACPIREGS *ar);
-void acpi_pm1_cnt_write(ACPIREGS *ar, uint16_t val, char s4);
+void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent);
 void acpi_pm1_cnt_update(ACPIREGS *ar,
                          bool sci_enable, bool sci_disable);
 void acpi_pm1_cnt_reset(ACPIREGS *ar);
 
 /* GPE0 */
 void acpi_gpe_init(ACPIREGS *ar, uint8_t len);
-void acpi_gpe_blk(ACPIREGS *ar, uint32_t blk);
 void acpi_gpe_reset(ACPIREGS *ar);
 
 void acpi_gpe_ioport_writeb(ACPIREGS *ar, uint32_t addr, uint32_t val);
diff --git a/hw/acpi_ich9.c b/hw/acpi_ich9.c
index 61034d3..d2f9808 100644
--- a/hw/acpi_ich9.c
+++ b/hw/acpi_ich9.c
@@ -2,6 +2,11 @@
  * ACPI implementation
  *
  * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on acpi.c.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -14,21 +19,18 @@
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-/*
- *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
- *                     VA Linux Systems Japan K.K.
- *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
  *
- *  This is based on acpi.c.
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
  */
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "pci/pci.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "acpi.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
 
 #include "ich9.h"
 
@@ -41,10 +43,6 @@ do { printf("%s "fmt, __func__, ## __VA_ARGS__); } while (0)
 #define ICH9_DEBUG(fmt, ...)    do { } while (0)
 #endif
 
-static void pm_ioport_write_fallback(void *opaque, uint32_t addr, int len,
-                                     uint32_t val);
-static uint32_t pm_ioport_read_fallback(void *opaque, uint32_t addr, int len);
-
 static void pm_update_sci(ICH9LPCPMRegs *pm)
 {
     int sci_level, pm1a_sts;
@@ -70,152 +68,60 @@ static void ich9_pm_update_sci_fn(ACPIREGS *regs)
     pm_update_sci(pm);
 }
 
-static void pm_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static uint64_t ich9_gpe_readb(void *opaque, hwaddr addr, unsigned width)
 {
     ICH9LPCPMRegs *pm = opaque;
-
-    switch (addr & ICH9_PMIO_MASK) {
-    case ICH9_PMIO_GPE0_STS ... (ICH9_PMIO_GPE0_STS + ICH9_PMIO_GPE0_LEN - 1):
-        acpi_gpe_ioport_writeb(&pm->acpi_regs, addr, val);
-        break;
-    default:
-        break;
-    }
-
-    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
+    return acpi_gpe_ioport_readb(&pm->acpi_regs, addr);
 }
 
-static uint32_t pm_ioport_readb(void *opaque, uint32_t addr)
+static void ich9_gpe_writeb(void *opaque, hwaddr addr, uint64_t val,
+                            unsigned width)
 {
     ICH9LPCPMRegs *pm = opaque;
-    uint32_t val = 0;
-
-    switch (addr & ICH9_PMIO_MASK) {
-    case ICH9_PMIO_GPE0_STS ... (ICH9_PMIO_GPE0_STS + ICH9_PMIO_GPE0_LEN - 1):
-        val = acpi_gpe_ioport_readb(&pm->acpi_regs, addr);
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
-    return val;
+    acpi_gpe_ioport_writeb(&pm->acpi_regs, addr, val);
 }
 
-static void pm_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
-{
-    ICH9LPCPMRegs *pm = opaque;
-
-    switch (addr & ICH9_PMIO_MASK) {
-    case ICH9_PMIO_PM1_STS:
-        acpi_pm1_evt_write_sts(&pm->acpi_regs, val);
-        pm_update_sci(pm);
-        break;
-    case ICH9_PMIO_PM1_EN:
-        pm->acpi_regs.pm1.evt.en = val;
-        pm_update_sci(pm);
-        break;
-    case ICH9_PMIO_PM1_CNT:
-        acpi_pm1_cnt_write(&pm->acpi_regs, val, 0);
-        break;
-    default:
-        pm_ioport_write_fallback(opaque, addr, 2, val);
-        break;
-    }
-    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
-}
+static const MemoryRegionOps ich9_gpe_ops = {
+    .read = ich9_gpe_readb,
+    .write = ich9_gpe_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static uint32_t pm_ioport_readw(void *opaque, uint32_t addr)
+static uint64_t ich9_smi_readl(void *opaque, hwaddr addr, unsigned width)
 {
     ICH9LPCPMRegs *pm = opaque;
-    uint32_t val;
-
-    switch (addr & ICH9_PMIO_MASK) {
-    case ICH9_PMIO_PM1_STS:
-        val = acpi_pm1_evt_get_sts(&pm->acpi_regs);
-        break;
-    case ICH9_PMIO_PM1_EN:
-        val = pm->acpi_regs.pm1.evt.en;
-        break;
-    case ICH9_PMIO_PM1_CNT:
-        val = pm->acpi_regs.pm1.cnt.cnt;
-        break;
+    switch (addr) {
+    case 0:
+        return pm->smi_en;
+    case 4:
+        return pm->smi_sts;
     default:
-        val = pm_ioport_read_fallback(opaque, addr, 2);
-        break;
+        return 0;
     }
-    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
-    return val;
 }
 
-static void pm_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
+static void ich9_smi_writel(void *opaque, hwaddr addr, uint64_t val,
+                            unsigned width)
 {
     ICH9LPCPMRegs *pm = opaque;
-
-    switch (addr & ICH9_PMIO_MASK) {
-    case ICH9_PMIO_SMI_EN:
+    switch (addr) {
+    case 0:
         pm->smi_en = val;
         break;
-    default:
-        pm_ioport_write_fallback(opaque, addr, 4, val);
-        break;
-    }
-    ICH9_DEBUG("port=0x%04x val=0x%08x\n", addr, val);
-}
-
-static uint32_t pm_ioport_readl(void *opaque, uint32_t addr)
-{
-    ICH9LPCPMRegs *pm = opaque;
-    uint32_t val;
-
-    switch (addr & ICH9_PMIO_MASK) {
-    case ICH9_PMIO_PM1_TMR:
-        val = acpi_pm_tmr_get(&pm->acpi_regs);
-        break;
-    case ICH9_PMIO_SMI_EN:
-        val = pm->smi_en;
-        break;
-
-    default:
-        val = pm_ioport_read_fallback(opaque, addr, 4);
-        break;
-    }
-    ICH9_DEBUG("port=0x%04x val=0x%08x\n", addr, val);
-    return val;
-}
-
-static void pm_ioport_write_fallback(void *opaque, uint32_t addr, int len,
-                                     uint32_t val)
- {
-    int subsize = (len == 4) ? 2 : 1;
-    IOPortWriteFunc *ioport_write =
-        (subsize == 2) ? pm_ioport_writew : pm_ioport_writeb;
-
-    int i;
-
-    for (i = 0; i < len; i += subsize) {
-        ioport_write(opaque, addr, val);
-        val >>= 8 * subsize;
     }
 }
 
-static uint32_t pm_ioport_read_fallback(void *opaque, uint32_t addr, int len)
-{
-    int subsize = (len == 4) ? 2 : 1;
-    IOPortReadFunc *ioport_read =
-        (subsize == 2) ? pm_ioport_readw : pm_ioport_readb;
-
-    uint32_t val;
-    int i;
-
-    val = 0;
-    for (i = 0; i < len; i += subsize) {
-        val <<= 8 * subsize;
-        val |= ioport_read(opaque, addr);
-    }
-
-    return val;
-}
+static const MemoryRegionOps ich9_smi_ops = {
+    .read = ich9_smi_readl,
+    .write = ich9_smi_writel,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
 void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base)
 {
@@ -223,24 +129,11 @@ void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base)
 
     assert((pm_io_base & ICH9_PMIO_MASK) == 0);
 
-    if (pm->pm_io_base != 0) {
-        isa_unassign_ioport(pm->pm_io_base, ICH9_PMIO_SIZE);
-    }
-
-    /* don't map at 0 */
-    if (pm_io_base == 0) {
-        return;
-    }
-
-    register_ioport_write(pm_io_base, ICH9_PMIO_SIZE, 1, pm_ioport_writeb, pm);
-    register_ioport_read(pm_io_base, ICH9_PMIO_SIZE, 1, pm_ioport_readb, pm);
-    register_ioport_write(pm_io_base, ICH9_PMIO_SIZE, 2, pm_ioport_writew, pm);
-    register_ioport_read(pm_io_base, ICH9_PMIO_SIZE, 2, pm_ioport_readw, pm);
-    register_ioport_write(pm_io_base, ICH9_PMIO_SIZE, 4, pm_ioport_writel, pm);
-    register_ioport_read(pm_io_base, ICH9_PMIO_SIZE, 4, pm_ioport_readl, pm);
-
     pm->pm_io_base = pm_io_base;
-    acpi_gpe_blk(&pm->acpi_regs, pm_io_base + ICH9_PMIO_GPE0_STS);
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&pm->io, pm->pm_io_base != 0);
+    memory_region_set_address(&pm->io, pm->pm_io_base);
+    memory_region_transaction_commit();
 }
 
 static int ich9_pm_post_load(void *opaque, int version_id)
@@ -309,11 +202,26 @@ static void pm_powerdown_req(Notifier *n, void *opaque)
     acpi_pm1_evt_power_down(&pm->acpi_regs);
 }
 
-void ich9_pm_init(ICH9LPCPMRegs *pm, qemu_irq sci_irq, qemu_irq cmos_s3)
+void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
+                  qemu_irq sci_irq, qemu_irq cmos_s3)
 {
-    acpi_pm_tmr_init(&pm->acpi_regs, ich9_pm_update_sci_fn);
-    acpi_pm1_cnt_init(&pm->acpi_regs);
+    memory_region_init(&pm->io, "ich9-pm", ICH9_PMIO_SIZE);
+    memory_region_set_enabled(&pm->io, false);
+    memory_region_add_subregion(pci_address_space_io(lpc_pci),
+                                0, &pm->io);
+
+    acpi_pm_tmr_init(&pm->acpi_regs, ich9_pm_update_sci_fn, &pm->io);
+    acpi_pm1_evt_init(&pm->acpi_regs, ich9_pm_update_sci_fn, &pm->io);
+    acpi_pm1_cnt_init(&pm->acpi_regs, &pm->io);
+
     acpi_gpe_init(&pm->acpi_regs, ICH9_PMIO_GPE0_LEN);
+    memory_region_init_io(&pm->io_gpe, &ich9_gpe_ops, pm, "apci-gpe0",
+                          ICH9_PMIO_GPE0_LEN);
+    memory_region_add_subregion(&pm->io, ICH9_PMIO_GPE0_STS, &pm->io_gpe);
+
+    memory_region_init_io(&pm->io_smi, &ich9_smi_ops, pm, "apci-smi",
+                          8);
+    memory_region_add_subregion(&pm->io, ICH9_PMIO_SMI_EN, &pm->io_smi);
 
     pm->irq = sci_irq;
     qemu_register_reset(pm_reset, pm);
diff --git a/hw/acpi_ich9.h b/hw/acpi_ich9.h
index 180c406..ecb82ab 100644
--- a/hw/acpi_ich9.h
+++ b/hw/acpi_ich9.h
@@ -30,6 +30,11 @@ typedef struct ICH9LPCPMRegs {
      * PM1a_CNT_BLK = 2 in FADT so it is defined as uint16_t.
      */
     ACPIREGS acpi_regs;
+
+    MemoryRegion io;
+    MemoryRegion io_gpe;
+    MemoryRegion io_smi;
+
     uint32_t smi_en;
     uint32_t smi_sts;
 
@@ -39,7 +44,7 @@ typedef struct ICH9LPCPMRegs {
     Notifier powerdown_notifier;
 } ICH9LPCPMRegs;
 
-void ich9_pm_init(ICH9LPCPMRegs *pm,
+void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
                   qemu_irq sci_irq, qemu_irq cmos_s3_resume);
 void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base);
 extern const VMStateDescription vmstate_ich9_pm;
diff --git a/hw/acpi_piix4.c b/hw/acpi_piix4.c
index 519269a..65b2601 100644
--- a/hw/acpi_piix4.c
+++ b/hw/acpi_piix4.c
@@ -22,12 +22,13 @@
 #include "pc.h"
 #include "apm.h"
 #include "pm_smbus.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "acpi.h"
-#include "sysemu.h"
-#include "range.h"
-#include "ioport.h"
+#include "sysemu/sysemu.h"
+#include "qemu/range.h"
+#include "exec/ioport.h"
 #include "fw_cfg.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG
 
@@ -37,10 +38,11 @@
 # define PIIX4_DPRINTF(format, ...)     do { } while (0)
 #endif
 
-#define ACPI_DBG_IO_ADDR  0xb044
-
 #define GPE_BASE 0xafe0
 #define GPE_LEN 4
+
+#define PCI_HOTPLUG_ADDR 0xae00
+#define PCI_HOTPLUG_SIZE 0x000f
 #define PCI_UP_BASE 0xae00
 #define PCI_DOWN_BASE 0xae04
 #define PCI_EJ_BASE 0xae08
@@ -55,7 +57,10 @@ struct pci_status {
 
 typedef struct PIIX4PMState {
     PCIDevice dev;
-    IORange ioport;
+
+    MemoryRegion io;
+    MemoryRegion io_gpe;
+    MemoryRegion io_pci;
     ACPIREGS ar;
 
     APMState apm;
@@ -79,7 +84,8 @@ typedef struct PIIX4PMState {
     uint8_t s4_val;
 } PIIX4PMState;
 
-static void piix4_acpi_system_hot_add_init(PCIBus *bus, PIIX4PMState *s);
+static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
+                                           PCIBus *bus, PIIX4PMState *s);
 
 #define ACPI_ENABLE 0xf1
 #define ACPI_DISABLE 0xf0
@@ -109,67 +115,6 @@ static void pm_tmr_timer(ACPIREGS *ar)
     pm_update_sci(s);
 }
 
-static void pm_ioport_write(IORange *ioport, uint64_t addr, unsigned width,
-                            uint64_t val)
-{
-    PIIX4PMState *s = container_of(ioport, PIIX4PMState, ioport);
-
-    if (width != 2) {
-        PIIX4_DPRINTF("PM write port=0x%04x width=%d val=0x%08x\n",
-                      (unsigned)addr, width, (unsigned)val);
-    }
-
-    switch(addr) {
-    case 0x00:
-        acpi_pm1_evt_write_sts(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x02:
-        acpi_pm1_evt_write_en(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x04:
-        acpi_pm1_cnt_write(&s->ar, val, s->s4_val);
-        break;
-    default:
-        break;
-    }
-    PIIX4_DPRINTF("PM writew port=0x%04x val=0x%04x\n", (unsigned int)addr,
-                  (unsigned int)val);
-}
-
-static void pm_ioport_read(IORange *ioport, uint64_t addr, unsigned width,
-                            uint64_t *data)
-{
-    PIIX4PMState *s = container_of(ioport, PIIX4PMState, ioport);
-    uint32_t val;
-
-    switch(addr) {
-    case 0x00:
-        val = acpi_pm1_evt_get_sts(&s->ar);
-        break;
-    case 0x02:
-        val = s->ar.pm1.evt.en;
-        break;
-    case 0x04:
-        val = s->ar.pm1.cnt.cnt;
-        break;
-    case 0x08:
-        val = acpi_pm_tmr_get(&s->ar);
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    PIIX4_DPRINTF("PM readw port=0x%04x val=0x%04x\n", (unsigned int)addr, val);
-    *data = val;
-}
-
-static const IORangeOps pm_iorange_ops = {
-    .read = pm_ioport_read,
-    .write = pm_ioport_write,
-};
-
 static void apm_ctrl_changed(uint32_t val, void *arg)
 {
     PIIX4PMState *s = arg;
@@ -184,32 +129,42 @@ static void apm_ctrl_changed(uint32_t val, void *arg)
     }
 }
 
-static void acpi_dbg_writel(void *opaque, uint32_t addr, uint32_t val)
-{
-    PIIX4_DPRINTF("ACPI: DBG: 0x%08x\n", val);
-}
-
 static void pm_io_space_update(PIIX4PMState *s)
 {
     uint32_t pm_io_base;
 
-    if (s->dev.config[0x80] & 1) {
-        pm_io_base = le32_to_cpu(*(uint32_t *)(s->dev.config + 0x40));
-        pm_io_base &= 0xffc0;
+    pm_io_base = le32_to_cpu(*(uint32_t *)(s->dev.config + 0x40));
+    pm_io_base &= 0xffc0;
 
-        /* XXX: need to improve memory and ioport allocation */
-        PIIX4_DPRINTF("PM: mapping to 0x%x\n", pm_io_base);
-        iorange_init(&s->ioport, &pm_iorange_ops, pm_io_base, 64);
-        ioport_register(&s->ioport);
-    }
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&s->io, s->dev.config[0x80] & 1);
+    memory_region_set_address(&s->io, pm_io_base);
+    memory_region_transaction_commit();
+}
+
+static void smbus_io_space_update(PIIX4PMState *s)
+{
+    s->smb_io_base = le32_to_cpu(*(uint32_t *)(s->dev.config + 0x90));
+    s->smb_io_base &= 0xffc0;
+
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&s->smb.io, s->dev.config[0xd2] & 1);
+    memory_region_set_address(&s->smb.io, s->smb_io_base);
+    memory_region_transaction_commit();
 }
 
 static void pm_write_config(PCIDevice *d,
                             uint32_t address, uint32_t val, int len)
 {
     pci_default_write_config(d, address, val, len);
-    if (range_covers_byte(address, len, 0x80))
+    if (range_covers_byte(address, len, 0x80) ||
+        ranges_overlap(address, len, 0x40, 4)) {
         pm_io_space_update((PIIX4PMState *)d);
+    }
+    if (range_covers_byte(address, len, 0xd2) ||
+        ranges_overlap(address, len, 0x90, 4)) {
+        smbus_io_space_update((PIIX4PMState *)d);
+    }
 }
 
 static void vmstate_pci_status_pre_save(void *opaque)
@@ -280,7 +235,7 @@ static int acpi_load_old(QEMUFile *f, void *opaque, int version_id)
     qemu_get_be16s(f, &s->ar.pm1.evt.en);
     qemu_get_be16s(f, &s->ar.pm1.cnt.cnt);
 
-    ret = vmstate_load_state(f, &vmstate_apm, opaque, 1);
+    ret = vmstate_load_state(f, &vmstate_apm, &s->apm, 1);
     if (ret) {
         return ret;
     }
@@ -298,7 +253,7 @@ static int acpi_load_old(QEMUFile *f, void *opaque, int version_id)
         qemu_get_be16s(f, &temp);
     }
 
-    ret = vmstate_load_state(f, &vmstate_pci_status, opaque, 1);
+    ret = vmstate_load_state(f, &vmstate_pci_status, &s->pci0_status, 1);
     return ret;
 }
 
@@ -438,9 +393,7 @@ static int piix4_pm_initfn(PCIDevice *dev)
     pci_conf[0x3d] = 0x01; // interrupt pin 1
 
     /* APM */
-    apm_init(&s->apm, apm_ctrl_changed, s);
-
-    register_ioport_write(ACPI_DBG_IO_ADDR, 4, 4, acpi_dbg_writel, s);
+    apm_init(dev, &s->apm, apm_ctrl_changed, s);
 
     if (s->kvm_enabled) {
         /* Mark SMM as already inited to prevent SMM from running.  KVM does not
@@ -453,20 +406,29 @@ static int piix4_pm_initfn(PCIDevice *dev)
     pci_conf[0x90] = s->smb_io_base | 1;
     pci_conf[0x91] = s->smb_io_base >> 8;
     pci_conf[0xd2] = 0x09;
-    register_ioport_write(s->smb_io_base, 64, 1, smb_ioport_writeb, &s->smb);
-    register_ioport_read(s->smb_io_base, 64, 1, smb_ioport_readb, &s->smb);
-
-    acpi_pm_tmr_init(&s->ar, pm_tmr_timer);
+    pm_smbus_init(&s->dev.qdev, &s->smb);
+    memory_region_set_enabled(&s->smb.io, pci_conf[0xd2] & 1);
+    memory_region_add_subregion(pci_address_space_io(dev),
+                                s->smb_io_base, &s->smb.io);
+
+    memory_region_init(&s->io, "piix4-pm", 64);
+    memory_region_set_enabled(&s->io, false);
+    memory_region_add_subregion(pci_address_space_io(dev),
+                                0, &s->io);
+
+    acpi_pm_tmr_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_evt_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_cnt_init(&s->ar, &s->io);
     acpi_gpe_init(&s->ar, GPE_LEN);
 
     s->powerdown_notifier.notify = piix4_pm_powerdown_req;
     qemu_register_powerdown_notifier(&s->powerdown_notifier);
 
-    pm_smbus_init(&s->dev.qdev, &s->smb);
     s->machine_ready.notify = piix4_pm_machine_ready;
     qemu_add_machine_init_done_notifier(&s->machine_ready);
     qemu_register_reset(piix4_reset, s);
-    piix4_acpi_system_hot_add_init(dev->bus, s);
+
+    piix4_acpi_system_hot_add_init(pci_address_space_io(dev), dev->bus, s);
 
     return 0;
 }
@@ -483,7 +445,6 @@ i2c_bus *piix4_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
 
     s = DO_UPCAST(PIIX4PMState, dev, dev);
     s->irq = sci_irq;
-    acpi_pm1_cnt_init(&s->ar);
     s->smi_irq = smi_irq;
     s->kvm_enabled = kvm_enabled;
 
@@ -526,7 +487,7 @@ static void piix4_pm_class_init(ObjectClass *klass, void *data)
     dc->props = piix4_pm_properties;
 }
 
-static TypeInfo piix4_pm_info = {
+static const TypeInfo piix4_pm_info = {
     .name          = "PIIX4_PM",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PIIX4PMState),
@@ -540,7 +501,7 @@ static void piix4_pm_register_types(void)
 
 type_init(piix4_pm_register_types)
 
-static uint32_t gpe_readb(void *opaque, uint32_t addr)
+static uint64_t gpe_readb(void *opaque, hwaddr addr, unsigned width)
 {
     PIIX4PMState *s = opaque;
     uint32_t val = acpi_gpe_ioport_readb(&s->ar, addr);
@@ -549,7 +510,8 @@ static uint32_t gpe_readb(void *opaque, uint32_t addr)
     return val;
 }
 
-static void gpe_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void gpe_writeb(void *opaque, hwaddr addr, uint64_t val,
+                       unsigned width)
 {
     PIIX4PMState *s = opaque;
 
@@ -559,67 +521,84 @@ static void gpe_writeb(void *opaque, uint32_t addr, uint32_t val)
     PIIX4_DPRINTF("gpe write %x <== %d\n", addr, val);
 }
 
-static uint32_t pci_up_read(void *opaque, uint32_t addr)
-{
-    PIIX4PMState *s = opaque;
-    uint32_t val;
-
-    /* Manufacture an "up" value to cause a device check on any hotplug
-     * slot with a device.  Extra device checks are harmless. */
-    val = s->pci0_slot_device_present & s->pci0_hotplug_enable;
-
-    PIIX4_DPRINTF("pci_up_read %x\n", val);
-    return val;
-}
+static const MemoryRegionOps piix4_gpe_ops = {
+    .read = gpe_readb,
+    .write = gpe_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static uint32_t pci_down_read(void *opaque, uint32_t addr)
+static uint64_t pci_read(void *opaque, hwaddr addr, unsigned int size)
 {
     PIIX4PMState *s = opaque;
-    uint32_t val = s->pci0_status.down;
+    uint32_t val = 0;
+
+    switch (addr) {
+    case PCI_UP_BASE - PCI_HOTPLUG_ADDR:
+        /* Manufacture an "up" value to cause a device check on any hotplug
+         * slot with a device.  Extra device checks are harmless. */
+        val = s->pci0_slot_device_present & s->pci0_hotplug_enable;
+        PIIX4_DPRINTF("pci_up_read %x\n", val);
+        break;
+    case PCI_DOWN_BASE - PCI_HOTPLUG_ADDR:
+        val = s->pci0_status.down;
+        PIIX4_DPRINTF("pci_down_read %x\n", val);
+        break;
+    case PCI_EJ_BASE - PCI_HOTPLUG_ADDR:
+        /* No feature defined yet */
+        PIIX4_DPRINTF("pci_features_read %x\n", val);
+        break;
+    case PCI_RMV_BASE - PCI_HOTPLUG_ADDR:
+        val = s->pci0_hotplug_enable;
+        break;
+    default:
+        break;
+    }
 
-    PIIX4_DPRINTF("pci_down_read %x\n", val);
     return val;
 }
 
-static uint32_t pci_features_read(void *opaque, uint32_t addr)
+static void pci_write(void *opaque, hwaddr addr, uint64_t data,
+                      unsigned int size)
 {
-    /* No feature defined yet */
-    PIIX4_DPRINTF("pci_features_read %x\n", 0);
-    return 0;
-}
-
-static void pciej_write(void *opaque, uint32_t addr, uint32_t val)
-{
-    acpi_piix_eject_slot(opaque, val);
-
-    PIIX4_DPRINTF("pciej write %x <== %d\n", addr, val);
+    switch (addr) {
+    case PCI_EJ_BASE - PCI_HOTPLUG_ADDR:
+        acpi_piix_eject_slot(opaque, (uint32_t)data);
+        PIIX4_DPRINTF("pciej write %" HWADDR_PRIx " <== % " PRIu64 "\n",
+                      addr, data);
+        break;
+    default:
+        break;
+    }
 }
 
-static uint32_t pcirmv_read(void *opaque, uint32_t addr)
-{
-    PIIX4PMState *s = opaque;
-
-    return s->pci0_hotplug_enable;
-}
+static const MemoryRegionOps piix4_pci_ops = {
+    .read = pci_read,
+    .write = pci_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
 static int piix4_device_hotplug(DeviceState *qdev, PCIDevice *dev,
                                 PCIHotplugState state);
 
-static void piix4_acpi_system_hot_add_init(PCIBus *bus, PIIX4PMState *s)
+static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
+                                           PCIBus *bus, PIIX4PMState *s)
 {
-
-    register_ioport_write(GPE_BASE, GPE_LEN, 1, gpe_writeb, s);
-    register_ioport_read(GPE_BASE, GPE_LEN, 1,  gpe_readb, s);
-    acpi_gpe_blk(&s->ar, GPE_BASE);
-
-    register_ioport_read(PCI_UP_BASE, 4, 4, pci_up_read, s);
-    register_ioport_read(PCI_DOWN_BASE, 4, 4, pci_down_read, s);
-
-    register_ioport_write(PCI_EJ_BASE, 4, 4, pciej_write, s);
-    register_ioport_read(PCI_EJ_BASE, 4, 4,  pci_features_read, s);
-
-    register_ioport_read(PCI_RMV_BASE, 4, 4,  pcirmv_read, s);
-
+    memory_region_init_io(&s->io_gpe, &piix4_gpe_ops, s, "apci-gpe0",
+                          GPE_LEN);
+    memory_region_add_subregion(parent, GPE_BASE, &s->io_gpe);
+
+    memory_region_init_io(&s->io_pci, &piix4_pci_ops, s, "apci-pci-hotplug",
+                          PCI_HOTPLUG_SIZE);
+    memory_region_add_subregion(parent, PCI_HOTPLUG_ADDR,
+                                &s->io_pci);
     pci_bus_hotplug(bus, piix4_device_hotplug, &s->dev.qdev);
 }
 
diff --git a/hw/adb.c b/hw/adb.c
index 3b547f0..6cf5465 100644
--- a/hw/adb.c
+++ b/hw/adb.c
@@ -23,7 +23,7 @@
  */
 #include "hw.h"
 #include "adb.h"
-#include "console.h"
+#include "ui/console.h"
 
 /* debug ADB */
 //#define DEBUG_ADB
@@ -48,16 +48,21 @@ do { printf("ADB: " fmt , ## __VA_ARGS__); } while (0)
 #define ADB_CMD_CHANGE_ID_AND_ENABLE	0x00
 
 /* ADB default device IDs (upper 4 bits of ADB command byte) */
-#define ADB_DONGLE	1
-#define ADB_KEYBOARD	2
-#define ADB_MOUSE	3
-#define ADB_TABLET	4
-#define ADB_MODEM	5
-#define ADB_MISC	7
+#define ADB_DEVID_DONGLE   1
+#define ADB_DEVID_KEYBOARD 2
+#define ADB_DEVID_MOUSE    3
+#define ADB_DEVID_TABLET   4
+#define ADB_DEVID_MODEM    5
+#define ADB_DEVID_MISC     7
 
 /* error codes */
 #define ADB_RET_NOTPRESENT (-2)
 
+static void adb_device_reset(ADBDevice *d)
+{
+    qdev_reset_all(DEVICE(d));
+}
+
 int adb_request(ADBBusState *s, uint8_t *obuf, const uint8_t *buf, int len)
 {
     ADBDevice *d;
@@ -66,18 +71,17 @@ int adb_request(ADBBusState *s, uint8_t *obuf, const uint8_t *buf, int len)
     cmd = buf[0] & 0xf;
     if (cmd == ADB_BUSRESET) {
         for(i = 0; i < s->nb_devices; i++) {
-            d = &s->devices[i];
-            if (d->devreset) {
-                d->devreset(d);
-            }
+            d = s->devices[i];
+            adb_device_reset(d);
         }
         return 0;
     }
     devaddr = buf[0] >> 4;
     for(i = 0; i < s->nb_devices; i++) {
-        d = &s->devices[i];
+        d = s->devices[i];
         if (d->devaddr == devaddr) {
-            return d->devreq(d, obuf, buf, len);
+            ADBDeviceClass *adc = ADB_DEVICE_GET_CLASS(d);
+            return adc->devreq(d, obuf, buf, len);
         }
     }
     return ADB_RET_NOTPRESENT;
@@ -94,7 +98,7 @@ int adb_poll(ADBBusState *s, uint8_t *obuf)
     for(i = 0; i < s->nb_devices; i++) {
         if (s->poll_index >= s->nb_devices)
             s->poll_index = 0;
-        d = &s->devices[s->poll_index];
+        d = s->devices[s->poll_index];
         buf[0] = ADB_READREG | (d->devaddr << 4);
         olen = adb_request(s, obuf + 1, buf, 1);
         /* if there is data, we poll again the same device */
@@ -108,32 +112,67 @@ int adb_poll(ADBBusState *s, uint8_t *obuf)
     return olen;
 }
 
-static ADBDevice *adb_register_device(ADBBusState *s, int devaddr,
-                                      ADBDeviceRequest *devreq,
-                                      ADBDeviceReset *devreset,
-                                      void *opaque)
+static const TypeInfo adb_bus_type_info = {
+    .name = TYPE_ADB_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(ADBBusState),
+};
+
+static void adb_device_realizefn(DeviceState *dev, Error **errp)
 {
-    ADBDevice *d;
-    if (s->nb_devices >= MAX_ADB_DEVICES)
-        return NULL;
-    d = &s->devices[s->nb_devices++];
-    d->bus = s;
-    d->devaddr = devaddr;
-    d->devreq = devreq;
-    d->devreset = devreset;
-    d->opaque = opaque;
-    qemu_register_reset((QEMUResetHandler *)devreset, d);
-    return d;
+    ADBDevice *d = ADB_DEVICE(dev);
+    ADBBusState *bus = ADB_BUS(qdev_get_parent_bus(dev));
+
+    if (bus->nb_devices >= MAX_ADB_DEVICES) {
+        return;
+    }
+
+    bus->devices[bus->nb_devices++] = d;
 }
 
+static void adb_device_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = adb_device_realizefn;
+    dc->bus_type = TYPE_ADB_BUS;
+}
+
+static const TypeInfo adb_device_type_info = {
+    .name = TYPE_ADB_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(ADBDevice),
+    .abstract = true,
+    .class_init = adb_device_class_init,
+};
+
 /***************************************************************/
 /* Keyboard ADB device */
 
+#define ADB_KEYBOARD(obj) OBJECT_CHECK(KBDState, (obj), TYPE_ADB_KEYBOARD)
+
 typedef struct KBDState {
+    /*< private >*/
+    ADBDevice parent_obj;
+    /*< public >*/
+
     uint8_t data[128];
     int rptr, wptr, count;
 } KBDState;
 
+#define ADB_KEYBOARD_CLASS(class) \
+    OBJECT_CLASS_CHECK(ADBKeyboardClass, (class), TYPE_ADB_KEYBOARD)
+#define ADB_KEYBOARD_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ADBKeyboardClass, (obj), TYPE_ADB_KEYBOARD)
+
+typedef struct ADBKeyboardClass {
+    /*< private >*/
+    ADBDeviceClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+} ADBKeyboardClass;
+
 static const uint8_t pc_to_adb_keycode[256] = {
   0, 53, 18, 19, 20, 21, 23, 22, 26, 28, 25, 29, 27, 24, 51, 48,
  12, 13, 14, 15, 17, 16, 32, 34, 31, 35, 33, 30, 36, 54,  0,  1,
@@ -155,8 +194,7 @@ static const uint8_t pc_to_adb_keycode[256] = {
 
 static void adb_kbd_put_keycode(void *opaque, int keycode)
 {
-    ADBDevice *d = opaque;
-    KBDState *s = d->opaque;
+    KBDState *s = opaque;
 
     if (s->count < sizeof(s->data)) {
         s->data[s->wptr] = keycode;
@@ -169,7 +207,7 @@ static void adb_kbd_put_keycode(void *opaque, int keycode)
 static int adb_kbd_poll(ADBDevice *d, uint8_t *obuf)
 {
     static int ext_keycode;
-    KBDState *s = d->opaque;
+    KBDState *s = ADB_KEYBOARD(d);
     int adb_keycode, keycode;
     int olen;
 
@@ -203,7 +241,7 @@ static int adb_kbd_poll(ADBDevice *d, uint8_t *obuf)
 static int adb_kbd_request(ADBDevice *d, uint8_t *obuf,
                            const uint8_t *buf, int len)
 {
-    KBDState *s = d->opaque;
+    KBDState *s = ADB_KEYBOARD(d);
     int cmd, reg, olen;
 
     if ((buf[0] & 0x0f) == ADB_FLUSH) {
@@ -275,41 +313,90 @@ static const VMStateDescription vmstate_adb_kbd = {
     }
 };
 
-static int adb_kbd_reset(ADBDevice *d)
+static void adb_kbd_reset(DeviceState *dev)
 {
-    KBDState *s = d->opaque;
+    ADBDevice *d = ADB_DEVICE(dev);
+    KBDState *s = ADB_KEYBOARD(dev);
 
     d->handler = 1;
-    d->devaddr = ADB_KEYBOARD;
-    memset(s, 0, sizeof(KBDState));
-
-    return 0;
+    d->devaddr = ADB_DEVID_KEYBOARD;
+    memset(s->data, 0, sizeof(s->data));
+    s->rptr = 0;
+    s->wptr = 0;
+    s->count = 0;
 }
 
-void adb_kbd_init(ADBBusState *bus)
+static void adb_kbd_realizefn(DeviceState *dev, Error **errp)
 {
-    ADBDevice *d;
-    KBDState *s;
-    s = g_malloc0(sizeof(KBDState));
-    d = adb_register_device(bus, ADB_KEYBOARD, adb_kbd_request,
-                            adb_kbd_reset, s);
+    ADBDevice *d = ADB_DEVICE(dev);
+    ADBKeyboardClass *akc = ADB_KEYBOARD_GET_CLASS(dev);
+
+    akc->parent_realize(dev, errp);
+
     qemu_add_kbd_event_handler(adb_kbd_put_keycode, d);
-    vmstate_register(NULL, -1, &vmstate_adb_kbd, s);
 }
 
+static void adb_kbd_initfn(Object *obj)
+{
+    ADBDevice *d = ADB_DEVICE(obj);
+
+    d->devaddr = ADB_DEVID_KEYBOARD;
+}
+
+static void adb_kbd_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    ADBDeviceClass *adc = ADB_DEVICE_CLASS(oc);
+    ADBKeyboardClass *akc = ADB_KEYBOARD_CLASS(oc);
+
+    akc->parent_realize = dc->realize;
+    dc->realize = adb_kbd_realizefn;
+
+    adc->devreq = adb_kbd_request;
+    dc->reset = adb_kbd_reset;
+    dc->vmsd = &vmstate_adb_kbd;
+}
+
+static const TypeInfo adb_kbd_type_info = {
+    .name = TYPE_ADB_KEYBOARD,
+    .parent = TYPE_ADB_DEVICE,
+    .instance_size = sizeof(KBDState),
+    .instance_init = adb_kbd_initfn,
+    .class_init = adb_kbd_class_init,
+    .class_size = sizeof(ADBKeyboardClass),
+};
+
 /***************************************************************/
 /* Mouse ADB device */
 
+#define ADB_MOUSE(obj) OBJECT_CHECK(MouseState, (obj), TYPE_ADB_MOUSE)
+
 typedef struct MouseState {
+    /*< public >*/
+    ADBDevice parent_obj;
+    /*< private >*/
+
     int buttons_state, last_buttons_state;
     int dx, dy, dz;
 } MouseState;
 
+#define ADB_MOUSE_CLASS(class) \
+    OBJECT_CLASS_CHECK(ADBMouseClass, (class), TYPE_ADB_MOUSE)
+#define ADB_MOUSE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ADBMouseClass, (obj), TYPE_ADB_MOUSE)
+
+typedef struct ADBMouseClass {
+    /*< public >*/
+    ADBDeviceClass parent_class;
+    /*< private >*/
+
+    DeviceRealize parent_realize;
+} ADBMouseClass;
+
 static void adb_mouse_event(void *opaque,
                             int dx1, int dy1, int dz1, int buttons_state)
 {
-    ADBDevice *d = opaque;
-    MouseState *s = d->opaque;
+    MouseState *s = opaque;
 
     s->dx += dx1;
     s->dy += dy1;
@@ -320,7 +407,7 @@ static void adb_mouse_event(void *opaque,
 
 static int adb_mouse_poll(ADBDevice *d, uint8_t *obuf)
 {
-    MouseState *s = d->opaque;
+    MouseState *s = ADB_MOUSE(d);
     int dx, dy;
 
     if (s->last_buttons_state == s->buttons_state &&
@@ -359,7 +446,7 @@ static int adb_mouse_poll(ADBDevice *d, uint8_t *obuf)
 static int adb_mouse_request(ADBDevice *d, uint8_t *obuf,
                              const uint8_t *buf, int len)
 {
-    MouseState *s = d->opaque;
+    MouseState *s = ADB_MOUSE(d);
     int cmd, reg, olen;
 
     if ((buf[0] & 0x0f) == ADB_FLUSH) {
@@ -416,15 +503,15 @@ static int adb_mouse_request(ADBDevice *d, uint8_t *obuf,
     return olen;
 }
 
-static int adb_mouse_reset(ADBDevice *d)
+static void adb_mouse_reset(DeviceState *dev)
 {
-    MouseState *s = d->opaque;
+    ADBDevice *d = ADB_DEVICE(dev);
+    MouseState *s = ADB_MOUSE(dev);
 
     d->handler = 2;
-    d->devaddr = ADB_MOUSE;
-    memset(s, 0, sizeof(MouseState));
-
-    return 0;
+    d->devaddr = ADB_DEVID_MOUSE;
+    s->last_buttons_state = s->buttons_state = 0;
+    s->dx = s->dy = s->dz = 0;
 }
 
 static const VMStateDescription vmstate_adb_mouse = {
@@ -442,14 +529,53 @@ static const VMStateDescription vmstate_adb_mouse = {
     }
 };
 
-void adb_mouse_init(ADBBusState *bus)
+static void adb_mouse_realizefn(DeviceState *dev, Error **errp)
 {
-    ADBDevice *d;
-    MouseState *s;
+    MouseState *s = ADB_MOUSE(dev);
+    ADBMouseClass *amc = ADB_MOUSE_GET_CLASS(dev);
+
+    amc->parent_realize(dev, errp);
+
+    qemu_add_mouse_event_handler(adb_mouse_event, s, 0, "QEMU ADB Mouse");
+}
+
+static void adb_mouse_initfn(Object *obj)
+{
+    ADBDevice *d = ADB_DEVICE(obj);
+
+    d->devaddr = ADB_DEVID_MOUSE;
+}
+
+static void adb_mouse_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    ADBDeviceClass *adc = ADB_DEVICE_CLASS(oc);
+    ADBMouseClass *amc = ADB_MOUSE_CLASS(oc);
+
+    amc->parent_realize = dc->realize;
+    dc->realize = adb_mouse_realizefn;
 
-    s = g_malloc0(sizeof(MouseState));
-    d = adb_register_device(bus, ADB_MOUSE, adb_mouse_request,
-                            adb_mouse_reset, s);
-    qemu_add_mouse_event_handler(adb_mouse_event, d, 0, "QEMU ADB Mouse");
-    vmstate_register(NULL, -1, &vmstate_adb_mouse, s);
+    adc->devreq = adb_mouse_request;
+    dc->reset = adb_mouse_reset;
+    dc->vmsd = &vmstate_adb_mouse;
 }
+
+static const TypeInfo adb_mouse_type_info = {
+    .name = TYPE_ADB_MOUSE,
+    .parent = TYPE_ADB_DEVICE,
+    .instance_size = sizeof(MouseState),
+    .instance_init = adb_mouse_initfn,
+    .class_init = adb_mouse_class_init,
+    .class_size = sizeof(ADBMouseClass),
+};
+
+
+static void adb_register_types(void)
+{
+    type_register_static(&adb_bus_type_info);
+    type_register_static(&adb_device_type_info);
+    type_register_static(&adb_kbd_type_info);
+    type_register_static(&adb_mouse_type_info);
+}
+
+type_init(adb_register_types)
diff --git a/hw/adb.h b/hw/adb.h
index 5b27da2..721f1ac 100644
--- a/hw/adb.h
+++ b/hw/adb.h
@@ -26,38 +26,62 @@
 #if !defined(__ADB_H__)
 #define __ADB_H__
 
+#include "qdev.h"
+
 #define MAX_ADB_DEVICES 16
 
 #define ADB_MAX_OUT_LEN 16
 
+typedef struct ADBBusState ADBBusState;
 typedef struct ADBDevice ADBDevice;
 
 /* buf = NULL means polling */
 typedef int ADBDeviceRequest(ADBDevice *d, uint8_t *buf_out,
                               const uint8_t *buf, int len);
-typedef int ADBDeviceReset(ADBDevice *d);
+
+#define TYPE_ADB_DEVICE "adb-device"
+#define ADB_DEVICE(obj) OBJECT_CHECK(ADBDevice, (obj), TYPE_ADB_DEVICE)
 
 struct ADBDevice {
-    struct ADBBusState *bus;
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
+
     int devaddr;
     int handler;
-    ADBDeviceRequest *devreq;
-    ADBDeviceReset *devreset;
-    void *opaque;
 };
 
-typedef struct ADBBusState {
-    ADBDevice devices[MAX_ADB_DEVICES];
+#define ADB_DEVICE_CLASS(cls) \
+    OBJECT_CLASS_CHECK(ADBDeviceClass, (cls), TYPE_ADB_DEVICE)
+#define ADB_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ADBDeviceClass, (obj), TYPE_ADB_DEVICE)
+
+typedef struct ADBDeviceClass {
+    /*< private >*/
+    DeviceClass parent_class;
+    /*< public >*/
+
+    ADBDeviceRequest *devreq;
+} ADBDeviceClass;
+
+#define TYPE_ADB_BUS "apple-desktop-bus"
+#define ADB_BUS(obj) OBJECT_CHECK(ADBBusState, (obj), TYPE_ADB_BUS)
+
+struct ADBBusState {
+    /*< private >*/
+    BusState parent_obj;
+    /*< public >*/
+
+    ADBDevice *devices[MAX_ADB_DEVICES];
     int nb_devices;
     int poll_index;
-} ADBBusState;
+};
 
 int adb_request(ADBBusState *s, uint8_t *buf_out,
                 const uint8_t *buf, int len);
 int adb_poll(ADBBusState *s, uint8_t *buf_out);
 
-void adb_kbd_init(ADBBusState *bus);
-void adb_mouse_init(ADBBusState *bus);
+#define TYPE_ADB_KEYBOARD "adb-keyboard"
+#define TYPE_ADB_MOUSE "adb-mouse"
 
-extern ADBBusState adb_bus;
 #endif /* !defined(__ADB_H__) */
diff --git a/hw/adlib.c b/hw/adlib.c
index d39cd97..07c69fc 100644
--- a/hw/adlib.c
+++ b/hw/adlib.c
@@ -32,7 +32,7 @@
 #define ADLIB_KILL_TIMERS 1
 
 #ifdef DEBUG
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #endif
 
 #define dolog(...) AUD_log ("adlib", __VA_ARGS__)
diff --git a/hw/ads7846.c b/hw/ads7846.c
index 2ea9e55..29e5585 100644
--- a/hw/ads7846.c
+++ b/hw/ads7846.c
@@ -11,7 +11,7 @@
  */
 
 #include "ssi.h"
-#include "console.h"
+#include "ui/console.h"
 
 typedef struct {
     SSISlave ssidev;
@@ -162,7 +162,7 @@ static void ads7846_class_init(ObjectClass *klass, void *data)
     k->transfer = ads7846_transfer;
 }
 
-static TypeInfo ads7846_info = {
+static const TypeInfo ads7846_info = {
     .name          = "ads7846",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ADS7846State),
diff --git a/hw/alpha_dp264.c b/hw/alpha_dp264.c
index 76d8ae8..1cd549c 100644
--- a/hw/alpha_dp264.c
+++ b/hw/alpha_dp264.c
@@ -11,7 +11,7 @@
 #include "loader.h"
 #include "boards.h"
 #include "alpha_sys.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "mc146818rtc.h"
 #include "ide.h"
 #include "i8254.h"
@@ -50,7 +50,7 @@ static void clipper_init(QEMUMachineInitArgs *args)
     const char *kernel_filename = args->kernel_filename;
     const char *kernel_cmdline = args->kernel_cmdline;
     const char *initrd_filename = args->initrd_filename;
-    CPUAlphaState *cpus[4];
+    AlphaCPU *cpus[4];
     PCIBus *pci_bus;
     ISABus *isa_bus;
     qemu_irq rtc_irq;
@@ -62,12 +62,12 @@ static void clipper_init(QEMUMachineInitArgs *args)
     /* Create up to 4 cpus.  */
     memset(cpus, 0, sizeof(cpus));
     for (i = 0; i < smp_cpus; ++i) {
-        cpus[i] = cpu_init(cpu_model ? cpu_model : "ev67");
+        cpus[i] = cpu_alpha_init(cpu_model ? cpu_model : "ev67");
     }
 
-    cpus[0]->trap_arg0 = ram_size;
-    cpus[0]->trap_arg1 = 0;
-    cpus[0]->trap_arg2 = smp_cpus;
+    cpus[0]->env.trap_arg0 = ram_size;
+    cpus[0]->env.trap_arg1 = 0;
+    cpus[0]->env.trap_arg2 = smp_cpus;
 
     /* Init the chipset.  */
     pci_bus = typhoon_init(ram_size, &isa_bus, &rtc_irq, cpus,
@@ -119,9 +119,9 @@ static void clipper_init(QEMUMachineInitArgs *args)
 
     /* Start all cpus at the PALcode RESET entry point.  */
     for (i = 0; i < smp_cpus; ++i) {
-        cpus[i]->pal_mode = 1;
-        cpus[i]->pc = palcode_entry;
-        cpus[i]->palbr = palcode_entry;
+        cpus[i]->env.pal_mode = 1;
+        cpus[i]->env.pc = palcode_entry;
+        cpus[i]->env.palbr = palcode_entry;
     }
 
     /* Load a kernel.  */
@@ -136,7 +136,7 @@ static void clipper_init(QEMUMachineInitArgs *args)
             exit(1);
         }
 
-        cpus[0]->trap_arg1 = kernel_entry;
+        cpus[0]->env.trap_arg1 = kernel_entry;
 
         param_offset = kernel_low - 0x6000;
 
@@ -171,6 +171,7 @@ static QEMUMachine clipper_machine = {
     .init = clipper_init,
     .max_cpus = 4,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void clipper_machine_init(void)
diff --git a/hw/alpha_pci.c b/hw/alpha_pci.c
index 7e7b1d2..7327d48 100644
--- a/hw/alpha_pci.c
+++ b/hw/alpha_pci.c
@@ -8,8 +8,8 @@
 
 #include "config.h"
 #include "alpha_sys.h"
-#include "qemu-log.h"
-#include "sysemu.h"
+#include "qemu/log.h"
+#include "sysemu/sysemu.h"
 
 
 /* PCI IO reads/writes, to byte-word addressable memory.  */
diff --git a/hw/alpha_sys.h b/hw/alpha_sys.h
index 7604d09..233a71e 100644
--- a/hw/alpha_sys.h
+++ b/hw/alpha_sys.h
@@ -3,15 +3,14 @@
 #ifndef HW_ALPHA_H
 #define HW_ALPHA_H 1
 
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "ide.h"
-#include "net.h"
 #include "pc.h"
 #include "irq.h"
 
 
-PCIBus *typhoon_init(ram_addr_t, ISABus **, qemu_irq *, CPUAlphaState *[4],
+PCIBus *typhoon_init(ram_addr_t, ISABus **, qemu_irq *, AlphaCPU *[4],
                      pci_map_irq_fn);
 
 /* alpha_pci.c.  */
diff --git a/hw/alpha_typhoon.c b/hw/alpha_typhoon.c
index 9b16d96..bf9aabf 100644
--- a/hw/alpha_typhoon.c
+++ b/hw/alpha_typhoon.c
@@ -7,12 +7,12 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 #include "hw.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "alpha_sys.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 
 #define TYPE_TYPHOON_PCI_HOST_BRIDGE "typhoon-pcihost"
@@ -23,7 +23,7 @@ typedef struct TyphoonCchip {
     uint64_t drir;
     uint64_t dim[4];
     uint32_t iic[4];
-    CPUAlphaState *cpu[4];
+    AlphaCPU *cpu[4];
 } TyphoonCchip;
 
 typedef struct TyphoonWindow {
@@ -58,10 +58,11 @@ typedef struct TyphoonState {
 } TyphoonState;
 
 /* Called when one of DRIR or DIM changes.  */
-static void cpu_irq_change(CPUAlphaState *env, uint64_t req)
+static void cpu_irq_change(AlphaCPU *cpu, uint64_t req)
 {
     /* If there are any non-masked interrupts, tell the cpu.  */
-    if (env) {
+    if (cpu != NULL) {
+        CPUAlphaState *env = &cpu->env;
         if (req) {
             cpu_interrupt(env, CPU_INTERRUPT_HARD);
         } else {
@@ -74,6 +75,7 @@ static uint64_t cchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     CPUAlphaState *env = cpu_single_env;
     TyphoonState *s = opaque;
+    CPUState *cpu;
     uint64_t ret = 0;
 
     if (addr & 4) {
@@ -94,7 +96,8 @@ static uint64_t cchip_read(void *opaque, hwaddr addr, unsigned size)
 
     case 0x0080:
         /* MISC: Miscellaneous Register.  */
-        ret = s->cchip.misc | (env->cpu_index & 3);
+        cpu = ENV_GET_CPU(env);
+        ret = s->cchip.misc | (cpu->cpu_index & 3);
         break;
 
     case 0x00c0:
@@ -353,8 +356,9 @@ static void cchip_write(void *opaque, hwaddr addr,
         if ((newval ^ oldval) & 0xff0) {
             int i;
             for (i = 0; i < 4; ++i) {
-                CPUAlphaState *env = s->cchip.cpu[i];
-                if (env) {
+                AlphaCPU *cpu = s->cchip.cpu[i];
+                if (cpu != NULL) {
+                    CPUAlphaState *env = &cpu->env;
                     /* IPI can be either cleared or set by the write.  */
                     if (newval & (1 << (i + 8))) {
                         cpu_interrupt(env, CPU_INTERRUPT_SMP);
@@ -661,8 +665,8 @@ static void typhoon_set_timer_irq(void *opaque, int irq, int level)
 
     /* Deliver the interrupt to each CPU, considering each CPU's IIC.  */
     for (i = 0; i < 4; ++i) {
-        CPUAlphaState *env = s->cchip.cpu[i];
-        if (env) {
+        AlphaCPU *cpu = s->cchip.cpu[i];
+        if (cpu != NULL) {
             uint32_t iic = s->cchip.iic[i];
 
             /* ??? The verbage in Section 10.2.2.10 isn't 100% clear.
@@ -681,7 +685,7 @@ static void typhoon_set_timer_irq(void *opaque, int irq, int level)
                 /* Set the ITI bit for this cpu.  */
                 s->cchip.misc |= 1 << (i + 4);
                 /* And signal the interrupt.  */
-                cpu_interrupt(env, CPU_INTERRUPT_TIMER);
+                cpu_interrupt(&cpu->env, CPU_INTERRUPT_TIMER);
             }
         }
     }
@@ -694,12 +698,12 @@ static void typhoon_alarm_timer(void *opaque)
 
     /* Set the ITI bit for this cpu.  */
     s->cchip.misc |= 1 << (cpu + 4);
-    cpu_interrupt(s->cchip.cpu[cpu], CPU_INTERRUPT_TIMER);
+    cpu_interrupt(&s->cchip.cpu[cpu]->env, CPU_INTERRUPT_TIMER);
 }
 
 PCIBus *typhoon_init(ram_addr_t ram_size, ISABus **isa_bus,
                      qemu_irq *p_rtc_irq,
-                     CPUAlphaState *cpus[4], pci_map_irq_fn sys_map_irq)
+                     AlphaCPU *cpus[4], pci_map_irq_fn sys_map_irq)
 {
     const uint64_t MB = 1024 * 1024;
     const uint64_t GB = 1024 * MB;
@@ -719,10 +723,10 @@ PCIBus *typhoon_init(ram_addr_t ram_size, ISABus **isa_bus,
 
     /* Remember the CPUs so that we can deliver interrupts to them.  */
     for (i = 0; i < 4; i++) {
-        CPUAlphaState *env = cpus[i];
-        s->cchip.cpu[i] = env;
-        if (env) {
-            env->alarm_timer = qemu_new_timer_ns(rtc_clock,
+        AlphaCPU *cpu = cpus[i];
+        s->cchip.cpu[i] = cpu;
+        if (cpu != NULL) {
+            cpu->alarm_timer = qemu_new_timer_ns(rtc_clock,
                                                  typhoon_alarm_timer,
                                                  (void *)((uintptr_t)s + i));
         }
diff --git a/hw/an5206.c b/hw/an5206.c
index d887c0e..750115a 100644
--- a/hw/an5206.c
+++ b/hw/an5206.c
@@ -11,7 +11,7 @@
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define KERNEL_LOAD_ADDR 0x10000
 #define AN5206_MBAR_ADDR 0x10000000
@@ -86,6 +86,7 @@ static QEMUMachine an5206_machine = {
     .name = "an5206",
     .desc = "Arnewsh 5206",
     .init = an5206_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void an5206_machine_init(void)
diff --git a/hw/apb_pci.c b/hw/apb_pci.c
index 054814f..7eb0c2b 100644
--- a/hw/apb_pci.c
+++ b/hw/apb_pci.c
@@ -27,13 +27,13 @@
    the secondary PCI bridge.  */
 
 #include "sysbus.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "pci_bridge.h"
-#include "pci_internals.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "pci/pci_bridge.h"
+#include "pci/pci_bus.h"
 #include "apb_pci.h"
-#include "sysemu.h"
-#include "exec-memory.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
 
 /* debug APB */
 //#define DEBUG_APB
@@ -365,7 +365,7 @@ PCIBus *pci_apb_init(hwaddr special_base,
     /* Ultrasparc PBM main bus */
     dev = qdev_create(NULL, "pbm");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     /* apb_config */
     sysbus_mmio_map(s, 0, special_base);
     /* PCI configuration space */
@@ -486,7 +486,7 @@ static void pbm_pci_host_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_BRIDGE_HOST;
 }
 
-static TypeInfo pbm_pci_host_info = {
+static const TypeInfo pbm_pci_host_info = {
     .name          = "pbm-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIDevice),
@@ -502,7 +502,7 @@ static void pbm_host_class_init(ObjectClass *klass, void *data)
     dc->reset = pci_pbm_reset;
 }
 
-static TypeInfo pbm_host_info = {
+static const TypeInfo pbm_host_info = {
     .name          = "pbm",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(APBState),
@@ -525,7 +525,7 @@ static void pbm_pci_bridge_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pci_device;
 }
 
-static TypeInfo pbm_pci_bridge_info = {
+static const TypeInfo pbm_pci_bridge_info = {
     .name          = "pbm-bridge",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIBridge),
diff --git a/hw/apic.c b/hw/apic.c
index f73fc87..fd14b73 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -16,12 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 #include "apic_internal.h"
 #include "apic.h"
 #include "ioapic.h"
-#include "msi.h"
-#include "host-utils.h"
+#include "pci/msi.h"
+#include "qemu/host-utils.h"
 #include "trace.h"
 #include "pc.h"
 #include "apic-msidef.h"
@@ -895,7 +895,7 @@ static void apic_class_init(ObjectClass *klass, void *data)
     k->post_load = apic_post_load;
 }
 
-static TypeInfo apic_info = {
+static const TypeInfo apic_info = {
     .name          = "apic",
     .instance_size = sizeof(APICCommonState),
     .parent        = TYPE_APIC_COMMON,
diff --git a/hw/apic_common.c b/hw/apic_common.c
index 5f54276..6e1b1e0 100644
--- a/hw/apic_common.c
+++ b/hw/apic_common.c
@@ -20,7 +20,7 @@
 #include "apic.h"
 #include "apic_internal.h"
 #include "trace.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 static int apic_irq_delivered;
 bool apic_report_tpr_access;
@@ -385,7 +385,7 @@ static void apic_common_class_init(ObjectClass *klass, void *data)
     sc->init = apic_init_common;
 }
 
-static TypeInfo apic_common_type = {
+static const TypeInfo apic_common_type = {
     .name = TYPE_APIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(APICCommonState),
diff --git a/hw/apic_internal.h b/hw/apic_internal.h
index 79e2de2..dcbbfd4 100644
--- a/hw/apic_internal.h
+++ b/hw/apic_internal.h
@@ -20,9 +20,9 @@
 #ifndef QEMU_APIC_INTERNAL_H
 #define QEMU_APIC_INTERNAL_H
 
-#include "memory.h"
+#include "exec/memory.h"
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /* APIC Local Vector Table */
 #define APIC_LVT_TIMER                  0
diff --git a/hw/apm.c b/hw/apm.c
index 2aead52..2e1b137 100644
--- a/hw/apm.c
+++ b/hw/apm.c
@@ -22,6 +22,7 @@
 
 #include "apm.h"
 #include "hw.h"
+#include "pci/pci.h"
 
 //#define DEBUG
 
@@ -35,7 +36,8 @@
 #define APM_CNT_IOPORT  0xb2
 #define APM_STS_IOPORT  0xb3
 
-static void apm_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void apm_ioport_writeb(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
 {
     APMState *apm = opaque;
     addr &= 1;
@@ -51,7 +53,7 @@ static void apm_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t apm_ioport_readb(void *opaque, uint32_t addr)
+static uint64_t apm_ioport_readb(void *opaque, hwaddr addr, unsigned size)
 {
     APMState *apm = opaque;
     uint32_t val;
@@ -78,12 +80,23 @@ const VMStateDescription vmstate_apm = {
     }
 };
 
-void apm_init(APMState *apm, apm_ctrl_changed_t callback, void *arg)
+static const MemoryRegionOps apm_ops = {
+    .read = apm_ioport_readb,
+    .write = apm_ioport_writeb,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+void apm_init(PCIDevice *dev, APMState *apm, apm_ctrl_changed_t callback,
+              void *arg)
 {
     apm->callback = callback;
     apm->arg = arg;
 
     /* ioport 0xb2, 0xb3 */
-    register_ioport_write(APM_CNT_IOPORT, 2, 1, apm_ioport_writeb, apm);
-    register_ioport_read(APM_CNT_IOPORT, 2, 1, apm_ioport_readb, apm);
+    memory_region_init_io(&apm->io, &apm_ops, apm, "apm-io", 2);
+    memory_region_add_subregion(pci_address_space_io(dev), APM_CNT_IOPORT,
+                                &apm->io);
 }
diff --git a/hw/apm.h b/hw/apm.h
index f7c741e..9abb47f 100644
--- a/hw/apm.h
+++ b/hw/apm.h
@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include "qemu-common.h"
 #include "hw.h"
+#include "exec/memory.h"
 
 typedef void (*apm_ctrl_changed_t)(uint32_t val, void *arg);
 
@@ -13,9 +14,11 @@ typedef struct APMState {
 
     apm_ctrl_changed_t callback;
     void *arg;
+    MemoryRegion io;
 } APMState;
 
-void apm_init(APMState *s, apm_ctrl_changed_t callback, void *arg);
+void apm_init(PCIDevice *dev, APMState *s, apm_ctrl_changed_t callback,
+              void *arg);
 
 extern const VMStateDescription vmstate_apm;
 
diff --git a/hw/applesmc.c b/hw/applesmc.c
index 8bedaad..5a8c4ff 100644
--- a/hw/applesmc.c
+++ b/hw/applesmc.c
@@ -32,8 +32,8 @@
 
 #include "hw.h"
 #include "isa.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 
 /* #define DEBUG_SMC */
 
@@ -236,7 +236,7 @@ static void qdev_applesmc_class_init(ObjectClass *klass, void *data)
     dc->props = applesmc_isa_properties;
 }
 
-static TypeInfo applesmc_isa_info = {
+static const TypeInfo applesmc_isa_info = {
     .name          = "isa-applesmc",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(struct AppleSMCStatus),
diff --git a/hw/arm-misc.h b/hw/arm-misc.h
index d129678..cba7553 100644
--- a/hw/arm-misc.h
+++ b/hw/arm-misc.h
@@ -11,7 +11,7 @@
 #ifndef ARM_MISC_H
 #define ARM_MISC_H 1
 
-#include "memory.h"
+#include "exec/memory.h"
 #include "hw/irq.h"
 
 /* The CPU is also modeled as an interrupt controller.  */
diff --git a/hw/arm11mpcore.c b/hw/arm11mpcore.c
index 640ed20..324e503 100644
--- a/hw/arm11mpcore.c
+++ b/hw/arm11mpcore.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /* MPCore private memory region.  */
 
@@ -83,8 +83,8 @@ static void mpcore_priv_set_irq(void *opaque, int irq, int level)
 static void mpcore_priv_map_setup(mpcore_priv_state *s)
 {
     int i;
-    SysBusDevice *gicbusdev = sysbus_from_qdev(s->gic);
-    SysBusDevice *busdev = sysbus_from_qdev(s->mptimer);
+    SysBusDevice *gicbusdev = SYS_BUS_DEVICE(s->gic);
+    SysBusDevice *busdev = SYS_BUS_DEVICE(s->mptimer);
     memory_region_init(&s->container, "mpcode-priv-container", 0x2000);
     memory_region_init_io(&s->iomem, &mpcore_scu_ops, s, "mpcore-scu", 0x100);
     memory_region_add_subregion(&s->container, 0, &s->iomem);
@@ -131,7 +131,7 @@ static int mpcore_priv_init(SysBusDevice *dev)
     qdev_init_nofail(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
-    sysbus_pass_irq(dev, sysbus_from_qdev(s->gic));
+    sysbus_pass_irq(dev, SYS_BUS_DEVICE(s->gic));
 
     /* Pass through inbound GPIO lines to the GIC */
     qdev_init_gpio_in(&s->busdev.qdev, mpcore_priv_set_irq, s->num_irq - 32);
@@ -190,7 +190,7 @@ static int realview_mpcore_init(SysBusDevice *dev)
     priv = qdev_create(NULL, "arm11mpcore_priv");
     qdev_prop_set_uint32(priv, "num-cpu", s->num_cpu);
     qdev_init_nofail(priv);
-    s->priv = sysbus_from_qdev(priv);
+    s->priv = SYS_BUS_DEVICE(priv);
     sysbus_pass_irq(dev, s->priv);
     for (i = 0; i < 32; i++) {
         s->cpuic[i] = qdev_get_gpio_in(priv, i);
@@ -222,7 +222,7 @@ static void mpcore_rirq_class_init(ObjectClass *klass, void *data)
     dc->props = mpcore_rirq_properties;
 }
 
-static TypeInfo mpcore_rirq_info = {
+static const TypeInfo mpcore_rirq_info = {
     .name          = "realview_mpcore",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mpcore_rirq_state),
@@ -252,7 +252,7 @@ static void mpcore_priv_class_init(ObjectClass *klass, void *data)
     dc->props = mpcore_priv_properties;
 }
 
-static TypeInfo mpcore_priv_info = {
+static const TypeInfo mpcore_priv_info = {
     .name          = "arm11mpcore_priv",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mpcore_priv_state),
diff --git a/hw/arm_boot.c b/hw/arm_boot.c
index ec3b8d5..4065424 100644
--- a/hw/arm_boot.c
+++ b/hw/arm_boot.c
@@ -10,11 +10,12 @@
 #include "config.h"
 #include "hw.h"
 #include "arm-misc.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
+#include "qemu/config-file.h"
 
 #define KERNEL_ARGS_ADDR 0x100
 #define KERNEL_LOAD_ADDR 0x00010000
@@ -440,9 +441,12 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
          * we point to the kernel args.
          */
         if (info->dtb_filename) {
-            /* Place the DTB after the initrd in memory */
-            hwaddr dtb_start = TARGET_PAGE_ALIGN(info->initrd_start +
-                                                 initrd_size);
+            /* Place the DTB after the initrd in memory. Note that some
+             * kernels will trash anything in the 4K page the initrd
+             * ends in, so make sure the DTB isn't caught up in that.
+             */
+            hwaddr dtb_start = QEMU_ALIGN_UP(info->initrd_start + initrd_size,
+                                             4096);
             if (load_dtb(dtb_start, info)) {
                 exit(1);
             }
diff --git a/hw/arm_gic.c b/hw/arm_gic.c
index 672d539..90e43d0 100644
--- a/hw/arm_gic.c
+++ b/hw/arm_gic.c
@@ -39,7 +39,8 @@ static const uint8_t gic_id[] = {
 static inline int gic_get_current_cpu(GICState *s)
 {
     if (s->num_cpu > 1) {
-        return cpu_single_env->cpu_index;
+        CPUState *cpu = ENV_GET_CPU(cpu_single_env);
+        return cpu->cpu_index;
     }
     return 0;
 }
@@ -76,7 +77,7 @@ void gic_update(GICState *s)
         if (best_prio < s->priority_mask[cpu]) {
             s->current_pending[cpu] = best_irq;
             if (best_prio < s->running_priority[cpu]) {
-                DPRINTF("Raised pending IRQ %d\n", best_irq);
+                DPRINTF("Raised pending IRQ %d (cpu %d)\n", best_irq, cpu);
                 level = 1;
             }
         }
@@ -374,7 +375,8 @@ static void gic_dist_writeb(void *opaque, hwaddr offset,
           value = 0xff;
         for (i = 0; i < 8; i++) {
             if (value & (1 << i)) {
-                int mask = (irq < GIC_INTERNAL) ? (1 << cpu) : GIC_TARGET(irq);
+                int mask =
+                    (irq < GIC_INTERNAL) ? (1 << cpu) : GIC_TARGET(irq + i);
                 int cm = (irq < GIC_INTERNAL) ? (1 << cpu) : ALL_CPU_MASK;
 
                 if (!GIC_TEST_ENABLED(irq + i, cm)) {
@@ -417,7 +419,7 @@ static void gic_dist_writeb(void *opaque, hwaddr offset,
 
         for (i = 0; i < 8; i++) {
             if (value & (1 << i)) {
-                GIC_SET_PENDING(irq + i, GIC_TARGET(irq));
+                GIC_SET_PENDING(irq + i, GIC_TARGET(irq + i));
             }
         }
     } else if (offset < 0x300) {
@@ -702,7 +704,7 @@ static void arm_gic_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo arm_gic_info = {
+static const TypeInfo arm_gic_info = {
     .name = TYPE_ARM_GIC,
     .parent = TYPE_ARM_GIC_COMMON,
     .instance_size = sizeof(GICState),
diff --git a/hw/arm_gic_common.c b/hw/arm_gic_common.c
index 670ecc5..2947622 100644
--- a/hw/arm_gic_common.c
+++ b/hw/arm_gic_common.c
@@ -133,7 +133,7 @@ static int arm_gic_common_init(SysBusDevice *dev)
 
 static void arm_gic_common_reset(DeviceState *dev)
 {
-    GICState *s = FROM_SYSBUS(GICState, sysbus_from_qdev(dev));
+    GICState *s = FROM_SYSBUS(GICState, SYS_BUS_DEVICE(dev));
     int i;
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
@@ -181,7 +181,7 @@ static void arm_gic_common_class_init(ObjectClass *klass, void *data)
     sc->init = arm_gic_common_init;
 }
 
-static TypeInfo arm_gic_common_type = {
+static const TypeInfo arm_gic_common_type = {
     .name = TYPE_ARM_GIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GICState),
diff --git a/hw/arm_l2x0.c b/hw/arm_l2x0.c
index 6abf0ee..ae1e51d 100644
--- a/hw/arm_l2x0.c
+++ b/hw/arm_l2x0.c
@@ -179,7 +179,7 @@ static void l2x0_class_init(ObjectClass *klass, void *data)
     dc->reset = l2x0_priv_reset;
 }
 
-static TypeInfo l2x0_info = {
+static const TypeInfo l2x0_info = {
     .name = "l2x0",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(l2x0_state),
diff --git a/hw/arm_mptimer.c b/hw/arm_mptimer.c
index 6790832..32817d3 100644
--- a/hw/arm_mptimer.c
+++ b/hw/arm_mptimer.c
@@ -20,7 +20,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /* This device implements the per-cpu private timer and watchdog block
  * which is used in both the ARM11MPCore and Cortex-A9MP.
@@ -49,11 +49,13 @@ typedef struct {
 
 static inline int get_current_cpu(arm_mptimer_state *s)
 {
-    if (cpu_single_env->cpu_index >= s->num_cpu) {
+    CPUState *cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
+
+    if (cpu_single_cpu->cpu_index >= s->num_cpu) {
         hw_error("arm_mptimer: num-cpu %d but this cpu is %d!\n",
-                 s->num_cpu, cpu_single_env->cpu_index);
+                 s->num_cpu, cpu_single_cpu->cpu_index);
     }
-    return cpu_single_env->cpu_index;
+    return cpu_single_cpu->cpu_index;
 }
 
 static inline void timerblock_update_irq(timerblock *tb)
@@ -236,7 +238,7 @@ static void timerblock_reset(timerblock *tb)
 static void arm_mptimer_reset(DeviceState *dev)
 {
     arm_mptimer_state *s =
-        FROM_SYSBUS(arm_mptimer_state, sysbus_from_qdev(dev));
+        FROM_SYSBUS(arm_mptimer_state, SYS_BUS_DEVICE(dev));
     int i;
     /* We reset every timer in the array, not just the ones we're using,
      * because vmsave will look at every array element.
@@ -329,7 +331,7 @@ static void arm_mptimer_class_init(ObjectClass *klass, void *data)
     dc->props = arm_mptimer_properties;
 }
 
-static TypeInfo arm_mptimer_info = {
+static const TypeInfo arm_mptimer_info = {
     .name          = "arm_mptimer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(arm_mptimer_state),
diff --git a/hw/arm_pic.c b/hw/arm_pic.c
index 874bbaf..45ccb9f 100644
--- a/hw/arm_pic.c
+++ b/hw/arm_pic.c
@@ -9,7 +9,7 @@
 
 #include "hw.h"
 #include "arm-misc.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 /* Input 0 is IRQ and input 1 is FIQ.  */
 static void arm_pic_cpu_handler(void *opaque, int irq, int level)
@@ -38,8 +38,8 @@ static void arm_pic_cpu_handler(void *opaque, int irq, int level)
 static void kvm_arm_pic_cpu_handler(void *opaque, int irq, int level)
 {
 #ifdef CONFIG_KVM
-    ARMCPU *cpu = opaque;
-    CPUARMState *env = &cpu->env;
+    ARMCPU *armcpu = opaque;
+    CPUState *cpu = CPU(armcpu);
     int kvm_irq = KVM_ARM_IRQ_TYPE_CPU << KVM_ARM_IRQ_TYPE_SHIFT;
 
     switch (irq) {
@@ -52,7 +52,7 @@ static void kvm_arm_pic_cpu_handler(void *opaque, int irq, int level)
     default:
         hw_error("kvm_arm_pic_cpu_handler: Bad interrupt line %d\n", irq);
     }
-    kvm_irq |= env->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
+    kvm_irq |= cpu->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
     kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
 #endif
 }
diff --git a/hw/arm_sysctl.c b/hw/arm_sysctl.c
index 58eb982..7ecb7da 100644
--- a/hw/arm_sysctl.c
+++ b/hw/arm_sysctl.c
@@ -8,10 +8,10 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sysbus.h"
 #include "primecell.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define LOCK_VALUE 0xa05f
 
@@ -75,7 +75,7 @@ static int board_id(arm_sysctl_state *s)
 
 static void arm_sysctl_reset(DeviceState *d)
 {
-    arm_sysctl_state *s = FROM_SYSBUS(arm_sysctl_state, sysbus_from_qdev(d));
+    arm_sysctl_state *s = FROM_SYSBUS(arm_sysctl_state, SYS_BUS_DEVICE(d));
 
     s->leds = 0;
     s->lockval = 0;
@@ -199,6 +199,7 @@ static void arm_sysctl_write(void *opaque, hwaddr offset,
     switch (offset) {
     case 0x08: /* LED */
         s->leds = val;
+        break;
     case 0x0c: /* OSC0 */
     case 0x10: /* OSC1 */
     case 0x14: /* OSC2 */
@@ -295,6 +296,7 @@ static void arm_sysctl_write(void *opaque, hwaddr offset,
             /* On VExpress this register is unimplemented and will RAZ/WI */
             break;
         }
+        break;
     case 0x54: /* CLCDSER */
     case 0x64: /* DMAPSR0 */
     case 0x68: /* DMAPSR1 */
@@ -332,6 +334,7 @@ static void arm_sysctl_write(void *opaque, hwaddr offset,
         default:
             s->sys_cfgstat |= 2;        /* error */
         }
+        s->sys_cfgctrl &= ~(1 << 31);
         return;
     case 0xa8: /* SYS_CFGSTAT */
         if (board_id(s) != BOARD_ID_VEXPRESS) {
@@ -410,7 +413,7 @@ static void arm_sysctl_class_init(ObjectClass *klass, void *data)
     dc->props = arm_sysctl_properties;
 }
 
-static TypeInfo arm_sysctl_info = {
+static const TypeInfo arm_sysctl_info = {
     .name          = "realview_sysctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(arm_sysctl_state),
diff --git a/hw/arm_timer.c b/hw/arm_timer.c
index af339d3..c1e56be 100644
--- a/hw/arm_timer.c
+++ b/hw/arm_timer.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "qdev.h"
 #include "ptimer.h"
@@ -361,7 +361,7 @@ static void icp_pit_class_init(ObjectClass *klass, void *data)
     sdc->init = icp_pit_init;
 }
 
-static TypeInfo icp_pit_info = {
+static const TypeInfo icp_pit_info = {
     .name          = "integrator_pit",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pit_state),
@@ -383,7 +383,7 @@ static void sp804_class_init(ObjectClass *klass, void *data)
     k->props = sp804_properties;
 }
 
-static TypeInfo sp804_info = {
+static const TypeInfo sp804_info = {
     .name          = "sp804",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(sp804_state),
diff --git a/hw/armv7m.c b/hw/armv7m.c
index ce2ec9b..904696c 100644
--- a/hw/armv7m.c
+++ b/hw/armv7m.c
@@ -137,12 +137,12 @@ static void armv7m_bitband_init(void)
     dev = qdev_create(NULL, "ARM,bitband-memory");
     qdev_prop_set_uint32(dev, "base", 0x20000000);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0x22000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0x22000000);
 
     dev = qdev_create(NULL, "ARM,bitband-memory");
     qdev_prop_set_uint32(dev, "base", 0x40000000);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0x42000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0x42000000);
 }
 
 /* Board init.  */
@@ -216,7 +216,7 @@ qemu_irq *armv7m_init(MemoryRegion *address_space_mem,
     env->nvic = nvic;
     qdev_init_nofail(nvic);
     cpu_pic = arm_pic_init_cpu(cpu);
-    sysbus_connect_irq(sysbus_from_qdev(nvic), 0, cpu_pic[ARM_PIC_CPU_IRQ]);
+    sysbus_connect_irq(SYS_BUS_DEVICE(nvic), 0, cpu_pic[ARM_PIC_CPU_IRQ]);
     for (i = 0; i < 64; i++) {
         pic[i] = qdev_get_gpio_in(nvic, i);
     }
@@ -269,7 +269,7 @@ static void bitband_class_init(ObjectClass *klass, void *data)
     dc->props = bitband_properties;
 }
 
-static TypeInfo bitband_info = {
+static const TypeInfo bitband_info = {
     .name          = "ARM,bitband-memory",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(BitBandState),
diff --git a/hw/armv7m_nvic.c b/hw/armv7m_nvic.c
index 4963678..d5798d0 100644
--- a/hw/armv7m_nvic.c
+++ b/hw/armv7m_nvic.c
@@ -11,9 +11,9 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "arm-misc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "arm_gic_internal.h"
 
 typedef struct {
@@ -535,7 +535,7 @@ static void armv7m_nvic_class_init(ObjectClass *klass, void *data)
     dc->reset = armv7m_nvic_reset;
 }
 
-static TypeInfo armv7m_nvic_info = {
+static const TypeInfo armv7m_nvic_info = {
     .name          = TYPE_NVIC,
     .parent        = TYPE_ARM_GIC_COMMON,
     .instance_init = armv7m_nvic_instance_init,
diff --git a/hw/audiodev.h b/hw/audiodev.h
index ed2790f..428274f 100644
--- a/hw/audiodev.h
+++ b/hw/audiodev.h
@@ -1,3 +1,6 @@
+#ifndef HW_AUDIODEV_H
+#define HW_AUDIODEV_H 1
+
 /* es1370.c */
 int es1370_init(PCIBus *bus);
 
@@ -18,3 +21,5 @@ int cs4231a_init(ISABus *bus);
 
 /* intel-hda.c + hda-audio.c */
 int intel_hda_and_codec_init(PCIBus *bus);
+
+#endif
diff --git a/hw/axis_dev88.c b/hw/axis_dev88.c
index aa1ac9e..dd37fa1 100644
--- a/hw/axis_dev88.c
+++ b/hw/axis_dev88.c
@@ -23,15 +23,15 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
 #include "boards.h"
 #include "etraxfs.h"
 #include "loader.h"
 #include "elf.h"
 #include "cris-boot.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define D(x)
 #define DNAND(x)
@@ -300,7 +300,7 @@ void axisdev88_init(QEMUMachineInitArgs *args)
     /* FIXME: Is there a proper way to signal vectors to the CPU core?  */
     qdev_prop_set_ptr(dev, "interrupt_vector", &env->interrupt_vector);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, 0x3001c000);
     sysbus_connect_irq(s, 0, cpu_irq[0]);
     sysbus_connect_irq(s, 1, cpu_irq[1]);
@@ -355,6 +355,7 @@ static QEMUMachine axisdev88_machine = {
     .desc = "AXIS devboard 88",
     .init = axisdev88_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void axisdev88_machine_init(void)
diff --git a/hw/baum.c b/hw/baum.c
index 3e94f84..09dcb9c 100644
--- a/hw/baum.c
+++ b/hw/baum.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "char/char.h"
+#include "qemu/timer.h"
 #include "usb.h"
 #include "baum.h"
 #include <brlapi.h>
diff --git a/hw/baum.h b/hw/baum.h
index 8af710f..7635884 100644
--- a/hw/baum.h
+++ b/hw/baum.h
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_BAUM_H
+#define HW_BAUM_H 1
 
 /* char device */
 CharDriverState *chr_baum_init(QemuOpts *opts);
+
+#endif
diff --git a/hw/beagle.c b/hw/beagle.c
index affefc0..8037247 100644
--- a/hw/beagle.c
+++ b/hw/beagle.c
@@ -20,17 +20,17 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "omap.h"
 #include "arm-misc.h"
 #include "boards.h"
 #include "i2c.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
 #include "flash.h"
 #include "sysbus.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define BEAGLE_NAND_CS       0
 #define BEAGLE_SMC_CS        1
diff --git a/hw/bitbang_i2c.c b/hw/bitbang_i2c.c
index 44ed7f4..114508f 100644
--- a/hw/bitbang_i2c.c
+++ b/hw/bitbang_i2c.c
@@ -230,7 +230,7 @@ static void gpio_i2c_class_init(ObjectClass *klass, void *data)
     dc->desc = "Virtual GPIO to I2C bridge";
 }
 
-static TypeInfo gpio_i2c_info = {
+static const TypeInfo gpio_i2c_info = {
     .name          = "gpio_i2c",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GPIOI2CState),
diff --git a/hw/blizzard.c b/hw/blizzard.c
index 1c8dadf..955e771 100644
--- a/hw/blizzard.c
+++ b/hw/blizzard.c
@@ -19,10 +19,10 @@
  */
 
 #include "qemu-common.h"
-#include "console.h"
+#include "ui/console.h"
 #include "devices.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
+#include "ui/pixel_ops.h"
 
 typedef void (*blizzard_fn_t)(uint8_t *, const uint8_t *, unsigned int);
 
diff --git a/hw/block-common.c b/hw/block-common.c
index f0196d7..d21ec3a 100644
--- a/hw/block-common.c
+++ b/hw/block-common.c
@@ -7,9 +7,9 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 void blkconf_serial(BlockConf *conf, char **serial)
 {
@@ -18,9 +18,7 @@ void blkconf_serial(BlockConf *conf, char **serial)
     if (!*serial) {
         /* try to fall back to value set with legacy -drive serial=... */
         dinfo = drive_get_by_blockdev(conf->bs);
-        if (dinfo->serial) {
-            *serial = g_strdup(dinfo->serial);
-        }
+        *serial = g_strdup(dinfo->serial);
     }
 }
 
diff --git a/hw/boards.h b/hw/boards.h
index 813d0e5..3813d4e 100644
--- a/hw/boards.h
+++ b/hw/boards.h
@@ -3,8 +3,12 @@
 #ifndef HW_BOARDS_H
 #define HW_BOARDS_H
 
+#include "sysemu/blockdev.h"
 #include "qdev.h"
 
+#define DEFAULT_MACHINE_OPTIONS \
+    .boot_order = "cad"
+
 typedef struct QEMUMachineInitArgs {
     ram_addr_t ram_size;
     const char *boot_device;
@@ -24,16 +28,18 @@ typedef struct QEMUMachine {
     const char *desc;
     QEMUMachineInitFunc *init;
     QEMUMachineResetFunc *reset;
-    int use_scsi;
+    BlockInterfaceType block_default_type;
     int max_cpus;
     unsigned int no_serial:1,
         no_parallel:1,
         use_virtcon:1,
+        use_sclp:1,
         no_floppy:1,
         no_cdrom:1,
         no_sdcard:1;
     int is_default;
     const char *default_machine_opts;
+    const char *boot_order;
     GlobalProperty *compat_props;
     struct QEMUMachine *next;
     const char *hw_version;
diff --git a/hw/bonito.c b/hw/bonito.c
index 0bf6d4a..0498c9b 100644
--- a/hw/bonito.c
+++ b/hw/bonito.c
@@ -40,12 +40,12 @@
 #include <assert.h>
 
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "pc.h"
 #include "mips.h"
-#include "pci_host.h"
-#include "sysemu.h"
-#include "exec-memory.h"
+#include "pci/pci_host.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_BONITO
 
diff --git a/hw/bt-hci-csr.c b/hw/bt-hci-csr.c
index 772b677..2070bb9 100644
--- a/hw/bt-hci-csr.c
+++ b/hw/bt-hci-csr.c
@@ -19,10 +19,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "char/char.h"
+#include "qemu/timer.h"
 #include "irq.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "bt.h"
 
 struct csrhci_s {
diff --git a/hw/bt-hci.c b/hw/bt-hci.c
index e54cfd7..69d2c73 100644
--- a/hw/bt-hci.c
+++ b/hw/bt-hci.c
@@ -19,9 +19,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "usb.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "bt.h"
 
 struct bt_hci_s {
diff --git a/hw/bt-hid.c b/hw/bt-hid.c
index 8d7a3da..cfa7c14 100644
--- a/hw/bt-hid.c
+++ b/hw/bt-hid.c
@@ -19,8 +19,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 #include "hid.h"
 #include "bt.h"
 
diff --git a/hw/bt-l2cap.c b/hw/bt-l2cap.c
index cb43ee7..ba061c0 100644
--- a/hw/bt-l2cap.c
+++ b/hw/bt-l2cap.c
@@ -18,7 +18,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "bt.h"
 
 #define L2CAP_CID_MAX	0x100	/* Between 0x40 and 0x10000 */
diff --git a/hw/bt.c b/hw/bt.c
index dc99fc2..4f2372d 100644
--- a/hw/bt.c
+++ b/hw/bt.c
@@ -18,7 +18,7 @@
  */
 
 #include "qemu-common.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "bt.h"
 
 /* Slave implementations can ignore this */
diff --git a/hw/bt.h b/hw/bt.h
index ebf6a37..830af94 100644
--- a/hw/bt.h
+++ b/hw/bt.h
@@ -23,6 +23,9 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifndef HW_BT_H
+#define HW_BT_H 1
+
 #include "hw/irq.h"
 
 /* BD Address */
@@ -2183,3 +2186,5 @@ enum bt_sdp_attribute_id {
     SDP_ATTR_NORMALLY_CONNECTABLE		= 0x020d,
     SDP_ATTR_BOOT_DEVICE			= 0x020e,
 };
+
+#endif
diff --git a/hw/cadence_gem.c b/hw/cadence_gem.c
index 0c037a2..ab86c17 100644
--- a/hw/cadence_gem.c
+++ b/hw/cadence_gem.c
@@ -25,7 +25,7 @@
 #include <zlib.h> /* For crc32 */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "net/checksum.h"
 
 #ifdef CADENCE_GEM_ERR_DEBUG
@@ -389,10 +389,10 @@ static void gem_init_register_masks(GemState *s)
  */
 static void phy_update_link(GemState *s)
 {
-    DB_PRINT("down %d\n", s->nic->nc.link_down);
+    DB_PRINT("down %d\n", qemu_get_queue(s->nic)->link_down);
 
     /* Autonegotiation status mirrors link status.  */
-    if (s->nic->nc.link_down) {
+    if (qemu_get_queue(s->nic)->link_down) {
         s->phy_regs[PHY_REG_STATUS] &= ~(PHY_REG_STATUS_ANEGCMPL |
                                          PHY_REG_STATUS_LINK);
         s->phy_regs[PHY_REG_INT_ST] |= PHY_REG_INT_ST_LINKC;
@@ -409,7 +409,7 @@ static int gem_can_receive(NetClientState *nc)
 {
     GemState *s;
 
-    s = DO_UPCAST(NICState, nc, nc)->opaque;
+    s = qemu_get_nic_opaque(nc);
 
     DB_PRINT("\n");
 
@@ -612,7 +612,7 @@ static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     uint8_t    rxbuf[2048];
     uint8_t   *rxbuf_ptr;
 
-    s = DO_UPCAST(NICState, nc, nc)->opaque;
+    s = qemu_get_nic_opaque(nc);
 
     /* Do nothing if receive is not enabled. */
     if (!(s->regs[GEM_NWCTRL] & GEM_NWCTRL_RXENA)) {
@@ -687,14 +687,15 @@ static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 
     packet_desc_addr = s->rx_desc_addr;
     while (1) {
-        DB_PRINT("read descriptor 0x%x\n", packet_desc_addr);
+        DB_PRINT("read descriptor 0x%x\n", (unsigned)packet_desc_addr);
         /* read current descriptor */
         cpu_physical_memory_read(packet_desc_addr,
                                  (uint8_t *)&desc[0], sizeof(desc));
 
         /* Descriptor owned by software ? */
         if (rx_desc_get_ownership(desc) == 1) {
-            DB_PRINT("descriptor 0x%x owned by sw.\n", packet_desc_addr);
+            DB_PRINT("descriptor 0x%x owned by sw.\n",
+                     (unsigned)packet_desc_addr);
             s->regs[GEM_RXSTATUS] |= GEM_RXSTATUS_NOBUF;
             /* Handle interrupt consequences */
             gem_update_int_status(s);
@@ -709,7 +710,7 @@ static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
          */
         if (rx_desc_get_buffer(desc) == 0) {
             DB_PRINT("Invalid RX buffer (NULL) for descriptor 0x%x\n",
-                       packet_desc_addr);
+                     (unsigned)packet_desc_addr);
             break;
         }
 
@@ -749,7 +750,7 @@ static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
         s->rx_desc_addr += 8;
     }
 
-    DB_PRINT("set SOF, OWN on descriptor 0x%08x\n", packet_desc_addr);
+    DB_PRINT("set SOF, OWN on descriptor 0x%08x\n", (unsigned)packet_desc_addr);
 
     /* Count it */
     gem_receive_updatestats(s, buf, size);
@@ -861,7 +862,8 @@ static void gem_transmit(GemState *s)
          */
         if ((tx_desc_get_buffer(desc) == 0) ||
             (tx_desc_get_length(desc) == 0)) {
-            DB_PRINT("Invalid TX descriptor @ 0x%x\n", packet_desc_addr);
+            DB_PRINT("Invalid TX descriptor @ 0x%x\n",
+                     (unsigned)packet_desc_addr);
             break;
         }
 
@@ -906,9 +908,10 @@ static void gem_transmit(GemState *s)
 
             /* Send the packet somewhere */
             if (s->phy_loop) {
-                gem_receive(&s->nic->nc, tx_packet, total_bytes);
+                gem_receive(qemu_get_queue(s->nic), tx_packet, total_bytes);
             } else {
-                qemu_send_packet(&s->nic->nc, tx_packet, total_bytes);
+                qemu_send_packet(qemu_get_queue(s->nic), tx_packet,
+                                 total_bytes);
             }
 
             /* Prepare for next packet */
@@ -959,7 +962,7 @@ static void gem_phy_reset(GemState *s)
 
 static void gem_reset(DeviceState *d)
 {
-    GemState *s = FROM_SYSBUS(GemState, sysbus_from_qdev(d));
+    GemState *s = FROM_SYSBUS(GemState, SYS_BUS_DEVICE(d));
 
     DB_PRINT("\n");
 
@@ -1031,10 +1034,11 @@ static uint64_t gem_read(void *opaque, hwaddr offset, unsigned size)
     offset >>= 2;
     retval = s->regs[offset];
 
-    DB_PRINT("offset: 0x%04x read: 0x%08x\n", offset*4, retval);
+    DB_PRINT("offset: 0x%04x read: 0x%08x\n", (unsigned)offset*4, retval);
 
     switch (offset) {
     case GEM_ISR:
+        DB_PRINT("lowering irq on ISR read\n");
         qemu_set_irq(s->irq, 0);
         break;
     case GEM_PHYMNTNC:
@@ -1073,7 +1077,7 @@ static void gem_write(void *opaque, hwaddr offset, uint64_t val,
     GemState *s = (GemState *)opaque;
     uint32_t readonly;
 
-    DB_PRINT("offset: 0x%04x write: 0x%08x ", offset, (unsigned)val);
+    DB_PRINT("offset: 0x%04x write: 0x%08x ", (unsigned)offset, (unsigned)val);
     offset >>= 2;
 
     /* Squash bits which are read only in write value */
@@ -1148,7 +1152,7 @@ static const MemoryRegionOps gem_ops = {
 
 static void gem_cleanup(NetClientState *nc)
 {
-    GemState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    GemState *s = qemu_get_nic_opaque(nc);
 
     DB_PRINT("\n");
     s->nic = NULL;
@@ -1157,7 +1161,7 @@ static void gem_cleanup(NetClientState *nc)
 static void gem_set_link(NetClientState *nc)
 {
     DB_PRINT("\n");
-    phy_update_link(DO_UPCAST(NICState, nc, nc)->opaque);
+    phy_update_link(qemu_get_nic_opaque(nc));
 }
 
 static NetClientInfo net_gem_info = {
@@ -1218,7 +1222,7 @@ static void gem_class_init(ObjectClass *klass, void *data)
     dc->reset = gem_reset;
 }
 
-static TypeInfo gem_info = {
+static const TypeInfo gem_info = {
     .class_init = gem_class_init,
     .name  = "cadence_gem",
     .parent = TYPE_SYS_BUS_DEVICE,
diff --git a/hw/cadence_ttc.c b/hw/cadence_ttc.c
index ec78a52..67028a3 100644
--- a/hw/cadence_ttc.c
+++ b/hw/cadence_ttc.c
@@ -17,7 +17,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #ifdef CADENCE_TTC_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -302,7 +302,7 @@ static uint64_t cadence_ttc_read(void *opaque, hwaddr offset,
 {
     uint32_t ret = cadence_ttc_read_imp(opaque, offset);
 
-    DB_PRINT("addr: %08x data: %08x\n", offset, ret);
+    DB_PRINT("addr: %08x data: %08x\n", (unsigned)offset, (unsigned)ret);
     return ret;
 }
 
@@ -311,7 +311,7 @@ static void cadence_ttc_write(void *opaque, hwaddr offset,
 {
     CadenceTimerState *s = cadence_timer_from_addr(opaque, offset);
 
-    DB_PRINT("addr: %08x data %08x\n", offset, (unsigned)value);
+    DB_PRINT("addr: %08x data %08x\n", (unsigned)offset, (unsigned)value);
 
     cadence_timer_sync(s);
 
@@ -474,7 +474,7 @@ static void cadence_ttc_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_cadence_ttc;
 }
 
-static TypeInfo cadence_ttc_info = {
+static const TypeInfo cadence_ttc_info = {
     .name  = "cadence_ttc",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size  = sizeof(CadenceTTCState),
diff --git a/hw/cadence_uart.c b/hw/cadence_uart.c
index 686e617..5766d38 100644
--- a/hw/cadence_uart.c
+++ b/hw/cadence_uart.c
@@ -17,8 +17,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "char/char.h"
+#include "qemu/timer.h"
 
 #ifdef CADENCE_UART_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -343,6 +343,7 @@ static void uart_read_rx_fifo(UartState *s, uint32_t *c)
         if (!s->rx_count) {
             s->r[R_SR] |= UART_SR_INTR_REMPTY;
         }
+        qemu_chr_accept_input(s->chr);
     } else {
         *c = 0;
         s->r[R_SR] |= UART_SR_INTR_REMPTY;
@@ -501,7 +502,7 @@ static void cadence_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_cadence_uart;
 }
 
-static TypeInfo cadence_uart_info = {
+static const TypeInfo cadence_uart_info = {
     .name          = "cadence_uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(UartState),
diff --git a/hw/cbus.c b/hw/cbus.c
index 7216899..6fd3905 100644
--- a/hw/cbus.c
+++ b/hw/cbus.c
@@ -23,7 +23,7 @@
 #include "qemu-common.h"
 #include "irq.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG
 
diff --git a/hw/ccid-card-emulated.c b/hw/ccid-card-emulated.c
index f4a6da4..c8f8ba3 100644
--- a/hw/ccid-card-emulated.c
+++ b/hw/ccid-card-emulated.c
@@ -31,9 +31,9 @@
 #include <vreader.h>
 #include <vcard_emul.h>
 
-#include "qemu-thread.h"
-#include "qemu-char.h"
-#include "monitor.h"
+#include "qemu/thread.h"
+#include "char/char.h"
+#include "monitor/monitor.h"
 #include "hw/ccid.h"
 
 #define DPRINTF(card, lvl, fmt, ...) \
@@ -587,7 +587,7 @@ static void emulated_class_initfn(ObjectClass *klass, void *data)
     dc->props = emulated_card_properties;
 }
 
-static TypeInfo emulated_card_info = {
+static const TypeInfo emulated_card_info = {
     .name          = EMULATED_DEV_NAME,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(EmulatedState),
diff --git a/hw/ccid-card-passthru.c b/hw/ccid-card-passthru.c
index bd6c777..984bd0b 100644
--- a/hw/ccid-card-passthru.c
+++ b/hw/ccid-card-passthru.c
@@ -8,9 +8,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu-char.h"
-#include "qemu_socket.h"
-#include "monitor.h"
+#include "char/char.h"
+#include "qemu/sockets.h"
+#include "monitor/monitor.h"
 #include "hw/ccid.h"
 #include "libcacard/vscard_common.h"
 
@@ -336,7 +336,7 @@ static void passthru_class_initfn(ObjectClass *klass, void *data)
     dc->props = passthru_card_properties;
 }
 
-static TypeInfo passthru_card_info = {
+static const TypeInfo passthru_card_info = {
     .name          = PASSTHRU_DEV_NAME,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(PassthruState),
diff --git a/hw/cirrus_vga.c b/hw/cirrus_vga.c
index 9bef96e..2a2c8da 100644
--- a/hw/cirrus_vga.c
+++ b/hw/cirrus_vga.c
@@ -27,8 +27,8 @@
  *   available at http://home.worldonline.dk/~finth/
  */
 #include "hw.h"
-#include "pci.h"
-#include "console.h"
+#include "pci/pci.h"
+#include "ui/console.h"
 #include "vga_int.h"
 #include "loader.h"
 
@@ -197,6 +197,7 @@ typedef void (*cirrus_fill_t)(struct CirrusVGAState *s,
 typedef struct CirrusVGAState {
     VGACommonState vga;
 
+    MemoryRegion cirrus_vga_io;
     MemoryRegion cirrus_linear_io;
     MemoryRegion cirrus_linear_bitblt_io;
     MemoryRegion cirrus_mmio_io;
@@ -2432,13 +2433,15 @@ static void cirrus_update_memory_access(CirrusVGAState *s)
 
 /* I/O ports */
 
-static uint32_t cirrus_vga_ioport_read(void *opaque, uint32_t addr)
+static uint64_t cirrus_vga_ioport_read(void *opaque, hwaddr addr,
+                                       unsigned size)
 {
     CirrusVGAState *c = opaque;
     VGACommonState *s = &c->vga;
     int val, index;
 
     qemu_flush_coalesced_mmio_buffer();
+    addr += 0x3b0;
 
     if (vga_ioport_invalid(s, addr)) {
 	val = 0xff;
@@ -2527,13 +2530,15 @@ static uint32_t cirrus_vga_ioport_read(void *opaque, uint32_t addr)
     return val;
 }
 
-static void cirrus_vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void cirrus_vga_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                    unsigned size)
 {
     CirrusVGAState *c = opaque;
     VGACommonState *s = &c->vga;
     int index;
 
     qemu_flush_coalesced_mmio_buffer();
+    addr += 0x3b0;
 
     /* check port range access depending on color/monochrome mode */
     if (vga_ioport_invalid(s, addr)) {
@@ -2646,7 +2651,7 @@ static uint64_t cirrus_mmio_read(void *opaque, hwaddr addr,
     if (addr >= 0x100) {
         return cirrus_mmio_blt_read(s, addr - 0x100);
     } else {
-        return cirrus_vga_ioport_read(s, addr + 0x3c0);
+        return cirrus_vga_ioport_read(s, addr + 0x10, size);
     }
 }
 
@@ -2658,7 +2663,7 @@ static void cirrus_mmio_write(void *opaque, hwaddr addr,
     if (addr >= 0x100) {
 	cirrus_mmio_blt_write(s, addr - 0x100, val);
     } else {
-        cirrus_vga_ioport_write(s, addr + 0x3c0, val);
+        cirrus_vga_ioport_write(s, addr + 0x10, val, size);
     }
 }
 
@@ -2784,8 +2789,19 @@ static const MemoryRegionOps cirrus_linear_io_ops = {
     },
 };
 
+static const MemoryRegionOps cirrus_vga_io_ops = {
+    .read = cirrus_vga_ioport_read,
+    .write = cirrus_vga_ioport_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
 static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
-                               MemoryRegion *system_memory)
+                               MemoryRegion *system_memory,
+                               MemoryRegion *system_io)
 {
     int i;
     static int inited;
@@ -2817,19 +2833,10 @@ static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
             s->bustype = CIRRUS_BUSTYPE_ISA;
     }
 
-    register_ioport_write(0x3c0, 16, 1, cirrus_vga_ioport_write, s);
-
-    register_ioport_write(0x3b4, 2, 1, cirrus_vga_ioport_write, s);
-    register_ioport_write(0x3d4, 2, 1, cirrus_vga_ioport_write, s);
-    register_ioport_write(0x3ba, 1, 1, cirrus_vga_ioport_write, s);
-    register_ioport_write(0x3da, 1, 1, cirrus_vga_ioport_write, s);
-
-    register_ioport_read(0x3c0, 16, 1, cirrus_vga_ioport_read, s);
-
-    register_ioport_read(0x3b4, 2, 1, cirrus_vga_ioport_read, s);
-    register_ioport_read(0x3d4, 2, 1, cirrus_vga_ioport_read, s);
-    register_ioport_read(0x3ba, 1, 1, cirrus_vga_ioport_read, s);
-    register_ioport_read(0x3da, 1, 1, cirrus_vga_ioport_read, s);
+    /* Register ioport 0x3b0 - 0x3df */
+    memory_region_init_io(&s->cirrus_vga_io, &cirrus_vga_io_ops, s,
+                          "cirrus-io", 0x30);
+    memory_region_add_subregion(system_io, 0x3b0, &s->cirrus_vga_io);
 
     memory_region_init(&s->low_mem_container,
                        "cirrus-lowmem-container",
@@ -2900,7 +2907,7 @@ static int vga_initfn(ISADevice *dev)
 
     vga_common_init(s);
     cirrus_init_common(&d->cirrus_vga, CIRRUS_ID_CLGD5430, 0,
-                       isa_address_space(dev));
+                       isa_address_space(dev), isa_address_space_io(dev));
     s->ds = graphic_console_init(s->update, s->invalidate,
                                  s->screen_dump, s->text_update,
                                  s);
@@ -2926,7 +2933,7 @@ static void isa_cirrus_vga_class_init(ObjectClass *klass, void *data)
     dc->props = isa_vga_cirrus_properties;
 }
 
-static TypeInfo isa_cirrus_vga_info = {
+static const TypeInfo isa_cirrus_vga_info = {
     .name          = "isa-cirrus-vga",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISACirrusVGAState),
@@ -2948,7 +2955,8 @@ static int pci_cirrus_vga_initfn(PCIDevice *dev)
 
      /* setup VGA */
      vga_common_init(&s->vga);
-     cirrus_init_common(s, device_id, 1, pci_address_space(dev));
+     cirrus_init_common(s, device_id, 1, pci_address_space(dev),
+                        pci_address_space_io(dev));
      s->vga.ds = graphic_console_init(s->vga.update, s->vga.invalidate,
                                       s->vga.screen_dump, s->vga.text_update,
                                       &s->vga);
@@ -2995,7 +3003,7 @@ static void cirrus_vga_class_init(ObjectClass *klass, void *data)
     dc->props = pci_vga_cirrus_properties;
 }
 
-static TypeInfo cirrus_vga_info = {
+static const TypeInfo cirrus_vga_info = {
     .name          = "cirrus-vga",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCICirrusVGAState),
diff --git a/hw/collie.c b/hw/collie.c
index 695982a..d19db59 100644
--- a/hw/collie.c
+++ b/hw/collie.c
@@ -15,8 +15,8 @@
 #include "strongarm.h"
 #include "arm-misc.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 static struct arm_boot_info collie_binfo = {
     .loader_start = SA_SDCS0,
@@ -62,6 +62,7 @@ static QEMUMachine collie_machine = {
     .name = "collie",
     .desc = "Collie PDA (SA-1110)",
     .init = collie_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void collie_machine_init(void)
diff --git a/hw/cris-boot.h b/hw/cris-boot.h
index 5b17d83..c4d3fa6 100644
--- a/hw/cris-boot.h
+++ b/hw/cris-boot.h
@@ -1,3 +1,5 @@
+#ifndef _CRIS_BOOT_H
+#define HW_CRIS_BOOT_H 1
 
 struct cris_load_info
 {
@@ -9,3 +11,5 @@ struct cris_load_info
 };
 
 void cris_load_image(CRISCPU *cpu, struct cris_load_info *li);
+
+#endif
diff --git a/hw/cs4231.c b/hw/cs4231.c
index 23570d5..ae384b9 100644
--- a/hw/cs4231.c
+++ b/hw/cs4231.c
@@ -166,7 +166,7 @@ static void cs4231_class_init(ObjectClass *klass, void *data)
     dc->props = cs4231_properties;
 }
 
-static TypeInfo cs4231_info = {
+static const TypeInfo cs4231_info = {
     .name          = "SUNW,CS4231",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(CSState),
diff --git a/hw/cs4231a.c b/hw/cs4231a.c
index 0257fd8..73f0859 100644
--- a/hw/cs4231a.c
+++ b/hw/cs4231a.c
@@ -26,7 +26,7 @@
 #include "audio/audio.h"
 #include "isa.h"
 #include "qdev.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /*
   Missing features:
@@ -682,7 +682,7 @@ static void cs4231a_class_initfn (ObjectClass *klass, void *data)
     dc->props = cs4231a_properties;
 }
 
-static TypeInfo cs4231a_info = {
+static const TypeInfo cs4231a_info = {
     .name          = "cs4231a",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof (CSState),
diff --git a/hw/cuda.c b/hw/cuda.c
index f1f408b..b36c535 100644
--- a/hw/cuda.c
+++ b/hw/cuda.c
@@ -23,10 +23,10 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
+#include "ppc/mac.h"
 #include "adb.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 /* XXX: implement all timer modes */
 
@@ -108,50 +108,6 @@
 /* CUDA returns time_t's offset from Jan 1, 1904, not 1970 */
 #define RTC_OFFSET                      2082844800
 
-typedef struct CUDATimer {
-    int index;
-    uint16_t latch;
-    uint16_t counter_value; /* counter value at load time */
-    int64_t load_time;
-    int64_t next_irq_time;
-    QEMUTimer *timer;
-} CUDATimer;
-
-typedef struct CUDAState {
-    MemoryRegion mem;
-    /* cuda registers */
-    uint8_t b;      /* B-side data */
-    uint8_t a;      /* A-side data */
-    uint8_t dirb;   /* B-side direction (1=output) */
-    uint8_t dira;   /* A-side direction (1=output) */
-    uint8_t sr;     /* Shift register */
-    uint8_t acr;    /* Auxiliary control register */
-    uint8_t pcr;    /* Peripheral control register */
-    uint8_t ifr;    /* Interrupt flag register */
-    uint8_t ier;    /* Interrupt enable register */
-    uint8_t anh;    /* A-side data, no handshake */
-
-    CUDATimer timers[2];
-
-    uint32_t tick_offset;
-
-    uint8_t last_b; /* last value of B register */
-    uint8_t last_acr; /* last value of B register */
-
-    int data_in_size;
-    int data_in_index;
-    int data_out_index;
-
-    qemu_irq irq;
-    uint8_t autopoll;
-    uint8_t data_in[128];
-    uint8_t data_out[16];
-    QEMUTimer *adb_poll_timer;
-} CUDAState;
-
-static CUDAState cuda_state;
-ADBBusState adb_bus;
-
 static void cuda_update(CUDAState *s);
 static void cuda_receive_packet_from_host(CUDAState *s,
                                           const uint8_t *data, int len);
@@ -501,7 +457,7 @@ static void cuda_adb_poll(void *opaque)
     uint8_t obuf[ADB_MAX_OUT_LEN + 2];
     int olen;
 
-    olen = adb_poll(&adb_bus, obuf + 2);
+    olen = adb_poll(&s->adb_bus, obuf + 2);
     if (olen > 0) {
         obuf[0] = ADB_PACKET;
         obuf[1] = 0x40; /* polled data */
@@ -597,7 +553,7 @@ static void cuda_receive_packet_from_host(CUDAState *s,
         {
             uint8_t obuf[ADB_MAX_OUT_LEN + 2];
             int olen;
-            olen = adb_request(&adb_bus, obuf + 2, data + 1, len - 1);
+            olen = adb_request(&s->adb_bus, obuf + 2, data + 1, len - 1);
             if (olen > 0) {
                 obuf[0] = ADB_PACKET;
                 obuf[1] = 0x00;
@@ -701,9 +657,9 @@ static const VMStateDescription vmstate_cuda = {
     }
 };
 
-static void cuda_reset(void *opaque)
+static void cuda_reset(DeviceState *dev)
 {
-    CUDAState *s = opaque;
+    CUDAState *s = CUDA(dev);
 
     s->b = 0;
     s->a = 0;
@@ -728,25 +684,57 @@ static void cuda_reset(void *opaque)
     set_counter(s, &s->timers[1], 0xffff);
 }
 
-void cuda_init (MemoryRegion **cuda_mem, qemu_irq irq)
+static void cuda_realizefn(DeviceState *dev, Error **errp)
 {
+    CUDAState *s = CUDA(dev);
     struct tm tm;
-    CUDAState *s = &cuda_state;
-
-    s->irq = irq;
 
-    s->timers[0].index = 0;
     s->timers[0].timer = qemu_new_timer_ns(vm_clock, cuda_timer1, s);
 
-    s->timers[1].index = 1;
-
     qemu_get_timedate(&tm, 0);
     s->tick_offset = (uint32_t)mktimegm(&tm) + RTC_OFFSET;
 
     s->adb_poll_timer = qemu_new_timer_ns(vm_clock, cuda_adb_poll, s);
+}
+
+static void cuda_initfn(Object *obj)
+{
+    SysBusDevice *d = SYS_BUS_DEVICE(obj);
+    CUDAState *s = CUDA(obj);
+    int i;
+
     memory_region_init_io(&s->mem, &cuda_ops, s, "cuda", 0x2000);
+    sysbus_init_mmio(d, &s->mem);
+    sysbus_init_irq(d, &s->irq);
+
+    for (i = 0; i < ARRAY_SIZE(s->timers); i++) {
+        s->timers[i].index = i;
+    }
 
-    *cuda_mem = &s->mem;
-    vmstate_register(NULL, -1, &vmstate_cuda, s);
-    qemu_register_reset(cuda_reset, s);
+    qbus_create_inplace((BusState *)&s->adb_bus, TYPE_ADB_BUS, DEVICE(obj),
+                        "adb.0");
 }
+
+static void cuda_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = cuda_realizefn;
+    dc->reset = cuda_reset;
+    dc->vmsd = &vmstate_cuda;
+}
+
+static const TypeInfo cuda_type_info = {
+    .name = TYPE_CUDA,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(CUDAState),
+    .instance_init = cuda_initfn,
+    .class_init = cuda_class_init,
+};
+
+static void cuda_register_types(void)
+{
+    type_register_static(&cuda_type_info);
+}
+
+type_init(cuda_register_types)
diff --git a/hw/dataplane/Makefile.objs b/hw/dataplane/Makefile.objs
new file mode 100644
index 0000000..3e47d05
--- /dev/null
+++ b/hw/dataplane/Makefile.objs
@@ -0,0 +1 @@
+obj-$(CONFIG_VIRTIO_BLK_DATA_PLANE) += hostmem.o vring.o event-poll.o ioq.o virtio-blk.o
diff --git a/hw/dataplane/event-poll.c b/hw/dataplane/event-poll.c
new file mode 100644
index 0000000..2b55c6e
--- /dev/null
+++ b/hw/dataplane/event-poll.c
@@ -0,0 +1,100 @@
+/*
+ * Event loop with file descriptor polling
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <sys/epoll.h>
+#include "hw/dataplane/event-poll.h"
+
+/* Add an event notifier and its callback for polling */
+void event_poll_add(EventPoll *poll, EventHandler *handler,
+                    EventNotifier *notifier, EventCallback *callback)
+{
+    struct epoll_event event = {
+        .events = EPOLLIN,
+        .data.ptr = handler,
+    };
+    handler->notifier = notifier;
+    handler->callback = callback;
+    if (epoll_ctl(poll->epoll_fd, EPOLL_CTL_ADD,
+                  event_notifier_get_fd(notifier), &event) != 0) {
+        fprintf(stderr, "failed to add event handler to epoll: %m\n");
+        exit(1);
+    }
+}
+
+/* Event callback for stopping event_poll() */
+static void handle_stop(EventHandler *handler)
+{
+    /* Do nothing */
+}
+
+void event_poll_init(EventPoll *poll)
+{
+    /* Create epoll file descriptor */
+    poll->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+    if (poll->epoll_fd < 0) {
+        fprintf(stderr, "epoll_create1 failed: %m\n");
+        exit(1);
+    }
+
+    /* Set up stop notifier */
+    if (event_notifier_init(&poll->stop_notifier, 0) < 0) {
+        fprintf(stderr, "failed to init stop notifier\n");
+        exit(1);
+    }
+    event_poll_add(poll, &poll->stop_handler,
+                   &poll->stop_notifier, handle_stop);
+}
+
+void event_poll_cleanup(EventPoll *poll)
+{
+    event_notifier_cleanup(&poll->stop_notifier);
+    close(poll->epoll_fd);
+    poll->epoll_fd = -1;
+}
+
+/* Block until the next event and invoke its callback */
+void event_poll(EventPoll *poll)
+{
+    EventHandler *handler;
+    struct epoll_event event;
+    int nevents;
+
+    /* Wait for the next event.  Only do one event per call to keep the
+     * function simple, this could be changed later. */
+    do {
+        nevents = epoll_wait(poll->epoll_fd, &event, 1, -1);
+    } while (nevents < 0 && errno == EINTR);
+    if (unlikely(nevents != 1)) {
+        fprintf(stderr, "epoll_wait failed: %m\n");
+        exit(1); /* should never happen */
+    }
+
+    /* Find out which event handler has become active */
+    handler = event.data.ptr;
+
+    /* Clear the eventfd */
+    event_notifier_test_and_clear(handler->notifier);
+
+    /* Handle the event */
+    handler->callback(handler);
+}
+
+/* Stop event_poll()
+ *
+ * This function can be used from another thread.
+ */
+void event_poll_notify(EventPoll *poll)
+{
+    event_notifier_set(&poll->stop_notifier);
+}
diff --git a/hw/dataplane/event-poll.h b/hw/dataplane/event-poll.h
new file mode 100644
index 0000000..3e8d3ec
--- /dev/null
+++ b/hw/dataplane/event-poll.h
@@ -0,0 +1,40 @@
+/*
+ * Event loop with file descriptor polling
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef EVENT_POLL_H
+#define EVENT_POLL_H
+
+#include "qemu/event_notifier.h"
+
+typedef struct EventHandler EventHandler;
+typedef void EventCallback(EventHandler *handler);
+struct EventHandler {
+    EventNotifier *notifier;        /* eventfd */
+    EventCallback *callback;        /* callback function */
+};
+
+typedef struct {
+    int epoll_fd;                   /* epoll(2) file descriptor */
+    EventNotifier stop_notifier;    /* stop poll notifier */
+    EventHandler stop_handler;      /* stop poll handler */
+} EventPoll;
+
+void event_poll_add(EventPoll *poll, EventHandler *handler,
+                    EventNotifier *notifier, EventCallback *callback);
+void event_poll_init(EventPoll *poll);
+void event_poll_cleanup(EventPoll *poll);
+void event_poll(EventPoll *poll);
+void event_poll_notify(EventPoll *poll);
+
+#endif /* EVENT_POLL_H */
diff --git a/hw/dataplane/hostmem.c b/hw/dataplane/hostmem.c
new file mode 100644
index 0000000..380537e
--- /dev/null
+++ b/hw/dataplane/hostmem.c
@@ -0,0 +1,176 @@
+/*
+ * Thread-safe guest to host memory mapping
+ *
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "exec/address-spaces.h"
+#include "hostmem.h"
+
+static int hostmem_lookup_cmp(const void *phys_, const void *region_)
+{
+    hwaddr phys = *(const hwaddr *)phys_;
+    const HostMemRegion *region = region_;
+
+    if (phys < region->guest_addr) {
+        return -1;
+    } else if (phys >= region->guest_addr + region->size) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+/**
+ * Map guest physical address to host pointer
+ */
+void *hostmem_lookup(HostMem *hostmem, hwaddr phys, hwaddr len, bool is_write)
+{
+    HostMemRegion *region;
+    void *host_addr = NULL;
+    hwaddr offset_within_region;
+
+    qemu_mutex_lock(&hostmem->current_regions_lock);
+    region = bsearch(&phys, hostmem->current_regions,
+                     hostmem->num_current_regions,
+                     sizeof(hostmem->current_regions[0]),
+                     hostmem_lookup_cmp);
+    if (!region) {
+        goto out;
+    }
+    if (is_write && region->readonly) {
+        goto out;
+    }
+    offset_within_region = phys - region->guest_addr;
+    if (len <= region->size - offset_within_region) {
+        host_addr = region->host_addr + offset_within_region;
+    }
+out:
+    qemu_mutex_unlock(&hostmem->current_regions_lock);
+
+    return host_addr;
+}
+
+/**
+ * Install new regions list
+ */
+static void hostmem_listener_commit(MemoryListener *listener)
+{
+    HostMem *hostmem = container_of(listener, HostMem, listener);
+
+    qemu_mutex_lock(&hostmem->current_regions_lock);
+    g_free(hostmem->current_regions);
+    hostmem->current_regions = hostmem->new_regions;
+    hostmem->num_current_regions = hostmem->num_new_regions;
+    qemu_mutex_unlock(&hostmem->current_regions_lock);
+
+    /* Reset new regions list */
+    hostmem->new_regions = NULL;
+    hostmem->num_new_regions = 0;
+}
+
+/**
+ * Add a MemoryRegionSection to the new regions list
+ */
+static void hostmem_append_new_region(HostMem *hostmem,
+                                      MemoryRegionSection *section)
+{
+    void *ram_ptr = memory_region_get_ram_ptr(section->mr);
+    size_t num = hostmem->num_new_regions;
+    size_t new_size = (num + 1) * sizeof(hostmem->new_regions[0]);
+
+    hostmem->new_regions = g_realloc(hostmem->new_regions, new_size);
+    hostmem->new_regions[num] = (HostMemRegion){
+        .host_addr = ram_ptr + section->offset_within_region,
+        .guest_addr = section->offset_within_address_space,
+        .size = section->size,
+        .readonly = section->readonly,
+    };
+    hostmem->num_new_regions++;
+}
+
+static void hostmem_listener_append_region(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    HostMem *hostmem = container_of(listener, HostMem, listener);
+
+    /* Ignore non-RAM regions, we may not be able to map them */
+    if (!memory_region_is_ram(section->mr)) {
+        return;
+    }
+
+    /* Ignore regions with dirty logging, we cannot mark them dirty */
+    if (memory_region_is_logging(section->mr)) {
+        return;
+    }
+
+    hostmem_append_new_region(hostmem, section);
+}
+
+/* We don't implement most MemoryListener callbacks, use these nop stubs */
+static void hostmem_listener_dummy(MemoryListener *listener)
+{
+}
+
+static void hostmem_listener_section_dummy(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+}
+
+static void hostmem_listener_eventfd_dummy(MemoryListener *listener,
+                                           MemoryRegionSection *section,
+                                           bool match_data, uint64_t data,
+                                           EventNotifier *e)
+{
+}
+
+static void hostmem_listener_coalesced_mmio_dummy(MemoryListener *listener,
+                                                  MemoryRegionSection *section,
+                                                  hwaddr addr, hwaddr len)
+{
+}
+
+void hostmem_init(HostMem *hostmem)
+{
+    memset(hostmem, 0, sizeof(*hostmem));
+
+    qemu_mutex_init(&hostmem->current_regions_lock);
+
+    hostmem->listener = (MemoryListener){
+        .begin = hostmem_listener_dummy,
+        .commit = hostmem_listener_commit,
+        .region_add = hostmem_listener_append_region,
+        .region_del = hostmem_listener_section_dummy,
+        .region_nop = hostmem_listener_append_region,
+        .log_start = hostmem_listener_section_dummy,
+        .log_stop = hostmem_listener_section_dummy,
+        .log_sync = hostmem_listener_section_dummy,
+        .log_global_start = hostmem_listener_dummy,
+        .log_global_stop = hostmem_listener_dummy,
+        .eventfd_add = hostmem_listener_eventfd_dummy,
+        .eventfd_del = hostmem_listener_eventfd_dummy,
+        .coalesced_mmio_add = hostmem_listener_coalesced_mmio_dummy,
+        .coalesced_mmio_del = hostmem_listener_coalesced_mmio_dummy,
+        .priority = 10,
+    };
+
+    memory_listener_register(&hostmem->listener, &address_space_memory);
+    if (hostmem->num_new_regions > 0) {
+        hostmem_listener_commit(&hostmem->listener);
+    }
+}
+
+void hostmem_finalize(HostMem *hostmem)
+{
+    memory_listener_unregister(&hostmem->listener);
+    g_free(hostmem->new_regions);
+    g_free(hostmem->current_regions);
+    qemu_mutex_destroy(&hostmem->current_regions_lock);
+}
diff --git a/hw/dataplane/hostmem.h b/hw/dataplane/hostmem.h
new file mode 100644
index 0000000..b2cf093
--- /dev/null
+++ b/hw/dataplane/hostmem.h
@@ -0,0 +1,57 @@
+/*
+ * Thread-safe guest to host memory mapping
+ *
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HOSTMEM_H
+#define HOSTMEM_H
+
+#include "exec/memory.h"
+#include "qemu/thread.h"
+
+typedef struct {
+    void *host_addr;
+    hwaddr guest_addr;
+    uint64_t size;
+    bool readonly;
+} HostMemRegion;
+
+typedef struct {
+    /* The listener is invoked when regions change and a new list of regions is
+     * built up completely before they are installed.
+     */
+    MemoryListener listener;
+    HostMemRegion *new_regions;
+    size_t num_new_regions;
+
+    /* Current regions are accessed from multiple threads either to lookup
+     * addresses or to install a new list of regions.  The lock protects the
+     * pointer and the regions.
+     */
+    QemuMutex current_regions_lock;
+    HostMemRegion *current_regions;
+    size_t num_current_regions;
+} HostMem;
+
+void hostmem_init(HostMem *hostmem);
+void hostmem_finalize(HostMem *hostmem);
+
+/**
+ * Map a guest physical address to a pointer
+ *
+ * Note that there is map/unmap mechanism here.  The caller must ensure that
+ * mapped memory is no longer used across events like hot memory unplug.  This
+ * can be done with other mechanisms like bdrv_drain_all() that quiesce
+ * in-flight I/O.
+ */
+void *hostmem_lookup(HostMem *hostmem, hwaddr phys, hwaddr len, bool is_write);
+
+#endif /* HOSTMEM_H */
diff --git a/hw/dataplane/ioq.c b/hw/dataplane/ioq.c
new file mode 100644
index 0000000..0c9f5c4
--- /dev/null
+++ b/hw/dataplane/ioq.c
@@ -0,0 +1,117 @@
+/*
+ * Linux AIO request queue
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "hw/dataplane/ioq.h"
+
+void ioq_init(IOQueue *ioq, int fd, unsigned int max_reqs)
+{
+    int rc;
+
+    ioq->fd = fd;
+    ioq->max_reqs = max_reqs;
+
+    memset(&ioq->io_ctx, 0, sizeof ioq->io_ctx);
+    rc = io_setup(max_reqs, &ioq->io_ctx);
+    if (rc != 0) {
+        fprintf(stderr, "ioq io_setup failed %d\n", rc);
+        exit(1);
+    }
+
+    rc = event_notifier_init(&ioq->io_notifier, 0);
+    if (rc != 0) {
+        fprintf(stderr, "ioq io event notifier creation failed %d\n", rc);
+        exit(1);
+    }
+
+    ioq->freelist = g_malloc0(sizeof ioq->freelist[0] * max_reqs);
+    ioq->freelist_idx = 0;
+
+    ioq->queue = g_malloc0(sizeof ioq->queue[0] * max_reqs);
+    ioq->queue_idx = 0;
+}
+
+void ioq_cleanup(IOQueue *ioq)
+{
+    g_free(ioq->freelist);
+    g_free(ioq->queue);
+
+    event_notifier_cleanup(&ioq->io_notifier);
+    io_destroy(ioq->io_ctx);
+}
+
+EventNotifier *ioq_get_notifier(IOQueue *ioq)
+{
+    return &ioq->io_notifier;
+}
+
+struct iocb *ioq_get_iocb(IOQueue *ioq)
+{
+    /* Underflow cannot happen since ioq is sized for max_reqs */
+    assert(ioq->freelist_idx != 0);
+
+    struct iocb *iocb = ioq->freelist[--ioq->freelist_idx];
+    ioq->queue[ioq->queue_idx++] = iocb;
+    return iocb;
+}
+
+void ioq_put_iocb(IOQueue *ioq, struct iocb *iocb)
+{
+    /* Overflow cannot happen since ioq is sized for max_reqs */
+    assert(ioq->freelist_idx != ioq->max_reqs);
+
+    ioq->freelist[ioq->freelist_idx++] = iocb;
+}
+
+struct iocb *ioq_rdwr(IOQueue *ioq, bool read, struct iovec *iov,
+                      unsigned int count, long long offset)
+{
+    struct iocb *iocb = ioq_get_iocb(ioq);
+
+    if (read) {
+        io_prep_preadv(iocb, ioq->fd, iov, count, offset);
+    } else {
+        io_prep_pwritev(iocb, ioq->fd, iov, count, offset);
+    }
+    io_set_eventfd(iocb, event_notifier_get_fd(&ioq->io_notifier));
+    return iocb;
+}
+
+int ioq_submit(IOQueue *ioq)
+{
+    int rc = io_submit(ioq->io_ctx, ioq->queue_idx, ioq->queue);
+    ioq->queue_idx = 0; /* reset */
+    return rc;
+}
+
+int ioq_run_completion(IOQueue *ioq, IOQueueCompletion *completion,
+                       void *opaque)
+{
+    struct io_event events[ioq->max_reqs];
+    int nevents, i;
+
+    do {
+        nevents = io_getevents(ioq->io_ctx, 0, ioq->max_reqs, events, NULL);
+    } while (nevents < 0 && errno == EINTR);
+    if (nevents < 0) {
+        return nevents;
+    }
+
+    for (i = 0; i < nevents; i++) {
+        ssize_t ret = ((uint64_t)events[i].res2 << 32) | events[i].res;
+
+        completion(events[i].obj, ret, opaque);
+        ioq_put_iocb(ioq, events[i].obj);
+    }
+    return nevents;
+}
diff --git a/hw/dataplane/ioq.h b/hw/dataplane/ioq.h
new file mode 100644
index 0000000..b49b5de
--- /dev/null
+++ b/hw/dataplane/ioq.h
@@ -0,0 +1,57 @@
+/*
+ * Linux AIO request queue
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef IOQ_H
+#define IOQ_H
+
+#include <libaio.h>
+#include "qemu/event_notifier.h"
+
+typedef struct {
+    int fd;                         /* file descriptor */
+    unsigned int max_reqs;          /* max length of freelist and queue */
+
+    io_context_t io_ctx;            /* Linux AIO context */
+    EventNotifier io_notifier;      /* Linux AIO eventfd */
+
+    /* Requests can complete in any order so a free list is necessary to manage
+     * available iocbs.
+     */
+    struct iocb **freelist;         /* free iocbs */
+    unsigned int freelist_idx;
+
+    /* Multiple requests are queued up before submitting them all in one go */
+    struct iocb **queue;            /* queued iocbs */
+    unsigned int queue_idx;
+} IOQueue;
+
+void ioq_init(IOQueue *ioq, int fd, unsigned int max_reqs);
+void ioq_cleanup(IOQueue *ioq);
+EventNotifier *ioq_get_notifier(IOQueue *ioq);
+struct iocb *ioq_get_iocb(IOQueue *ioq);
+void ioq_put_iocb(IOQueue *ioq, struct iocb *iocb);
+struct iocb *ioq_rdwr(IOQueue *ioq, bool read, struct iovec *iov,
+                      unsigned int count, long long offset);
+int ioq_submit(IOQueue *ioq);
+
+static inline unsigned int ioq_num_queued(IOQueue *ioq)
+{
+    return ioq->queue_idx;
+}
+
+typedef void IOQueueCompletion(struct iocb *iocb, ssize_t ret, void *opaque);
+int ioq_run_completion(IOQueue *ioq, IOQueueCompletion *completion,
+                       void *opaque);
+
+#endif /* IOQ_H */
diff --git a/hw/dataplane/virtio-blk.c b/hw/dataplane/virtio-blk.c
new file mode 100644
index 0000000..3f2da22
--- /dev/null
+++ b/hw/dataplane/virtio-blk.c
@@ -0,0 +1,517 @@
+/*
+ * Dedicated thread for virtio-blk I/O processing
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "qemu/iov.h"
+#include "event-poll.h"
+#include "qemu/thread.h"
+#include "vring.h"
+#include "ioq.h"
+#include "migration/migration.h"
+#include "hw/virtio-blk.h"
+#include "hw/dataplane/virtio-blk.h"
+
+enum {
+    SEG_MAX = 126,                  /* maximum number of I/O segments */
+    VRING_MAX = SEG_MAX + 2,        /* maximum number of vring descriptors */
+    REQ_MAX = VRING_MAX,            /* maximum number of requests in the vring,
+                                     * is VRING_MAX / 2 with traditional and
+                                     * VRING_MAX with indirect descriptors */
+};
+
+typedef struct {
+    struct iocb iocb;               /* Linux AIO control block */
+    QEMUIOVector *inhdr;            /* iovecs for virtio_blk_inhdr */
+    unsigned int head;              /* vring descriptor index */
+    struct iovec *bounce_iov;       /* used if guest buffers are unaligned */
+    QEMUIOVector *read_qiov;        /* for read completion /w bounce buffer */
+} VirtIOBlockRequest;
+
+struct VirtIOBlockDataPlane {
+    bool started;
+    bool stopping;
+    QEMUBH *start_bh;
+    QemuThread thread;
+
+    VirtIOBlkConf *blk;
+    int fd;                         /* image file descriptor */
+
+    VirtIODevice *vdev;
+    Vring vring;                    /* virtqueue vring */
+    EventNotifier *guest_notifier;  /* irq */
+
+    EventPoll event_poll;           /* event poller */
+    EventHandler io_handler;        /* Linux AIO completion handler */
+    EventHandler notify_handler;    /* virtqueue notify handler */
+
+    IOQueue ioqueue;                /* Linux AIO queue (should really be per
+                                       dataplane thread) */
+    VirtIOBlockRequest requests[REQ_MAX]; /* pool of requests, managed by the
+                                             queue */
+
+    unsigned int num_reqs;
+
+    Error *migration_blocker;
+};
+
+/* Raise an interrupt to signal guest, if necessary */
+static void notify_guest(VirtIOBlockDataPlane *s)
+{
+    if (!vring_should_notify(s->vdev, &s->vring)) {
+        return;
+    }
+
+    event_notifier_set(s->guest_notifier);
+}
+
+static void complete_request(struct iocb *iocb, ssize_t ret, void *opaque)
+{
+    VirtIOBlockDataPlane *s = opaque;
+    VirtIOBlockRequest *req = container_of(iocb, VirtIOBlockRequest, iocb);
+    struct virtio_blk_inhdr hdr;
+    int len;
+
+    if (likely(ret >= 0)) {
+        hdr.status = VIRTIO_BLK_S_OK;
+        len = ret;
+    } else {
+        hdr.status = VIRTIO_BLK_S_IOERR;
+        len = 0;
+    }
+
+    trace_virtio_blk_data_plane_complete_request(s, req->head, ret);
+
+    if (req->read_qiov) {
+        assert(req->bounce_iov);
+        qemu_iovec_from_buf(req->read_qiov, 0, req->bounce_iov->iov_base, len);
+        qemu_iovec_destroy(req->read_qiov);
+        g_slice_free(QEMUIOVector, req->read_qiov);
+    }
+
+    if (req->bounce_iov) {
+        qemu_vfree(req->bounce_iov->iov_base);
+        g_slice_free(struct iovec, req->bounce_iov);
+    }
+
+    qemu_iovec_from_buf(req->inhdr, 0, &hdr, sizeof(hdr));
+    qemu_iovec_destroy(req->inhdr);
+    g_slice_free(QEMUIOVector, req->inhdr);
+
+    /* According to the virtio specification len should be the number of bytes
+     * written to, but for virtio-blk it seems to be the number of bytes
+     * transferred plus the status bytes.
+     */
+    vring_push(&s->vring, req->head, len + sizeof(hdr));
+
+    s->num_reqs--;
+}
+
+static void complete_request_early(VirtIOBlockDataPlane *s, unsigned int head,
+                                   QEMUIOVector *inhdr, unsigned char status)
+{
+    struct virtio_blk_inhdr hdr = {
+        .status = status,
+    };
+
+    qemu_iovec_from_buf(inhdr, 0, &hdr, sizeof(hdr));
+    qemu_iovec_destroy(inhdr);
+    g_slice_free(QEMUIOVector, inhdr);
+
+    vring_push(&s->vring, head, sizeof(hdr));
+    notify_guest(s);
+}
+
+/* Get disk serial number */
+static void do_get_id_cmd(VirtIOBlockDataPlane *s,
+                          struct iovec *iov, unsigned int iov_cnt,
+                          unsigned int head, QEMUIOVector *inhdr)
+{
+    char id[VIRTIO_BLK_ID_BYTES];
+
+    /* Serial number not NUL-terminated when shorter than buffer */
+    strncpy(id, s->blk->serial ? s->blk->serial : "", sizeof(id));
+    iov_from_buf(iov, iov_cnt, 0, id, sizeof(id));
+    complete_request_early(s, head, inhdr, VIRTIO_BLK_S_OK);
+}
+
+static int do_rdwr_cmd(VirtIOBlockDataPlane *s, bool read,
+                       struct iovec *iov, unsigned int iov_cnt,
+                       long long offset, unsigned int head,
+                       QEMUIOVector *inhdr)
+{
+    struct iocb *iocb;
+    QEMUIOVector qiov;
+    struct iovec *bounce_iov = NULL;
+    QEMUIOVector *read_qiov = NULL;
+
+    qemu_iovec_init_external(&qiov, iov, iov_cnt);
+    if (!bdrv_qiov_is_aligned(s->blk->conf.bs, &qiov)) {
+        void *bounce_buffer = qemu_blockalign(s->blk->conf.bs, qiov.size);
+
+        if (read) {
+            /* Need to copy back from bounce buffer on completion */
+            read_qiov = g_slice_new(QEMUIOVector);
+            qemu_iovec_init(read_qiov, iov_cnt);
+            qemu_iovec_concat_iov(read_qiov, iov, iov_cnt, 0, qiov.size);
+        } else {
+            qemu_iovec_to_buf(&qiov, 0, bounce_buffer, qiov.size);
+        }
+
+        /* Redirect I/O to aligned bounce buffer */
+        bounce_iov = g_slice_new(struct iovec);
+        bounce_iov->iov_base = bounce_buffer;
+        bounce_iov->iov_len = qiov.size;
+        iov = bounce_iov;
+        iov_cnt = 1;
+    }
+
+    iocb = ioq_rdwr(&s->ioqueue, read, iov, iov_cnt, offset);
+
+    /* Fill in virtio block metadata needed for completion */
+    VirtIOBlockRequest *req = container_of(iocb, VirtIOBlockRequest, iocb);
+    req->head = head;
+    req->inhdr = inhdr;
+    req->bounce_iov = bounce_iov;
+    req->read_qiov = read_qiov;
+    return 0;
+}
+
+static int process_request(IOQueue *ioq, struct iovec iov[],
+                           unsigned int out_num, unsigned int in_num,
+                           unsigned int head)
+{
+    VirtIOBlockDataPlane *s = container_of(ioq, VirtIOBlockDataPlane, ioqueue);
+    struct iovec *in_iov = &iov[out_num];
+    struct virtio_blk_outhdr outhdr;
+    QEMUIOVector *inhdr;
+    size_t in_size;
+
+    /* Copy in outhdr */
+    if (unlikely(iov_to_buf(iov, out_num, 0, &outhdr,
+                            sizeof(outhdr)) != sizeof(outhdr))) {
+        error_report("virtio-blk request outhdr too short");
+        return -EFAULT;
+    }
+    iov_discard_front(&iov, &out_num, sizeof(outhdr));
+
+    /* Grab inhdr for later */
+    in_size = iov_size(in_iov, in_num);
+    if (in_size < sizeof(struct virtio_blk_inhdr)) {
+        error_report("virtio_blk request inhdr too short");
+        return -EFAULT;
+    }
+    inhdr = g_slice_new(QEMUIOVector);
+    qemu_iovec_init(inhdr, 1);
+    qemu_iovec_concat_iov(inhdr, in_iov, in_num,
+            in_size - sizeof(struct virtio_blk_inhdr),
+            sizeof(struct virtio_blk_inhdr));
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+    /* TODO Linux sets the barrier bit even when not advertised! */
+    outhdr.type &= ~VIRTIO_BLK_T_BARRIER;
+
+    switch (outhdr.type) {
+    case VIRTIO_BLK_T_IN:
+        do_rdwr_cmd(s, true, in_iov, in_num, outhdr.sector * 512, head, inhdr);
+        return 0;
+
+    case VIRTIO_BLK_T_OUT:
+        do_rdwr_cmd(s, false, iov, out_num, outhdr.sector * 512, head, inhdr);
+        return 0;
+
+    case VIRTIO_BLK_T_SCSI_CMD:
+        /* TODO support SCSI commands */
+        complete_request_early(s, head, inhdr, VIRTIO_BLK_S_UNSUPP);
+        return 0;
+
+    case VIRTIO_BLK_T_FLUSH:
+        /* TODO fdsync not supported by Linux AIO, do it synchronously here! */
+        if (qemu_fdatasync(s->fd) < 0) {
+            complete_request_early(s, head, inhdr, VIRTIO_BLK_S_IOERR);
+        } else {
+            complete_request_early(s, head, inhdr, VIRTIO_BLK_S_OK);
+        }
+        return 0;
+
+    case VIRTIO_BLK_T_GET_ID:
+        do_get_id_cmd(s, in_iov, in_num, head, inhdr);
+        return 0;
+
+    default:
+        error_report("virtio-blk unsupported request type %#x", outhdr.type);
+        qemu_iovec_destroy(inhdr);
+        g_slice_free(QEMUIOVector, inhdr);
+        return -EFAULT;
+    }
+}
+
+static void handle_notify(EventHandler *handler)
+{
+    VirtIOBlockDataPlane *s = container_of(handler, VirtIOBlockDataPlane,
+                                           notify_handler);
+
+    /* There is one array of iovecs into which all new requests are extracted
+     * from the vring.  Requests are read from the vring and the translated
+     * descriptors are written to the iovecs array.  The iovecs do not have to
+     * persist across handle_notify() calls because the kernel copies the
+     * iovecs on io_submit().
+     *
+     * Handling io_submit() EAGAIN may require storing the requests across
+     * handle_notify() calls until the kernel has sufficient resources to
+     * accept more I/O.  This is not implemented yet.
+     */
+    struct iovec iovec[VRING_MAX];
+    struct iovec *end = &iovec[VRING_MAX];
+    struct iovec *iov = iovec;
+
+    /* When a request is read from the vring, the index of the first descriptor
+     * (aka head) is returned so that the completed request can be pushed onto
+     * the vring later.
+     *
+     * The number of hypervisor read-only iovecs is out_num.  The number of
+     * hypervisor write-only iovecs is in_num.
+     */
+    int head;
+    unsigned int out_num = 0, in_num = 0;
+    unsigned int num_queued;
+
+    for (;;) {
+        /* Disable guest->host notifies to avoid unnecessary vmexits */
+        vring_disable_notification(s->vdev, &s->vring);
+
+        for (;;) {
+            head = vring_pop(s->vdev, &s->vring, iov, end, &out_num, &in_num);
+            if (head < 0) {
+                break; /* no more requests */
+            }
+
+            trace_virtio_blk_data_plane_process_request(s, out_num, in_num,
+                                                        head);
+
+            if (process_request(&s->ioqueue, iov, out_num, in_num, head) < 0) {
+                vring_set_broken(&s->vring);
+                break;
+            }
+            iov += out_num + in_num;
+        }
+
+        if (likely(head == -EAGAIN)) { /* vring emptied */
+            /* Re-enable guest->host notifies and stop processing the vring.
+             * But if the guest has snuck in more descriptors, keep processing.
+             */
+            if (vring_enable_notification(s->vdev, &s->vring)) {
+                break;
+            }
+        } else { /* head == -ENOBUFS or fatal error, iovecs[] is depleted */
+            /* Since there are no iovecs[] left, stop processing for now.  Do
+             * not re-enable guest->host notifies since the I/O completion
+             * handler knows to check for more vring descriptors anyway.
+             */
+            break;
+        }
+    }
+
+    num_queued = ioq_num_queued(&s->ioqueue);
+    if (num_queued > 0) {
+        s->num_reqs += num_queued;
+
+        int rc = ioq_submit(&s->ioqueue);
+        if (unlikely(rc < 0)) {
+            fprintf(stderr, "ioq_submit failed %d\n", rc);
+            exit(1);
+        }
+    }
+}
+
+static void handle_io(EventHandler *handler)
+{
+    VirtIOBlockDataPlane *s = container_of(handler, VirtIOBlockDataPlane,
+                                           io_handler);
+
+    if (ioq_run_completion(&s->ioqueue, complete_request, s) > 0) {
+        notify_guest(s);
+    }
+
+    /* If there were more requests than iovecs, the vring will not be empty yet
+     * so check again.  There should now be enough resources to process more
+     * requests.
+     */
+    if (unlikely(vring_more_avail(&s->vring))) {
+        handle_notify(&s->notify_handler);
+    }
+}
+
+static void *data_plane_thread(void *opaque)
+{
+    VirtIOBlockDataPlane *s = opaque;
+
+    do {
+        event_poll(&s->event_poll);
+    } while (!s->stopping || s->num_reqs > 0);
+    return NULL;
+}
+
+static void start_data_plane_bh(void *opaque)
+{
+    VirtIOBlockDataPlane *s = opaque;
+
+    qemu_bh_delete(s->start_bh);
+    s->start_bh = NULL;
+    qemu_thread_create(&s->thread, data_plane_thread,
+                       s, QEMU_THREAD_JOINABLE);
+}
+
+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
+                                  VirtIOBlockDataPlane **dataplane)
+{
+    VirtIOBlockDataPlane *s;
+    int fd;
+
+    *dataplane = NULL;
+
+    if (!blk->data_plane) {
+        return true;
+    }
+
+    if (blk->scsi) {
+        error_report("device is incompatible with x-data-plane, use scsi=off");
+        return false;
+    }
+
+    if (blk->config_wce) {
+        error_report("device is incompatible with x-data-plane, "
+                     "use config-wce=off");
+        return false;
+    }
+
+    fd = raw_get_aio_fd(blk->conf.bs);
+    if (fd < 0) {
+        error_report("drive is incompatible with x-data-plane, "
+                     "use format=raw,cache=none,aio=native");
+        return false;
+    }
+
+    s = g_new0(VirtIOBlockDataPlane, 1);
+    s->vdev = vdev;
+    s->fd = fd;
+    s->blk = blk;
+
+    /* Prevent block operations that conflict with data plane thread */
+    bdrv_set_in_use(blk->conf.bs, 1);
+
+    error_setg(&s->migration_blocker,
+            "x-data-plane does not support migration");
+    migrate_add_blocker(s->migration_blocker);
+
+    *dataplane = s;
+    return true;
+}
+
+void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
+{
+    if (!s) {
+        return;
+    }
+
+    virtio_blk_data_plane_stop(s);
+    migrate_del_blocker(s->migration_blocker);
+    error_free(s->migration_blocker);
+    bdrv_set_in_use(s->blk->conf.bs, 0);
+    g_free(s);
+}
+
+void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
+{
+    VirtQueue *vq;
+    int i;
+
+    if (s->started) {
+        return;
+    }
+
+    vq = virtio_get_queue(s->vdev, 0);
+    if (!vring_setup(&s->vring, s->vdev, 0)) {
+        return;
+    }
+
+    event_poll_init(&s->event_poll);
+
+    /* Set up guest notifier (irq) */
+    if (s->vdev->binding->set_guest_notifiers(s->vdev->binding_opaque, 1,
+                                              true) != 0) {
+        fprintf(stderr, "virtio-blk failed to set guest notifier, "
+                "ensure -enable-kvm is set\n");
+        exit(1);
+    }
+    s->guest_notifier = virtio_queue_get_guest_notifier(vq);
+
+    /* Set up virtqueue notify */
+    if (s->vdev->binding->set_host_notifier(s->vdev->binding_opaque,
+                                            0, true) != 0) {
+        fprintf(stderr, "virtio-blk failed to set host notifier\n");
+        exit(1);
+    }
+    event_poll_add(&s->event_poll, &s->notify_handler,
+                   virtio_queue_get_host_notifier(vq),
+                   handle_notify);
+
+    /* Set up ioqueue */
+    ioq_init(&s->ioqueue, s->fd, REQ_MAX);
+    for (i = 0; i < ARRAY_SIZE(s->requests); i++) {
+        ioq_put_iocb(&s->ioqueue, &s->requests[i].iocb);
+    }
+    event_poll_add(&s->event_poll, &s->io_handler,
+                   ioq_get_notifier(&s->ioqueue), handle_io);
+
+    s->started = true;
+    trace_virtio_blk_data_plane_start(s);
+
+    /* Kick right away to begin processing requests already in vring */
+    event_notifier_set(virtio_queue_get_host_notifier(vq));
+
+    /* Spawn thread in BH so it inherits iothread cpusets */
+    s->start_bh = qemu_bh_new(start_data_plane_bh, s);
+    qemu_bh_schedule(s->start_bh);
+}
+
+void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
+{
+    if (!s->started || s->stopping) {
+        return;
+    }
+    s->stopping = true;
+    trace_virtio_blk_data_plane_stop(s);
+
+    /* Stop thread or cancel pending thread creation BH */
+    if (s->start_bh) {
+        qemu_bh_delete(s->start_bh);
+        s->start_bh = NULL;
+    } else {
+        event_poll_notify(&s->event_poll);
+        qemu_thread_join(&s->thread);
+    }
+
+    ioq_cleanup(&s->ioqueue);
+
+    s->vdev->binding->set_host_notifier(s->vdev->binding_opaque, 0, false);
+
+    event_poll_cleanup(&s->event_poll);
+
+    /* Clean up guest notifier (irq) */
+    s->vdev->binding->set_guest_notifiers(s->vdev->binding_opaque, 1, false);
+
+    vring_teardown(&s->vring);
+    s->started = false;
+    s->stopping = false;
+}
diff --git a/hw/dataplane/virtio-blk.h b/hw/dataplane/virtio-blk.h
new file mode 100644
index 0000000..1e8fdfe
--- /dev/null
+++ b/hw/dataplane/virtio-blk.h
@@ -0,0 +1,29 @@
+/*
+ * Dedicated thread for virtio-blk I/O processing
+ *
+ * Copyright 2012 IBM, Corp.
+ * Copyright 2012 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *   Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_DATAPLANE_VIRTIO_BLK_H
+#define HW_DATAPLANE_VIRTIO_BLK_H
+
+#include "hw/virtio.h"
+
+typedef struct VirtIOBlockDataPlane VirtIOBlockDataPlane;
+
+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *blk,
+                                  VirtIOBlockDataPlane **dataplane);
+void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_drain(VirtIOBlockDataPlane *s);
+
+#endif /* HW_DATAPLANE_VIRTIO_BLK_H */
diff --git a/hw/dataplane/vring.c b/hw/dataplane/vring.c
new file mode 100644
index 0000000..d5d4ef4
--- /dev/null
+++ b/hw/dataplane/vring.c
@@ -0,0 +1,362 @@
+/* Copyright 2012 Red Hat, Inc.
+ * Copyright IBM, Corp. 2012
+ *
+ * Based on Linux 2.6.39 vhost code:
+ * Copyright (C) 2009 Red Hat, Inc.
+ * Copyright (C) 2006 Rusty Russell IBM Corporation
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *         Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * Inspiration, some code, and most witty comments come from
+ * Documentation/virtual/lguest/lguest.c, by Rusty Russell
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+
+#include "trace.h"
+#include "hw/dataplane/vring.h"
+
+/* Map the guest's vring to host memory */
+bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
+{
+    hwaddr vring_addr = virtio_queue_get_ring_addr(vdev, n);
+    hwaddr vring_size = virtio_queue_get_ring_size(vdev, n);
+    void *vring_ptr;
+
+    vring->broken = false;
+
+    hostmem_init(&vring->hostmem);
+    vring_ptr = hostmem_lookup(&vring->hostmem, vring_addr, vring_size, true);
+    if (!vring_ptr) {
+        error_report("Failed to map vring "
+                     "addr %#" HWADDR_PRIx " size %" HWADDR_PRIu,
+                     vring_addr, vring_size);
+        vring->broken = true;
+        return false;
+    }
+
+    vring_init(&vring->vr, virtio_queue_get_num(vdev, n), vring_ptr, 4096);
+
+    vring->last_avail_idx = 0;
+    vring->last_used_idx = 0;
+    vring->signalled_used = 0;
+    vring->signalled_used_valid = false;
+
+    trace_vring_setup(virtio_queue_get_ring_addr(vdev, n),
+                      vring->vr.desc, vring->vr.avail, vring->vr.used);
+    return true;
+}
+
+void vring_teardown(Vring *vring)
+{
+    hostmem_finalize(&vring->hostmem);
+}
+
+/* Disable guest->host notifies */
+void vring_disable_notification(VirtIODevice *vdev, Vring *vring)
+{
+    if (!(vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX))) {
+        vring->vr.used->flags |= VRING_USED_F_NO_NOTIFY;
+    }
+}
+
+/* Enable guest->host notifies
+ *
+ * Return true if the vring is empty, false if there are more requests.
+ */
+bool vring_enable_notification(VirtIODevice *vdev, Vring *vring)
+{
+    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
+        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
+    } else {
+        vring->vr.used->flags &= ~VRING_USED_F_NO_NOTIFY;
+    }
+    smp_mb(); /* ensure update is seen before reading avail_idx */
+    return !vring_more_avail(vring);
+}
+
+/* This is stolen from linux/drivers/vhost/vhost.c:vhost_notify() */
+bool vring_should_notify(VirtIODevice *vdev, Vring *vring)
+{
+    uint16_t old, new;
+    bool v;
+    /* Flush out used index updates. This is paired
+     * with the barrier that the Guest executes when enabling
+     * interrupts. */
+    smp_mb();
+
+    if ((vdev->guest_features & VIRTIO_F_NOTIFY_ON_EMPTY) &&
+        unlikely(vring->vr.avail->idx == vring->last_avail_idx)) {
+        return true;
+    }
+
+    if (!(vdev->guest_features & VIRTIO_RING_F_EVENT_IDX)) {
+        return !(vring->vr.avail->flags & VRING_AVAIL_F_NO_INTERRUPT);
+    }
+    old = vring->signalled_used;
+    v = vring->signalled_used_valid;
+    new = vring->signalled_used = vring->last_used_idx;
+    vring->signalled_used_valid = true;
+
+    if (unlikely(!v)) {
+        return true;
+    }
+
+    return vring_need_event(vring_used_event(&vring->vr), new, old);
+}
+
+/* This is stolen from linux/drivers/vhost/vhost.c. */
+static int get_indirect(Vring *vring,
+                        struct iovec iov[], struct iovec *iov_end,
+                        unsigned int *out_num, unsigned int *in_num,
+                        struct vring_desc *indirect)
+{
+    struct vring_desc desc;
+    unsigned int i = 0, count, found = 0;
+
+    /* Sanity check */
+    if (unlikely(indirect->len % sizeof(desc))) {
+        error_report("Invalid length in indirect descriptor: "
+                     "len %#x not multiple of %#zx",
+                     indirect->len, sizeof(desc));
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    count = indirect->len / sizeof(desc);
+    /* Buffers are chained via a 16 bit next field, so
+     * we can have at most 2^16 of these. */
+    if (unlikely(count > USHRT_MAX + 1)) {
+        error_report("Indirect buffer length too big: %d", indirect->len);
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    do {
+        struct vring_desc *desc_ptr;
+
+        /* Translate indirect descriptor */
+        desc_ptr = hostmem_lookup(&vring->hostmem,
+                                  indirect->addr + found * sizeof(desc),
+                                  sizeof(desc), false);
+        if (!desc_ptr) {
+            error_report("Failed to map indirect descriptor "
+                         "addr %#" PRIx64 " len %zu",
+                         (uint64_t)indirect->addr + found * sizeof(desc),
+                         sizeof(desc));
+            vring->broken = true;
+            return -EFAULT;
+        }
+        desc = *desc_ptr;
+
+        /* Ensure descriptor has been loaded before accessing fields */
+        barrier(); /* read_barrier_depends(); */
+
+        if (unlikely(++found > count)) {
+            error_report("Loop detected: last one at %u "
+                         "indirect size %u", i, count);
+            vring->broken = true;
+            return -EFAULT;
+        }
+
+        if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
+            error_report("Nested indirect descriptor");
+            vring->broken = true;
+            return -EFAULT;
+        }
+
+        /* Stop for now if there are not enough iovecs available. */
+        if (iov >= iov_end) {
+            return -ENOBUFS;
+        }
+
+        iov->iov_base = hostmem_lookup(&vring->hostmem, desc.addr, desc.len,
+                                       desc.flags & VRING_DESC_F_WRITE);
+        if (!iov->iov_base) {
+            error_report("Failed to map indirect descriptor"
+                         "addr %#" PRIx64 " len %u",
+                         (uint64_t)desc.addr, desc.len);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        iov->iov_len = desc.len;
+        iov++;
+
+        /* If this is an input descriptor, increment that count. */
+        if (desc.flags & VRING_DESC_F_WRITE) {
+            *in_num += 1;
+        } else {
+            /* If it's an output descriptor, they're all supposed
+             * to come before any input descriptors. */
+            if (unlikely(*in_num)) {
+                error_report("Indirect descriptor "
+                             "has out after in: idx %u", i);
+                vring->broken = true;
+                return -EFAULT;
+            }
+            *out_num += 1;
+        }
+        i = desc.next;
+    } while (desc.flags & VRING_DESC_F_NEXT);
+    return 0;
+}
+
+/* This looks in the virtqueue and for the first available buffer, and converts
+ * it to an iovec for convenient access.  Since descriptors consist of some
+ * number of output then some number of input descriptors, it's actually two
+ * iovecs, but we pack them into one and note how many of each there were.
+ *
+ * This function returns the descriptor number found, or vq->num (which is
+ * never a valid descriptor number) if none was found.  A negative code is
+ * returned on error.
+ *
+ * Stolen from linux/drivers/vhost/vhost.c.
+ */
+int vring_pop(VirtIODevice *vdev, Vring *vring,
+              struct iovec iov[], struct iovec *iov_end,
+              unsigned int *out_num, unsigned int *in_num)
+{
+    struct vring_desc desc;
+    unsigned int i, head, found = 0, num = vring->vr.num;
+    uint16_t avail_idx, last_avail_idx;
+
+    /* If there was a fatal error then refuse operation */
+    if (vring->broken) {
+        return -EFAULT;
+    }
+
+    /* Check it isn't doing very strange things with descriptor numbers. */
+    last_avail_idx = vring->last_avail_idx;
+    avail_idx = vring->vr.avail->idx;
+    barrier(); /* load indices now and not again later */
+
+    if (unlikely((uint16_t)(avail_idx - last_avail_idx) > num)) {
+        error_report("Guest moved used index from %u to %u",
+                     last_avail_idx, avail_idx);
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    /* If there's nothing new since last we looked. */
+    if (avail_idx == last_avail_idx) {
+        return -EAGAIN;
+    }
+
+    /* Only get avail ring entries after they have been exposed by guest. */
+    smp_rmb();
+
+    /* Grab the next descriptor number they're advertising, and increment
+     * the index we've seen. */
+    head = vring->vr.avail->ring[last_avail_idx % num];
+
+    /* If their number is silly, that's an error. */
+    if (unlikely(head >= num)) {
+        error_report("Guest says index %u > %u is available", head, num);
+        vring->broken = true;
+        return -EFAULT;
+    }
+
+    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
+        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
+    }
+
+    /* When we start there are none of either input nor output. */
+    *out_num = *in_num = 0;
+
+    i = head;
+    do {
+        if (unlikely(i >= num)) {
+            error_report("Desc index is %u > %u, head = %u", i, num, head);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        if (unlikely(++found > num)) {
+            error_report("Loop detected: last one at %u vq size %u head %u",
+                         i, num, head);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        desc = vring->vr.desc[i];
+
+        /* Ensure descriptor is loaded before accessing fields */
+        barrier();
+
+        if (desc.flags & VRING_DESC_F_INDIRECT) {
+            int ret = get_indirect(vring, iov, iov_end, out_num, in_num, &desc);
+            if (ret < 0) {
+                return ret;
+            }
+            continue;
+        }
+
+        /* If there are not enough iovecs left, stop for now.  The caller
+         * should check if there are more descs available once they have dealt
+         * with the current set.
+         */
+        if (iov >= iov_end) {
+            return -ENOBUFS;
+        }
+
+        /* TODO handle non-contiguous memory across region boundaries */
+        iov->iov_base = hostmem_lookup(&vring->hostmem, desc.addr, desc.len,
+                                       desc.flags & VRING_DESC_F_WRITE);
+        if (!iov->iov_base) {
+            error_report("Failed to map vring desc addr %#" PRIx64 " len %u",
+                         (uint64_t)desc.addr, desc.len);
+            vring->broken = true;
+            return -EFAULT;
+        }
+        iov->iov_len  = desc.len;
+        iov++;
+
+        if (desc.flags & VRING_DESC_F_WRITE) {
+            /* If this is an input descriptor,
+             * increment that count. */
+            *in_num += 1;
+        } else {
+            /* If it's an output descriptor, they're all supposed
+             * to come before any input descriptors. */
+            if (unlikely(*in_num)) {
+                error_report("Descriptor has out after in: idx %d", i);
+                vring->broken = true;
+                return -EFAULT;
+            }
+            *out_num += 1;
+        }
+        i = desc.next;
+    } while (desc.flags & VRING_DESC_F_NEXT);
+
+    /* On success, increment avail index. */
+    vring->last_avail_idx++;
+    return head;
+}
+
+/* After we've used one of their buffers, we tell them about it.
+ *
+ * Stolen from linux/drivers/vhost/vhost.c.
+ */
+void vring_push(Vring *vring, unsigned int head, int len)
+{
+    struct vring_used_elem *used;
+    uint16_t new;
+
+    /* Don't touch vring if a fatal error occurred */
+    if (vring->broken) {
+        return;
+    }
+
+    /* The virtqueue contains a ring of used buffers.  Get a pointer to the
+     * next entry in that used ring. */
+    used = &vring->vr.used->ring[vring->last_used_idx % vring->vr.num];
+    used->id = head;
+    used->len = len;
+
+    /* Make sure buffer is written before we update index. */
+    smp_wmb();
+
+    new = vring->vr.used->idx = ++vring->last_used_idx;
+    if (unlikely((int16_t)(new - vring->signalled_used) < (uint16_t)1)) {
+        vring->signalled_used_valid = false;
+    }
+}
diff --git a/hw/dataplane/vring.h b/hw/dataplane/vring.h
new file mode 100644
index 0000000..3274f62
--- /dev/null
+++ b/hw/dataplane/vring.h
@@ -0,0 +1,62 @@
+/* Copyright 2012 Red Hat, Inc. and/or its affiliates
+ * Copyright IBM, Corp. 2012
+ *
+ * Based on Linux 2.6.39 vhost code:
+ * Copyright (C) 2009 Red Hat, Inc.
+ * Copyright (C) 2006 Rusty Russell IBM Corporation
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *         Stefan Hajnoczi <stefanha@redhat.com>
+ *
+ * Inspiration, some code, and most witty comments come from
+ * Documentation/virtual/lguest/lguest.c, by Rusty Russell
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+
+#ifndef VRING_H
+#define VRING_H
+
+#include <linux/virtio_ring.h>
+#include "qemu-common.h"
+#include "hw/dataplane/hostmem.h"
+#include "hw/virtio.h"
+
+typedef struct {
+    HostMem hostmem;                /* guest memory mapper */
+    struct vring vr;                /* virtqueue vring mapped to host memory */
+    uint16_t last_avail_idx;        /* last processed avail ring index */
+    uint16_t last_used_idx;         /* last processed used ring index */
+    uint16_t signalled_used;        /* EVENT_IDX state */
+    bool signalled_used_valid;
+    bool broken;                    /* was there a fatal error? */
+} Vring;
+
+static inline unsigned int vring_get_num(Vring *vring)
+{
+    return vring->vr.num;
+}
+
+/* Are there more descriptors available? */
+static inline bool vring_more_avail(Vring *vring)
+{
+    return vring->vr.avail->idx != vring->last_avail_idx;
+}
+
+/* Fail future vring_pop() and vring_push() calls until reset */
+static inline void vring_set_broken(Vring *vring)
+{
+    vring->broken = true;
+}
+
+bool vring_setup(Vring *vring, VirtIODevice *vdev, int n);
+void vring_teardown(Vring *vring);
+void vring_disable_notification(VirtIODevice *vdev, Vring *vring);
+bool vring_enable_notification(VirtIODevice *vdev, Vring *vring);
+bool vring_should_notify(VirtIODevice *vdev, Vring *vring);
+int vring_pop(VirtIODevice *vdev, Vring *vring,
+              struct iovec iov[], struct iovec *iov_end,
+              unsigned int *out_num, unsigned int *in_num);
+void vring_push(Vring *vring, unsigned int head, int len);
+
+#endif /* VRING_H */
diff --git a/hw/debugcon.c b/hw/debugcon.c
index 14ab326..81b2bb0 100644
--- a/hw/debugcon.c
+++ b/hw/debugcon.c
@@ -25,24 +25,31 @@
  */
 
 #include "hw.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "isa.h"
 #include "pc.h"
 
+#define TYPE_ISA_DEBUGCON_DEVICE "isa-debugcon"
+#define ISA_DEBUGCON_DEVICE(obj) \
+     OBJECT_CHECK(ISADebugconState, (obj), TYPE_ISA_DEBUGCON_DEVICE)
+
 //#define DEBUG_DEBUGCON
 
 typedef struct DebugconState {
+    MemoryRegion io;
     CharDriverState *chr;
     uint32_t readback;
 } DebugconState;
 
 typedef struct ISADebugconState {
-    ISADevice dev;
+    ISADevice parent_obj;
+
     uint32_t iobase;
     DebugconState state;
 } ISADebugconState;
 
-static void debugcon_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void debugcon_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                  unsigned width)
 {
     DebugconState *s = opaque;
     unsigned char ch = val;
@@ -55,7 +62,7 @@ static void debugcon_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 }
 
 
-static uint32_t debugcon_ioport_read(void *opaque, uint32_t addr)
+static uint64_t debugcon_ioport_read(void *opaque, hwaddr addr, unsigned width)
 {
     DebugconState *s = opaque;
 
@@ -66,6 +73,14 @@ static uint32_t debugcon_ioport_read(void *opaque, uint32_t addr)
     return s->readback;
 }
 
+static const MemoryRegionOps debugcon_ops = {
+    .read = debugcon_ioport_read,
+    .write = debugcon_ioport_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
 static void debugcon_init_core(DebugconState *s)
 {
     if (!s->chr) {
@@ -78,12 +93,14 @@ static void debugcon_init_core(DebugconState *s)
 
 static int debugcon_isa_initfn(ISADevice *dev)
 {
-    ISADebugconState *isa = DO_UPCAST(ISADebugconState, dev, dev);
+    ISADebugconState *isa = ISA_DEBUGCON_DEVICE(dev);
     DebugconState *s = &isa->state;
 
     debugcon_init_core(s);
-    register_ioport_write(isa->iobase, 1, 1, debugcon_ioport_write, s);
-    register_ioport_read(isa->iobase, 1, 1, debugcon_ioport_read, s);
+    memory_region_init_io(&s->io, &debugcon_ops, s,
+                          TYPE_ISA_DEBUGCON_DEVICE, 1);
+    memory_region_add_subregion(isa_address_space_io(dev),
+                                isa->iobase, &s->io);
     return 0;
 }
 
@@ -102,8 +119,8 @@ static void debugcon_isa_class_initfn(ObjectClass *klass, void *data)
     dc->props = debugcon_isa_properties;
 }
 
-static TypeInfo debugcon_isa_info = {
-    .name          = "isa-debugcon",
+static const TypeInfo debugcon_isa_info = {
+    .name          = TYPE_ISA_DEBUGCON_DEVICE,
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISADebugconState),
     .class_init    = debugcon_isa_class_initfn,
diff --git a/hw/debugexit.c b/hw/debugexit.c
new file mode 100644
index 0000000..c1b489d
--- /dev/null
+++ b/hw/debugexit.c
@@ -0,0 +1,75 @@
+/*
+ * debug exit port emulation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) any later version.
+ */
+
+#include "hw.h"
+#include "isa.h"
+
+#define TYPE_ISA_DEBUG_EXIT_DEVICE "isa-debug-exit"
+#define ISA_DEBUG_EXIT_DEVICE(obj) \
+     OBJECT_CHECK(ISADebugExitState, (obj), TYPE_ISA_DEBUG_EXIT_DEVICE)
+
+typedef struct ISADebugExitState {
+    ISADevice parent_obj;
+
+    uint32_t iobase;
+    uint32_t iosize;
+    MemoryRegion io;
+} ISADebugExitState;
+
+static void debug_exit_write(void *opaque, hwaddr addr, uint64_t val,
+                             unsigned width)
+{
+    exit((val << 1) | 1);
+}
+
+static const MemoryRegionOps debug_exit_ops = {
+    .write = debug_exit_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int debug_exit_initfn(ISADevice *dev)
+{
+    ISADebugExitState *isa = ISA_DEBUG_EXIT_DEVICE(dev);
+
+    memory_region_init_io(&isa->io, &debug_exit_ops, isa,
+                          TYPE_ISA_DEBUG_EXIT_DEVICE, isa->iosize);
+    memory_region_add_subregion(isa_address_space_io(dev),
+                                isa->iobase, &isa->io);
+    return 0;
+}
+
+static Property debug_exit_properties[] = {
+    DEFINE_PROP_HEX32("iobase", ISADebugExitState, iobase, 0x501),
+    DEFINE_PROP_HEX32("iosize", ISADebugExitState, iosize, 0x02),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void debug_exit_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
+    ic->init = debug_exit_initfn;
+    dc->props = debug_exit_properties;
+}
+
+static const TypeInfo debug_exit_info = {
+    .name          = TYPE_ISA_DEBUG_EXIT_DEVICE,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISADebugExitState),
+    .class_init    = debug_exit_class_initfn,
+};
+
+static void debug_exit_register_types(void)
+{
+    type_register_static(&debug_exit_info);
+}
+
+type_init(debug_exit_register_types)
diff --git a/hw/dec_pci.c b/hw/dec_pci.c
index c30ade3..ee3f4ca 100644
--- a/hw/dec_pci.c
+++ b/hw/dec_pci.c
@@ -25,10 +25,10 @@
 
 #include "dec_pci.h"
 #include "sysbus.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "pci_bridge.h"
-#include "pci_internals.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "pci/pci_bridge.h"
+#include "pci/pci_bus.h"
 
 /* debug DEC */
 //#define DEBUG_DEC
diff --git a/hw/device-hotplug.c b/hw/device-hotplug.c
index eec0fe3..88da145 100644
--- a/hw/device-hotplug.c
+++ b/hw/device-hotplug.c
@@ -24,11 +24,10 @@
 
 #include "hw.h"
 #include "boards.h"
-#include "net.h"
-#include "blockdev.h"
-#include "qemu-config.h"
-#include "sysemu.h"
-#include "monitor.h"
+#include "sysemu/blockdev.h"
+#include "qemu/config-file.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
 
 DriveInfo *add_init_drive(const char *optstr)
 {
@@ -39,7 +38,7 @@ DriveInfo *add_init_drive(const char *optstr)
     if (!opts)
         return NULL;
 
-    dinfo = drive_init(opts, current_machine->use_scsi);
+    dinfo = drive_init(opts, current_machine->block_default_type);
     if (!dinfo) {
         qemu_opts_del(opts);
         return NULL;
@@ -49,18 +48,16 @@ DriveInfo *add_init_drive(const char *optstr)
 }
 
 #if !defined(TARGET_I386)
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
-                      DriveInfo *dinfo, int type)
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo)
 {
     /* On non-x86 we don't do PCI hotplug */
-    monitor_printf(mon, "Can't hot-add drive to type %d\n", type);
+    monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
     return -1;
 }
 #endif
 
 void drive_hot_add(Monitor *mon, const QDict *qdict)
 {
-    int type;
     DriveInfo *dinfo = NULL;
     const char *opts = qdict_get_str(qdict, "opts");
 
@@ -72,14 +69,13 @@ void drive_hot_add(Monitor *mon, const QDict *qdict)
         monitor_printf(mon, "Parameter addr not supported\n");
         goto err;
     }
-    type = dinfo->type;
 
-    switch (type) {
+    switch (dinfo->type) {
     case IF_NONE:
         monitor_printf(mon, "OK\n");
         break;
     default:
-        if (pci_drive_hot_add(mon, qdict, dinfo, type)) {
+        if (pci_drive_hot_add(mon, qdict, dinfo)) {
             goto err;
         }
     }
diff --git a/hw/dma.c b/hw/dma.c
index d6aeac2..5bdf435 100644
--- a/hw/dma.c
+++ b/hw/dma.c
@@ -23,6 +23,7 @@
  */
 #include "hw.h"
 #include "isa.h"
+#include "qemu/main-loop.h"
 
 /* #define DEBUG_DMA */
 
@@ -58,6 +59,8 @@ static struct dma_cont {
     int dshift;
     struct dma_regs regs[4];
     qemu_irq *cpu_request_exit;
+    MemoryRegion channel_io;
+    MemoryRegion cont_io;
 } dma_controllers[2];
 
 enum {
@@ -149,7 +152,7 @@ static inline int getff (struct dma_cont *d)
     return ff;
 }
 
-static uint32_t read_chan (void *opaque, uint32_t nport)
+static uint64_t read_chan(void *opaque, hwaddr nport, unsigned size)
 {
     struct dma_cont *d = opaque;
     int ichan, nreg, iport, ff, val, dir;
@@ -171,7 +174,8 @@ static uint32_t read_chan (void *opaque, uint32_t nport)
     return (val >> (d->dshift + (ff << 3))) & 0xff;
 }
 
-static void write_chan (void *opaque, uint32_t nport, uint32_t data)
+static void write_chan(void *opaque, hwaddr nport, uint64_t data,
+                       unsigned size)
 {
     struct dma_cont *d = opaque;
     int iport, ichan, nreg;
@@ -189,22 +193,23 @@ static void write_chan (void *opaque, uint32_t nport, uint32_t data)
     }
 }
 
-static void write_cont (void *opaque, uint32_t nport, uint32_t data)
+static void write_cont(void *opaque, hwaddr nport, uint64_t data,
+                       unsigned size)
 {
     struct dma_cont *d = opaque;
     int iport, ichan = 0;
 
     iport = (nport >> d->dshift) & 0x0f;
     switch (iport) {
-    case 0x08:                  /* command */
+    case 0x00:                  /* command */
         if ((data != 0) && (data & CMD_NOT_SUPPORTED)) {
-            dolog ("command %#x not supported\n", data);
+            dolog("command %"PRIx64" not supported\n", data);
             return;
         }
         d->command = data;
         break;
 
-    case 0x09:
+    case 0x01:
         ichan = data & 3;
         if (data & 4) {
             d->status |= 1 << (ichan + 4);
@@ -216,7 +221,7 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
         DMA_run();
         break;
 
-    case 0x0a:                  /* single mask */
+    case 0x02:                  /* single mask */
         if (data & 4)
             d->mask |= 1 << (data & 3);
         else
@@ -224,7 +229,7 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
         DMA_run();
         break;
 
-    case 0x0b:                  /* mode */
+    case 0x03:                  /* mode */
         {
             ichan = data & 3;
 #ifdef DEBUG_DMA
@@ -243,23 +248,23 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
             break;
         }
 
-    case 0x0c:                  /* clear flip flop */
+    case 0x04:                  /* clear flip flop */
         d->flip_flop = 0;
         break;
 
-    case 0x0d:                  /* reset */
+    case 0x05:                  /* reset */
         d->flip_flop = 0;
         d->mask = ~0;
         d->status = 0;
         d->command = 0;
         break;
 
-    case 0x0e:                  /* clear mask for all channels */
+    case 0x06:                  /* clear mask for all channels */
         d->mask = 0;
         DMA_run();
         break;
 
-    case 0x0f:                  /* write mask for all channels */
+    case 0x07:                  /* write mask for all channels */
         d->mask = data;
         DMA_run();
         break;
@@ -277,18 +282,18 @@ static void write_cont (void *opaque, uint32_t nport, uint32_t data)
 #endif
 }
 
-static uint32_t read_cont (void *opaque, uint32_t nport)
+static uint64_t read_cont(void *opaque, hwaddr nport, unsigned size)
 {
     struct dma_cont *d = opaque;
     int iport, val;
 
     iport = (nport >> d->dshift) & 0x0f;
     switch (iport) {
-    case 0x08:                  /* status */
+    case 0x00:                  /* status */
         val = d->status;
         d->status &= 0xf0;
         break;
-    case 0x0f:                  /* mask */
+    case 0x01:                  /* mask */
         val = d->mask;
         break;
     default:
@@ -463,7 +468,7 @@ void DMA_schedule(int nchan)
 static void dma_reset(void *opaque)
 {
     struct dma_cont *d = opaque;
-    write_cont (d, (0x0d << d->dshift), 0);
+    write_cont(d, (0x05 << d->dshift), 0, 1);
 }
 
 static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
@@ -473,38 +478,68 @@ static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
     return dma_pos;
 }
 
+
+static const MemoryRegionOps channel_io_ops = {
+    .read = read_chan,
+    .write = write_chan,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+/* IOport from page_base */
+static const MemoryRegionPortio page_portio_list[] = {
+    { 0x01, 3, 1, .write = write_page, .read = read_page, },
+    { 0x07, 1, 1, .write = write_page, .read = read_page, },
+    PORTIO_END_OF_LIST(),
+};
+
+/* IOport from pageh_base */
+static const MemoryRegionPortio pageh_portio_list[] = {
+    { 0x01, 3, 1, .write = write_pageh, .read = read_pageh, },
+    { 0x07, 3, 1, .write = write_pageh, .read = read_pageh, },
+    PORTIO_END_OF_LIST(),
+};
+
+static const MemoryRegionOps cont_io_ops = {
+    .read = read_cont,
+    .write = write_cont,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
 /* dshift = 0: 8 bit DMA, 1 = 16 bit DMA */
 static void dma_init2(struct dma_cont *d, int base, int dshift,
                       int page_base, int pageh_base,
                       qemu_irq *cpu_request_exit)
 {
-    static const int page_port_list[] = { 0x1, 0x2, 0x3, 0x7 };
     int i;
 
     d->dshift = dshift;
     d->cpu_request_exit = cpu_request_exit;
-    for (i = 0; i < 8; i++) {
-        register_ioport_write (base + (i << dshift), 1, 1, write_chan, d);
-        register_ioport_read (base + (i << dshift), 1, 1, read_chan, d);
-    }
-    for (i = 0; i < ARRAY_SIZE (page_port_list); i++) {
-        register_ioport_write (page_base + page_port_list[i], 1, 1,
-                               write_page, d);
-        register_ioport_read (page_base + page_port_list[i], 1, 1,
-                              read_page, d);
-        if (pageh_base >= 0) {
-            register_ioport_write (pageh_base + page_port_list[i], 1, 1,
-                                   write_pageh, d);
-            register_ioport_read (pageh_base + page_port_list[i], 1, 1,
-                                  read_pageh, d);
-        }
-    }
-    for (i = 0; i < 8; i++) {
-        register_ioport_write (base + ((i + 8) << dshift), 1, 1,
-                               write_cont, d);
-        register_ioport_read (base + ((i + 8) << dshift), 1, 1,
-                              read_cont, d);
+
+    memory_region_init_io(&d->channel_io, &channel_io_ops, d,
+                          "dma-chan", 8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(NULL),
+                                base, &d->channel_io);
+
+    isa_register_portio_list(NULL, page_base, page_portio_list, d,
+                             "dma-page");
+    if (pageh_base >= 0) {
+        isa_register_portio_list(NULL, pageh_base, pageh_portio_list, d,
+                                 "dma-pageh");
     }
+
+    memory_region_init_io(&d->cont_io, &cont_io_ops, d, "dma-cont",
+                          8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(NULL),
+                                base + (8 << d->dshift), &d->cont_io);
+
     qemu_register_reset(dma_reset, d);
     dma_reset(d);
     for (i = 0; i < ARRAY_SIZE (d->regs); ++i) {
diff --git a/hw/dp8393x.c b/hw/dp8393x.c
index 3f6386e..808157b 100644
--- a/hw/dp8393x.c
+++ b/hw/dp8393x.c
@@ -18,8 +18,8 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "net.h"
+#include "qemu/timer.h"
+#include "net/net.h"
 #include "mips.h"
 
 //#define DEBUG_SONIC
@@ -339,6 +339,7 @@ static void do_receiver_disable(dp8393xState *s)
 
 static void do_transmit_packets(dp8393xState *s)
 {
+    NetClientState *nc = qemu_get_queue(s->nic);
     uint16_t data[12];
     int width, size;
     int tx_len, len;
@@ -408,13 +409,13 @@ static void do_transmit_packets(dp8393xState *s)
         if (s->regs[SONIC_RCR] & (SONIC_RCR_LB1 | SONIC_RCR_LB0)) {
             /* Loopback */
             s->regs[SONIC_TCR] |= SONIC_TCR_CRSL;
-            if (s->nic->nc.info->can_receive(&s->nic->nc)) {
+            if (nc->info->can_receive(nc)) {
                 s->loopback_packet = 1;
-                s->nic->nc.info->receive(&s->nic->nc, s->tx_buffer, tx_len);
+                nc->info->receive(nc, s->tx_buffer, tx_len);
             }
         } else {
             /* Transmit packet */
-            qemu_send_packet(&s->nic->nc, s->tx_buffer, tx_len);
+            qemu_send_packet(nc, s->tx_buffer, tx_len);
         }
         s->regs[SONIC_TCR] |= SONIC_TCR_PTX;
 
@@ -675,7 +676,7 @@ static const MemoryRegionOps dp8393x_ops = {
 
 static int nic_can_receive(NetClientState *nc)
 {
-    dp8393xState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    dp8393xState *s = qemu_get_nic_opaque(nc);
 
     if (!(s->regs[SONIC_CR] & SONIC_CR_RXEN))
         return 0;
@@ -724,7 +725,7 @@ static int receive_filter(dp8393xState *s, const uint8_t * buf, int size)
 
 static ssize_t nic_receive(NetClientState *nc, const uint8_t * buf, size_t size)
 {
-    dp8393xState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    dp8393xState *s = qemu_get_nic_opaque(nc);
     uint16_t data[10];
     int packet_type;
     uint32_t available, address;
@@ -860,7 +861,7 @@ static void nic_reset(void *opaque)
 
 static void nic_cleanup(NetClientState *nc)
 {
-    dp8393xState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    dp8393xState *s = qemu_get_nic_opaque(nc);
 
     memory_region_del_subregion(s->address_space, &s->mmio);
     memory_region_destroy(&s->mmio);
@@ -899,11 +900,11 @@ void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
     s->regs[SONIC_SR] = 0x0004; /* only revision recognized by Linux */
 
     s->conf.macaddr = nd->macaddr;
-    s->conf.peer = nd->netdev;
+    s->conf.peers.ncs[0] = nd->netdev;
 
     s->nic = qemu_new_nic(&net_dp83932_info, &s->conf, nd->model, nd->name, s);
 
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
     qemu_register_reset(nic_reset, s);
     nic_reset(s);
 
diff --git a/hw/ds1225y.c b/hw/ds1225y.c
index 4b3f69b..a6219a7 100644
--- a/hw/ds1225y.c
+++ b/hw/ds1225y.c
@@ -150,7 +150,7 @@ static void nvram_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = nvram_sysbus_properties;
 }
 
-static TypeInfo nvram_sysbus_info = {
+static const TypeInfo nvram_sysbus_info = {
     .name          = "ds1225y",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusNvRamState),
diff --git a/hw/ds1338.c b/hw/ds1338.c
index b576d56..6f70538 100644
--- a/hw/ds1338.c
+++ b/hw/ds1338.c
@@ -17,9 +17,16 @@
  */
 #define NVRAM_SIZE 64
 
+/* Flags definitions */
+#define SECONDS_CH 0x80
+#define HOURS_12   0x40
+#define HOURS_PM   0x20
+#define CTRL_OSF   0x20
+
 typedef struct {
     I2CSlave i2c;
     int64_t offset;
+    uint8_t wday_offset;
     uint8_t nvram[NVRAM_SIZE];
     int32_t ptr;
     bool addr_byte;
@@ -27,12 +34,13 @@ typedef struct {
 
 static const VMStateDescription vmstate_ds1338 = {
     .name = "ds1338",
-    .version_id = 1,
+    .version_id = 2,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .fields = (VMStateField[]) {
         VMSTATE_I2C_SLAVE(i2c, DS1338State),
         VMSTATE_INT64(offset, DS1338State),
+        VMSTATE_UINT8_V(wday_offset, DS1338State, 2),
         VMSTATE_UINT8_ARRAY(nvram, DS1338State, NVRAM_SIZE),
         VMSTATE_INT32(ptr, DS1338State),
         VMSTATE_BOOL(addr_byte, DS1338State),
@@ -49,17 +57,22 @@ static void capture_current_time(DS1338State *s)
     qemu_get_timedate(&now, s->offset);
     s->nvram[0] = to_bcd(now.tm_sec);
     s->nvram[1] = to_bcd(now.tm_min);
-    if (s->nvram[2] & 0x40) {
-        s->nvram[2] = (to_bcd((now.tm_hour % 12)) + 1) | 0x40;
-        if (now.tm_hour >= 12) {
-            s->nvram[2] |= 0x20;
+    if (s->nvram[2] & HOURS_12) {
+        int tmp = now.tm_hour;
+        if (tmp == 0) {
+            tmp = 24;
+        }
+        if (tmp <= 12) {
+            s->nvram[2] = HOURS_12 | to_bcd(tmp);
+        } else {
+            s->nvram[2] = HOURS_12 | HOURS_PM | to_bcd(tmp - 12);
         }
     } else {
         s->nvram[2] = to_bcd(now.tm_hour);
     }
-    s->nvram[3] = to_bcd(now.tm_wday) + 1;
+    s->nvram[3] = (now.tm_wday + s->wday_offset) % 7 + 1;
     s->nvram[4] = to_bcd(now.tm_mday);
-    s->nvram[5] = to_bcd(now.tm_mon) + 1;
+    s->nvram[5] = to_bcd(now.tm_mon + 1);
     s->nvram[6] = to_bcd(now.tm_year - 100);
 }
 
@@ -114,7 +127,8 @@ static int ds1338_send(I2CSlave *i2c, uint8_t data)
         s->addr_byte = false;
         return 0;
     }
-    if (s->ptr < 8) {
+    if (s->ptr < 7) {
+        /* Time register. */
         struct tm now;
         qemu_get_timedate(&now, s->offset);
         switch(s->ptr) {
@@ -126,19 +140,27 @@ static int ds1338_send(I2CSlave *i2c, uint8_t data)
             now.tm_min = from_bcd(data & 0x7f);
             break;
         case 2:
-            if (data & 0x40) {
-                if (data & 0x20) {
-                    data = from_bcd(data & 0x4f) + 11;
-                } else {
-                    data = from_bcd(data & 0x1f) - 1;
+            if (data & HOURS_12) {
+                int tmp = from_bcd(data & (HOURS_PM - 1));
+                if (data & HOURS_PM) {
+                    tmp += 12;
+                }
+                if (tmp == 24) {
+                    tmp = 0;
                 }
+                now.tm_hour = tmp;
             } else {
-                data = from_bcd(data);
+                now.tm_hour = from_bcd(data & (HOURS_12 - 1));
             }
-            now.tm_hour = data;
             break;
         case 3:
-            now.tm_wday = from_bcd(data & 7) - 1;
+            {
+                /* The day field is supposed to contain a value in
+                   the range 1-7. Otherwise behavior is undefined.
+                 */
+                int user_wday = (data & 7) - 1;
+                s->wday_offset = (user_wday - now.tm_wday + 7) % 7;
+            }
             break;
         case 4:
             now.tm_mday = from_bcd(data & 0x3f);
@@ -149,11 +171,19 @@ static int ds1338_send(I2CSlave *i2c, uint8_t data)
         case 6:
             now.tm_year = from_bcd(data) + 100;
             break;
-        case 7:
-            /* Control register. Currently ignored.  */
-            break;
         }
         s->offset = qemu_timedate_diff(&now);
+    } else if (s->ptr == 7) {
+        /* Control register. */
+
+        /* Ensure bits 2, 3 and 6 will read back as zero. */
+        data &= 0xB3;
+
+        /* Attempting to write the OSF flag to logic 1 leaves the
+           value unchanged. */
+        data = (data & ~CTRL_OSF) | (data & s->nvram[s->ptr] & CTRL_OSF);
+
+        s->nvram[s->ptr] = data;
     } else {
         s->nvram[s->ptr] = data;
     }
@@ -166,6 +196,18 @@ static int ds1338_init(I2CSlave *i2c)
     return 0;
 }
 
+static void ds1338_reset(DeviceState *dev)
+{
+    DS1338State *s = FROM_I2C_SLAVE(DS1338State, I2C_SLAVE(dev));
+
+    /* The clock is running and synchronized with the host */
+    s->offset = 0;
+    s->wday_offset = 0;
+    memset(s->nvram, 0, NVRAM_SIZE);
+    s->ptr = 0;
+    s->addr_byte = false;
+}
+
 static void ds1338_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -175,10 +217,11 @@ static void ds1338_class_init(ObjectClass *klass, void *data)
     k->event = ds1338_event;
     k->recv = ds1338_recv;
     k->send = ds1338_send;
+    dc->reset = ds1338_reset;
     dc->vmsd = &vmstate_ds1338;
 }
 
-static TypeInfo ds1338_info = {
+static const TypeInfo ds1338_info = {
     .name          = "ds1338",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(DS1338State),
diff --git a/hw/dummy_m68k.c b/hw/dummy_m68k.c
index 20f790b..3a88805 100644
--- a/hw/dummy_m68k.c
+++ b/hw/dummy_m68k.c
@@ -10,7 +10,7 @@
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define KERNEL_LOAD_ADDR 0x10000
 
@@ -73,6 +73,7 @@ static QEMUMachine dummy_m68k_machine = {
     .name = "dummy",
     .desc = "Dummy board",
     .init = dummy_m68k_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void dummy_m68k_machine_init(void)
diff --git a/hw/e1000.c b/hw/e1000.c
index 5537ad2..d6fe815 100644
--- a/hw/e1000.c
+++ b/hw/e1000.c
@@ -26,12 +26,12 @@
 
 
 #include "hw.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "net/checksum.h"
 #include "loader.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include "e1000_hw.h"
 
@@ -61,6 +61,8 @@ static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 
 /* this is the size past which hardware will drop packets when setting LPE=0 */
 #define MAXIMUM_ETHERNET_VLAN_SIZE 1522
+/* this is the size past which hardware will drop packets when setting LPE=1 */
+#define MAXIMUM_ETHERNET_LPE_SIZE 16384
 
 /*
  * HW models:
@@ -164,7 +166,6 @@ static void
 set_phy_ctrl(E1000State *s, int index, uint16_t val)
 {
     if ((val & MII_CR_AUTO_NEG_EN) && (val & MII_CR_RESTART_AUTO_NEG)) {
-        s->nic->nc.link_down = true;
         e1000_link_down(s);
         s->phy_reg[PHY_STATUS] &= ~MII_SR_AUTONEG_COMPLETE;
         DBGOUT(PHY, "Start link auto negotiation\n");
@@ -176,8 +177,9 @@ static void
 e1000_autoneg_timer(void *opaque)
 {
     E1000State *s = opaque;
-    s->nic->nc.link_down = false;
-    e1000_link_up(s);
+    if (!qemu_get_queue(s->nic)->link_down) {
+        e1000_link_up(s);
+    }
     s->phy_reg[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE;
     DBGOUT(PHY, "Auto negotiation is completed\n");
 }
@@ -230,7 +232,17 @@ set_interrupt_cause(E1000State *s, int index, uint32_t val)
         val |= E1000_ICR_INT_ASSERTED;
     }
     s->mac_reg[ICR] = val;
+
+    /*
+     * Make sure ICR and ICS registers have the same value.
+     * The spec says that the ICS register is write-only.  However in practice,
+     * on real hardware ICS is readable, and for reads it has the same value as
+     * ICR (except that ICS does not have the clear on read behaviour of ICR).
+     *
+     * The VxWorks PRO/1000 driver uses this behaviour.
+     */
     s->mac_reg[ICS] = val;
+
     qemu_set_irq(s->dev.irq[0], (s->mac_reg[IMS] & s->mac_reg[ICR]) != 0);
 }
 
@@ -279,7 +291,7 @@ static void e1000_reset(void *opaque)
     d->rxbuf_min_shift = 1;
     memset(&d->tx, 0, sizeof d->tx);
 
-    if (d->nic->nc.link_down) {
+    if (qemu_get_queue(d->nic)->link_down) {
         e1000_link_down(d);
     }
 
@@ -307,7 +319,7 @@ set_rx_control(E1000State *s, int index, uint32_t val)
     s->rxbuf_min_shift = ((val / E1000_RCTL_RDMTS_QUAT) & 3) + 1;
     DBGOUT(RX, "RCTL: %d, mac_reg[RCTL] = 0x%x\n", s->mac_reg[RDT],
            s->mac_reg[RCTL]);
-    qemu_flush_queued_packets(&s->nic->nc);
+    qemu_flush_queued_packets(qemu_get_queue(s->nic));
 }
 
 static void
@@ -458,10 +470,11 @@ fcs_len(E1000State *s)
 static void
 e1000_send_packet(E1000State *s, const uint8_t *buf, int size)
 {
+    NetClientState *nc = qemu_get_queue(s->nic);
     if (s->phy_reg[PHY_CTRL] & MII_CR_LOOPBACK) {
-        s->nic->nc.info->receive(&s->nic->nc, buf, size);
+        nc->info->receive(nc, buf, size);
     } else {
-        qemu_send_packet(&s->nic->nc, buf, size);
+        qemu_send_packet(nc, buf, size);
     }
 }
 
@@ -735,7 +748,7 @@ receive_filter(E1000State *s, const uint8_t *buf, int size)
 static void
 e1000_set_link_status(NetClientState *nc)
 {
-    E1000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    E1000State *s = qemu_get_nic_opaque(nc);
     uint32_t old_status = s->mac_reg[STATUS];
 
     if (nc->link_down) {
@@ -769,9 +782,10 @@ static bool e1000_has_rxbufs(E1000State *s, size_t total_size)
 static int
 e1000_can_receive(NetClientState *nc)
 {
-    E1000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    E1000State *s = qemu_get_nic_opaque(nc);
 
-    return (s->mac_reg[RCTL] & E1000_RCTL_EN) && e1000_has_rxbufs(s, 1);
+    return (s->mac_reg[STATUS] & E1000_STATUS_LU) &&
+        (s->mac_reg[RCTL] & E1000_RCTL_EN) && e1000_has_rxbufs(s, 1);
 }
 
 static uint64_t rx_desc_base(E1000State *s)
@@ -785,7 +799,7 @@ static uint64_t rx_desc_base(E1000State *s)
 static ssize_t
 e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    E1000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    E1000State *s = qemu_get_nic_opaque(nc);
     struct e1000_rx_desc desc;
     dma_addr_t base;
     unsigned int n, rdt;
@@ -797,8 +811,13 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     size_t desc_size;
     size_t total_size;
 
-    if (!(s->mac_reg[RCTL] & E1000_RCTL_EN))
+    if (!(s->mac_reg[STATUS] & E1000_STATUS_LU)) {
         return -1;
+    }
+
+    if (!(s->mac_reg[RCTL] & E1000_RCTL_EN)) {
+        return -1;
+    }
 
     /* Pad to minimum Ethernet frame length */
     if (size < sizeof(min_buf)) {
@@ -809,8 +828,9 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     }
 
     /* Discard oversized packets if !LPE and !SBP. */
-    if (size > MAXIMUM_ETHERNET_VLAN_SIZE
-        && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)
+    if ((size > MAXIMUM_ETHERNET_LPE_SIZE ||
+        (size > MAXIMUM_ETHERNET_VLAN_SIZE
+        && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)))
         && !(s->mac_reg[RCTL] & E1000_RCTL_SBP)) {
         return size;
     }
@@ -945,7 +965,7 @@ set_rdt(E1000State *s, int index, uint32_t val)
 {
     s->mac_reg[index] = val & 0xffff;
     if (e1000_has_rxbufs(s, 1)) {
-        qemu_flush_queued_packets(&s->nic->nc);
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
     }
 }
 
@@ -1096,13 +1116,37 @@ static bool is_version_1(void *opaque, int version_id)
     return version_id == 1;
 }
 
+static void e1000_pre_save(void *opaque)
+{
+    E1000State *s = opaque;
+    NetClientState *nc = qemu_get_queue(s->nic);
+    /*
+     * If link is down and auto-negotiation is ongoing, complete
+     * auto-negotiation immediately.  This allows is to look at
+     * MII_SR_AUTONEG_COMPLETE to infer link status on load.
+     */
+    if (nc->link_down &&
+        s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN &&
+        s->phy_reg[PHY_CTRL] & MII_CR_RESTART_AUTO_NEG) {
+         s->phy_reg[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE;
+    }
+}
+
 static int e1000_post_load(void *opaque, int version_id)
 {
     E1000State *s = opaque;
+    NetClientState *nc = qemu_get_queue(s->nic);
 
     /* nc.link_down can't be migrated, so infer link_down according
-     * to link status bit in mac_reg[STATUS] */
-    s->nic->nc.link_down = (s->mac_reg[STATUS] & E1000_STATUS_LU) == 0;
+     * to link status bit in mac_reg[STATUS].
+     * Alternatively, restart link negotiation if it was in progress. */
+    nc->link_down = (s->mac_reg[STATUS] & E1000_STATUS_LU) == 0;
+    if (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN &&
+        s->phy_reg[PHY_CTRL] & MII_CR_RESTART_AUTO_NEG &&
+        !(s->phy_reg[PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) {
+        nc->link_down = false;
+        qemu_mod_timer(s->autoneg_timer, qemu_get_clock_ms(vm_clock) + 500);
+    }
 
     return 0;
 }
@@ -1112,6 +1156,7 @@ static const VMStateDescription vmstate_e1000 = {
     .version_id = 2,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
+    .pre_save = e1000_pre_save,
     .post_load = e1000_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_PCI_DEVICE(dev, E1000State),
@@ -1220,7 +1265,7 @@ e1000_mmio_setup(E1000State *d)
 static void
 e1000_cleanup(NetClientState *nc)
 {
-    E1000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    E1000State *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -1234,7 +1279,7 @@ pci_e1000_uninit(PCIDevice *dev)
     qemu_free_timer(d->autoneg_timer);
     memory_region_destroy(&d->mmio);
     memory_region_destroy(&d->io);
-    qemu_del_net_client(&d->nic->nc);
+    qemu_del_nic(d->nic);
 }
 
 static NetClientInfo net_e1000_info = {
@@ -1281,7 +1326,7 @@ static int pci_e1000_init(PCIDevice *pci_dev)
     d->nic = qemu_new_nic(&net_e1000_info, &d->conf,
                           object_get_typename(OBJECT(d)), d->dev.qdev.id, d);
 
-    qemu_format_nic_info_str(&d->nic->nc, macaddr);
+    qemu_format_nic_info_str(qemu_get_queue(d->nic), macaddr);
 
     add_boot_device_path(d->conf.bootindex, &pci_dev->qdev, "/ethernet-phy@0");
 
@@ -1319,7 +1364,7 @@ static void e1000_class_init(ObjectClass *klass, void *data)
     dc->props = e1000_properties;
 }
 
-static TypeInfo e1000_info = {
+static const TypeInfo e1000_info = {
     .name          = "e1000",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(E1000State),
diff --git a/hw/eccmemctl.c b/hw/eccmemctl.c
index 000bd08..dbac2c2 100644
--- a/hw/eccmemctl.c
+++ b/hw/eccmemctl.c
@@ -324,7 +324,7 @@ static void ecc_class_init(ObjectClass *klass, void *data)
     dc->props = ecc_properties;
 }
 
-static TypeInfo ecc_info = {
+static const TypeInfo ecc_info = {
     .name          = "eccmemctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ECCState),
diff --git a/hw/eepro100.c b/hw/eepro100.c
index a189474..5d23796 100644
--- a/hw/eepro100.c
+++ b/hw/eepro100.c
@@ -42,11 +42,11 @@
 
 #include <stddef.h>             /* offsetof */
 #include "hw.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "eeprom93xx.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 /* QEMU sends frames smaller than 60 bytes to ethernet nics.
  * Such frames are rejected by real nics and their emulations.
@@ -828,7 +828,7 @@ static void tx_command(EEPRO100State *s)
         }
     }
     TRACE(RXTX, logout("%p sending frame, len=%d,%s\n", s, size, nic_dump(buf, size)));
-    qemu_send_packet(&s->nic->nc, buf, size);
+    qemu_send_packet(qemu_get_queue(s->nic), buf, size);
     s->statistics.tx_good_frames++;
     /* Transmit with bad status would raise an CX/TNO interrupt.
      * (82557 only). Emulation never has bad status. */
@@ -1036,7 +1036,7 @@ static void eepro100_ru_command(EEPRO100State * s, uint8_t val)
         }
         set_ru_state(s, ru_ready);
         s->ru_offset = e100_read_reg4(s, SCBPointer);
-        qemu_flush_queued_packets(&s->nic->nc);
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
         TRACE(OTHER, logout("val=0x%02x (rx start)\n", val));
         break;
     case RX_RESUME:
@@ -1619,7 +1619,7 @@ static const MemoryRegionOps eepro100_ops = {
 
 static int nic_can_receive(NetClientState *nc)
 {
-    EEPRO100State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    EEPRO100State *s = qemu_get_nic_opaque(nc);
     TRACE(RXTX, logout("%p\n", s));
     return get_ru_state(s) == ru_ready;
 #if 0
@@ -1633,7 +1633,7 @@ static ssize_t nic_receive(NetClientState *nc, const uint8_t * buf, size_t size)
      * - Magic packets should set bit 30 in power management driver register.
      * - Interesting packets should set bit 29 in power management driver register.
      */
-    EEPRO100State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    EEPRO100State *s = qemu_get_nic_opaque(nc);
     uint16_t rfd_status = 0xa000;
 #if defined(CONFIG_PAD_RECEIVED_FRAMES)
     uint8_t min_buf[60];
@@ -1835,7 +1835,7 @@ static const VMStateDescription vmstate_eepro100 = {
 
 static void nic_cleanup(NetClientState *nc)
 {
-    EEPRO100State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    EEPRO100State *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -1849,7 +1849,7 @@ static void pci_nic_uninit(PCIDevice *pci_dev)
     memory_region_destroy(&s->flash_bar);
     vmstate_unregister(&pci_dev->qdev, s->vmstate, s);
     eeprom93xx_free(&pci_dev->qdev, s->eeprom);
-    qemu_del_net_client(&s->nic->nc);
+    qemu_del_nic(s->nic);
 }
 
 static NetClientInfo net_eepro100_info = {
@@ -1895,14 +1895,14 @@ static int e100_nic_init(PCIDevice *pci_dev)
     s->nic = qemu_new_nic(&net_eepro100_info, &s->conf,
                           object_get_typename(OBJECT(pci_dev)), pci_dev->qdev.id, s);
 
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
-    TRACE(OTHER, logout("%s\n", s->nic->nc.info_str));
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
+    TRACE(OTHER, logout("%s\n", qemu_get_queue(s->nic)->info_str));
 
     qemu_register_reset(nic_reset, s);
 
     s->vmstate = g_malloc(sizeof(vmstate_eepro100));
     memcpy(s->vmstate, &vmstate_eepro100, sizeof(vmstate_eepro100));
-    s->vmstate->name = s->nic->nc.model;
+    s->vmstate->name = qemu_get_queue(s->nic)->model;
     vmstate_register(&pci_dev->qdev, -1, s->vmstate, s);
 
     add_boot_device_path(s->conf.bootindex, &pci_dev->qdev, "/ethernet-phy@0");
diff --git a/hw/empty_slot.c b/hw/empty_slot.c
index 23978eb..d7b5497 100644
--- a/hw/empty_slot.c
+++ b/hw/empty_slot.c
@@ -56,7 +56,7 @@ void empty_slot_init(hwaddr addr, uint64_t slot_size)
         EmptySlot *e;
 
         dev = qdev_create(NULL, "empty_slot");
-        s = sysbus_from_qdev(dev);
+        s = SYS_BUS_DEVICE(dev);
         e = FROM_SYSBUS(EmptySlot, s);
         e->size = slot_size;
 
@@ -83,7 +83,7 @@ static void empty_slot_class_init(ObjectClass *klass, void *data)
     k->init = empty_slot_init1;
 }
 
-static TypeInfo empty_slot_info = {
+static const TypeInfo empty_slot_info = {
     .name          = "empty_slot",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(EmptySlot),
diff --git a/hw/empty_slot.h b/hw/empty_slot.h
index 4e9e460..6079602 100644
--- a/hw/empty_slot.h
+++ b/hw/empty_slot.h
@@ -1,2 +1,7 @@
+#ifndef HW_EMPTY_SLOT_H
+#define HW_EMPTY_SLOT_H 1
+
 /* empty_slot.c */
 void empty_slot_init(hwaddr addr, uint64_t slot_size);
+
+#endif
diff --git a/hw/es1370.c b/hw/es1370.c
index e0c9729..977d2e3 100644
--- a/hw/es1370.c
+++ b/hw/es1370.c
@@ -29,8 +29,8 @@
 #include "hw.h"
 #include "audiodev.h"
 #include "audio/audio.h"
-#include "pci.h"
-#include "dma.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
 
 /* Missing stuff:
    SCTRL_P[12](END|ST)INC
@@ -1073,7 +1073,7 @@ static void es1370_class_init (ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_es1370;
 }
 
-static TypeInfo es1370_info = {
+static const TypeInfo es1370_info = {
     .name          = "ES1370",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof (ES1370State),
diff --git a/hw/escc.c b/hw/escc.c
index a356613..18c0292 100644
--- a/hw/escc.c
+++ b/hw/escc.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "escc.h"
-#include "qemu-char.h"
-#include "console.h"
+#include "char/char.h"
+#include "ui/console.h"
 #include "trace.h"
 
 /*
@@ -700,7 +700,7 @@ MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
     qdev_prop_set_uint32(dev, "chnBtype", ser);
     qdev_prop_set_uint32(dev, "chnAtype", ser);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irqB);
     sysbus_connect_irq(s, 1, irqA);
     if (base) {
@@ -861,7 +861,7 @@ void slavio_serial_ms_kbd_init(hwaddr base, qemu_irq irq,
     qdev_prop_set_uint32(dev, "chnBtype", mouse);
     qdev_prop_set_uint32(dev, "chnAtype", kbd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_connect_irq(s, 1, irq);
     sysbus_mmio_map(s, 0, base);
@@ -923,7 +923,7 @@ static void escc_class_init(ObjectClass *klass, void *data)
     dc->props = escc_properties;
 }
 
-static TypeInfo escc_info = {
+static const TypeInfo escc_info = {
     .name          = "escc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SerialState),
diff --git a/hw/escc.h b/hw/escc.h
index def2894..bda3213 100644
--- a/hw/escc.h
+++ b/hw/escc.h
@@ -1,3 +1,6 @@
+#ifndef HW_ESCC_H
+#define HW_ESCC_H 1
+
 /* escc.c */
 #define ESCC_SIZE 4
 MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
@@ -6,3 +9,5 @@ MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
 
 void slavio_serial_ms_kbd_init(hwaddr base, qemu_irq irq,
                                int disabled, int clock, int it_shift);
+
+#endif
diff --git a/hw/esp-pci.c b/hw/esp-pci.c
index d9a8e59..c949e6e 100644
--- a/hw/esp-pci.c
+++ b/hw/esp-pci.c
@@ -23,11 +23,11 @@
  * THE SOFTWARE.
  */
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "eeprom93xx.h"
 #include "esp.h"
 #include "trace.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #define TYPE_AM53C974_DEVICE "am53c974"
 
diff --git a/hw/esp.c b/hw/esp.c
index 6d01624..2af48aa 100644
--- a/hw/esp.c
+++ b/hw/esp.c
@@ -26,7 +26,7 @@
 #include "sysbus.h"
 #include "esp.h"
 #include "trace.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 /*
  * On Sparc32, this is the ESP (NCR53C90) part of chip STP2000 (Master I/O),
@@ -633,7 +633,7 @@ void esp_init(hwaddr espaddr, int it_shift,
     /* XXX for now until rc4030 has been changed to use DMA enable signal */
     esp->dma_enabled = 1;
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_mmio_map(s, 0, espaddr);
     *reset = qdev_get_gpio_in(dev, 0);
diff --git a/hw/etraxfs.h b/hw/etraxfs.h
index 725bb9e..180de5a 100644
--- a/hw/etraxfs.h
+++ b/hw/etraxfs.h
@@ -22,7 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include "net.h"
+#ifndef HW_EXTRAXFS_H
+#define HW_EXTRAXFS_H 1
+
+#include "net/net.h"
 #include "etraxfs_dma.h"
 
 qemu_irq *cris_pic_init_cpu(CPUCRISState *env);
@@ -41,6 +44,8 @@ etraxfs_eth_init(NICInfo *nd, hwaddr base, int phyaddr,
     qdev_prop_set_ptr(dev, "dma_out", dma_out);
     qdev_prop_set_ptr(dev, "dma_in", dma_in);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     return dev;
 }
+
+#endif
diff --git a/hw/etraxfs_dma.c b/hw/etraxfs_dma.c
index 49221ab..d415003 100644
--- a/hw/etraxfs_dma.c
+++ b/hw/etraxfs_dma.c
@@ -24,9 +24,9 @@
 #include <stdio.h>
 #include <sys/time.h>
 #include "hw.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #include "etraxfs_dma.h"
 
diff --git a/hw/etraxfs_dma.h b/hw/etraxfs_dma.h
index 3fef80f..38104a6 100644
--- a/hw/etraxfs_dma.h
+++ b/hw/etraxfs_dma.h
@@ -1,3 +1,6 @@
+#ifndef HW_ETRAXFS_DMA_H
+#define HW_ETRAXFS_DMA_H 1
+
 struct dma_context_metadata {
 	/* data descriptor md */
 	uint16_t metadata;
@@ -27,3 +30,5 @@ void etraxfs_dmac_connect_client(void *opaque, int c,
 				 struct etraxfs_dma_client *cl);
 int etraxfs_dmac_input(struct etraxfs_dma_client *client, 
 		       void *buf, int len, int eop);
+
+#endif
diff --git a/hw/etraxfs_eth.c b/hw/etraxfs_eth.c
index 3d42426..ad36411 100644
--- a/hw/etraxfs_eth.c
+++ b/hw/etraxfs_eth.c
@@ -24,7 +24,7 @@
 
 #include <stdio.h>
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "etraxfs.h"
 
 #define D(x)
@@ -35,582 +35,593 @@
 #define ADVERTISE_100HALF       0x0080  /* Try for 100mbps half-duplex */
 #define ADVERTISE_100FULL       0x0100  /* Try for 100mbps full-duplex */
 
-/* 
- * The MDIO extensions in the TDK PHY model were reversed engineered from the 
+/*
+ * The MDIO extensions in the TDK PHY model were reversed engineered from the
  * linux driver (PHYID and Diagnostics reg).
  * TODO: Add friendly names for the register nums.
  */
 struct qemu_phy
 {
-	uint32_t regs[32];
+    uint32_t regs[32];
 
-	int link;
+    int link;
 
-	unsigned int (*read)(struct qemu_phy *phy, unsigned int req);
-	void (*write)(struct qemu_phy *phy, unsigned int req, 
-		      unsigned int data);
+    unsigned int (*read)(struct qemu_phy *phy, unsigned int req);
+    void (*write)(struct qemu_phy *phy, unsigned int req, unsigned int data);
 };
 
 static unsigned int tdk_read(struct qemu_phy *phy, unsigned int req)
 {
-	int regnum;
-	unsigned r = 0;
-
-	regnum = req & 0x1f;
-
-	switch (regnum) {
-		case 1:
-			if (!phy->link)
-				break;
-			/* MR1.	 */
-			/* Speeds and modes.  */
-			r |= (1 << 13) | (1 << 14);
-			r |= (1 << 11) | (1 << 12);
-			r |= (1 << 5); /* Autoneg complete.  */
-			r |= (1 << 3); /* Autoneg able.	 */
-			r |= (1 << 2); /* link.	 */
-			break;
-		case 5:
-			/* Link partner ability.
-			   We are kind; always agree with whatever best mode
-			   the guest advertises.  */
-			r = 1 << 14; /* Success.  */
-			/* Copy advertised modes.  */
-			r |= phy->regs[4] & (15 << 5);
-			/* Autoneg support.  */
-			r |= 1;
-			break;
-		case 18:
-		{
-			/* Diagnostics reg.  */
-			int duplex = 0;
-			int speed_100 = 0;
-
-			if (!phy->link)
-				break;
-
-			/* Are we advertising 100 half or 100 duplex ? */
-			speed_100 = !!(phy->regs[4] & ADVERTISE_100HALF);
-			speed_100 |= !!(phy->regs[4] & ADVERTISE_100FULL);
-
-			/* Are we advertising 10 duplex or 100 duplex ? */
-			duplex = !!(phy->regs[4] & ADVERTISE_100FULL);
-			duplex |= !!(phy->regs[4] & ADVERTISE_10FULL);
-			r = (speed_100 << 10) | (duplex << 11);
-		}
-		break;
-
-		default:
-			r = phy->regs[regnum];
-			break;
-	}
-	D(printf("\n%s %x = reg[%d]\n", __func__, r, regnum));
-	return r;
+    int regnum;
+    unsigned r = 0;
+
+    regnum = req & 0x1f;
+
+    switch (regnum) {
+    case 1:
+        if (!phy->link) {
+            break;
+        }
+        /* MR1.     */
+        /* Speeds and modes.  */
+        r |= (1 << 13) | (1 << 14);
+        r |= (1 << 11) | (1 << 12);
+        r |= (1 << 5); /* Autoneg complete.  */
+        r |= (1 << 3); /* Autoneg able.     */
+        r |= (1 << 2); /* link.     */
+        break;
+    case 5:
+        /* Link partner ability.
+           We are kind; always agree with whatever best mode
+           the guest advertises.  */
+        r = 1 << 14; /* Success.  */
+        /* Copy advertised modes.  */
+        r |= phy->regs[4] & (15 << 5);
+        /* Autoneg support.  */
+        r |= 1;
+        break;
+    case 18:
+    {
+        /* Diagnostics reg.  */
+        int duplex = 0;
+        int speed_100 = 0;
+
+        if (!phy->link) {
+            break;
+        }
+
+        /* Are we advertising 100 half or 100 duplex ? */
+        speed_100 = !!(phy->regs[4] & ADVERTISE_100HALF);
+        speed_100 |= !!(phy->regs[4] & ADVERTISE_100FULL);
+
+        /* Are we advertising 10 duplex or 100 duplex ? */
+        duplex = !!(phy->regs[4] & ADVERTISE_100FULL);
+        duplex |= !!(phy->regs[4] & ADVERTISE_10FULL);
+        r = (speed_100 << 10) | (duplex << 11);
+    }
+    break;
+
+    default:
+        r = phy->regs[regnum];
+        break;
+    }
+    D(printf("\n%s %x = reg[%d]\n", __func__, r, regnum));
+    return r;
 }
 
-static void 
+static void
 tdk_write(struct qemu_phy *phy, unsigned int req, unsigned int data)
 {
-	int regnum;
-
-	regnum = req & 0x1f;
-	D(printf("%s reg[%d] = %x\n", __func__, regnum, data));
-	switch (regnum) {
-		default:
-			phy->regs[regnum] = data;
-			break;
-	}
+    int regnum;
+
+    regnum = req & 0x1f;
+    D(printf("%s reg[%d] = %x\n", __func__, regnum, data));
+    switch (regnum) {
+    default:
+        phy->regs[regnum] = data;
+        break;
+    }
 }
 
-static void 
+static void
 tdk_init(struct qemu_phy *phy)
 {
-	phy->regs[0] = 0x3100;
-	/* PHY Id.  */
-	phy->regs[2] = 0x0300;
-	phy->regs[3] = 0xe400;
-	/* Autonegotiation advertisement reg.  */
-	phy->regs[4] = 0x01E1;
-	phy->link = 1;
-
-	phy->read = tdk_read;
-	phy->write = tdk_write;
+    phy->regs[0] = 0x3100;
+    /* PHY Id.  */
+    phy->regs[2] = 0x0300;
+    phy->regs[3] = 0xe400;
+    /* Autonegotiation advertisement reg.  */
+    phy->regs[4] = 0x01E1;
+    phy->link = 1;
+
+    phy->read = tdk_read;
+    phy->write = tdk_write;
 }
 
 struct qemu_mdio
 {
-	/* bus.	 */
-	int mdc;
-	int mdio;
-
-	/* decoder.  */
-	enum {
-		PREAMBLE,
-		SOF,
-		OPC,
-		ADDR,
-		REQ,
-		TURNAROUND,
-		DATA
-	} state;
-	unsigned int drive;
-
-	unsigned int cnt;
-	unsigned int addr;
-	unsigned int opc;
-	unsigned int req;
-	unsigned int data;
-
-	struct qemu_phy *devs[32];
+    /* bus.     */
+    int mdc;
+    int mdio;
+
+    /* decoder.  */
+    enum {
+        PREAMBLE,
+        SOF,
+        OPC,
+        ADDR,
+        REQ,
+        TURNAROUND,
+        DATA
+    } state;
+    unsigned int drive;
+
+    unsigned int cnt;
+    unsigned int addr;
+    unsigned int opc;
+    unsigned int req;
+    unsigned int data;
+
+    struct qemu_phy *devs[32];
 };
 
-static void 
+static void
 mdio_attach(struct qemu_mdio *bus, struct qemu_phy *phy, unsigned int addr)
 {
-	bus->devs[addr & 0x1f] = phy;
+    bus->devs[addr & 0x1f] = phy;
 }
 
 #ifdef USE_THIS_DEAD_CODE
-static void 
+static void
 mdio_detach(struct qemu_mdio *bus, struct qemu_phy *phy, unsigned int addr)
 {
-	bus->devs[addr & 0x1f] = NULL;	
+    bus->devs[addr & 0x1f] = NULL;
 }
 #endif
 
 static void mdio_read_req(struct qemu_mdio *bus)
 {
-	struct qemu_phy *phy;
-
-	phy = bus->devs[bus->addr];
-	if (phy && phy->read)
-		bus->data = phy->read(phy, bus->req);
-	else 
-		bus->data = 0xffff;
+    struct qemu_phy *phy;
+
+    phy = bus->devs[bus->addr];
+    if (phy && phy->read) {
+        bus->data = phy->read(phy, bus->req);
+    } else {
+        bus->data = 0xffff;
+    }
 }
 
 static void mdio_write_req(struct qemu_mdio *bus)
 {
-	struct qemu_phy *phy;
+    struct qemu_phy *phy;
 
-	phy = bus->devs[bus->addr];
-	if (phy && phy->write)
-		phy->write(phy, bus->req, bus->data);
+    phy = bus->devs[bus->addr];
+    if (phy && phy->write) {
+        phy->write(phy, bus->req, bus->data);
+    }
 }
 
 static void mdio_cycle(struct qemu_mdio *bus)
 {
-	bus->cnt++;
+    bus->cnt++;
 
-	D(printf("mdc=%d mdio=%d state=%d cnt=%d drv=%d\n",
-		bus->mdc, bus->mdio, bus->state, bus->cnt, bus->drive));
+    D(printf("mdc=%d mdio=%d state=%d cnt=%d drv=%d\n",
+        bus->mdc, bus->mdio, bus->state, bus->cnt, bus->drive));
 #if 0
-	if (bus->mdc)
-		printf("%d", bus->mdio);
+    if (bus->mdc) {
+        printf("%d", bus->mdio);
+    }
 #endif
-	switch (bus->state)
-	{
-		case PREAMBLE:
-			if (bus->mdc) {
-				if (bus->cnt >= (32 * 2) && !bus->mdio) {
-					bus->cnt = 0;
-					bus->state = SOF;
-					bus->data = 0;
-				}
-			}
-			break;
-		case SOF:
-			if (bus->mdc) {
-				if (bus->mdio != 1)
-					printf("WARNING: no SOF\n");
-				if (bus->cnt == 1*2) {
-					bus->cnt = 0;
-					bus->opc = 0;
-					bus->state = OPC;
-				}
-			}
-			break;
-		case OPC:
-			if (bus->mdc) {
-				bus->opc <<= 1;
-				bus->opc |= bus->mdio & 1;
-				if (bus->cnt == 2*2) {
-					bus->cnt = 0;
-					bus->addr = 0;
-					bus->state = ADDR;
-				}
-			}
-			break;
-		case ADDR:
-			if (bus->mdc) {
-				bus->addr <<= 1;
-				bus->addr |= bus->mdio & 1;
-
-				if (bus->cnt == 5*2) {
-					bus->cnt = 0;
-					bus->req = 0;
-					bus->state = REQ;
-				}
-			}
-			break;
-		case REQ:
-			if (bus->mdc) {
-				bus->req <<= 1;
-				bus->req |= bus->mdio & 1;
-				if (bus->cnt == 5*2) {
-					bus->cnt = 0;
-					bus->state = TURNAROUND;
-				}
-			}
-			break;
-		case TURNAROUND:
-			if (bus->mdc && bus->cnt == 2*2) {
-				bus->mdio = 0;
-				bus->cnt = 0;
-
-				if (bus->opc == 2) {
-					bus->drive = 1;
-					mdio_read_req(bus);
-					bus->mdio = bus->data & 1;
-				}
-				bus->state = DATA;
-			}
-			break;
-		case DATA:			
-			if (!bus->mdc) {
-				if (bus->drive) {
-					bus->mdio = !!(bus->data & (1 << 15));
-					bus->data <<= 1;
-				}
-			} else {
-				if (!bus->drive) {
-					bus->data <<= 1;
-					bus->data |= bus->mdio;
-				}
-				if (bus->cnt == 16 * 2) {
-					bus->cnt = 0;
-					bus->state = PREAMBLE;
-					if (!bus->drive)
-						mdio_write_req(bus);
-					bus->drive = 0;
-				}
-			}
-			break;
-		default:
-			break;
-	}
+    switch (bus->state) {
+    case PREAMBLE:
+        if (bus->mdc) {
+            if (bus->cnt >= (32 * 2) && !bus->mdio) {
+                bus->cnt = 0;
+                bus->state = SOF;
+                bus->data = 0;
+            }
+        }
+        break;
+    case SOF:
+        if (bus->mdc) {
+            if (bus->mdio != 1) {
+                printf("WARNING: no SOF\n");
+            }
+            if (bus->cnt == 1*2) {
+                bus->cnt = 0;
+                bus->opc = 0;
+                bus->state = OPC;
+            }
+        }
+        break;
+    case OPC:
+        if (bus->mdc) {
+            bus->opc <<= 1;
+            bus->opc |= bus->mdio & 1;
+            if (bus->cnt == 2*2) {
+                bus->cnt = 0;
+                bus->addr = 0;
+                bus->state = ADDR;
+            }
+        }
+        break;
+    case ADDR:
+        if (bus->mdc) {
+            bus->addr <<= 1;
+            bus->addr |= bus->mdio & 1;
+
+            if (bus->cnt == 5*2) {
+                bus->cnt = 0;
+                bus->req = 0;
+                bus->state = REQ;
+            }
+        }
+        break;
+    case REQ:
+        if (bus->mdc) {
+            bus->req <<= 1;
+            bus->req |= bus->mdio & 1;
+            if (bus->cnt == 5*2) {
+                bus->cnt = 0;
+                bus->state = TURNAROUND;
+            }
+        }
+        break;
+    case TURNAROUND:
+        if (bus->mdc && bus->cnt == 2*2) {
+            bus->mdio = 0;
+            bus->cnt = 0;
+
+            if (bus->opc == 2) {
+                bus->drive = 1;
+                mdio_read_req(bus);
+                bus->mdio = bus->data & 1;
+            }
+            bus->state = DATA;
+        }
+        break;
+    case DATA:
+        if (!bus->mdc) {
+            if (bus->drive) {
+                bus->mdio = !!(bus->data & (1 << 15));
+                bus->data <<= 1;
+            }
+        } else {
+            if (!bus->drive) {
+                bus->data <<= 1;
+                bus->data |= bus->mdio;
+            }
+            if (bus->cnt == 16 * 2) {
+                bus->cnt = 0;
+                bus->state = PREAMBLE;
+                if (!bus->drive) {
+                    mdio_write_req(bus);
+                }
+                bus->drive = 0;
+            }
+        }
+        break;
+    default:
+        break;
+    }
 }
 
 /* ETRAX-FS Ethernet MAC block starts here.  */
 
-#define RW_MA0_LO	  0x00
-#define RW_MA0_HI	  0x01
-#define RW_MA1_LO	  0x02
-#define RW_MA1_HI	  0x03
-#define RW_GA_LO	  0x04
-#define RW_GA_HI	  0x05
-#define RW_GEN_CTRL	  0x06
-#define RW_REC_CTRL	  0x07
-#define RW_TR_CTRL	  0x08
-#define RW_CLR_ERR	  0x09
-#define RW_MGM_CTRL	  0x0a
-#define R_STAT		  0x0b
-#define FS_ETH_MAX_REGS	  0x17
+#define RW_MA0_LO      0x00
+#define RW_MA0_HI      0x01
+#define RW_MA1_LO      0x02
+#define RW_MA1_HI      0x03
+#define RW_GA_LO      0x04
+#define RW_GA_HI      0x05
+#define RW_GEN_CTRL      0x06
+#define RW_REC_CTRL      0x07
+#define RW_TR_CTRL      0x08
+#define RW_CLR_ERR      0x09
+#define RW_MGM_CTRL      0x0a
+#define R_STAT          0x0b
+#define FS_ETH_MAX_REGS      0x17
 
 struct fs_eth
 {
-	SysBusDevice busdev;
-	MemoryRegion mmio;
-	NICState *nic;
-	NICConf conf;
-
-	/* Two addrs in the filter.  */
-	uint8_t macaddr[2][6];
-	uint32_t regs[FS_ETH_MAX_REGS];
-
-	union {
-		void *vdma_out;
-		struct etraxfs_dma_client *dma_out;
-	};
-	union {
-		void *vdma_in;
-		struct etraxfs_dma_client *dma_in;
-	};
-
-	/* MDIO bus.  */
-	struct qemu_mdio mdio_bus;
-	unsigned int phyaddr;
-	int duplex_mismatch;
-
-	/* PHY.	 */
-	struct qemu_phy phy;
+    SysBusDevice busdev;
+    MemoryRegion mmio;
+    NICState *nic;
+    NICConf conf;
+
+    /* Two addrs in the filter.  */
+    uint8_t macaddr[2][6];
+    uint32_t regs[FS_ETH_MAX_REGS];
+
+    union {
+        void *vdma_out;
+        struct etraxfs_dma_client *dma_out;
+    };
+    union {
+        void *vdma_in;
+        struct etraxfs_dma_client *dma_in;
+    };
+
+    /* MDIO bus.  */
+    struct qemu_mdio mdio_bus;
+    unsigned int phyaddr;
+    int duplex_mismatch;
+
+    /* PHY.     */
+    struct qemu_phy phy;
 };
 
 static void eth_validate_duplex(struct fs_eth *eth)
 {
-	struct qemu_phy *phy;
-	unsigned int phy_duplex;
-	unsigned int mac_duplex;
-	int new_mm = 0;
-
-	phy = eth->mdio_bus.devs[eth->phyaddr];
-	phy_duplex = !!(phy->read(phy, 18) & (1 << 11));
-	mac_duplex = !!(eth->regs[RW_REC_CTRL] & 128);
-
-	if (mac_duplex != phy_duplex)
-		new_mm = 1;
-
-	if (eth->regs[RW_GEN_CTRL] & 1) {
-		if (new_mm != eth->duplex_mismatch) {
-			if (new_mm)
-				printf("HW: WARNING "
-				       "ETH duplex mismatch MAC=%d PHY=%d\n",
-				       mac_duplex, phy_duplex);
-			else
-				printf("HW: ETH duplex ok.\n");
-		}
-		eth->duplex_mismatch = new_mm;
-	}
+    struct qemu_phy *phy;
+    unsigned int phy_duplex;
+    unsigned int mac_duplex;
+    int new_mm = 0;
+
+    phy = eth->mdio_bus.devs[eth->phyaddr];
+    phy_duplex = !!(phy->read(phy, 18) & (1 << 11));
+    mac_duplex = !!(eth->regs[RW_REC_CTRL] & 128);
+
+    if (mac_duplex != phy_duplex) {
+        new_mm = 1;
+    }
+
+    if (eth->regs[RW_GEN_CTRL] & 1) {
+        if (new_mm != eth->duplex_mismatch) {
+            if (new_mm) {
+                printf("HW: WARNING ETH duplex mismatch MAC=%d PHY=%d\n",
+                       mac_duplex, phy_duplex);
+            } else {
+                printf("HW: ETH duplex ok.\n");
+            }
+        }
+        eth->duplex_mismatch = new_mm;
+    }
 }
 
 static uint64_t
 eth_read(void *opaque, hwaddr addr, unsigned int size)
 {
-	struct fs_eth *eth = opaque;
-	uint32_t r = 0;
-
-	addr >>= 2;
-
-	switch (addr) {
-		case R_STAT:
-			r = eth->mdio_bus.mdio & 1;
-			break;
-	default:
-		r = eth->regs[addr];
-		D(printf ("%s %x\n", __func__, addr * 4));
-		break;
-	}
-	return r;
+    struct fs_eth *eth = opaque;
+    uint32_t r = 0;
+
+    addr >>= 2;
+
+    switch (addr) {
+    case R_STAT:
+        r = eth->mdio_bus.mdio & 1;
+        break;
+    default:
+        r = eth->regs[addr];
+        D(printf("%s %x\n", __func__, addr * 4));
+        break;
+    }
+    return r;
 }
 
 static void eth_update_ma(struct fs_eth *eth, int ma)
 {
-	int reg;
-	int i = 0;
-
-	ma &= 1;
-
-	reg = RW_MA0_LO;
-	if (ma)
-		reg = RW_MA1_LO;
-
-	eth->macaddr[ma][i++] = eth->regs[reg];
-	eth->macaddr[ma][i++] = eth->regs[reg] >> 8;
-	eth->macaddr[ma][i++] = eth->regs[reg] >> 16;
-	eth->macaddr[ma][i++] = eth->regs[reg] >> 24;
-	eth->macaddr[ma][i++] = eth->regs[reg + 1];
-	eth->macaddr[ma][i] = eth->regs[reg + 1] >> 8;
-
-	D(printf("set mac%d=%x.%x.%x.%x.%x.%x\n", ma,
-		 eth->macaddr[ma][0], eth->macaddr[ma][1],
-		 eth->macaddr[ma][2], eth->macaddr[ma][3],
-		 eth->macaddr[ma][4], eth->macaddr[ma][5]));
+    int reg;
+    int i = 0;
+
+    ma &= 1;
+
+    reg = RW_MA0_LO;
+    if (ma) {
+        reg = RW_MA1_LO;
+    }
+
+    eth->macaddr[ma][i++] = eth->regs[reg];
+    eth->macaddr[ma][i++] = eth->regs[reg] >> 8;
+    eth->macaddr[ma][i++] = eth->regs[reg] >> 16;
+    eth->macaddr[ma][i++] = eth->regs[reg] >> 24;
+    eth->macaddr[ma][i++] = eth->regs[reg + 1];
+    eth->macaddr[ma][i] = eth->regs[reg + 1] >> 8;
+
+    D(printf("set mac%d=%x.%x.%x.%x.%x.%x\n", ma,
+             eth->macaddr[ma][0], eth->macaddr[ma][1],
+             eth->macaddr[ma][2], eth->macaddr[ma][3],
+             eth->macaddr[ma][4], eth->macaddr[ma][5]));
 }
 
 static void
 eth_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
-	struct fs_eth *eth = opaque;
-	uint32_t value = val64;
-
-	addr >>= 2;
-	switch (addr)
-	{
-		case RW_MA0_LO:
-		case RW_MA0_HI:
-			eth->regs[addr] = value;
-			eth_update_ma(eth, 0);
-			break;
-		case RW_MA1_LO:
-		case RW_MA1_HI:
-			eth->regs[addr] = value;
-			eth_update_ma(eth, 1);
-			break;
-
-		case RW_MGM_CTRL:
-			/* Attach an MDIO/PHY abstraction.  */
-			if (value & 2)
-				eth->mdio_bus.mdio = value & 1;
-			if (eth->mdio_bus.mdc != (value & 4)) {
-				mdio_cycle(&eth->mdio_bus);
-				eth_validate_duplex(eth);
-			}
-			eth->mdio_bus.mdc = !!(value & 4);
-			eth->regs[addr] = value;
-			break;
-
-		case RW_REC_CTRL:
-			eth->regs[addr] = value;
-			eth_validate_duplex(eth);
-			break;
-
-		default:
-			eth->regs[addr] = value;
-			D(printf ("%s %x %x\n",
-				  __func__, addr, value));
-			break;
-	}
+    struct fs_eth *eth = opaque;
+    uint32_t value = val64;
+
+    addr >>= 2;
+    switch (addr) {
+    case RW_MA0_LO:
+    case RW_MA0_HI:
+        eth->regs[addr] = value;
+        eth_update_ma(eth, 0);
+        break;
+    case RW_MA1_LO:
+    case RW_MA1_HI:
+        eth->regs[addr] = value;
+        eth_update_ma(eth, 1);
+        break;
+
+    case RW_MGM_CTRL:
+        /* Attach an MDIO/PHY abstraction.  */
+        if (value & 2) {
+            eth->mdio_bus.mdio = value & 1;
+        }
+        if (eth->mdio_bus.mdc != (value & 4)) {
+            mdio_cycle(&eth->mdio_bus);
+            eth_validate_duplex(eth);
+        }
+        eth->mdio_bus.mdc = !!(value & 4);
+        eth->regs[addr] = value;
+        break;
+
+    case RW_REC_CTRL:
+        eth->regs[addr] = value;
+        eth_validate_duplex(eth);
+        break;
+
+    default:
+        eth->regs[addr] = value;
+        D(printf("%s %x %x\n", __func__, addr, value));
+        break;
+    }
 }
 
 /* The ETRAX FS has a groupt address table (GAT) which works like a k=1 bloom
-   filter dropping group addresses we have not joined.	The filter has 64
-   bits (m). The has function is a simple nible xor of the group addr.	*/
+   filter dropping group addresses we have not joined.    The filter has 64
+   bits (m). The has function is a simple nible xor of the group addr.    */
 static int eth_match_groupaddr(struct fs_eth *eth, const unsigned char *sa)
 {
-	unsigned int hsh;
-	int m_individual = eth->regs[RW_REC_CTRL] & 4;
-	int match;
-
-	/* First bit on the wire of a MAC address signals multicast or
-	   physical address.  */
-	if (!m_individual && !(sa[0] & 1))
-		return 0;
-
-	/* Calculate the hash index for the GA registers. */
-	hsh = 0;
-	hsh ^= (*sa) & 0x3f;
-	hsh ^= ((*sa) >> 6) & 0x03;
-	++sa;
-	hsh ^= ((*sa) << 2) & 0x03c;
-	hsh ^= ((*sa) >> 4) & 0xf;
-	++sa;
-	hsh ^= ((*sa) << 4) & 0x30;
-	hsh ^= ((*sa) >> 2) & 0x3f;
-	++sa;
-	hsh ^= (*sa) & 0x3f;
-	hsh ^= ((*sa) >> 6) & 0x03;
-	++sa;
-	hsh ^= ((*sa) << 2) & 0x03c;
-	hsh ^= ((*sa) >> 4) & 0xf;
-	++sa;
-	hsh ^= ((*sa) << 4) & 0x30;
-	hsh ^= ((*sa) >> 2) & 0x3f;
-
-	hsh &= 63;
-	if (hsh > 31)
-		match = eth->regs[RW_GA_HI] & (1 << (hsh - 32));
-	else
-		match = eth->regs[RW_GA_LO] & (1 << hsh);
-	D(printf("hsh=%x ga=%x.%x mtch=%d\n", hsh,
-		 eth->regs[RW_GA_HI], eth->regs[RW_GA_LO], match));
-	return match;
+    unsigned int hsh;
+    int m_individual = eth->regs[RW_REC_CTRL] & 4;
+    int match;
+
+    /* First bit on the wire of a MAC address signals multicast or
+       physical address.  */
+    if (!m_individual && !(sa[0] & 1)) {
+        return 0;
+    }
+
+    /* Calculate the hash index for the GA registers. */
+    hsh = 0;
+    hsh ^= (*sa) & 0x3f;
+    hsh ^= ((*sa) >> 6) & 0x03;
+    ++sa;
+    hsh ^= ((*sa) << 2) & 0x03c;
+    hsh ^= ((*sa) >> 4) & 0xf;
+    ++sa;
+    hsh ^= ((*sa) << 4) & 0x30;
+    hsh ^= ((*sa) >> 2) & 0x3f;
+    ++sa;
+    hsh ^= (*sa) & 0x3f;
+    hsh ^= ((*sa) >> 6) & 0x03;
+    ++sa;
+    hsh ^= ((*sa) << 2) & 0x03c;
+    hsh ^= ((*sa) >> 4) & 0xf;
+    ++sa;
+    hsh ^= ((*sa) << 4) & 0x30;
+    hsh ^= ((*sa) >> 2) & 0x3f;
+
+    hsh &= 63;
+    if (hsh > 31) {
+        match = eth->regs[RW_GA_HI] & (1 << (hsh - 32));
+    } else {
+        match = eth->regs[RW_GA_LO] & (1 << hsh);
+    }
+    D(printf("hsh=%x ga=%x.%x mtch=%d\n", hsh,
+             eth->regs[RW_GA_HI], eth->regs[RW_GA_LO], match));
+    return match;
 }
 
 static int eth_can_receive(NetClientState *nc)
 {
-	return 1;
+    return 1;
 }
 
 static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-	unsigned char sa_bcast[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
-	struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
-	int use_ma0 = eth->regs[RW_REC_CTRL] & 1;
-	int use_ma1 = eth->regs[RW_REC_CTRL] & 2;
-	int r_bcast = eth->regs[RW_REC_CTRL] & 8;
-
-	if (size < 12)
-		return -1;
-
-	D(printf("%x.%x.%x.%x.%x.%x ma=%d %d bc=%d\n",
-		 buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
-		 use_ma0, use_ma1, r_bcast));
-	       
-	/* Does the frame get through the address filters?  */
-	if ((!use_ma0 || memcmp(buf, eth->macaddr[0], 6))
-	    && (!use_ma1 || memcmp(buf, eth->macaddr[1], 6))
-	    && (!r_bcast || memcmp(buf, sa_bcast, 6))
-	    && !eth_match_groupaddr(eth, buf))
-		return size;
-
-	/* FIXME: Find another way to pass on the fake csum.  */
-	etraxfs_dmac_input(eth->dma_in, (void *)buf, size + 4, 1);
+    unsigned char sa_bcast[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+    struct fs_eth *eth = qemu_get_nic_opaque(nc);
+    int use_ma0 = eth->regs[RW_REC_CTRL] & 1;
+    int use_ma1 = eth->regs[RW_REC_CTRL] & 2;
+    int r_bcast = eth->regs[RW_REC_CTRL] & 8;
+
+    if (size < 12) {
+        return -1;
+    }
+
+    D(printf("%x.%x.%x.%x.%x.%x ma=%d %d bc=%d\n",
+         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
+         use_ma0, use_ma1, r_bcast));
+
+    /* Does the frame get through the address filters?  */
+    if ((!use_ma0 || memcmp(buf, eth->macaddr[0], 6))
+        && (!use_ma1 || memcmp(buf, eth->macaddr[1], 6))
+        && (!r_bcast || memcmp(buf, sa_bcast, 6))
+        && !eth_match_groupaddr(eth, buf)) {
+        return size;
+    }
+
+    /* FIXME: Find another way to pass on the fake csum.  */
+    etraxfs_dmac_input(eth->dma_in, (void *)buf, size + 4, 1);
 
         return size;
 }
 
 static int eth_tx_push(void *opaque, unsigned char *buf, int len, bool eop)
 {
-	struct fs_eth *eth = opaque;
+    struct fs_eth *eth = opaque;
 
-	D(printf("%s buf=%p len=%d\n", __func__, buf, len));
-	qemu_send_packet(&eth->nic->nc, buf, len);
-	return len;
+    D(printf("%s buf=%p len=%d\n", __func__, buf, len));
+    qemu_send_packet(qemu_get_queue(eth->nic), buf, len);
+    return len;
 }
 
 static void eth_set_link(NetClientState *nc)
 {
-	struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
-	D(printf("%s %d\n", __func__, nc->link_down));
-	eth->phy.link = !nc->link_down;
+    struct fs_eth *eth = qemu_get_nic_opaque(nc);
+    D(printf("%s %d\n", __func__, nc->link_down));
+    eth->phy.link = !nc->link_down;
 }
 
 static const MemoryRegionOps eth_ops = {
-	.read = eth_read,
-	.write = eth_write,
-	.endianness = DEVICE_LITTLE_ENDIAN,
-	.valid = {
-		.min_access_size = 4,
-		.max_access_size = 4
-	}
+    .read = eth_read,
+    .write = eth_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4
+    }
 };
 
 static void eth_cleanup(NetClientState *nc)
 {
-	struct fs_eth *eth = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct fs_eth *eth = qemu_get_nic_opaque(nc);
 
-	/* Disconnect the client.  */
-	eth->dma_out->client.push = NULL;
-	eth->dma_out->client.opaque = NULL;
-	eth->dma_in->client.opaque = NULL;
-	eth->dma_in->client.pull = NULL;
+    /* Disconnect the client.  */
+    eth->dma_out->client.push = NULL;
+    eth->dma_out->client.opaque = NULL;
+    eth->dma_in->client.opaque = NULL;
+    eth->dma_in->client.pull = NULL;
         g_free(eth);
 }
 
 static NetClientInfo net_etraxfs_info = {
-	.type = NET_CLIENT_OPTIONS_KIND_NIC,
-	.size = sizeof(NICState),
-	.can_receive = eth_can_receive,
-	.receive = eth_receive,
-	.cleanup = eth_cleanup,
-	.link_status_changed = eth_set_link,
+    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .size = sizeof(NICState),
+    .can_receive = eth_can_receive,
+    .receive = eth_receive,
+    .cleanup = eth_cleanup,
+    .link_status_changed = eth_set_link,
 };
 
 static int fs_eth_init(SysBusDevice *dev)
 {
-	struct fs_eth *s = FROM_SYSBUS(typeof(*s), dev);
+    struct fs_eth *s = FROM_SYSBUS(typeof(*s), dev);
+
+    if (!s->dma_out || !s->dma_in) {
+        hw_error("Unconnected ETRAX-FS Ethernet MAC.\n");
+    }
 
-	if (!s->dma_out || !s->dma_in) {
-		hw_error("Unconnected ETRAX-FS Ethernet MAC.\n");
-	}
+    s->dma_out->client.push = eth_tx_push;
+    s->dma_out->client.opaque = s;
+    s->dma_in->client.opaque = s;
+    s->dma_in->client.pull = NULL;
 
-	s->dma_out->client.push = eth_tx_push;
-	s->dma_out->client.opaque = s;
-	s->dma_in->client.opaque = s;
-	s->dma_in->client.pull = NULL;
+    memory_region_init_io(&s->mmio, &eth_ops, s, "etraxfs-eth", 0x5c);
+    sysbus_init_mmio(dev, &s->mmio);
 
-	memory_region_init_io(&s->mmio, &eth_ops, s, "etraxfs-eth", 0x5c);
-	sysbus_init_mmio(dev, &s->mmio);
+    qemu_macaddr_default_if_unset(&s->conf.macaddr);
+    s->nic = qemu_new_nic(&net_etraxfs_info, &s->conf,
+                          object_get_typename(OBJECT(s)), dev->qdev.id, s);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
-	qemu_macaddr_default_if_unset(&s->conf.macaddr);
-	s->nic = qemu_new_nic(&net_etraxfs_info, &s->conf,
-			      object_get_typename(OBJECT(s)), dev->qdev.id, s);
-	qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
 
-	tdk_init(&s->phy);
-	mdio_attach(&s->mdio_bus, &s->phy, s->phyaddr);
-	return 0;
+    tdk_init(&s->phy);
+    mdio_attach(&s->mdio_bus, &s->phy, s->phyaddr);
+    return 0;
 }
 
 static Property etraxfs_eth_properties[] = {
@@ -630,7 +641,7 @@ static void etraxfs_eth_class_init(ObjectClass *klass, void *data)
     dc->props = etraxfs_eth_properties;
 }
 
-static TypeInfo etraxfs_eth_info = {
+static const TypeInfo etraxfs_eth_info = {
     .name          = "etraxfs-eth",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct fs_eth),
diff --git a/hw/etraxfs_pic.c b/hw/etraxfs_pic.c
index 62a62a3..64af31c 100644
--- a/hw/etraxfs_pic.c
+++ b/hw/etraxfs_pic.c
@@ -165,7 +165,7 @@ static void etraxfs_pic_class_init(ObjectClass *klass, void *data)
     dc->props = etraxfs_pic_properties;
 }
 
-static TypeInfo etraxfs_pic_info = {
+static const TypeInfo etraxfs_pic_info = {
     .name          = "etraxfs,pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct etrax_pic),
diff --git a/hw/etraxfs_ser.c b/hw/etraxfs_ser.c
index ee0d72b..72c8868 100644
--- a/hw/etraxfs_ser.c
+++ b/hw/etraxfs_ser.c
@@ -23,8 +23,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
+#include "char/char.h"
+#include "qemu/log.h"
 
 #define D(x)
 
@@ -233,7 +233,7 @@ static void etraxfs_ser_class_init(ObjectClass *klass, void *data)
     dc->reset = etraxfs_ser_reset;
 }
 
-static TypeInfo etraxfs_ser_info = {
+static const TypeInfo etraxfs_ser_info = {
     .name          = "etraxfs,serial",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct etrax_serial),
diff --git a/hw/etraxfs_timer.c b/hw/etraxfs_timer.c
index f5601dc..d3dac52 100644
--- a/hw/etraxfs_timer.c
+++ b/hw/etraxfs_timer.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 
 #define D(x)
@@ -336,7 +336,7 @@ static void etraxfs_timer_class_init(ObjectClass *klass, void *data)
     sdc->init = etraxfs_timer_init;
 }
 
-static TypeInfo etraxfs_timer_info = {
+static const TypeInfo etraxfs_timer_info = {
     .name          = "etraxfs,timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof (struct etrax_timer),
diff --git a/hw/exynos4210.c b/hw/exynos4210.c
index 22148cd..fa54e42 100644
--- a/hw/exynos4210.c
+++ b/hw/exynos4210.c
@@ -22,11 +22,12 @@
  */
 
 #include "boards.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 #include "arm-misc.h"
 #include "loader.h"
 #include "exynos4210.h"
+#include "usb/hcd-ehci.h"
 
 #define EXYNOS4210_CHIPID_ADDR         0x10000000
 
@@ -72,6 +73,9 @@
 /* Display controllers (FIMD) */
 #define EXYNOS4210_FIMD0_BASE_ADDR          0x11C00000
 
+/* EHCI */
+#define EXYNOS4210_EHCI_BASE_ADDR           0x12580000
+
 static uint8_t chipid_and_omr[] = { 0x11, 0x02, 0x21, 0x43,
                                     0x09, 0x00, 0x00, 0x00 };
 
@@ -150,7 +154,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
         for (n = 0; n < EXYNOS4210_IRQ_GATE_NINPUTS; n++) {
             gate_irq[i][n] = qdev_get_gpio_in(dev, n);
         }
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
 
         /* Connect IRQ Gate output to cpu_irq */
         sysbus_connect_irq(busdev, 0, cpu_irq[i]);
@@ -160,7 +164,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     dev = qdev_create(NULL, "a9mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", EXYNOS4210_NCPUS);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, EXYNOS4210_SMP_PRIVATE_BASE_ADDR);
     for (n = 0; n < EXYNOS4210_NCPUS; n++) {
         sysbus_connect_irq(busdev, n, gate_irq[n][0]);
@@ -176,7 +180,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     dev = qdev_create(NULL, "exynos4210.gic");
     qdev_prop_set_uint32(dev, "num-cpu", EXYNOS4210_NCPUS);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     /* Map CPU interface */
     sysbus_mmio_map(busdev, 0, EXYNOS4210_EXT_GIC_CPU_BASE_ADDR);
     /* Map Distributer interface */
@@ -191,7 +195,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     /* Internal Interrupt Combiner */
     dev = qdev_create(NULL, "exynos4210.combiner");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     for (n = 0; n < EXYNOS4210_MAX_INT_COMBINER_OUT_IRQ; n++) {
         sysbus_connect_irq(busdev, n, s->irqs.int_gic_irq[n]);
     }
@@ -202,7 +206,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     dev = qdev_create(NULL, "exynos4210.combiner");
     qdev_prop_set_uint32(dev, "external", 1);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     for (n = 0; n < EXYNOS4210_MAX_INT_COMBINER_OUT_IRQ; n++) {
         sysbus_connect_irq(busdev, n, s->irqs.ext_gic_irq[n]);
     }
@@ -281,7 +285,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     /* Multi Core Timer */
     dev = qdev_create(NULL, "exynos4210.mct");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     for (n = 0; n < 4; n++) {
         /* Connect global timer interrupts to Combiner gpio_in */
         sysbus_connect_irq(busdev, n,
@@ -307,7 +311,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
         dev = qdev_create(NULL, "exynos4210.i2c");
         qdev_init_nofail(dev);
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         sysbus_connect_irq(busdev, 0, i2c_irq);
         sysbus_mmio_map(busdev, 0, addr);
         s->i2c_if[n] = (i2c_bus *)qdev_get_child_bus(dev, "i2c");
@@ -338,5 +342,8 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
             s->irq_table[exynos4210_get_irq(11, 2)],
             NULL);
 
+    sysbus_create_simple(TYPE_EXYNOS4210_EHCI, EXYNOS4210_EHCI_BASE_ADDR,
+            s->irq_table[exynos4210_get_irq(28, 3)]);
+
     return s;
 }
diff --git a/hw/exynos4210.h b/hw/exynos4210.h
index 777f0f5..bb9a1dd 100644
--- a/hw/exynos4210.h
+++ b/hw/exynos4210.h
@@ -27,7 +27,7 @@
 #define EXYNOS4210_H_
 
 #include "qemu-common.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 #define EXYNOS4210_NCPUS                    2
 
diff --git a/hw/exynos4210_combiner.c b/hw/exynos4210_combiner.c
index 84d36ed..ba644b4 100644
--- a/hw/exynos4210_combiner.c
+++ b/hw/exynos4210_combiner.c
@@ -440,7 +440,7 @@ static void exynos4210_combiner_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_combiner;
 }
 
-static TypeInfo exynos4210_combiner_info = {
+static const TypeInfo exynos4210_combiner_info = {
     .name          = "exynos4210.combiner",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210CombinerState),
diff --git a/hw/exynos4210_fimd.c b/hw/exynos4210_fimd.c
index f2443ca..3d498b7 100644
--- a/hw/exynos4210_fimd.c
+++ b/hw/exynos4210_fimd.c
@@ -23,11 +23,11 @@
  */
 
 #include "qemu-common.h"
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "sysbus.h"
-#include "console.h"
-#include "pixel_ops.h"
-#include "bswap.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
+#include "qemu/bswap.h"
 
 /* Debug messages configuration */
 #define EXYNOS4210_FIMD_DEBUG              0
@@ -1913,7 +1913,7 @@ static void exynos4210_fimd_class_init(ObjectClass *klass, void *data)
     k->init = exynos4210_fimd_init;
 }
 
-static TypeInfo exynos4210_fimd_info = {
+static const TypeInfo exynos4210_fimd_info = {
     .name = "exynos4210.fimd",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210fimdState),
diff --git a/hw/exynos4210_gic.c b/hw/exynos4210_gic.c
index 4fea098..94b138f 100644
--- a/hw/exynos4210_gic.c
+++ b/hw/exynos4210_gic.c
@@ -140,7 +140,7 @@ combiner_grp_to_gic_id[64-EXYNOS4210_MAX_EXT_COMBINER_OUT_IRQ][8] = {
             EXT_GIC_ID_I2C4, EXT_GIC_ID_I2C5, EXT_GIC_ID_I2C6,
             EXT_GIC_ID_I2C7 },
     /* int combiner group 28 */
-    { EXT_GIC_ID_SPI0, EXT_GIC_ID_SPI1, EXT_GIC_ID_SPI2 },
+    { EXT_GIC_ID_SPI0, EXT_GIC_ID_SPI1, EXT_GIC_ID_SPI2 , EXT_GIC_ID_USB_HOST},
     /* int combiner group 29 */
     { EXT_GIC_ID_HSMMC0, EXT_GIC_ID_HSMMC1, EXT_GIC_ID_HSMMC2,
      EXT_GIC_ID_HSMMC3, EXT_GIC_ID_SDMMC },
@@ -290,7 +290,7 @@ static int exynos4210_gic_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(s->gic, "num-irq", EXYNOS4210_GIC_NIRQ);
     qdev_init_nofail(s->gic);
-    busdev = sysbus_from_qdev(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, busdev);
@@ -346,7 +346,7 @@ static void exynos4210_gic_class_init(ObjectClass *klass, void *data)
     dc->props = exynos4210_gic_properties;
 }
 
-static TypeInfo exynos4210_gic_info = {
+static const TypeInfo exynos4210_gic_info = {
     .name          = "exynos4210.gic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210GicState),
@@ -447,7 +447,7 @@ static void exynos4210_irq_gate_class_init(ObjectClass *klass, void *data)
     dc->props = exynos4210_irq_gate_properties;
 }
 
-static TypeInfo exynos4210_irq_gate_info = {
+static const TypeInfo exynos4210_irq_gate_info = {
     .name          = "exynos4210.irq_gate",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210IRQGateState),
diff --git a/hw/exynos4210_i2c.c b/hw/exynos4210_i2c.c
index 1e11d9b..cefd736 100644
--- a/hw/exynos4210_i2c.c
+++ b/hw/exynos4210_i2c.c
@@ -20,7 +20,7 @@
  *
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sysbus.h"
 #include "i2c.h"
 
diff --git a/hw/exynos4210_mct.c b/hw/exynos4210_mct.c
index e79cd6a..d7d5904 100644
--- a/hw/exynos4210_mct.c
+++ b/hw/exynos4210_mct.c
@@ -53,7 +53,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "ptimer.h"
 
@@ -568,7 +568,7 @@ static void exynos4210_gfrc_event(void *opaque)
     /* Reload FRC to reach nearest comparator */
     s->g_timer.curr_comp = exynos4210_gcomp_find(s);
     distance = exynos4210_gcomp_get_distance(s, s->g_timer.curr_comp);
-    if (distance > MCT_GT_COUNTER_STEP) {
+    if (distance > MCT_GT_COUNTER_STEP || !distance) {
         distance = MCT_GT_COUNTER_STEP;
     }
     exynos4210_gfrc_set_count(&s->g_timer, distance);
@@ -1467,7 +1467,7 @@ static void exynos4210_mct_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_mct_state;
 }
 
-static TypeInfo exynos4210_mct_info = {
+static const TypeInfo exynos4210_mct_info = {
     .name          = "exynos4210.mct",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210MCTState),
diff --git a/hw/exynos4210_pmu.c b/hw/exynos4210_pmu.c
index a22b8f1..7c81a1b 100644
--- a/hw/exynos4210_pmu.c
+++ b/hw/exynos4210_pmu.c
@@ -484,7 +484,7 @@ static void exynos4210_pmu_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &exynos4210_pmu_vmstate;
 }
 
-static TypeInfo exynos4210_pmu_info = {
+static const TypeInfo exynos4210_pmu_info = {
     .name          = "exynos4210.pmu",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PmuState),
diff --git a/hw/exynos4210_pwm.c b/hw/exynos4210_pwm.c
index 5e2872f..c865624 100644
--- a/hw/exynos4210_pwm.c
+++ b/hw/exynos4210_pwm.c
@@ -21,7 +21,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "ptimer.h"
 
@@ -407,7 +407,7 @@ static void exynos4210_pwm_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_pwm_state;
 }
 
-static TypeInfo exynos4210_pwm_info = {
+static const TypeInfo exynos4210_pwm_info = {
     .name          = "exynos4210.pwm",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PWMState),
diff --git a/hw/exynos4210_rtc.c b/hw/exynos4210_rtc.c
index c4fbd49..5694a62 100644
--- a/hw/exynos4210_rtc.c
+++ b/hw/exynos4210_rtc.c
@@ -26,13 +26,13 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "ptimer.h"
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 #include "exynos4210.h"
 
diff --git a/hw/exynos4210_uart.c b/hw/exynos4210_uart.c
index 20dcd9f..bdf797a 100644
--- a/hw/exynos4210_uart.c
+++ b/hw/exynos4210_uart.c
@@ -20,8 +20,8 @@
  */
 
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 
 #include "exynos4210.h"
 
@@ -615,7 +615,7 @@ DeviceState *exynos4210_uart_create(hwaddr addr,
     qdev_prop_set_uint32(dev, "rx-size", fifo_size);
     qdev_prop_set_uint32(dev, "tx-size", fifo_size);
 
-    bus = sysbus_from_qdev(dev);
+    bus = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     if (addr != (hwaddr)-1) {
         sysbus_mmio_map(bus, 0, addr);
@@ -661,7 +661,7 @@ static void exynos4210_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_exynos4210_uart;
 }
 
-static TypeInfo exynos4210_uart_info = {
+static const TypeInfo exynos4210_uart_info = {
     .name          = "exynos4210.uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210UartState),
diff --git a/hw/exynos4_boards.c b/hw/exynos4_boards.c
index bc815bb..b59e6aa 100644
--- a/hw/exynos4_boards.c
+++ b/hw/exynos4_boards.c
@@ -21,11 +21,11 @@
  *
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "arm-misc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "exynos4210.h"
 #include "boards.h"
 
@@ -87,7 +87,7 @@ static void lan9215_init(uint32_t base, qemu_irq irq)
         qdev_set_nic_properties(dev, &nd_table[0]);
         qdev_prop_set_uint32(dev, "mode_16bit", 1);
         qdev_init_nofail(dev);
-        s = sysbus_from_qdev(dev);
+        s = SYS_BUS_DEVICE(dev);
         sysbus_mmio_map(s, 0, base);
         sysbus_connect_irq(s, 0, irq);
     }
@@ -150,12 +150,14 @@ static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS] = {
         .desc = "Samsung NURI board (Exynos4210)",
         .init = nuri_init,
         .max_cpus = EXYNOS4210_NCPUS,
+        DEFAULT_MACHINE_OPTIONS,
     },
     [EXYNOS4_BOARD_SMDKC210] = {
         .name = "smdkc210",
         .desc = "Samsung SMDKC210 board (Exynos4210)",
         .init = smdkc210_init,
         .max_cpus = EXYNOS4210_NCPUS,
+        DEFAULT_MACHINE_OPTIONS,
     },
 };
 
diff --git a/hw/fdc.c b/hw/fdc.c
index 29b5449..976a587 100644
--- a/hw/fdc.c
+++ b/hw/fdc.c
@@ -29,14 +29,14 @@
 
 #include "hw.h"
 #include "fdc.h"
-#include "qemu-error.h"
-#include "qemu-timer.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "qdev-addr.h"
-#include "blockdev.h"
-#include "sysemu.h"
-#include "qemu-log.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
 
 /********************************************************/
 /* debug Floppy devices */
@@ -2210,7 +2210,7 @@ static void isabus_fdc_class_init1(ObjectClass *klass, void *data)
     dc->props = isa_fdc_properties;
 }
 
-static TypeInfo isa_fdc_info = {
+static const TypeInfo isa_fdc_info = {
     .name          = "isa-fdc",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(FDCtrlISABus),
@@ -2244,7 +2244,7 @@ static void sysbus_fdc_class_init(ObjectClass *klass, void *data)
     dc->props = sysbus_fdc_properties;
 }
 
-static TypeInfo sysbus_fdc_info = {
+static const TypeInfo sysbus_fdc_info = {
     .name          = "sysbus-fdc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(FDCtrlSysBus),
@@ -2267,7 +2267,7 @@ static void sun4m_fdc_class_init(ObjectClass *klass, void *data)
     dc->props = sun4m_fdc_properties;
 }
 
-static TypeInfo sun4m_fdc_info = {
+static const TypeInfo sun4m_fdc_info = {
     .name          = "SUNW,fdtwo",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(FDCtrlSysBus),
diff --git a/hw/flash.h b/hw/flash.h
index d790f3c..920d759 100644
--- a/hw/flash.h
+++ b/hw/flash.h
@@ -1,6 +1,9 @@
+#ifndef HW_FLASH_H
+#define HW_FLASH_H 1
+
 /* NOR flash devices */
 
-#include "memory.h"
+#include "exec/memory.h"
 
 typedef struct pflash_t pflash_t;
 
@@ -57,3 +60,5 @@ typedef struct {
 uint8_t ecc_digest(ECCState *s, uint8_t sample);
 void ecc_reset(ECCState *s);
 extern VMStateDescription vmstate_ecc_state;
+
+#endif
diff --git a/hw/framebuffer.c b/hw/framebuffer.c
index fa0f786..2a87096 100644
--- a/hw/framebuffer.c
+++ b/hw/framebuffer.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "framebuffer.h"
 
 /* Render an image from a shared memory framebuffer.  */
diff --git a/hw/framebuffer.h b/hw/framebuffer.h
index 46e375b..11f53ed 100644
--- a/hw/framebuffer.h
+++ b/hw/framebuffer.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_FRAMEBUFFER_H
 #define QEMU_FRAMEBUFFER_H
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* Framebuffer device helper routines.  */
 
diff --git a/hw/fw_cfg.c b/hw/fw_cfg.c
index 2b92cda..02618f2 100644
--- a/hw/fw_cfg.c
+++ b/hw/fw_cfg.c
@@ -22,21 +22,13 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "isa.h"
 #include "fw_cfg.h"
 #include "sysbus.h"
-#include "qemu-error.h"
-
-/* debug firmware config */
-//#define DEBUG_FW_CFG
-
-#ifdef DEBUG_FW_CFG
-#define FW_CFG_DPRINTF(fmt, ...)                        \
-    do { printf("FW_CFG: " fmt , ## __VA_ARGS__); } while (0)
-#else
-#define FW_CFG_DPRINTF(fmt, ...)
-#endif
+#include "trace.h"
+#include "qemu/error-report.h"
+#include "qemu/config-file.h"
 
 #define FW_CFG_SIZE 2
 #define FW_CFG_DATA_SIZE 1
@@ -62,16 +54,17 @@ struct FWCfgState {
 #define JPG_FILE 0
 #define BMP_FILE 1
 
-static char *read_splashfile(char *filename, int *file_sizep, int *file_typep)
+static char *read_splashfile(char *filename, size_t *file_sizep,
+                             int *file_typep)
 {
     GError *err = NULL;
     gboolean res;
     gchar *content;
-    int file_type = -1;
-    unsigned int filehead = 0;
+    int file_type;
+    unsigned int filehead;
     int bmp_bpp;
 
-    res = g_file_get_contents(filename, &content, (gsize *)file_sizep, &err);
+    res = g_file_get_contents(filename, &content, file_sizep, &err);
     if (res == FALSE) {
         error_report("failed to read splash file '%s'", filename);
         g_error_free(err);
@@ -119,8 +112,8 @@ static void fw_cfg_bootsplash(FWCfgState *s)
     const char *boot_splash_filename = NULL;
     char *p;
     char *filename, *file_data;
-    int file_size;
-    int file_type = -1;
+    size_t file_size;
+    int file_type;
     const char *temp;
 
     /* get user configuration */
@@ -212,7 +205,7 @@ static void fw_cfg_write(FWCfgState *s, uint8_t value)
     int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
     FWCfgEntry *e = &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
 
-    FW_CFG_DPRINTF("write %d\n", value);
+    trace_fw_cfg_write(s, value);
 
     if (s->cur_entry & FW_CFG_WRITE_CHANNEL && e->callback &&
         s->cur_offset < e->len) {
@@ -237,8 +230,7 @@ static int fw_cfg_select(FWCfgState *s, uint16_t key)
         ret = 1;
     }
 
-    FW_CFG_DPRINTF("select key %d (%sfound)\n", key, ret ? "" : "not ");
-
+    trace_fw_cfg_select(s, key, ret);
     return ret;
 }
 
@@ -253,8 +245,7 @@ static uint8_t fw_cfg_read(FWCfgState *s)
     else
         ret = e->data[s->cur_offset++];
 
-    FW_CFG_DPRINTF("read %d\n", ret);
-
+    trace_fw_cfg_read(s, ret);
     return ret;
 }
 
@@ -383,85 +374,83 @@ static const VMStateDescription vmstate_fw_cfg = {
     }
 };
 
-int fw_cfg_add_bytes(FWCfgState *s, uint16_t key, uint8_t *data, uint32_t len)
+void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len)
 {
     int arch = !!(key & FW_CFG_ARCH_LOCAL);
 
     key &= FW_CFG_ENTRY_MASK;
 
-    if (key >= FW_CFG_MAX_ENTRY)
-        return 0;
+    assert(key < FW_CFG_MAX_ENTRY && len < UINT32_MAX);
 
     s->entries[arch][key].data = data;
-    s->entries[arch][key].len = len;
+    s->entries[arch][key].len = (uint32_t)len;
+}
 
-    return 1;
+void fw_cfg_add_string(FWCfgState *s, uint16_t key, const char *value)
+{
+    size_t sz = strlen(value) + 1;
+
+    return fw_cfg_add_bytes(s, key, g_memdup(value, sz), sz);
 }
 
-int fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value)
+void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value)
 {
     uint16_t *copy;
 
     copy = g_malloc(sizeof(value));
     *copy = cpu_to_le16(value);
-    return fw_cfg_add_bytes(s, key, (uint8_t *)copy, sizeof(value));
+    fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-int fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value)
+void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value)
 {
     uint32_t *copy;
 
     copy = g_malloc(sizeof(value));
     *copy = cpu_to_le32(value);
-    return fw_cfg_add_bytes(s, key, (uint8_t *)copy, sizeof(value));
+    fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-int fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value)
+void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value)
 {
     uint64_t *copy;
 
     copy = g_malloc(sizeof(value));
     *copy = cpu_to_le64(value);
-    return fw_cfg_add_bytes(s, key, (uint8_t *)copy, sizeof(value));
+    fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-int fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
-                        void *callback_opaque, uint8_t *data, size_t len)
+void fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
+                         void *callback_opaque, void *data, size_t len)
 {
     int arch = !!(key & FW_CFG_ARCH_LOCAL);
 
-    if (!(key & FW_CFG_WRITE_CHANNEL))
-        return 0;
+    assert(key & FW_CFG_WRITE_CHANNEL);
 
     key &= FW_CFG_ENTRY_MASK;
 
-    if (key >= FW_CFG_MAX_ENTRY || len > 65535)
-        return 0;
+    assert(key < FW_CFG_MAX_ENTRY && len <= UINT32_MAX);
 
     s->entries[arch][key].data = data;
-    s->entries[arch][key].len = len;
+    s->entries[arch][key].len = (uint32_t)len;
     s->entries[arch][key].callback_opaque = callback_opaque;
     s->entries[arch][key].callback = callback;
-
-    return 1;
 }
 
-int fw_cfg_add_file(FWCfgState *s,  const char *filename, uint8_t *data,
-                    uint32_t len)
+void fw_cfg_add_file(FWCfgState *s,  const char *filename,
+                     void *data, size_t len)
 {
     int i, index;
+    size_t dsize;
 
     if (!s->files) {
-        int dsize = sizeof(uint32_t) + sizeof(FWCfgFile) * FW_CFG_FILE_SLOTS;
+        dsize = sizeof(uint32_t) + sizeof(FWCfgFile) * FW_CFG_FILE_SLOTS;
         s->files = g_malloc0(dsize);
-        fw_cfg_add_bytes(s, FW_CFG_FILE_DIR, (uint8_t*)s->files, dsize);
+        fw_cfg_add_bytes(s, FW_CFG_FILE_DIR, s->files, dsize);
     }
 
     index = be32_to_cpu(s->files->count);
-    if (index == FW_CFG_FILE_SLOTS) {
-        fprintf(stderr, "fw_cfg: out of file slots\n");
-        return 0;
-    }
+    assert(index < FW_CFG_FILE_SLOTS);
 
     fw_cfg_add_bytes(s, FW_CFG_FILE_FIRST + index, data, len);
 
@@ -469,24 +458,21 @@ int fw_cfg_add_file(FWCfgState *s,  const char *filename, uint8_t *data,
             filename);
     for (i = 0; i < index; i++) {
         if (strcmp(s->files->f[index].name, s->files->f[i].name) == 0) {
-            FW_CFG_DPRINTF("%s: skip duplicate: %s\n", __FUNCTION__,
-                           s->files->f[index].name);
-            return 1;
+            trace_fw_cfg_add_file_dupe(s, s->files->f[index].name);
+            return;
         }
     }
 
     s->files->f[index].size   = cpu_to_be32(len);
     s->files->f[index].select = cpu_to_be16(FW_CFG_FILE_FIRST + index);
-    FW_CFG_DPRINTF("%s: #%d: %s (%d bytes)\n", __FUNCTION__,
-                   index, s->files->f[index].name, len);
+    trace_fw_cfg_add_file(s, index, s->files->f[index].name, len);
 
     s->files->count = cpu_to_be32(index+1);
-    return 1;
 }
 
 static void fw_cfg_machine_ready(struct Notifier *n, void *data)
 {
-    uint32_t len;
+    size_t len;
     FWCfgState *s = container_of(n, FWCfgState, machine_ready);
     char *bootindex = get_boot_devices_list(&len);
 
@@ -504,7 +490,7 @@ FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
     qdev_prop_set_uint32(dev, "ctl_iobase", ctl_port);
     qdev_prop_set_uint32(dev, "data_iobase", data_port);
     qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
+    d = SYS_BUS_DEVICE(dev);
 
     s = DO_UPCAST(FWCfgState, busdev.qdev, dev);
 
@@ -514,11 +500,10 @@ FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
     if (data_addr) {
         sysbus_mmio_map(d, 1, data_addr);
     }
-    fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (uint8_t *)"QEMU", 4);
+    fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (char *)"QEMU", 4);
     fw_cfg_add_bytes(s, FW_CFG_UUID, qemu_uuid, 16);
     fw_cfg_add_i16(s, FW_CFG_NOGRAPHIC, (uint16_t)(display_type == DT_NOGRAPHIC));
     fw_cfg_add_i16(s, FW_CFG_NB_CPUS, (uint16_t)smp_cpus);
-    fw_cfg_add_i16(s, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i16(s, FW_CFG_BOOT_MENU, (uint16_t)boot_menu);
     fw_cfg_bootsplash(s);
     fw_cfg_reboot(s);
@@ -574,7 +559,7 @@ static void fw_cfg_class_init(ObjectClass *klass, void *data)
     dc->props = fw_cfg_properties;
 }
 
-static TypeInfo fw_cfg_info = {
+static const TypeInfo fw_cfg_info = {
     .name          = "fw_cfg",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(FWCfgState),
diff --git a/hw/fw_cfg.h b/hw/fw_cfg.h
index 619a394..05c8df1 100644
--- a/hw/fw_cfg.h
+++ b/hw/fw_cfg.h
@@ -54,14 +54,15 @@ typedef struct FWCfgFiles {
 typedef void (*FWCfgCallback)(void *opaque, uint8_t *data);
 
 typedef struct FWCfgState FWCfgState;
-int fw_cfg_add_bytes(FWCfgState *s, uint16_t key, uint8_t *data, uint32_t len);
-int fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value);
-int fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value);
-int fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value);
-int fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
-                        void *callback_opaque, uint8_t *data, size_t len);
-int fw_cfg_add_file(FWCfgState *s, const char *filename, uint8_t *data,
-                    uint32_t len);
+void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len);
+void fw_cfg_add_string(FWCfgState *s, uint16_t key, const char *value);
+void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value);
+void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value);
+void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value);
+void fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
+                         void *callback_opaque, void *data, size_t len);
+void fw_cfg_add_file(FWCfgState *s, const char *filename, void *data,
+                     size_t len);
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
                         hwaddr crl_addr, hwaddr data_addr);
 
diff --git a/hw/g364fb.c b/hw/g364fb.c
index 8192baf..0c0c8ba 100644
--- a/hw/g364fb.c
+++ b/hw/g364fb.c
@@ -18,8 +18,8 @@
  */
 
 #include "hw.h"
-#include "console.h"
-#include "pixel_ops.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "trace.h"
 #include "sysbus.h"
 
@@ -597,7 +597,7 @@ static void g364fb_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = g364fb_sysbus_properties;
 }
 
-static TypeInfo g364fb_sysbus_info = {
+static const TypeInfo g364fb_sysbus_info = {
     .name          = "sysbus-g364",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(G364SysBusState),
diff --git a/hw/grackle_pci.c b/hw/grackle_pci.c
index 67da307..95639d5 100644
--- a/hw/grackle_pci.c
+++ b/hw/grackle_pci.c
@@ -23,9 +23,9 @@
  * THE SOFTWARE.
  */
 
-#include "pci_host.h"
-#include "ppc_mac.h"
-#include "pci.h"
+#include "pci/pci_host.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
 
 /* debug Grackle */
 //#define DEBUG_GRACKLE
diff --git a/hw/grlib.h b/hw/grlib.h
index 35c22f5..afd5389 100644
--- a/hw/grlib.h
+++ b/hw/grlib.h
@@ -61,7 +61,7 @@ DeviceState *grlib_irqmp_create(hwaddr   base,
 
     env->irq_manager = dev;
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     *cpu_irqs = qemu_allocate_irqs(grlib_irqmp_set_irq,
                                    dev,
@@ -91,10 +91,10 @@ DeviceState *grlib_gptimer_create(hwaddr  base,
         return NULL;
     }
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     for (i = 0; i < nr_timers; i++) {
-        sysbus_connect_irq(sysbus_from_qdev(dev), i, cpu_irqs[base_irq + i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, cpu_irqs[base_irq + i]);
     }
 
     return dev;
@@ -116,9 +116,9 @@ DeviceState *grlib_apbuart_create(hwaddr  base,
         return NULL;
     }
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
diff --git a/hw/grlib_apbuart.c b/hw/grlib_apbuart.c
index 0865764..760bed0 100644
--- a/hw/grlib_apbuart.c
+++ b/hw/grlib_apbuart.c
@@ -23,7 +23,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 #include "trace.h"
 
@@ -256,7 +256,7 @@ static void grlib_gptimer_class_init(ObjectClass *klass, void *data)
     dc->props = grlib_gptimer_properties;
 }
 
-static TypeInfo grlib_gptimer_info = {
+static const TypeInfo grlib_gptimer_info = {
     .name          = "grlib,apbuart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(UART),
diff --git a/hw/grlib_gptimer.c b/hw/grlib_gptimer.c
index 2fdccfb..7962b74 100644
--- a/hw/grlib_gptimer.c
+++ b/hw/grlib_gptimer.c
@@ -23,7 +23,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 
 #include "trace.h"
@@ -389,7 +389,7 @@ static void grlib_gptimer_class_init(ObjectClass *klass, void *data)
     dc->props = grlib_gptimer_properties;
 }
 
-static TypeInfo grlib_gptimer_info = {
+static const TypeInfo grlib_gptimer_info = {
     .name          = "grlib,gptimer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GPTimerUnit),
diff --git a/hw/grlib_irqmp.c b/hw/grlib_irqmp.c
index 23a6a02..ef8dd95 100644
--- a/hw/grlib_irqmp.c
+++ b/hw/grlib_irqmp.c
@@ -109,7 +109,7 @@ void grlib_irqmp_ack(DeviceState *dev, int intno)
 
     assert(dev != NULL);
 
-    sdev = sysbus_from_qdev(dev);
+    sdev = SYS_BUS_DEVICE(dev);
     assert(sdev != NULL);
 
     irqmp = FROM_SYSBUS(typeof(*irqmp), sdev);
@@ -138,7 +138,7 @@ void grlib_irqmp_set_irq(void *opaque, int irq, int level)
 
     assert(opaque != NULL);
 
-    irqmp = FROM_SYSBUS(typeof(*irqmp), sysbus_from_qdev(opaque));
+    irqmp = FROM_SYSBUS(typeof(*irqmp), SYS_BUS_DEVICE(opaque));
     assert(irqmp != NULL);
 
     s = irqmp->state;
@@ -370,7 +370,7 @@ static void grlib_irqmp_class_init(ObjectClass *klass, void *data)
     dc->props = grlib_irqmp_properties;
 }
 
-static TypeInfo grlib_irqmp_info = {
+static const TypeInfo grlib_irqmp_info = {
     .name          = "grlib,irqmp",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IRQMP),
diff --git a/hw/gt64xxx.c b/hw/gt64xxx.c
index 95d491d..977a2c5 100644
--- a/hw/gt64xxx.c
+++ b/hw/gt64xxx.c
@@ -24,10 +24,10 @@
 
 #include "hw.h"
 #include "mips.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "pc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG
 
diff --git a/hw/gumstix.c b/hw/gumstix.c
index 4103a88..bea1605 100644
--- a/hw/gumstix.c
+++ b/hw/gumstix.c
@@ -36,12 +36,12 @@
 
 #include "hw.h"
 #include "pxa.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
 #include "devices.h"
 #include "boards.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 static const int sector_len = 128 * 1024;
 
@@ -122,12 +122,14 @@ static QEMUMachine connex_machine = {
     .name = "connex",
     .desc = "Gumstix Connex (PXA255)",
     .init = connex_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine verdex_machine = {
     .name = "verdex",
     .desc = "Gumstix Verdex (PXA270)",
     .init = verdex_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void gumstix_machine_init(void)
diff --git a/hw/gus.c b/hw/gus.c
index 840d098..aa13fcc 100644
--- a/hw/gus.c
+++ b/hw/gus.c
@@ -317,7 +317,7 @@ static void gus_class_initfn (ObjectClass *klass, void *data)
     dc->props = gus_properties;
 }
 
-static TypeInfo gus_info = {
+static const TypeInfo gus_info = {
     .name          = "gus",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof (GUSState),
diff --git a/hw/hd-geometry.c b/hw/hd-geometry.c
index 1cdb9fb..c305143 100644
--- a/hw/hd-geometry.c
+++ b/hw/hd-geometry.c
@@ -30,7 +30,7 @@
  * THE SOFTWARE.
  */
 
-#include "block.h"
+#include "block/block.h"
 #include "hw/block-common.h"
 #include "trace.h"
 
diff --git a/hw/hda-audio.c b/hw/hda-audio.c
index 36761dd..3190bd1 100644
--- a/hw/hda-audio.c
+++ b/hw/hda-audio.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "intel-hda.h"
 #include "intel-hda-defs.h"
 #include "audio/audio.h"
@@ -1039,7 +1039,7 @@ static void hda_audio_output_class_init(ObjectClass *klass, void *data)
     dc->props = hda_audio_properties;
 }
 
-static TypeInfo hda_audio_output_info = {
+static const TypeInfo hda_audio_output_info = {
     .name          = "hda-output",
     .parent        = TYPE_HDA_CODEC_DEVICE,
     .instance_size = sizeof(HDAAudioState),
@@ -1060,7 +1060,7 @@ static void hda_audio_duplex_class_init(ObjectClass *klass, void *data)
     dc->props = hda_audio_properties;
 }
 
-static TypeInfo hda_audio_duplex_info = {
+static const TypeInfo hda_audio_duplex_info = {
     .name          = "hda-duplex",
     .parent        = TYPE_HDA_CODEC_DEVICE,
     .instance_size = sizeof(HDAAudioState),
@@ -1081,7 +1081,7 @@ static void hda_audio_micro_class_init(ObjectClass *klass, void *data)
     dc->props = hda_audio_properties;
 }
 
-static TypeInfo hda_audio_micro_info = {
+static const TypeInfo hda_audio_micro_info = {
     .name          = "hda-micro",
     .parent        = TYPE_HDA_CODEC_DEVICE,
     .instance_size = sizeof(HDAAudioState),
diff --git a/hw/heathrow_pic.c b/hw/heathrow_pic.c
index b9ec8e7..c0a71c3 100644
--- a/hw/heathrow_pic.c
+++ b/hw/heathrow_pic.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
+#include "ppc/mac.h"
 
 /* debug PIC */
 //#define DEBUG_PIC
diff --git a/hw/hid.c b/hw/hid.c
index e17776d..d159460 100644
--- a/hw/hid.c
+++ b/hw/hid.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 #include "hid.h"
 
 #define HID_USAGE_ERROR_ROLLOVER        0x01
@@ -71,12 +71,38 @@ static const uint8_t hid_usage_keys[0x100] = {
 
 bool hid_has_events(HIDState *hs)
 {
-    return hs->n > 0;
+    return hs->n > 0 || hs->idle_pending;
 }
 
-void hid_set_next_idle(HIDState *hs, int64_t curtime)
+static void hid_idle_timer(void *opaque)
 {
-    hs->next_idle_clock = curtime + (get_ticks_per_sec() * hs->idle * 4) / 1000;
+    HIDState *hs = opaque;
+
+    hs->idle_pending = true;
+    hs->event(hs);
+}
+
+static void hid_del_idle_timer(HIDState *hs)
+{
+    if (hs->idle_timer) {
+        qemu_del_timer(hs->idle_timer);
+        qemu_free_timer(hs->idle_timer);
+        hs->idle_timer = NULL;
+    }
+}
+
+void hid_set_next_idle(HIDState *hs)
+{
+    if (hs->idle) {
+        uint64_t expire_time = qemu_get_clock_ns(vm_clock) +
+                               get_ticks_per_sec() * hs->idle * 4 / 1000;
+        if (!hs->idle_timer) {
+            hs->idle_timer = qemu_new_timer_ns(vm_clock, hid_idle_timer, hs);
+        }
+        qemu_mod_timer_ns(hs->idle_timer, expire_time);
+    } else {
+        hid_del_idle_timer(hs);
+    }
 }
 
 static void hid_pointer_event_clear(HIDPointerEvent *e, int buttons)
@@ -232,6 +258,8 @@ int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len)
     int index;
     HIDPointerEvent *e;
 
+    hs->idle_pending = false;
+
     hid_pointer_activate(hs);
 
     /* When the buffer is empty, return the last event.  Relative
@@ -319,6 +347,8 @@ int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len)
 
 int hid_keyboard_poll(HIDState *hs, uint8_t *buf, int len)
 {
+    hs->idle_pending = false;
+
     if (len < 2) {
         return 0;
     }
@@ -377,6 +407,8 @@ void hid_reset(HIDState *hs)
     hs->n = 0;
     hs->protocol = 1;
     hs->idle = 0;
+    hs->idle_pending = false;
+    hid_del_idle_timer(hs);
 }
 
 void hid_free(HIDState *hs)
@@ -390,6 +422,7 @@ void hid_free(HIDState *hs)
         qemu_remove_mouse_event_handler(hs->ptr.eh_entry);
         break;
     }
+    hid_del_idle_timer(hs);
 }
 
 void hid_init(HIDState *hs, int kind, HIDEventFunc event)
@@ -412,9 +445,7 @@ static int hid_post_load(void *opaque, int version_id)
 {
     HIDState *s = opaque;
 
-    if (s->idle) {
-        hid_set_next_idle(s, qemu_get_clock_ns(vm_clock));
-    }
+    hid_set_next_idle(s);
     return 0;
 }
 
diff --git a/hw/hid.h b/hw/hid.h
index 5315cf7..56c71ed 100644
--- a/hw/hid.h
+++ b/hw/hid.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_HID_H
 #define QEMU_HID_H
 
-#include "vmstate.h"
+#include "migration/vmstate.h"
 
 #define HID_MOUSE     1
 #define HID_TABLET    2
@@ -43,7 +43,8 @@ struct HIDState {
     int kind;
     int32_t protocol;
     uint8_t idle;
-    int64_t next_idle_clock;
+    bool idle_pending;
+    QEMUTimer *idle_timer;
     HIDEventFunc event;
 };
 
@@ -52,7 +53,7 @@ void hid_reset(HIDState *hs);
 void hid_free(HIDState *hs);
 
 bool hid_has_events(HIDState *hs);
-void hid_set_next_idle(HIDState *hs, int64_t curtime);
+void hid_set_next_idle(HIDState *hs);
 void hid_pointer_activate(HIDState *hs);
 int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len);
 int hid_keyboard_poll(HIDState *hs, uint8_t *buf, int len);
diff --git a/hw/highbank.c b/hw/highbank.c
index 447e57d..defcc09 100644
--- a/hw/highbank.c
+++ b/hw/highbank.c
@@ -21,12 +21,12 @@
 #include "arm-misc.h"
 #include "devices.h"
 #include "loader.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "sysbus.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define SMP_BOOT_ADDR 0x100
 #define SMP_BOOT_REG  0x40
@@ -136,7 +136,7 @@ static VMStateDescription vmstate_highbank_regs = {
 
 static void highbank_regs_reset(DeviceState *dev)
 {
-    SysBusDevice *sys_dev = sysbus_from_qdev(dev);
+    SysBusDevice *sys_dev = SYS_BUS_DEVICE(dev);
     HighbankRegsState *s = FROM_SYSBUS(HighbankRegsState, sys_dev);
 
     s->regs[0x40] = 0x05F20121;
@@ -168,7 +168,7 @@ static void highbank_regs_class_init(ObjectClass *klass, void *data)
     dc->reset = highbank_regs_reset;
 }
 
-static TypeInfo highbank_regs_info = {
+static const TypeInfo highbank_regs_info = {
     .name          = "highbank-regs",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(HighbankRegsState),
@@ -251,7 +251,7 @@ static void highbank_init(QEMUMachineInitArgs *args)
     qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
     qdev_prop_set_uint32(dev, "num-irq", NIRQ_GIC);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, GIC_BASE_ADDR);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
@@ -263,21 +263,21 @@ static void highbank_init(QEMUMachineInitArgs *args)
 
     dev = qdev_create(NULL, "l2x0");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff12000);
 
     dev = qdev_create(NULL, "sp804");
     qdev_prop_set_uint32(dev, "freq0", 150000000);
     qdev_prop_set_uint32(dev, "freq1", 150000000);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff34000);
     sysbus_connect_irq(busdev, 0, pic[18]);
     sysbus_create_simple("pl011", 0xfff36000, pic[20]);
 
     dev = qdev_create(NULL, "highbank-regs");
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff3c000);
 
     sysbus_create_simple("pl061", 0xfff30000, pic[14]);
@@ -294,19 +294,19 @@ static void highbank_init(QEMUMachineInitArgs *args)
         dev = qdev_create(NULL, "xgmac");
         qdev_set_nic_properties(dev, &nd_table[0]);
         qdev_init_nofail(dev);
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0xfff50000);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 0, pic[77]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 1, pic[78]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 2, pic[79]);
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xfff50000);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[77]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, pic[78]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, pic[79]);
 
         qemu_check_nic_model(&nd_table[1], "xgmac");
         dev = qdev_create(NULL, "xgmac");
         qdev_set_nic_properties(dev, &nd_table[1]);
         qdev_init_nofail(dev);
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0xfff51000);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 0, pic[80]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 1, pic[81]);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 2, pic[82]);
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xfff51000);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[80]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, pic[81]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, pic[82]);
     }
 
     highbank_binfo.ram_size = ram_size;
@@ -329,8 +329,9 @@ static QEMUMachine highbank_machine = {
     .name = "highbank",
     .desc = "Calxeda Highbank (ECX-1000)",
     .init = highbank_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void highbank_machine_init(void)
diff --git a/hw/hpet.c b/hw/hpet.c
index 50ac067..97eaa2f 100644
--- a/hw/hpet.c
+++ b/hw/hpet.c
@@ -26,8 +26,8 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 #include "hpet_emul.h"
 #include "sysbus.h"
 #include "mc146818rtc.h"
@@ -634,7 +634,7 @@ static const MemoryRegionOps hpet_ram_ops = {
 
 static void hpet_reset(DeviceState *d)
 {
-    HPETState *s = FROM_SYSBUS(HPETState, sysbus_from_qdev(d));
+    HPETState *s = FROM_SYSBUS(HPETState, SYS_BUS_DEVICE(d));
     int i;
 
     for (i = 0; i < s->num_timers; i++) {
@@ -657,7 +657,7 @@ static void hpet_reset(DeviceState *d)
     s->hpet_offset = 0ULL;
     s->config = 0ULL;
     hpet_cfg.hpet[s->hpet_id].event_timer_block_id = (uint32_t)s->capability;
-    hpet_cfg.hpet[s->hpet_id].address = sysbus_from_qdev(d)->mmio[0].addr;
+    hpet_cfg.hpet[s->hpet_id].address = SYS_BUS_DEVICE(d)->mmio[0].addr;
 
     /* to document that the RTC lowers its output on reset as well */
     s->rtc_irq_level = 0;
@@ -745,7 +745,7 @@ static void hpet_device_class_init(ObjectClass *klass, void *data)
     dc->props = hpet_device_properties;
 }
 
-static TypeInfo hpet_device_info = {
+static const TypeInfo hpet_device_info = {
     .name          = "hpet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(HPETState),
diff --git a/hw/hw.h b/hw/hw.h
index f530f6f..dfced97 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -5,15 +5,15 @@
 #include "qemu-common.h"
 
 #if !defined(CONFIG_USER_ONLY) && !defined(NEED_CPU_H)
-#include "cpu-common.h"
+#include "exec/cpu-common.h"
 #endif
 
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "irq.h"
-#include "qemu-aio.h"
-#include "qemu-file.h"
-#include "vmstate.h"
-#include "qemu-log.h"
+#include "block/aio.h"
+#include "migration/qemu-file.h"
+#include "migration/vmstate.h"
+#include "qemu/log.h"
 
 #ifdef NEED_CPU_H
 #if TARGET_LONG_BITS == 64
diff --git a/hw/i2c-ddc.c b/hw/i2c-ddc.c
index b7d7a52..028aff4 100644
--- a/hw/i2c-ddc.c
+++ b/hw/i2c-ddc.c
@@ -226,7 +226,7 @@ typedef struct I2CDDCState_s {
 
 static void i2c_ddc_reset(DeviceState *ds)
 {
-    I2CDDCState *s = FROM_I2C_SLAVE(I2CDDCState, I2C_SLAVE_FROM_QDEV(ds));
+    I2CDDCState *s = FROM_I2C_SLAVE(I2CDDCState, I2C_SLAVE(ds));
     s->firstbyte = 0;
     s->reg = 0;
 }
diff --git a/hw/i2c.c b/hw/i2c.c
index 296bece..ec314a4 100644
--- a/hw/i2c.c
+++ b/hw/i2c.c
@@ -92,7 +92,7 @@ int i2c_start_transfer(i2c_bus *bus, uint8_t address, int recv)
 
     QTAILQ_FOREACH(kid, &bus->qbus.children, sibling) {
         DeviceState *qdev = kid->child;
-        I2CSlave *candidate = I2C_SLAVE_FROM_QDEV(qdev);
+        I2CSlave *candidate = I2C_SLAVE(qdev);
         if (candidate->address == address) {
             slave = candidate;
             break;
@@ -204,7 +204,7 @@ const VMStateDescription vmstate_i2c_slave = {
 
 static int i2c_slave_qdev_init(DeviceState *dev)
 {
-    I2CSlave *s = I2C_SLAVE_FROM_QDEV(dev);
+    I2CSlave *s = I2C_SLAVE(dev);
     I2CSlaveClass *sc = I2C_SLAVE_GET_CLASS(s);
 
     return sc->init(s);
@@ -228,7 +228,7 @@ static void i2c_slave_class_init(ObjectClass *klass, void *data)
     k->props = i2c_props;
 }
 
-static TypeInfo i2c_slave_type_info = {
+static const TypeInfo i2c_slave_type_info = {
     .name = TYPE_I2C_SLAVE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(I2CSlave),
diff --git a/hw/i2c.h b/hw/i2c.h
index 6a90334..0eba883 100644
--- a/hw/i2c.h
+++ b/hw/i2c.h
@@ -59,7 +59,6 @@ void i2c_nack(i2c_bus *bus);
 int i2c_send(i2c_bus *bus, uint8_t data);
 int i2c_recv(i2c_bus *bus);
 
-#define I2C_SLAVE_FROM_QDEV(dev) DO_UPCAST(I2CSlave, qdev, dev)
 #define FROM_I2C_SLAVE(type, dev) DO_UPCAST(type, i2c, dev)
 
 DeviceState *i2c_create_slave(i2c_bus *bus, const char *name, uint8_t addr);
@@ -73,9 +72,6 @@ void *wm8750_dac_buffer(void *opaque, int samples);
 void wm8750_dac_commit(void *opaque);
 void wm8750_set_bclk_in(void *opaque, int new_hz);
 
-/* tmp105.c */
-void tmp105_set(I2CSlave *i2c, int temp);
-
 /* lm832x.c */
 void lm832x_key_event(DeviceState *dev, int key, int state);
 
diff --git a/hw/i386/Makefile.objs b/hw/i386/Makefile.objs
index 0d3f6a8..025803a 100644
--- a/hw/i386/Makefile.objs
+++ b/hw/i386/Makefile.objs
@@ -2,8 +2,8 @@ obj-y += mc146818rtc.o pc.o
 obj-y += apic_common.o apic.o kvmvapic.o
 obj-y += sga.o ioapic_common.o ioapic.o piix_pci.o
 obj-y += vmport.o
-obj-y += pci-hotplug.o smbios.o wdt_ib700.o
-obj-y += debugcon.o multiboot.o
+obj-y += pci/pci-hotplug.o smbios.o wdt_ib700.o
+obj-y += debugcon.o debugexit.o multiboot.o
 obj-y += pc_piix.o
 obj-y += pc_sysfw.o
 obj-y += lpc_ich9.o q35.o pc_q35.o
@@ -12,5 +12,6 @@ obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen-host-pci-device.o
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o
 obj-y += kvm/
 obj-$(CONFIG_SPICE) += qxl.o qxl-logger.o qxl-render.o
+obj-y += pc-testdev.o
 
 obj-y := $(addprefix ../,$(obj-y))
diff --git a/hw/i82374.c b/hw/i82374.c
index 4a922c3..6a62ba2 100644
--- a/hw/i82374.c
+++ b/hw/i82374.c
@@ -153,7 +153,7 @@ static void i82374_class_init(ObjectClass *klass, void *data)
     dc->props = i82374_properties;
 }
 
-static TypeInfo i82374_isa_info = {
+static const TypeInfo i82374_isa_info = {
     .name  = "i82374",
     .parent = TYPE_ISA_DEVICE,
     .instance_size  = sizeof(ISAi82374State),
diff --git a/hw/i82378.c b/hw/i82378.c
index 99f35d4..0914d7b 100644
--- a/hw/i82378.c
+++ b/hw/i82378.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "pc.h"
 #include "i8254.h"
 #include "pcspk.h"
@@ -262,7 +262,7 @@ static void pci_i82378_class_init(ObjectClass *klass, void *data)
     dc->props = i82378_properties;
 }
 
-static TypeInfo pci_i82378_info = {
+static const TypeInfo pci_i82378_info = {
     .name = "i82378",
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIi82378State),
diff --git a/hw/i8254.c b/hw/i8254.c
index bea5f92..394b2e8 100644
--- a/hw/i8254.c
+++ b/hw/i8254.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "pc.h"
 #include "isa.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i8254.h"
 #include "i8254_internal.h"
 
@@ -347,7 +347,7 @@ static void pit_class_initfn(ObjectClass *klass, void *data)
     dc->props = pit_properties;
 }
 
-static TypeInfo pit_info = {
+static const TypeInfo pit_info = {
     .name          = "isa-pit",
     .parent        = TYPE_PIT_COMMON,
     .instance_size = sizeof(PITCommonState),
diff --git a/hw/i8254_common.c b/hw/i8254_common.c
index a03d7cd..8c2e45a 100644
--- a/hw/i8254_common.c
+++ b/hw/i8254_common.c
@@ -25,7 +25,7 @@
 #include "hw.h"
 #include "pc.h"
 #include "isa.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i8254.h"
 #include "i8254_internal.h"
 
@@ -294,7 +294,7 @@ static void pit_common_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo pit_common_type = {
+static const TypeInfo pit_common_type = {
     .name          = TYPE_PIT_COMMON,
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(PITCommonState),
diff --git a/hw/i8259.c b/hw/i8259.c
index af0ba4d..54fe144 100644
--- a/hw/i8259.c
+++ b/hw/i8259.c
@@ -24,8 +24,8 @@
 #include "hw.h"
 #include "pc.h"
 #include "isa.h"
-#include "monitor.h"
-#include "qemu-timer.h"
+#include "monitor/monitor.h"
+#include "qemu/timer.h"
 #include "i8259_internal.h"
 
 /* debug PIC */
@@ -407,7 +407,7 @@ static void pic_init(PICCommonState *s)
     qdev_init_gpio_in(&s->dev.qdev, pic_set_irq, 8);
 }
 
-void pic_info(Monitor *mon)
+void pic_info(Monitor *mon, const QDict *qdict)
 {
     int i;
     PICCommonState *s;
@@ -425,7 +425,7 @@ void pic_info(Monitor *mon)
     }
 }
 
-void irq_info(Monitor *mon)
+void irq_info(Monitor *mon, const QDict *qdict)
 {
 #ifndef DEBUG_IRQ_COUNT
     monitor_printf(mon, "irq statistic code not compiled.\n");
@@ -481,7 +481,7 @@ static void i8259_class_init(ObjectClass *klass, void *data)
     dc->reset = pic_reset;
 }
 
-static TypeInfo i8259_info = {
+static const TypeInfo i8259_info = {
     .name       = "isa-i8259",
     .instance_size = sizeof(PICCommonState),
     .parent     = TYPE_PIC_COMMON,
diff --git a/hw/i8259_common.c b/hw/i8259_common.c
index ab3d98b..fc91056 100644
--- a/hw/i8259_common.c
+++ b/hw/i8259_common.c
@@ -144,7 +144,7 @@ static void pic_common_class_init(ObjectClass *klass, void *data)
     ic->init = pic_init_common;
 }
 
-static TypeInfo pic_common_type = {
+static const TypeInfo pic_common_type = {
     .name = TYPE_PIC_COMMON,
     .parent = TYPE_ISA_DEVICE,
     .instance_size = sizeof(PICCommonState),
diff --git a/hw/i82801b11.c b/hw/i82801b11.c
index 3d1f996..3dc1000 100644
--- a/hw/i82801b11.c
+++ b/hw/i82801b11.c
@@ -41,7 +41,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "ich9.h"
 
 
diff --git a/hw/ich9.h b/hw/ich9.h
index de49135..d4509bb 100644
--- a/hw/ich9.h
+++ b/hw/ich9.h
@@ -2,22 +2,23 @@
 #define HW_ICH9_H
 
 #include "hw.h"
-#include "range.h"
+#include "qemu/range.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "pc.h"
 #include "apm.h"
 #include "ioapic.h"
-#include "pci.h"
-#include "pcie_host.h"
-#include "pci_bridge.h"
+#include "pci/pci.h"
+#include "pci/pcie_host.h"
+#include "pci/pci_bridge.h"
 #include "acpi.h"
 #include "acpi_ich9.h"
 #include "pam.h"
-#include "pci_internals.h"
+#include "pci/pci_bus.h"
 
 void ich9_lpc_set_irq(void *opaque, int irq_num, int level);
 int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx);
+PCIINTxRoute ich9_route_intx_pin_to_irq(void *opaque, int pirq_pin);
 void ich9_lpc_pm_init(PCIDevice *pci_lpc, qemu_irq cmos_s3);
 PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus);
 i2c_bus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base);
@@ -51,6 +52,7 @@ typedef struct ICH9LPCState {
     /* isa bus */
     ISABus *isa_bus;
     MemoryRegion rbca_mem;
+    Notifier machine_ready;
 
     qemu_irq *pic;
     qemu_irq *ioapic;
diff --git a/hw/ide.h b/hw/ide.h
index add742c..0eb3a74 100644
--- a/hw/ide.h
+++ b/hw/ide.h
@@ -2,8 +2,8 @@
 #define HW_IDE_H
 
 #include "isa.h"
-#include "pci.h"
-#include "memory.h"
+#include "pci/pci.h"
+#include "exec/memory.h"
 
 #define MAX_IDE_DEVS	2
 
@@ -19,15 +19,8 @@ PCIDevice *pci_piix3_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn);
 PCIDevice *pci_piix4_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn);
 void vt82c686b_ide_init(PCIBus *bus, DriveInfo **hd_table, int devfn);
 
-/* ide-macio.c */
-MemoryRegion *pmac_ide_init (DriveInfo **hd_table, qemu_irq irq,
-		   void *dbdma, int channel, qemu_irq dma_irq);
-
 /* ide-mmio.c */
-void mmio_ide_init (hwaddr membase, hwaddr membase2,
-                    MemoryRegion *address_space,
-                    qemu_irq irq, int shift,
-                    DriveInfo *hd0, DriveInfo *hd1);
+void mmio_ide_init_drives(DeviceState *dev, DriveInfo *hd0, DriveInfo *hd1);
 
 int ide_get_geometry(BusState *bus, int unit,
                      int16_t *cyls, int8_t *heads, int8_t *secs);
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 67562db..ad0094f 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -22,14 +22,14 @@
  */
 
 #include <hw/hw.h>
-#include <hw/msi.h>
+#include <hw/pci/msi.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/sysbus.h>
 
-#include "monitor.h"
-#include "dma.h"
-#include "cpu-common.h"
+#include "monitor/monitor.h"
+#include "sysemu/dma.h"
+#include "exec/cpu-common.h"
 #include "internal.h"
 #include <hw/ide/pci.h>
 #include <hw/ide/ahci.h>
@@ -241,7 +241,7 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
             if ((pr->cmd & PORT_CMD_FIS_ON) &&
                 !s->dev[port].init_d2h_sent) {
                 ahci_init_d2h(&s->dev[port]);
-                s->dev[port].init_d2h_sent = 1;
+                s->dev[port].init_d2h_sent = true;
             }
 
             check_cmd(s, port);
@@ -494,7 +494,7 @@ static void ahci_reset_port(AHCIState *s, int port)
     pr->scr_err = 0;
     pr->scr_act = 0;
     d->busy_slot = -1;
-    d->init_d2h_sent = 0;
+    d->init_d2h_sent = false;
 
     ide_state = &s->dev[port].port.ifs[0];
     if (!ide_state->bs) {
@@ -946,7 +946,7 @@ static int handle_cmd(AHCIState *s, int port, int slot)
             ide_state->hcyl = 0xeb;
             debug_print_fis(ide_state->io_buffer, 0x10);
             ide_state->feature = IDE_FEATURE_DMA;
-            s->dev[port].done_atapi_packet = 0;
+            s->dev[port].done_atapi_packet = false;
             /* XXX send PIO setup FIS */
         }
 
@@ -991,7 +991,7 @@ static int ahci_start_transfer(IDEDMA *dma)
 
     if (is_atapi && !ad->done_atapi_packet) {
         /* already prepopulated iobuffer */
-        ad->done_atapi_packet = 1;
+        ad->done_atapi_packet = true;
         goto out;
     }
 
@@ -1035,11 +1035,10 @@ out:
 static void ahci_start_dma(IDEDMA *dma, IDEState *s,
                            BlockDriverCompletionFunc *dma_cb)
 {
+#ifdef DEBUG_AHCI
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-
+#endif
     DPRINTF(ad->port_no, "\n");
-    ad->dma_cb = dma_cb;
-    ad->dma_status |= BM_STATUS_DMAING;
     s->io_buffer_offset = 0;
     dma_cb(s, 0);
 }
@@ -1095,7 +1094,6 @@ static int ahci_dma_set_unit(IDEDMA *dma, int unit)
 static int ahci_dma_add_status(IDEDMA *dma, int status)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-    ad->dma_status |= status;
     DPRINTF(ad->port_no, "set status: %x\n", status);
 
     if (status & BM_STATUS_INT) {
@@ -1114,8 +1112,6 @@ static int ahci_dma_set_inactive(IDEDMA *dma)
     /* update d2h status */
     ahci_write_fis_d2h(ad, NULL);
 
-    ad->dma_cb = NULL;
-
     if (!ad->check_bh) {
         /* maybe we still have something to process, check later */
         ad->check_bh = qemu_bh_new(ahci_check_cmd_bh, ad);
@@ -1203,6 +1199,82 @@ void ahci_reset(AHCIState *s)
     }
 }
 
+static const VMStateDescription vmstate_ahci_device = {
+    .name = "ahci port",
+    .version_id = 1,
+    .fields = (VMStateField []) {
+        VMSTATE_IDE_BUS(port, AHCIDevice),
+        VMSTATE_UINT32(port_state, AHCIDevice),
+        VMSTATE_UINT32(finished, AHCIDevice),
+        VMSTATE_UINT32(port_regs.lst_addr, AHCIDevice),
+        VMSTATE_UINT32(port_regs.lst_addr_hi, AHCIDevice),
+        VMSTATE_UINT32(port_regs.fis_addr, AHCIDevice),
+        VMSTATE_UINT32(port_regs.fis_addr_hi, AHCIDevice),
+        VMSTATE_UINT32(port_regs.irq_stat, AHCIDevice),
+        VMSTATE_UINT32(port_regs.irq_mask, AHCIDevice),
+        VMSTATE_UINT32(port_regs.cmd, AHCIDevice),
+        VMSTATE_UINT32(port_regs.tfdata, AHCIDevice),
+        VMSTATE_UINT32(port_regs.sig, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_stat, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_ctl, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_err, AHCIDevice),
+        VMSTATE_UINT32(port_regs.scr_act, AHCIDevice),
+        VMSTATE_UINT32(port_regs.cmd_issue, AHCIDevice),
+        VMSTATE_BOOL(done_atapi_packet, AHCIDevice),
+        VMSTATE_INT32(busy_slot, AHCIDevice),
+        VMSTATE_BOOL(init_d2h_sent, AHCIDevice),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static int ahci_state_post_load(void *opaque, int version_id)
+{
+    int i;
+    struct AHCIDevice *ad;
+    AHCIState *s = opaque;
+
+    for (i = 0; i < s->ports; i++) {
+        ad = &s->dev[i];
+        AHCIPortRegs *pr = &ad->port_regs;
+
+        map_page(&ad->lst,
+                 ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
+        map_page(&ad->res_fis,
+                 ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
+        /*
+         * All pending i/o should be flushed out on a migrate. However,
+         * we might not have cleared the busy_slot since this is done
+         * in a bh. Also, issue i/o against any slots that are pending.
+         */
+        if ((ad->busy_slot != -1) &&
+            !(ad->port.ifs[0].status & (BUSY_STAT|DRQ_STAT))) {
+            pr->cmd_issue &= ~(1 << ad->busy_slot);
+            ad->busy_slot = -1;
+        }
+        check_cmd(s, i);
+    }
+
+    return 0;
+}
+
+const VMStateDescription vmstate_ahci = {
+    .name = "ahci",
+    .version_id = 1,
+    .post_load = ahci_state_post_load,
+    .fields = (VMStateField []) {
+        VMSTATE_STRUCT_VARRAY_POINTER_INT32(dev, AHCIState, ports,
+                                     vmstate_ahci_device, AHCIDevice),
+        VMSTATE_UINT32(control_regs.cap, AHCIState),
+        VMSTATE_UINT32(control_regs.ghc, AHCIState),
+        VMSTATE_UINT32(control_regs.irqstatus, AHCIState),
+        VMSTATE_UINT32(control_regs.impl, AHCIState),
+        VMSTATE_UINT32(control_regs.version, AHCIState),
+        VMSTATE_UINT32(idp_index, AHCIState),
+        VMSTATE_INT32(ports, AHCIState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 typedef struct SysbusAHCIState {
     SysBusDevice busdev;
     AHCIState ahci;
@@ -1211,7 +1283,11 @@ typedef struct SysbusAHCIState {
 
 static const VMStateDescription vmstate_sysbus_ahci = {
     .name = "sysbus-ahci",
-    .unmigratable = 1,
+    .unmigratable = 1, /* Still buggy under I/O load */
+    .fields = (VMStateField []) {
+        VMSTATE_AHCI(ahci, AHCIPCIState),
+        VMSTATE_END_OF_LIST()
+    },
 };
 
 static void sysbus_ahci_reset(DeviceState *dev)
@@ -1247,7 +1323,7 @@ static void sysbus_ahci_class_init(ObjectClass *klass, void *data)
     dc->reset = sysbus_ahci_reset;
 }
 
-static TypeInfo sysbus_ahci_info = {
+static const TypeInfo sysbus_ahci_info = {
     .name          = "sysbus-ahci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysbusAHCIState),
diff --git a/hw/ide/ahci.h b/hw/ide/ahci.h
index 1200a56..85f37fe 100644
--- a/hw/ide/ahci.h
+++ b/hw/ide/ahci.h
@@ -281,11 +281,9 @@ struct AHCIDevice {
     QEMUBH *check_bh;
     uint8_t *lst;
     uint8_t *res_fis;
-    int dma_status;
-    int done_atapi_packet;
-    int busy_slot;
-    int init_d2h_sent;
-    BlockDriverCompletionFunc *dma_cb;
+    bool done_atapi_packet;
+    int32_t busy_slot;
+    bool init_d2h_sent;
     AHCICmdHdr *cur_cmd;
     NCQTransferState ncq_tfs[AHCI_MAX_CMDS];
 };
@@ -297,7 +295,7 @@ typedef struct AHCIState {
     MemoryRegion idp;       /* Index-Data Pair I/O port space */
     unsigned idp_offset;    /* Offset of index in I/O port space */
     uint32_t idp_index;     /* Current IDP index */
-    int ports;
+    int32_t ports;
     qemu_irq irq;
     DMAContext *dma;
 } AHCIState;
@@ -307,6 +305,16 @@ typedef struct AHCIPCIState {
     AHCIState ahci;
 } AHCIPCIState;
 
+extern const VMStateDescription vmstate_ahci;
+
+#define VMSTATE_AHCI(_field, _state) {                               \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(AHCIState),                                 \
+    .vmsd       = &vmstate_ahci,                                     \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, AHCIState),   \
+}
+
 typedef struct NCQFrame {
     uint8_t fis_type;
     uint8_t c;
diff --git a/hw/ide/cmd646.c b/hw/ide/cmd646.c
index 804db60..745ef94 100644
--- a/hw/ide/cmd646.c
+++ b/hw/ide/cmd646.c
@@ -24,11 +24,11 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
@@ -342,7 +342,7 @@ static void cmd646_ide_class_init(ObjectClass *klass, void *data)
     dc->props = cmd646_ide_properties;
 }
 
-static TypeInfo cmd646_ide_info = {
+static const TypeInfo cmd646_ide_info = {
     .name          = "cmd646-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
diff --git a/hw/ide/core.c b/hw/ide/core.c
index c4f93d0..3743dc3 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -24,14 +24,14 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "qemu-error.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 #include "hw/block-common.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 #include <hw/ide/internal.h>
 
@@ -325,14 +325,26 @@ typedef struct TrimAIOCB {
     BlockDriverAIOCB common;
     QEMUBH *bh;
     int ret;
+    QEMUIOVector *qiov;
+    BlockDriverAIOCB *aiocb;
+    int i, j;
 } TrimAIOCB;
 
 static void trim_aio_cancel(BlockDriverAIOCB *acb)
 {
     TrimAIOCB *iocb = container_of(acb, TrimAIOCB, common);
 
+    /* Exit the loop in case bdrv_aio_cancel calls ide_issue_trim_cb again.  */
+    iocb->j = iocb->qiov->niov - 1;
+    iocb->i = (iocb->qiov->iov[iocb->j].iov_len / 8) - 1;
+
+    /* Tell ide_issue_trim_cb not to trigger the completion, too.  */
     qemu_bh_delete(iocb->bh);
     iocb->bh = NULL;
+
+    if (iocb->aiocb) {
+        bdrv_aio_cancel(iocb->aiocb);
+    }
     qemu_aio_release(iocb);
 }
 
@@ -349,43 +361,60 @@ static void ide_trim_bh_cb(void *opaque)
 
     qemu_bh_delete(iocb->bh);
     iocb->bh = NULL;
-
     qemu_aio_release(iocb);
 }
 
+static void ide_issue_trim_cb(void *opaque, int ret)
+{
+    TrimAIOCB *iocb = opaque;
+    if (ret >= 0) {
+        while (iocb->j < iocb->qiov->niov) {
+            int j = iocb->j;
+            while (++iocb->i < iocb->qiov->iov[j].iov_len / 8) {
+                int i = iocb->i;
+                uint64_t *buffer = iocb->qiov->iov[j].iov_base;
+
+                /* 6-byte LBA + 2-byte range per entry */
+                uint64_t entry = le64_to_cpu(buffer[i]);
+                uint64_t sector = entry & 0x0000ffffffffffffULL;
+                uint16_t count = entry >> 48;
+
+                if (count == 0) {
+                    continue;
+                }
+
+                /* Got an entry! Submit and exit.  */
+                iocb->aiocb = bdrv_aio_discard(iocb->common.bs, sector, count,
+                                               ide_issue_trim_cb, opaque);
+                return;
+            }
+
+            iocb->j++;
+            iocb->i = -1;
+        }
+    } else {
+        iocb->ret = ret;
+    }
+
+    iocb->aiocb = NULL;
+    if (iocb->bh) {
+        qemu_bh_schedule(iocb->bh);
+    }
+}
+
 BlockDriverAIOCB *ide_issue_trim(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque)
 {
     TrimAIOCB *iocb;
-    int i, j, ret;
 
     iocb = qemu_aio_get(&trim_aiocb_info, bs, cb, opaque);
     iocb->bh = qemu_bh_new(ide_trim_bh_cb, iocb);
     iocb->ret = 0;
-
-    for (j = 0; j < qiov->niov; j++) {
-        uint64_t *buffer = qiov->iov[j].iov_base;
-
-        for (i = 0; i < qiov->iov[j].iov_len / 8; i++) {
-            /* 6-byte LBA + 2-byte range per entry */
-            uint64_t entry = le64_to_cpu(buffer[i]);
-            uint64_t sector = entry & 0x0000ffffffffffffULL;
-            uint16_t count = entry >> 48;
-
-            if (count == 0) {
-                break;
-            }
-
-            ret = bdrv_discard(bs, sector, count);
-            if (!iocb->ret) {
-                iocb->ret = ret;
-            }
-        }
-    }
-
-    qemu_bh_schedule(iocb->bh);
-
+    iocb->qiov = qiov;
+    iocb->i = -1;
+    iocb->j = 0;
+    ide_issue_trim_cb(iocb, 0);
     return &iocb->common;
 }
 
@@ -1120,8 +1149,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         }
         ide_set_irq(s->bus);
         break;
+
     case WIN_VERIFY_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_VERIFY:
     case WIN_VERIFY_ONCE:
         /* do sector number check ? */
@@ -1129,8 +1160,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         s->status = READY_STAT | SEEK_STAT;
         ide_set_irq(s->bus);
         break;
+
     case WIN_READ_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_READ:
     case WIN_READ_ONCE:
         if (s->drive_kind == IDE_CD) {
@@ -1144,8 +1177,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         s->req_nb_sectors = 1;
         ide_sector_read(s);
         break;
+
     case WIN_WRITE_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_WRITE:
     case WIN_WRITE_ONCE:
     case CFA_WRITE_SECT_WO_ERASE:
@@ -1160,8 +1195,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         ide_transfer_start(s, s->io_buffer, 512, ide_sector_write);
         s->media_changed = 1;
         break;
+
     case WIN_MULTREAD_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_MULTREAD:
         if (!s->bs) {
             goto abort_cmd;
@@ -1173,8 +1210,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         s->req_nb_sectors = s->mult_sectors;
         ide_sector_read(s);
         break;
+
     case WIN_MULTWRITE_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_MULTWRITE:
     case CFA_WRITE_MULTI_WO_ERASE:
         if (!s->bs) {
@@ -1193,8 +1232,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         ide_transfer_start(s, s->io_buffer, 512 * n, ide_sector_write);
         s->media_changed = 1;
         break;
+
     case WIN_READDMA_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_READDMA:
     case WIN_READDMA_ONCE:
         if (!s->bs) {
@@ -1203,8 +1244,10 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 	ide_cmd_lba48_transform(s, lba48);
         ide_sector_start_dma(s, IDE_DMA_READ);
         break;
+
     case WIN_WRITEDMA_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_WRITEDMA:
     case WIN_WRITEDMA_ONCE:
         if (!s->bs) {
@@ -1214,14 +1257,17 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         ide_sector_start_dma(s, IDE_DMA_WRITE);
         s->media_changed = 1;
         break;
+
     case WIN_READ_NATIVE_MAX_EXT:
-	lba48 = 1;
+        lba48 = 1;
+        /* fall through */
     case WIN_READ_NATIVE_MAX:
 	ide_cmd_lba48_transform(s, lba48);
         ide_set_sector(s, s->nb_sectors - 1);
         s->status = READY_STAT | SEEK_STAT;
         ide_set_irq(s->bus);
         break;
+
     case WIN_CHECKPOWERMODE1:
     case WIN_CHECKPOWERMODE2:
         s->error = 0;
@@ -1869,6 +1915,8 @@ static void ide_reset(IDEState *s)
     s->io_buffer_index = 0;
     s->cd_sector_size = 0;
     s->atapi_dma = 0;
+    s->tray_locked = 0;
+    s->tray_open = 0;
     /* ATA DMA state */
     s->io_buffer_size = 0;
     s->req_nb_sectors = 0;
diff --git a/hw/ide/ich.c b/hw/ide/ich.c
index 272b773..cc30adc 100644
--- a/hw/ide/ich.c
+++ b/hw/ide/ich.c
@@ -61,12 +61,12 @@
  */
 
 #include <hw/hw.h>
-#include <hw/msi.h>
+#include <hw/pci/msi.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 #include <hw/ide/ahci.h>
@@ -79,9 +79,15 @@
 #define ICH9_IDP_INDEX          0x10
 #define ICH9_IDP_INDEX_LOG2     0x04
 
-static const VMStateDescription vmstate_ahci = {
-    .name = "ahci",
-    .unmigratable = 1,
+static const VMStateDescription vmstate_ich9_ahci = {
+    .name = "ich9_ahci",
+    .unmigratable = 1, /* Still buggy under I/O load */
+    .version_id = 1,
+    .fields = (VMStateField []) {
+        VMSTATE_PCI_DEVICE(card, AHCIPCIState),
+        VMSTATE_AHCI(ahci, AHCIPCIState),
+        VMSTATE_END_OF_LIST()
+    },
 };
 
 static void pci_ich9_reset(DeviceState *dev)
@@ -152,11 +158,11 @@ static void ich_ahci_class_init(ObjectClass *klass, void *data)
     k->device_id = PCI_DEVICE_ID_INTEL_82801IR;
     k->revision = 0x02;
     k->class_id = PCI_CLASS_STORAGE_SATA;
-    dc->vmsd = &vmstate_ahci;
+    dc->vmsd = &vmstate_ich9_ahci;
     dc->reset = pci_ich9_reset;
 }
 
-static TypeInfo ich_ahci_info = {
+static const TypeInfo ich_ahci_info = {
     .name          = "ich9-ahci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(AHCIPCIState),
diff --git a/hw/ide/internal.h b/hw/ide/internal.h
index bf7d313..d80360e 100644
--- a/hw/ide/internal.h
+++ b/hw/ide/internal.h
@@ -8,9 +8,9 @@
  */
 #include <hw/ide.h>
 #include <hw/isa.h>
-#include "iorange.h"
-#include "dma.h"
-#include "sysemu.h"
+#include "exec/iorange.h"
+#include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
 #include "hw/block-common.h"
 #include "hw/scsi-defs.h"
 
diff --git a/hw/ide/isa.c b/hw/ide/isa.c
index 8ab2718..fb7bb82 100644
--- a/hw/ide/isa.c
+++ b/hw/ide/isa.c
@@ -25,8 +25,8 @@
 #include <hw/hw.h>
 #include <hw/pc.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
@@ -111,7 +111,7 @@ static void isa_ide_class_initfn(ObjectClass *klass, void *data)
     dc->props = isa_ide_properties;
 }
 
-static TypeInfo isa_ide_info = {
+static const TypeInfo isa_ide_info = {
     .name          = "isa-ide",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAIDEState),
diff --git a/hw/ide/macio.c b/hw/ide/macio.c
index d2edcc0..375c46f 100644
--- a/hw/ide/macio.c
+++ b/hw/ide/macio.c
@@ -22,23 +22,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/ppc_mac.h>
-#include <hw/mac_dbdma.h>
-#include "block.h"
-#include "dma.h"
+#include "hw/hw.h"
+#include "hw/ppc/mac.h"
+#include "hw/mac_dbdma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
 /***********************************************************/
 /* MacIO based PowerPC IDE */
 
-typedef struct MACIOIDEState {
-    MemoryRegion mem;
-    IDEBus bus;
-    BlockDriverAIOCB *aiocb;
-} MACIOIDEState;
-
 #define MACIO_PAGE_SIZE 4096
 
 static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
@@ -321,30 +315,70 @@ static const VMStateDescription vmstate_pmac = {
     }
 };
 
-static void pmac_ide_reset(void *opaque)
+static void macio_ide_reset(DeviceState *dev)
 {
-    MACIOIDEState *d = opaque;
+    MACIOIDEState *d = MACIO_IDE(dev);
 
     ide_bus_reset(&d->bus);
 }
 
-/* hd_table must contain 4 block drivers */
-/* PowerMac uses memory mapped registers, not I/O. Return the memory
-   I/O index to access the ide. */
-MemoryRegion *pmac_ide_init (DriveInfo **hd_table, qemu_irq irq,
-                             void *dbdma, int channel, qemu_irq dma_irq)
+static void macio_ide_realizefn(DeviceState *dev, Error **errp)
 {
-    MACIOIDEState *d;
+    MACIOIDEState *s = MACIO_IDE(dev);
+
+    ide_init2(&s->bus, s->irq);
+}
+
+static void macio_ide_initfn(Object *obj)
+{
+    SysBusDevice *d = SYS_BUS_DEVICE(obj);
+    MACIOIDEState *s = MACIO_IDE(obj);
+
+    ide_bus_new(&s->bus, DEVICE(obj), 0);
+    memory_region_init_io(&s->mem, &pmac_ide_ops, s, "pmac-ide", 0x1000);
+    sysbus_init_mmio(d, &s->mem);
+    sysbus_init_irq(d, &s->irq);
+    sysbus_init_irq(d, &s->dma_irq);
+}
+
+static void macio_ide_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = macio_ide_realizefn;
+    dc->reset = macio_ide_reset;
+    dc->vmsd = &vmstate_pmac;
+}
 
-    d = g_malloc0(sizeof(MACIOIDEState));
-    ide_init2_with_non_qdev_drives(&d->bus, hd_table[0], hd_table[1], irq);
+static const TypeInfo macio_ide_type_info = {
+    .name = TYPE_MACIO_IDE,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MACIOIDEState),
+    .instance_init = macio_ide_initfn,
+    .class_init = macio_ide_class_init,
+};
 
-    if (dbdma)
-        DBDMA_register_channel(dbdma, channel, dma_irq, pmac_ide_transfer, pmac_ide_flush, d);
+static void macio_ide_register_types(void)
+{
+    type_register_static(&macio_ide_type_info);
+}
 
-    memory_region_init_io(&d->mem, &pmac_ide_ops, d, "pmac-ide", 0x1000);
-    vmstate_register(NULL, 0, &vmstate_pmac, d);
-    qemu_register_reset(pmac_ide_reset, d);
+/* hd_table must contain 4 block drivers */
+void macio_ide_init_drives(MACIOIDEState *s, DriveInfo **hd_table)
+{
+    int i;
 
-    return &d->mem;
+    for (i = 0; i < 2; i++) {
+        if (hd_table[i]) {
+            ide_create_drive(&s->bus, i, hd_table[i]);
+        }
+    }
 }
+
+void macio_ide_register_dma(MACIOIDEState *s, void *dbdma, int channel)
+{
+    DBDMA_register_channel(dbdma, channel, s->dma_irq,
+                           pmac_ide_transfer, pmac_ide_flush, s);
+}
+
+type_init(macio_ide_register_types)
diff --git a/hw/ide/microdrive.c b/hw/ide/microdrive.c
index 9eee5b5..642774e 100644
--- a/hw/ide/microdrive.c
+++ b/hw/ide/microdrive.c
@@ -25,8 +25,8 @@
 #include <hw/hw.h>
 #include <hw/pc.h>
 #include <hw/pcmcia.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
diff --git a/hw/ide/mmio.c b/hw/ide/mmio.c
index bcb26c8..ce88c3a 100644
--- a/hw/ide/mmio.c
+++ b/hw/ide/mmio.c
@@ -22,9 +22,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include "block.h"
-#include "dma.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/internal.h>
 
@@ -34,15 +35,24 @@
  * dedicated ide controller, which is often seen on embedded boards.
  */
 
-typedef struct {
+#define TYPE_MMIO_IDE "mmio-ide"
+#define MMIO_IDE(obj) OBJECT_CHECK(MMIOState, (obj), TYPE_MMIO_IDE)
+
+typedef struct MMIOIDEState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
     IDEBus bus;
-    int shift;
+
+    uint32_t shift;
+    qemu_irq irq;
     MemoryRegion iomem1, iomem2;
 } MMIOState;
 
-static void mmio_ide_reset(void *opaque)
+static void mmio_ide_reset(DeviceState *dev)
 {
-    MMIOState *s = opaque;
+    MMIOState *s = MMIO_IDE(dev);
 
     ide_bus_reset(&s->bus);
 }
@@ -107,24 +117,68 @@ static const VMStateDescription vmstate_ide_mmio = {
     }
 };
 
-void mmio_ide_init (hwaddr membase, hwaddr membase2,
-                    MemoryRegion *address_space,
-                    qemu_irq irq, int shift,
-                    DriveInfo *hd0, DriveInfo *hd1)
+static void mmio_ide_realizefn(DeviceState *dev, Error **errp)
 {
-    MMIOState *s = g_malloc0(sizeof(MMIOState));
+    SysBusDevice *d = SYS_BUS_DEVICE(dev);
+    MMIOState *s = MMIO_IDE(dev);
 
-    ide_init2_with_non_qdev_drives(&s->bus, hd0, hd1, irq);
-
-    s->shift = shift;
+    ide_init2(&s->bus, s->irq);
 
     memory_region_init_io(&s->iomem1, &mmio_ide_ops, s,
-                          "ide-mmio.1", 16 << shift);
+                          "ide-mmio.1", 16 << s->shift);
     memory_region_init_io(&s->iomem2, &mmio_ide_cs_ops, s,
-                          "ide-mmio.2", 2 << shift);
-    memory_region_add_subregion(address_space, membase, &s->iomem1);
-    memory_region_add_subregion(address_space, membase2, &s->iomem2);
-    vmstate_register(NULL, 0, &vmstate_ide_mmio, s);
-    qemu_register_reset(mmio_ide_reset, s);
+                          "ide-mmio.2", 2 << s->shift);
+    sysbus_init_mmio(d, &s->iomem1);
+    sysbus_init_mmio(d, &s->iomem2);
+}
+
+static void mmio_ide_initfn(Object *obj)
+{
+    SysBusDevice *d = SYS_BUS_DEVICE(obj);
+    MMIOState *s = MMIO_IDE(obj);
+
+    ide_bus_new(&s->bus, DEVICE(obj), 0);
+    sysbus_init_irq(d, &s->irq);
+}
+
+static Property mmio_ide_properties[] = {
+    DEFINE_PROP_UINT32("shift", MMIOState, shift, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void mmio_ide_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = mmio_ide_realizefn;
+    dc->reset = mmio_ide_reset;
+    dc->props = mmio_ide_properties;
+    dc->vmsd = &vmstate_ide_mmio;
+}
+
+static const TypeInfo mmio_ide_type_info = {
+    .name = TYPE_MMIO_IDE,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MMIOState),
+    .instance_init = mmio_ide_initfn,
+    .class_init = mmio_ide_class_init,
+};
+
+static void mmio_ide_register_types(void)
+{
+    type_register_static(&mmio_ide_type_info);
+}
+
+void mmio_ide_init_drives(DeviceState *dev, DriveInfo *hd0, DriveInfo *hd1)
+{
+    MMIOState *s = MMIO_IDE(dev);
+
+    if (hd0 != NULL) {
+        ide_create_drive(&s->bus, 0, hd0);
+    }
+    if (hd1 != NULL) {
+        ide_create_drive(&s->bus, 1, hd1);
+    }
 }
 
+type_init(mmio_ide_register_types)
diff --git a/hw/ide/pci.c b/hw/ide/pci.c
index bcdd70e..59fd539 100644
--- a/hw/ide/pci.c
+++ b/hw/ide/pci.c
@@ -24,10 +24,10 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
@@ -311,7 +311,6 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
             if (bm->bus->dma->aiocb) {
                 bdrv_drain_all();
                 assert(bm->bus->dma->aiocb == NULL);
-                assert((bm->status & BM_STATUS_DMAING) == 0);
             }
         } else {
             bm->cur_addr = bm->addr;
diff --git a/hw/ide/piix.c b/hw/ide/piix.c
index 9431bad..4d3e822 100644
--- a/hw/ide/piix.c
+++ b/hw/ide/piix.c
@@ -25,11 +25,11 @@
 
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "blockdev.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
@@ -251,7 +251,7 @@ static void piix3_ide_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo piix3_ide_info = {
+static const TypeInfo piix3_ide_info = {
     .name          = "piix3-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
@@ -271,7 +271,7 @@ static void piix3_ide_xen_class_init(ObjectClass *klass, void *data)
     dc->unplug = pci_piix3_xen_ide_unplug;
 }
 
-static TypeInfo piix3_ide_xen_info = {
+static const TypeInfo piix3_ide_xen_info = {
     .name          = "piix3-ide-xen",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
@@ -292,7 +292,7 @@ static void piix4_ide_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo piix4_ide_info = {
+static const TypeInfo piix4_ide_info = {
     .name          = "piix4-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index f2e4ea4..c436b38 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -17,12 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include <hw/hw.h>
-#include "dma.h"
-#include "qemu-error.h"
+#include "sysemu/dma.h"
+#include "qemu/error-report.h"
 #include <hw/ide/internal.h>
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* --------------------------------- */
 
@@ -216,7 +216,7 @@ static void ide_hd_class_init(ObjectClass *klass, void *data)
     dc->props = ide_hd_properties;
 }
 
-static TypeInfo ide_hd_info = {
+static const TypeInfo ide_hd_info = {
     .name          = "ide-hd",
     .parent        = TYPE_IDE_DEVICE,
     .instance_size = sizeof(IDEDrive),
@@ -238,7 +238,7 @@ static void ide_cd_class_init(ObjectClass *klass, void *data)
     dc->props = ide_cd_properties;
 }
 
-static TypeInfo ide_cd_info = {
+static const TypeInfo ide_cd_info = {
     .name          = "ide-cd",
     .parent        = TYPE_IDE_DEVICE,
     .instance_size = sizeof(IDEDrive),
@@ -260,7 +260,7 @@ static void ide_drive_class_init(ObjectClass *klass, void *data)
     dc->props = ide_drive_properties;
 }
 
-static TypeInfo ide_drive_info = {
+static const TypeInfo ide_drive_info = {
     .name          = "ide-drive",
     .parent        = TYPE_IDE_DEVICE,
     .instance_size = sizeof(IDEDrive),
@@ -275,7 +275,7 @@ static void ide_device_class_init(ObjectClass *klass, void *data)
     k->props = ide_props;
 }
 
-static TypeInfo ide_device_type_info = {
+static const TypeInfo ide_device_type_info = {
     .name = TYPE_IDE_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(IDEDevice),
diff --git a/hw/ide/via.c b/hw/ide/via.c
index efda173..f40c1ad 100644
--- a/hw/ide/via.c
+++ b/hw/ide/via.c
@@ -25,11 +25,11 @@
  */
 #include <hw/hw.h>
 #include <hw/pc.h>
-#include <hw/pci.h>
+#include <hw/pci/pci.h>
 #include <hw/isa.h>
-#include "block.h"
-#include "sysemu.h"
-#include "dma.h"
+#include "block/block.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 
 #include <hw/ide/pci.h>
 
@@ -226,7 +226,7 @@ static void via_ide_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo via_ide_info = {
+static const TypeInfo via_ide_info = {
     .name          = "via-ide",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIIDEState),
diff --git a/hw/imx_avic.c b/hw/imx_avic.c
index 8109793..f1f066c 100644
--- a/hw/imx_avic.c
+++ b/hw/imx_avic.c
@@ -16,7 +16,7 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #define DEBUG_INT 1
 #undef DEBUG_INT /* comment out for debugging */
diff --git a/hw/imx_ccm.c b/hw/imx_ccm.c
index f2e623c..477903a 100644
--- a/hw/imx_ccm.c
+++ b/hw/imx_ccm.c
@@ -12,7 +12,7 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "imx.h"
 
 #define CKIH_FREQ 26000000 /* 26MHz crystal input */
@@ -306,7 +306,7 @@ static void imx_ccm_class_init(ObjectClass *klass, void *data)
     dc->desc = "i.MX Clock Control Module";
 }
 
-static TypeInfo imx_ccm_info = {
+static const TypeInfo imx_ccm_info = {
     .name = "imx_ccm",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXCCMState),
diff --git a/hw/imx_serial.c b/hw/imx_serial.c
index dcd125f..2d8253e 100644
--- a/hw/imx_serial.c
+++ b/hw/imx_serial.c
@@ -19,8 +19,8 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 #include "imx.h"
 
 //#define DEBUG_SERIAL 1
@@ -425,7 +425,7 @@ void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq)
     }
 
     qdev_prop_set_chr(dev, "chardev", chr);
-    bus = sysbus_from_qdev(dev);
+    bus = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     if (addr != (hwaddr)-1) {
         sysbus_mmio_map(bus, 0, addr);
@@ -452,7 +452,7 @@ static void imx_serial_class_init(ObjectClass *klass, void *data)
     dc->props = imx32_serial_properties;
 }
 
-static TypeInfo imx_serial_info = {
+static const TypeInfo imx_serial_info = {
     .name = "imx-serial",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXSerialState),
diff --git a/hw/imx_timer.c b/hw/imx_timer.c
index 33f33fb..e924c74 100644
--- a/hw/imx_timer.c
+++ b/hw/imx_timer.c
@@ -12,7 +12,7 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 #include "sysbus.h"
 #include "imx.h"
diff --git a/hw/integratorcp.c b/hw/integratorcp.c
index 77807c3..9e3630a 100644
--- a/hw/integratorcp.c
+++ b/hw/integratorcp.c
@@ -11,9 +11,9 @@
 #include "devices.h"
 #include "boards.h"
 #include "arm-misc.h"
-#include "net.h"
-#include "exec-memory.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 
 typedef struct {
     SysBusDevice busdev;
@@ -512,6 +512,7 @@ static QEMUMachine integratorcp_machine = {
     .desc = "ARM Integrator/CP (ARM926EJ-S)",
     .init = integratorcp_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void integratorcp_machine_init(void)
@@ -535,7 +536,7 @@ static void core_class_init(ObjectClass *klass, void *data)
     dc->props = core_properties;
 }
 
-static TypeInfo core_info = {
+static const TypeInfo core_info = {
     .name          = "integrator_core",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(integratorcm_state),
@@ -549,7 +550,7 @@ static void icp_pic_class_init(ObjectClass *klass, void *data)
     sdc->init = icp_pic_init;
 }
 
-static TypeInfo icp_pic_info = {
+static const TypeInfo icp_pic_info = {
     .name          = "integrator_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pic_state),
diff --git a/hw/intel-hda.c b/hw/intel-hda.c
index a68c368..784c229 100644
--- a/hw/intel-hda.c
+++ b/hw/intel-hda.c
@@ -18,13 +18,13 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "msi.h"
-#include "qemu-timer.h"
+#include "pci/pci.h"
+#include "pci/msi.h"
+#include "qemu/timer.h"
 #include "audiodev.h"
 #include "intel-hda.h"
 #include "intel-hda-defs.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 /* --------------------------------------------------------------------- */
 /* hda bus                                                               */
@@ -1232,7 +1232,7 @@ static Property intel_hda_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void intel_hda_class_init(ObjectClass *klass, void *data)
+static void intel_hda_class_init_common(ObjectClass *klass)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
@@ -1240,20 +1240,46 @@ static void intel_hda_class_init(ObjectClass *klass, void *data)
     k->init = intel_hda_init;
     k->exit = intel_hda_exit;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = 0x2668;
-    k->revision = 1;
     k->class_id = PCI_CLASS_MULTIMEDIA_HD_AUDIO;
-    dc->desc = "Intel HD Audio Controller";
     dc->reset = intel_hda_reset;
     dc->vmsd = &vmstate_intel_hda;
     dc->props = intel_hda_properties;
 }
 
-static TypeInfo intel_hda_info = {
+static void intel_hda_class_init_ich6(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    intel_hda_class_init_common(klass);
+    k->device_id = 0x2668;
+    k->revision = 1;
+    dc->desc = "Intel HD Audio Controller (ich6)";
+}
+
+static void intel_hda_class_init_ich9(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    intel_hda_class_init_common(klass);
+    k->device_id = 0x293e;
+    k->revision = 3;
+    dc->desc = "Intel HD Audio Controller (ich9)";
+}
+
+static const TypeInfo intel_hda_info_ich6 = {
     .name          = "intel-hda",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(IntelHDAState),
-    .class_init    = intel_hda_class_init,
+    .class_init    = intel_hda_class_init_ich6,
+};
+
+static const TypeInfo intel_hda_info_ich9 = {
+    .name          = "ich9-intel-hda",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(IntelHDAState),
+    .class_init    = intel_hda_class_init_ich9,
 };
 
 static void hda_codec_device_class_init(ObjectClass *klass, void *data)
@@ -1265,7 +1291,7 @@ static void hda_codec_device_class_init(ObjectClass *klass, void *data)
     k->props = hda_props;
 }
 
-static TypeInfo hda_codec_device_type_info = {
+static const TypeInfo hda_codec_device_type_info = {
     .name = TYPE_HDA_CODEC_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(HDACodecDevice),
@@ -1277,7 +1303,8 @@ static TypeInfo hda_codec_device_type_info = {
 static void intel_hda_register_types(void)
 {
     type_register_static(&hda_codec_bus_info);
-    type_register_static(&intel_hda_info);
+    type_register_static(&intel_hda_info_ich6);
+    type_register_static(&intel_hda_info_ich9);
     type_register_static(&hda_codec_device_type_info);
 }
 
diff --git a/hw/ioapic.c b/hw/ioapic.c
index 7273095..f06c2dc 100644
--- a/hw/ioapic.c
+++ b/hw/ioapic.c
@@ -244,7 +244,7 @@ static void ioapic_class_init(ObjectClass *klass, void *data)
     dc->reset = ioapic_reset_common;
 }
 
-static TypeInfo ioapic_info = {
+static const TypeInfo ioapic_info = {
     .name          = "ioapic",
     .parent        = TYPE_IOAPIC_COMMON,
     .instance_size = sizeof(IOAPICCommonState),
diff --git a/hw/ioapic_common.c b/hw/ioapic_common.c
index 653eef2..7dc552f 100644
--- a/hw/ioapic_common.c
+++ b/hw/ioapic_common.c
@@ -103,7 +103,7 @@ static void ioapic_common_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo ioapic_common_type = {
+static const TypeInfo ioapic_common_type = {
     .name = TYPE_IOAPIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IOAPICCommonState),
diff --git a/hw/ioapic_internal.h b/hw/ioapic_internal.h
index e04c9f3..c8447d7 100644
--- a/hw/ioapic_internal.h
+++ b/hw/ioapic_internal.h
@@ -23,7 +23,7 @@
 #define QEMU_IOAPIC_INTERNAL_H
 
 #include "hw.h"
-#include "memory.h"
+#include "exec/memory.h"
 #include "sysbus.h"
 
 #define MAX_IOAPICS                     1
diff --git a/hw/ioh3420.c b/hw/ioh3420.c
index 4d31473..95bceb5 100644
--- a/hw/ioh3420.c
+++ b/hw/ioh3420.c
@@ -20,9 +20,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_ids.h"
-#include "msi.h"
-#include "pcie.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/pcie.h"
 #include "ioh3420.h"
 
 #define PCI_DEVICE_ID_IOH_EPORT         0x3420  /* D0:F0 express mode */
@@ -226,7 +226,7 @@ static void ioh3420_class_init(ObjectClass *klass, void *data)
     dc->props = ioh3420_properties;
 }
 
-static TypeInfo ioh3420_info = {
+static const TypeInfo ioh3420_info = {
     .name          = "ioh3420",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIESlot),
diff --git a/hw/ioh3420.h b/hw/ioh3420.h
index 68c523a..046cf2c 100644
--- a/hw/ioh3420.h
+++ b/hw/ioh3420.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_IOH3420_H
 #define QEMU_IOH3420_H
 
-#include "pcie_port.h"
+#include "pci/pcie_port.h"
 
 PCIESlot *ioh3420_init(PCIBus *bus, int devfn, bool multifunction,
                        const char *bus_name, pci_map_irq_fn map_irq,
diff --git a/hw/ipack.c b/hw/ipack.c
new file mode 100644
index 0000000..e15540d
--- /dev/null
+++ b/hw/ipack.c
@@ -0,0 +1,115 @@
+/*
+ * QEMU IndustryPack emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#include "ipack.h"
+
+IPackDevice *ipack_device_find(IPackBus *bus, int32_t slot)
+{
+    BusChild *kid;
+
+    QTAILQ_FOREACH(kid, &BUS(bus)->children, sibling) {
+        DeviceState *qdev = kid->child;
+        IPackDevice *ip = IPACK_DEVICE(qdev);
+        if (ip->slot == slot) {
+            return ip;
+        }
+    }
+    return NULL;
+}
+
+void ipack_bus_new_inplace(IPackBus *bus, DeviceState *parent,
+                           const char *name, uint8_t n_slots,
+                           qemu_irq_handler handler)
+{
+    qbus_create_inplace(&bus->qbus, TYPE_IPACK_BUS, parent, name);
+    bus->n_slots = n_slots;
+    bus->set_irq = handler;
+}
+
+static int ipack_device_dev_init(DeviceState *qdev)
+{
+    IPackBus *bus = IPACK_BUS(qdev_get_parent_bus(qdev));
+    IPackDevice *dev = IPACK_DEVICE(qdev);
+    IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(dev);
+
+    if (dev->slot < 0) {
+        dev->slot = bus->free_slot;
+    }
+    if (dev->slot >= bus->n_slots) {
+        return -1;
+    }
+    bus->free_slot = dev->slot + 1;
+
+    dev->irq = qemu_allocate_irqs(bus->set_irq, dev, 2);
+
+    return k->init(dev);
+}
+
+static int ipack_device_dev_exit(DeviceState *qdev)
+{
+    IPackDevice *dev = IPACK_DEVICE(qdev);
+    IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(dev);
+
+    if (k->exit) {
+        k->exit(dev);
+    }
+
+    qemu_free_irqs(dev->irq);
+
+    return 0;
+}
+
+static Property ipack_device_props[] = {
+    DEFINE_PROP_INT32("slot", IPackDevice, slot, -1),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void ipack_device_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *k = DEVICE_CLASS(klass);
+    k->bus_type = TYPE_IPACK_BUS;
+    k->init = ipack_device_dev_init;
+    k->exit = ipack_device_dev_exit;
+    k->props = ipack_device_props;
+}
+
+const VMStateDescription vmstate_ipack_device = {
+    .name = "ipack_device",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_INT32(slot, IPackDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const TypeInfo ipack_device_info = {
+    .name          = TYPE_IPACK_DEVICE,
+    .parent        = TYPE_DEVICE,
+    .instance_size = sizeof(IPackDevice),
+    .class_size    = sizeof(IPackDeviceClass),
+    .class_init    = ipack_device_class_init,
+    .abstract      = true,
+};
+
+static const TypeInfo ipack_bus_info = {
+    .name = TYPE_IPACK_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(IPackBus),
+};
+
+static void ipack_register_types(void)
+{
+    type_register_static(&ipack_device_info);
+    type_register_static(&ipack_bus_info);
+}
+
+type_init(ipack_register_types)
diff --git a/hw/ipack.h b/hw/ipack.h
new file mode 100644
index 0000000..69e2628
--- /dev/null
+++ b/hw/ipack.h
@@ -0,0 +1,79 @@
+/*
+ * QEMU IndustryPack emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#ifndef QEMU_IPACK_H
+#define QEMU_IPACK_H
+
+#include "qdev.h"
+
+typedef struct IPackBus IPackBus;
+
+#define TYPE_IPACK_BUS "IndustryPack"
+#define IPACK_BUS(obj) OBJECT_CHECK(IPackBus, (obj), TYPE_IPACK_BUS)
+
+struct IPackBus {
+    BusState qbus;
+    /* All fields are private */
+    uint8_t n_slots;
+    uint8_t free_slot;
+    qemu_irq_handler set_irq;
+};
+
+typedef struct IPackDevice IPackDevice;
+typedef struct IPackDeviceClass IPackDeviceClass;
+
+#define TYPE_IPACK_DEVICE "ipack-device"
+#define IPACK_DEVICE(obj) \
+     OBJECT_CHECK(IPackDevice, (obj), TYPE_IPACK_DEVICE)
+#define IPACK_DEVICE_CLASS(klass)                                        \
+     OBJECT_CLASS_CHECK(IPackDeviceClass, (klass), TYPE_IPACK_DEVICE)
+#define IPACK_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IPackDeviceClass, (obj), TYPE_IPACK_DEVICE)
+
+struct IPackDeviceClass {
+    DeviceClass parent_class;
+
+    int (*init)(IPackDevice *dev);
+    int (*exit)(IPackDevice *dev);
+
+    uint16_t (*io_read)(IPackDevice *dev, uint8_t addr);
+    void (*io_write)(IPackDevice *dev, uint8_t addr, uint16_t val);
+
+    uint16_t (*id_read)(IPackDevice *dev, uint8_t addr);
+    void (*id_write)(IPackDevice *dev, uint8_t addr, uint16_t val);
+
+    uint16_t (*int_read)(IPackDevice *dev, uint8_t addr);
+    void (*int_write)(IPackDevice *dev, uint8_t addr, uint16_t val);
+
+    uint16_t (*mem_read16)(IPackDevice *dev, uint32_t addr);
+    void (*mem_write16)(IPackDevice *dev, uint32_t addr, uint16_t val);
+
+    uint8_t (*mem_read8)(IPackDevice *dev, uint32_t addr);
+    void (*mem_write8)(IPackDevice *dev, uint32_t addr, uint8_t val);
+};
+
+struct IPackDevice {
+    DeviceState qdev;
+    int32_t slot;
+    /* IRQ objects for the IndustryPack INT0# and INT1# */
+    qemu_irq *irq;
+};
+
+extern const VMStateDescription vmstate_ipack_device;
+
+#define VMSTATE_IPACK_DEVICE(_field, _state)                            \
+    VMSTATE_STRUCT(_field, _state, 1, vmstate_ipack_device, IPackDevice)
+
+IPackDevice *ipack_device_find(IPackBus *bus, int32_t slot);
+void ipack_bus_new_inplace(IPackBus *bus, DeviceState *parent,
+                           const char *name, uint8_t n_slots,
+                           qemu_irq_handler handler);
+
+#endif
diff --git a/hw/ipoctal232.c b/hw/ipoctal232.c
new file mode 100644
index 0000000..c1e3b19
--- /dev/null
+++ b/hw/ipoctal232.c
@@ -0,0 +1,619 @@
+/*
+ * QEMU GE IP-Octal 232 IndustryPack emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#include "ipack.h"
+#include "qemu/bitops.h"
+#include "char/char.h"
+
+/* #define DEBUG_IPOCTAL */
+
+#ifdef DEBUG_IPOCTAL
+#define DPRINTF2(fmt, ...) \
+    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF2(fmt, ...) do { } while (0)
+#endif
+
+#define DPRINTF(fmt, ...) DPRINTF2("IP-Octal: " fmt, ## __VA_ARGS__)
+
+#define RX_FIFO_SIZE 3
+
+/* The IP-Octal has 8 channels (a-h)
+   divided into 4 blocks (A-D) */
+#define N_CHANNELS 8
+#define N_BLOCKS   4
+
+#define REG_MRa  0x01
+#define REG_MRb  0x11
+#define REG_SRa  0x03
+#define REG_SRb  0x13
+#define REG_CSRa 0x03
+#define REG_CSRb 0x13
+#define REG_CRa  0x05
+#define REG_CRb  0x15
+#define REG_RHRa 0x07
+#define REG_RHRb 0x17
+#define REG_THRa 0x07
+#define REG_THRb 0x17
+#define REG_ACR  0x09
+#define REG_ISR  0x0B
+#define REG_IMR  0x0B
+#define REG_OPCR 0x1B
+
+#define CR_ENABLE_RX    BIT(0)
+#define CR_DISABLE_RX   BIT(1)
+#define CR_ENABLE_TX    BIT(2)
+#define CR_DISABLE_TX   BIT(3)
+#define CR_CMD(cr)      ((cr) >> 4)
+#define CR_NO_OP        0
+#define CR_RESET_MR     1
+#define CR_RESET_RX     2
+#define CR_RESET_TX     3
+#define CR_RESET_ERR    4
+#define CR_RESET_BRKINT 5
+#define CR_START_BRK    6
+#define CR_STOP_BRK     7
+#define CR_ASSERT_RTSN  8
+#define CR_NEGATE_RTSN  9
+#define CR_TIMEOUT_ON   10
+#define CR_TIMEOUT_OFF  12
+
+#define SR_RXRDY   BIT(0)
+#define SR_FFULL   BIT(1)
+#define SR_TXRDY   BIT(2)
+#define SR_TXEMT   BIT(3)
+#define SR_OVERRUN BIT(4)
+#define SR_PARITY  BIT(5)
+#define SR_FRAMING BIT(6)
+#define SR_BREAK   BIT(7)
+
+#define ISR_TXRDYA BIT(0)
+#define ISR_RXRDYA BIT(1)
+#define ISR_BREAKA BIT(2)
+#define ISR_CNTRDY BIT(3)
+#define ISR_TXRDYB BIT(4)
+#define ISR_RXRDYB BIT(5)
+#define ISR_BREAKB BIT(6)
+#define ISR_MPICHG BIT(7)
+#define ISR_TXRDY(CH) (((CH) & 1) ? BIT(4) : BIT(0))
+#define ISR_RXRDY(CH) (((CH) & 1) ? BIT(5) : BIT(1))
+#define ISR_BREAK(CH) (((CH) & 1) ? BIT(6) : BIT(2))
+
+typedef struct IPOctalState IPOctalState;
+typedef struct SCC2698Channel SCC2698Channel;
+typedef struct SCC2698Block SCC2698Block;
+
+struct SCC2698Channel {
+    IPOctalState *ipoctal;
+    CharDriverState *dev;
+    char *devpath;
+    bool rx_enabled;
+    uint8_t mr[2];
+    uint8_t mr_idx;
+    uint8_t sr;
+    uint8_t rhr[RX_FIFO_SIZE];
+    uint8_t rhr_idx;
+    uint8_t rx_pending;
+};
+
+struct SCC2698Block {
+    uint8_t imr;
+    uint8_t isr;
+};
+
+struct IPOctalState {
+    IPackDevice dev;
+    SCC2698Channel ch[N_CHANNELS];
+    SCC2698Block blk[N_BLOCKS];
+    uint8_t irq_vector;
+};
+
+#define TYPE_IPOCTAL "ipoctal232"
+
+#define IPOCTAL(obj) \
+    OBJECT_CHECK(IPOctalState, (obj), TYPE_IPOCTAL)
+
+static const VMStateDescription vmstate_scc2698_channel = {
+    .name = "scc2698_channel",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(rx_enabled, SCC2698Channel),
+        VMSTATE_UINT8_ARRAY(mr, SCC2698Channel, 2),
+        VMSTATE_UINT8(mr_idx, SCC2698Channel),
+        VMSTATE_UINT8(sr, SCC2698Channel),
+        VMSTATE_UINT8_ARRAY(rhr, SCC2698Channel, RX_FIFO_SIZE),
+        VMSTATE_UINT8(rhr_idx, SCC2698Channel),
+        VMSTATE_UINT8(rx_pending, SCC2698Channel),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_scc2698_block = {
+    .name = "scc2698_block",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT8(imr, SCC2698Block),
+        VMSTATE_UINT8(isr, SCC2698Block),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_ipoctal = {
+    .name = "ipoctal232",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_IPACK_DEVICE(dev, IPOctalState),
+        VMSTATE_STRUCT_ARRAY(ch, IPOctalState, N_CHANNELS, 1,
+                             vmstate_scc2698_channel, SCC2698Channel),
+        VMSTATE_STRUCT_ARRAY(blk, IPOctalState, N_BLOCKS, 1,
+                             vmstate_scc2698_block, SCC2698Block),
+        VMSTATE_UINT8(irq_vector, IPOctalState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* data[10] is 0x0C, not 0x0B as the doc says */
+static const uint8_t id_prom_data[] = {
+    0x49, 0x50, 0x41, 0x43, 0xF0, 0x22,
+    0xA1, 0x00, 0x00, 0x00, 0x0C, 0xCC
+};
+
+static void update_irq(IPOctalState *dev, unsigned block)
+{
+    /* Blocks A and B interrupt on INT0#, C and D on INT1#.
+       Thus, to get the status we have to check two blocks. */
+    SCC2698Block *blk0 = &dev->blk[block];
+    SCC2698Block *blk1 = &dev->blk[block^1];
+    unsigned intno = block / 2;
+
+    if ((blk0->isr & blk0->imr) || (blk1->isr & blk1->imr)) {
+        qemu_irq_raise(dev->dev.irq[intno]);
+    } else {
+        qemu_irq_lower(dev->dev.irq[intno]);
+    }
+}
+
+static void write_cr(IPOctalState *dev, unsigned channel, uint8_t val)
+{
+    SCC2698Channel *ch = &dev->ch[channel];
+    SCC2698Block *blk = &dev->blk[channel / 2];
+
+    DPRINTF("Write CR%c %u: ", channel + 'a', val);
+
+    /* The lower 4 bits are used to enable and disable Tx and Rx */
+    if (val & CR_ENABLE_RX) {
+        DPRINTF2("Rx on, ");
+        ch->rx_enabled = true;
+    }
+    if (val & CR_DISABLE_RX) {
+        DPRINTF2("Rx off, ");
+        ch->rx_enabled = false;
+    }
+    if (val & CR_ENABLE_TX) {
+        DPRINTF2("Tx on, ");
+        ch->sr |= SR_TXRDY | SR_TXEMT;
+        blk->isr |= ISR_TXRDY(channel);
+    }
+    if (val & CR_DISABLE_TX) {
+        DPRINTF2("Tx off, ");
+        ch->sr &= ~(SR_TXRDY | SR_TXEMT);
+        blk->isr &= ~ISR_TXRDY(channel);
+    }
+
+    DPRINTF2("cmd: ");
+
+    /* The rest of the bits implement different commands */
+    switch (CR_CMD(val)) {
+    case CR_NO_OP:
+        DPRINTF2("none");
+        break;
+    case CR_RESET_MR:
+        DPRINTF2("reset MR");
+        ch->mr_idx = 0;
+        break;
+    case CR_RESET_RX:
+        DPRINTF2("reset Rx");
+        ch->rx_enabled = false;
+        ch->rx_pending = 0;
+        ch->sr &= ~SR_RXRDY;
+        blk->isr &= ~ISR_RXRDY(channel);
+        break;
+    case CR_RESET_TX:
+        DPRINTF2("reset Tx");
+        ch->sr &= ~(SR_TXRDY | SR_TXEMT);
+        blk->isr &= ~ISR_TXRDY(channel);
+        break;
+    case CR_RESET_ERR:
+        DPRINTF2("reset err");
+        ch->sr &= ~(SR_OVERRUN | SR_PARITY | SR_FRAMING | SR_BREAK);
+        break;
+    case CR_RESET_BRKINT:
+        DPRINTF2("reset brk ch int");
+        blk->isr &= ~(ISR_BREAKA | ISR_BREAKB);
+        break;
+    default:
+        DPRINTF2("unsupported 0x%x", CR_CMD(val));
+    }
+
+    DPRINTF2("\n");
+}
+
+static uint16_t io_read(IPackDevice *ip, uint8_t addr)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    uint16_t ret = 0;
+    /* addr[7:6]: block   (A-D)
+       addr[7:5]: channel (a-h)
+       addr[5:0]: register */
+    unsigned block = addr >> 5;
+    unsigned channel = addr >> 4;
+    /* Big endian, accessed using 8-bit bytes at odd locations */
+    unsigned offset = (addr & 0x1F) ^ 1;
+    SCC2698Channel *ch = &dev->ch[channel];
+    SCC2698Block *blk = &dev->blk[block];
+    uint8_t old_isr = blk->isr;
+
+    switch (offset) {
+
+    case REG_MRa:
+    case REG_MRb:
+        ret = ch->mr[ch->mr_idx];
+        DPRINTF("Read MR%u%c: 0x%x\n", ch->mr_idx + 1, channel + 'a', ret);
+        ch->mr_idx = 1;
+        break;
+
+    case REG_SRa:
+    case REG_SRb:
+        ret = ch->sr;
+        DPRINTF("Read SR%c: 0x%x\n", channel + 'a', ret);
+        break;
+
+    case REG_RHRa:
+    case REG_RHRb:
+        ret = ch->rhr[ch->rhr_idx];
+        if (ch->rx_pending > 0) {
+            ch->rx_pending--;
+            if (ch->rx_pending == 0) {
+                ch->sr &= ~SR_RXRDY;
+                blk->isr &= ~ISR_RXRDY(channel);
+                if (ch->dev) {
+                    qemu_chr_accept_input(ch->dev);
+                }
+            } else {
+                ch->rhr_idx = (ch->rhr_idx + 1) % RX_FIFO_SIZE;
+            }
+            if (ch->sr & SR_BREAK) {
+                ch->sr &= ~SR_BREAK;
+                blk->isr |= ISR_BREAK(channel);
+            }
+        }
+        DPRINTF("Read RHR%c (0x%x)\n", channel + 'a', ret);
+        break;
+
+    case REG_ISR:
+        ret = blk->isr;
+        DPRINTF("Read ISR%c: 0x%x\n", block + 'A', ret);
+        break;
+
+    default:
+        DPRINTF("Read unknown/unsupported register 0x%02x\n", offset);
+    }
+
+    if (old_isr != blk->isr) {
+        update_irq(dev, block);
+    }
+
+    return ret;
+}
+
+static void io_write(IPackDevice *ip, uint8_t addr, uint16_t val)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    unsigned reg = val & 0xFF;
+    /* addr[7:6]: block   (A-D)
+       addr[7:5]: channel (a-h)
+       addr[5:0]: register */
+    unsigned block = addr >> 5;
+    unsigned channel = addr >> 4;
+    /* Big endian, accessed using 8-bit bytes at odd locations */
+    unsigned offset = (addr & 0x1F) ^ 1;
+    SCC2698Channel *ch = &dev->ch[channel];
+    SCC2698Block *blk = &dev->blk[block];
+    uint8_t old_isr = blk->isr;
+    uint8_t old_imr = blk->imr;
+
+    switch (offset) {
+
+    case REG_MRa:
+    case REG_MRb:
+        ch->mr[ch->mr_idx] = reg;
+        DPRINTF("Write MR%u%c 0x%x\n", ch->mr_idx + 1, channel + 'a', reg);
+        ch->mr_idx = 1;
+        break;
+
+    /* Not implemented */
+    case REG_CSRa:
+    case REG_CSRb:
+        DPRINTF("Write CSR%c: 0x%x\n", channel + 'a', reg);
+        break;
+
+    case REG_CRa:
+    case REG_CRb:
+        write_cr(dev, channel, reg);
+        break;
+
+    case REG_THRa:
+    case REG_THRb:
+        if (ch->sr & SR_TXRDY) {
+            DPRINTF("Write THR%c (0x%x)\n", channel + 'a', reg);
+            if (ch->dev) {
+                uint8_t thr = reg;
+                qemu_chr_fe_write(ch->dev, &thr, 1);
+            }
+        } else {
+            DPRINTF("Write THR%c (0x%x), Tx disabled\n", channel + 'a', reg);
+        }
+        break;
+
+    /* Not implemented */
+    case REG_ACR:
+        DPRINTF("Write ACR%c 0x%x\n", block + 'A', val);
+        break;
+
+    case REG_IMR:
+        DPRINTF("Write IMR%c 0x%x\n", block + 'A', val);
+        blk->imr = reg;
+        break;
+
+    /* Not implemented */
+    case REG_OPCR:
+        DPRINTF("Write OPCR%c 0x%x\n", block + 'A', val);
+        break;
+
+    default:
+        DPRINTF("Write unknown/unsupported register 0x%02x %u\n", offset, val);
+    }
+
+    if (old_isr != blk->isr || old_imr != blk->imr) {
+        update_irq(dev, block);
+    }
+}
+
+static uint16_t id_read(IPackDevice *ip, uint8_t addr)
+{
+    uint16_t ret = 0;
+    unsigned pos = addr / 2; /* The ID PROM data is stored every other byte */
+
+    if (pos < ARRAY_SIZE(id_prom_data)) {
+        ret = id_prom_data[pos];
+    } else {
+        DPRINTF("Attempt to read unavailable PROM data at 0x%x\n",  addr);
+    }
+
+    return ret;
+}
+
+static void id_write(IPackDevice *ip, uint8_t addr, uint16_t val)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    if (addr == 1) {
+        DPRINTF("Write IRQ vector: %u\n", (unsigned) val);
+        dev->irq_vector = val; /* Undocumented, but the hw works like that */
+    } else {
+        DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+    }
+}
+
+static uint16_t int_read(IPackDevice *ip, uint8_t addr)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    /* Read address 0 to ACK INT0# and address 2 to ACK INT1# */
+    if (addr != 0 && addr != 2) {
+        DPRINTF("Attempt to read from 0x%x\n", addr);
+        return 0;
+    } else {
+        /* Update interrupts if necessary */
+        update_irq(dev, addr);
+        return dev->irq_vector;
+    }
+}
+
+static void int_write(IPackDevice *ip, uint8_t addr, uint16_t val)
+{
+    DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+}
+
+static uint16_t mem_read16(IPackDevice *ip, uint32_t addr)
+{
+    DPRINTF("Attempt to read from 0x%x\n", addr);
+    return 0;
+}
+
+static void mem_write16(IPackDevice *ip, uint32_t addr, uint16_t val)
+{
+    DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+}
+
+static uint8_t mem_read8(IPackDevice *ip, uint32_t addr)
+{
+    DPRINTF("Attempt to read from 0x%x\n", addr);
+    return 0;
+}
+
+static void mem_write8(IPackDevice *ip, uint32_t addr, uint8_t val)
+{
+    IPOctalState *dev = IPOCTAL(ip);
+    if (addr == 1) {
+        DPRINTF("Write IRQ vector: %u\n", (unsigned) val);
+        dev->irq_vector = val;
+    } else {
+        DPRINTF("Attempt to write 0x%x to 0x%x\n", val, addr);
+    }
+}
+
+static int hostdev_can_receive(void *opaque)
+{
+    SCC2698Channel *ch = opaque;
+    int available_bytes = RX_FIFO_SIZE - ch->rx_pending;
+    return ch->rx_enabled ? available_bytes : 0;
+}
+
+static void hostdev_receive(void *opaque, const uint8_t *buf, int size)
+{
+    SCC2698Channel *ch = opaque;
+    IPOctalState *dev = ch->ipoctal;
+    unsigned pos = ch->rhr_idx + ch->rx_pending;
+    int i;
+
+    assert(size + ch->rx_pending <= RX_FIFO_SIZE);
+
+    /* Copy data to the RxFIFO */
+    for (i = 0; i < size; i++) {
+        pos %= RX_FIFO_SIZE;
+        ch->rhr[pos++] = buf[i];
+    }
+
+    ch->rx_pending += size;
+
+    /* If the RxFIFO was empty raise an interrupt */
+    if (!(ch->sr & SR_RXRDY)) {
+        unsigned block, channel = 0;
+        /* Find channel number to update the ISR register */
+        while (&dev->ch[channel] != ch) {
+            channel++;
+        }
+        block = channel / 2;
+        dev->blk[block].isr |= ISR_RXRDY(channel);
+        ch->sr |= SR_RXRDY;
+        update_irq(dev, block);
+    }
+}
+
+static void hostdev_event(void *opaque, int event)
+{
+    SCC2698Channel *ch = opaque;
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        DPRINTF("Device %s opened\n", ch->dev->label);
+        break;
+    case CHR_EVENT_BREAK: {
+        uint8_t zero = 0;
+        DPRINTF("Device %s received break\n", ch->dev->label);
+
+        if (!(ch->sr & SR_BREAK)) {
+            IPOctalState *dev = ch->ipoctal;
+            unsigned block, channel = 0;
+
+            while (&dev->ch[channel] != ch) {
+                channel++;
+            }
+            block = channel / 2;
+
+            ch->sr |= SR_BREAK;
+            dev->blk[block].isr |= ISR_BREAK(channel);
+        }
+
+        /* Put a zero character in the buffer */
+        hostdev_receive(ch, &zero, 1);
+    }
+        break;
+    default:
+        DPRINTF("Device %s received event %d\n", ch->dev->label, event);
+    }
+}
+
+static int ipoctal_init(IPackDevice *ip)
+{
+    IPOctalState *s = IPOCTAL(ip);
+    unsigned i;
+
+    for (i = 0; i < N_CHANNELS; i++) {
+        SCC2698Channel *ch = &s->ch[i];
+        ch->ipoctal = s;
+
+        /* Redirect IP-Octal channels to host character devices */
+        if (ch->devpath) {
+            const char chr_name[] = "ipoctal";
+            char label[ARRAY_SIZE(chr_name) + 2];
+            static int index;
+
+            snprintf(label, sizeof(label), "%s%d", chr_name, index);
+
+            ch->dev = qemu_chr_new(label, ch->devpath, NULL);
+
+            if (ch->dev) {
+                index++;
+                qemu_chr_add_handlers(ch->dev, hostdev_can_receive,
+                                      hostdev_receive, hostdev_event, ch);
+                DPRINTF("Redirecting channel %u to %s (%s)\n",
+                        i, ch->devpath, label);
+            } else {
+                DPRINTF("Could not redirect channel %u to %s\n",
+                        i, ch->devpath);
+            }
+        }
+    }
+
+    return 0;
+}
+
+static Property ipoctal_properties[] = {
+    DEFINE_PROP_STRING("serial0", IPOctalState, ch[0].devpath),
+    DEFINE_PROP_STRING("serial1", IPOctalState, ch[1].devpath),
+    DEFINE_PROP_STRING("serial2", IPOctalState, ch[2].devpath),
+    DEFINE_PROP_STRING("serial3", IPOctalState, ch[3].devpath),
+    DEFINE_PROP_STRING("serial4", IPOctalState, ch[4].devpath),
+    DEFINE_PROP_STRING("serial5", IPOctalState, ch[5].devpath),
+    DEFINE_PROP_STRING("serial6", IPOctalState, ch[6].devpath),
+    DEFINE_PROP_STRING("serial7", IPOctalState, ch[7].devpath),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ipoctal_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IPackDeviceClass *ic = IPACK_DEVICE_CLASS(klass);
+
+    ic->init        = ipoctal_init;
+    ic->io_read     = io_read;
+    ic->io_write    = io_write;
+    ic->id_read     = id_read;
+    ic->id_write    = id_write;
+    ic->int_read    = int_read;
+    ic->int_write   = int_write;
+    ic->mem_read16  = mem_read16;
+    ic->mem_write16 = mem_write16;
+    ic->mem_read8   = mem_read8;
+    ic->mem_write8  = mem_write8;
+
+    dc->desc    = "GE IP-Octal 232 8-channel RS-232 IndustryPack";
+    dc->props   = ipoctal_properties;
+    dc->vmsd    = &vmstate_ipoctal;
+}
+
+static const TypeInfo ipoctal_info = {
+    .name          = TYPE_IPOCTAL,
+    .parent        = TYPE_IPACK_DEVICE,
+    .instance_size = sizeof(IPOctalState),
+    .class_init    = ipoctal_class_init,
+};
+
+static void ipoctal_register_types(void)
+{
+    type_register_static(&ipoctal_info);
+}
+
+type_init(ipoctal_register_types)
diff --git a/hw/isa-bus.c b/hw/isa-bus.c
index 685fdc0..fce311b 100644
--- a/hw/isa-bus.c
+++ b/hw/isa-bus.c
@@ -17,11 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "isa.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static ISABus *isabus;
 hwaddr isa_mem_base = 0;
@@ -215,7 +215,7 @@ static void isabus_bridge_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo isabus_bridge_info = {
+static const TypeInfo isabus_bridge_info = {
     .name          = "isabus-bridge",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
@@ -229,7 +229,7 @@ static void isa_device_class_init(ObjectClass *klass, void *data)
     k->bus_type = TYPE_ISA_BUS;
 }
 
-static TypeInfo isa_device_type_info = {
+static const TypeInfo isa_device_type_info = {
     .name = TYPE_ISA_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(ISADevice),
@@ -264,4 +264,13 @@ MemoryRegion *isa_address_space(ISADevice *dev)
     return get_system_memory();
 }
 
+MemoryRegion *isa_address_space_io(ISADevice *dev)
+{
+    if (dev) {
+        return isa_bus_from_device(dev)->address_space_io;
+    }
+
+    return isabus->address_space_io;
+}
+
 type_init(isabus_register_types)
diff --git a/hw/isa.h b/hw/isa.h
index f9382e8..7a8874a 100644
--- a/hw/isa.h
+++ b/hw/isa.h
@@ -3,8 +3,8 @@
 
 /* ISA bus */
 
-#include "ioport.h"
-#include "memory.h"
+#include "exec/ioport.h"
+#include "exec/memory.h"
 #include "qdev.h"
 
 #define ISA_NUM_IRQS 16
@@ -43,6 +43,7 @@ void isa_bus_irqs(ISABus *bus, qemu_irq *irqs);
 qemu_irq isa_get_irq(ISADevice *dev, int isairq);
 void isa_init_irq(ISADevice *dev, qemu_irq *p, int isairq);
 MemoryRegion *isa_address_space(ISADevice *dev);
+MemoryRegion *isa_address_space_io(ISADevice *dev);
 ISADevice *isa_create(ISABus *bus, const char *name);
 ISADevice *isa_try_create(ISABus *bus, const char *name);
 ISADevice *isa_create_simple(ISABus *bus, const char *name);
@@ -81,7 +82,7 @@ void isa_register_portio_list(ISADevice *dev, uint16_t start,
 
 static inline ISABus *isa_bus_from_device(ISADevice *d)
 {
-    return DO_UPCAST(ISABus, qbus, d->qdev.parent_bus);
+    return ISA_BUS(qdev_get_parent_bus(DEVICE(d)));
 }
 
 extern hwaddr isa_mem_base;
diff --git a/hw/isa_mmio.c b/hw/isa_mmio.c
index 1405396..487cf6a 100644
--- a/hw/isa_mmio.c
+++ b/hw/isa_mmio.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "isa.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static void isa_mmio_writeb (void *opaque, hwaddr addr,
                                   uint32_t val)
diff --git a/hw/ivshmem.c b/hw/ivshmem.c
index f6dbb21..afaf9b3 100644
--- a/hw/ivshmem.c
+++ b/hw/ivshmem.c
@@ -18,16 +18,20 @@
  */
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
-#include "msix.h"
-#include "kvm.h"
-#include "migration.h"
-#include "qerror.h"
-#include "event_notifier.h"
+#include "pci/pci.h"
+#include "pci/msix.h"
+#include "sysemu/kvm.h"
+#include "migration/migration.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/event_notifier.h"
+#include "char/char.h"
 
 #include <sys/mman.h>
 #include <sys/types.h>
 
+#define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
+#define PCI_DEVICE_ID_IVSHMEM   0x1110
+
 #define IVSHMEM_IOEVENTFD   0
 #define IVSHMEM_MSI     1
 
@@ -799,14 +803,14 @@ static void ivshmem_class_init(ObjectClass *klass, void *data)
 
     k->init = pci_ivshmem_init;
     k->exit = pci_ivshmem_uninit;
-    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
-    k->device_id = 0x1110;
+    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
+    k->device_id = PCI_DEVICE_ID_IVSHMEM;
     k->class_id = PCI_CLASS_MEMORY_RAM;
     dc->reset = ivshmem_reset;
     dc->props = ivshmem_properties;
 }
 
-static TypeInfo ivshmem_info = {
+static const TypeInfo ivshmem_info = {
     .name          = "ivshmem",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(IVShmemState),
diff --git a/hw/jazz_led.c b/hw/jazz_led.c
index 640e75e..4822c48 100644
--- a/hw/jazz_led.c
+++ b/hw/jazz_led.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "console.h"
-#include "pixel_ops.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "trace.h"
 #include "sysbus.h"
 
@@ -276,7 +277,7 @@ static void jazz_led_class_init(ObjectClass *klass, void *data)
     dc->reset = jazz_led_reset;
 }
 
-static TypeInfo jazz_led_info = {
+static const TypeInfo jazz_led_info = {
     .name          = "jazz-led",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LedState),
diff --git a/hw/kvm/apic.c b/hw/kvm/apic.c
index 8b65d51..d994ea7 100644
--- a/hw/kvm/apic.c
+++ b/hw/kvm/apic.c
@@ -10,8 +10,8 @@
  * See the COPYING file in the top-level directory.
  */
 #include "hw/apic_internal.h"
-#include "hw/msi.h"
-#include "kvm.h"
+#include "hw/pci/msi.h"
+#include "sysemu/kvm.h"
 
 static inline void kvm_apic_set_reg(struct kvm_lapic_state *kapic,
                                     int reg_id, uint32_t val)
@@ -104,7 +104,7 @@ static void kvm_apic_enable_tpr_reporting(APICCommonState *s, bool enable)
         .enabled = enable
     };
 
-    kvm_vcpu_ioctl(&s->cpu->env, KVM_TPR_ACCESS_REPORTING, &ctl);
+    kvm_vcpu_ioctl(CPU(s->cpu), KVM_TPR_ACCESS_REPORTING, &ctl);
 }
 
 static void kvm_apic_vapic_base_update(APICCommonState *s)
@@ -114,7 +114,7 @@ static void kvm_apic_vapic_base_update(APICCommonState *s)
     };
     int ret;
 
-    ret = kvm_vcpu_ioctl(&s->cpu->env, KVM_SET_VAPIC_ADDR, &vapid_addr);
+    ret = kvm_vcpu_ioctl(CPU(s->cpu), KVM_SET_VAPIC_ADDR, &vapid_addr);
     if (ret < 0) {
         fprintf(stderr, "KVM: setting VAPIC address failed (%s)\n",
                 strerror(-ret));
@@ -125,15 +125,15 @@ static void kvm_apic_vapic_base_update(APICCommonState *s)
 static void do_inject_external_nmi(void *data)
 {
     APICCommonState *s = data;
-    CPUX86State *env = &s->cpu->env;
+    CPUState *cpu = CPU(s->cpu);
     uint32_t lvt;
     int ret;
 
-    cpu_synchronize_state(env);
+    cpu_synchronize_state(&s->cpu->env);
 
     lvt = s->lvt[APIC_LVT_LINT1];
     if (!(lvt & APIC_LVT_MASKED) && ((lvt >> 8) & 7) == APIC_DM_NMI) {
-        ret = kvm_vcpu_ioctl(env, KVM_NMI);
+        ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
         if (ret < 0) {
             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
                     strerror(-ret));
@@ -194,7 +194,7 @@ static void kvm_apic_class_init(ObjectClass *klass, void *data)
     k->external_nmi = kvm_apic_external_nmi;
 }
 
-static TypeInfo kvm_apic_info = {
+static const TypeInfo kvm_apic_info = {
     .name = "kvm-apic",
     .parent = TYPE_APIC_COMMON,
     .instance_size = sizeof(APICCommonState),
diff --git a/hw/kvm/arm_gic.c b/hw/kvm/arm_gic.c
index 0ad1b8b..c3b73c4 100644
--- a/hw/kvm/arm_gic.c
+++ b/hw/kvm/arm_gic.c
@@ -19,7 +19,7 @@
  */
 
 #include "hw/sysbus.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "hw/arm_gic_internal.h"
 
@@ -119,8 +119,8 @@ static int kvm_arm_gic_init(SysBusDevice *dev)
     memory_region_init_reservation(&s->iomem, "kvm-gic_dist", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
     kvm_arm_register_device(&s->iomem,
-                            (KVM_ARM_DEVICE_VGIC_V2 << KVM_DEVICE_ID_SHIFT) |
-                            KVM_VGIC_V2_ADDR_TYPE_DIST);
+                            (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
+                            | KVM_VGIC_V2_ADDR_TYPE_DIST);
     /* CPU interface for current core. Unlike arm_gic, we don't
      * provide the "interface for core #N" memory regions, because
      * cores with a VGIC don't have those.
@@ -128,8 +128,8 @@ static int kvm_arm_gic_init(SysBusDevice *dev)
     memory_region_init_reservation(&s->cpuiomem[0], "kvm-gic_cpu", 0x1000);
     sysbus_init_mmio(dev, &s->cpuiomem[0]);
     kvm_arm_register_device(&s->cpuiomem[0],
-                            (KVM_ARM_DEVICE_VGIC_V2 << KVM_DEVICE_ID_SHIFT) |
-                            KVM_VGIC_V2_ADDR_TYPE_CPU);
+                            (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
+                            | KVM_VGIC_V2_ADDR_TYPE_CPU);
     /* TODO: we should tell the kernel at some point the address
      * of the private peripheral base. However we don't currently have
      * any convenient infrastructure to do that, and in any case the
diff --git a/hw/kvm/clock.c b/hw/kvm/clock.c
index 824b978..fa40e28 100644
--- a/hw/kvm/clock.c
+++ b/hw/kvm/clock.c
@@ -14,8 +14,8 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -76,7 +76,7 @@ static void kvmclock_vm_state_change(void *opaque, int running,
             return;
         }
         for (penv = first_cpu; penv != NULL; penv = penv->next_cpu) {
-            ret = kvm_vcpu_ioctl(penv, KVM_KVMCLOCK_CTRL, 0);
+            ret = kvm_vcpu_ioctl(ENV_GET_CPU(penv), KVM_KVMCLOCK_CTRL, 0);
             if (ret) {
                 if (ret != -EINVAL) {
                     fprintf(stderr, "%s: %s\n", __func__, strerror(-ret));
@@ -118,7 +118,7 @@ static void kvmclock_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &kvmclock_vmsd;
 }
 
-static TypeInfo kvmclock_info = {
+static const TypeInfo kvmclock_info = {
     .name          = "kvmclock",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(KVMClockState),
diff --git a/hw/kvm/i8254.c b/hw/kvm/i8254.c
index 53d13e3..04ad649 100644
--- a/hw/kvm/i8254.c
+++ b/hw/kvm/i8254.c
@@ -22,11 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "hw/i8254.h"
 #include "hw/i8254_internal.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #define KVM_PIT_REINJECT_BIT 0
 
@@ -302,7 +302,7 @@ static void kvm_pit_class_init(ObjectClass *klass, void *data)
     dc->props = kvm_pit_properties;
 }
 
-static TypeInfo kvm_pit_info = {
+static const TypeInfo kvm_pit_info = {
     .name          = "kvm-pit",
     .parent        = TYPE_PIT_COMMON,
     .instance_size = sizeof(KVMPITState),
diff --git a/hw/kvm/i8259.c b/hw/kvm/i8259.c
index 1e24cd4..5ae8b68 100644
--- a/hw/kvm/i8259.c
+++ b/hw/kvm/i8259.c
@@ -11,7 +11,7 @@
  */
 #include "hw/i8259_internal.h"
 #include "hw/apic_internal.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 static void kvm_pic_get(PICCommonState *s)
 {
@@ -123,7 +123,7 @@ static void kvm_i8259_class_init(ObjectClass *klass, void *data)
     k->post_load  = kvm_pic_put;
 }
 
-static TypeInfo kvm_i8259_info = {
+static const TypeInfo kvm_i8259_info = {
     .name  = "kvm-i8259",
     .parent = TYPE_PIC_COMMON,
     .instance_size = sizeof(PICCommonState),
diff --git a/hw/kvm/ioapic.c b/hw/kvm/ioapic.c
index f95c157..23877d4 100644
--- a/hw/kvm/ioapic.c
+++ b/hw/kvm/ioapic.c
@@ -13,7 +13,7 @@
 #include "hw/pc.h"
 #include "hw/ioapic_internal.h"
 #include "hw/apic_internal.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 /* PC Utility function */
 void kvm_pc_setup_irq_routing(bool pci_enabled)
@@ -150,7 +150,7 @@ static void kvm_ioapic_class_init(ObjectClass *klass, void *data)
     dc->props    = kvm_ioapic_properties;
 }
 
-static TypeInfo kvm_ioapic_info = {
+static const TypeInfo kvm_ioapic_info = {
     .name  = "kvm-ioapic",
     .parent = TYPE_IOAPIC_COMMON,
     .instance_size = sizeof(KVMIOAPICState),
diff --git a/hw/kvm/pci-assign.c b/hw/kvm/pci-assign.c
index e80dad0..da64b5b 100644
--- a/hw/kvm/pci-assign.c
+++ b/hw/kvm/pci-assign.c
@@ -28,14 +28,14 @@
 #include <sys/stat.h>
 #include "hw/hw.h"
 #include "hw/pc.h"
-#include "qemu-error.h"
-#include "console.h"
+#include "qemu/error-report.h"
+#include "ui/console.h"
 #include "hw/loader.h"
-#include "monitor.h"
-#include "range.h"
-#include "sysemu.h"
-#include "hw/pci.h"
-#include "hw/msi.h"
+#include "monitor/monitor.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
 #include "kvm_i386.h"
 
 #define MSIX_PAGE_SIZE 0x1000
@@ -46,6 +46,7 @@
 #define IORESOURCE_IRQ      0x00000400
 #define IORESOURCE_DMA      0x00000800
 #define IORESOURCE_PREFETCH 0x00002000  /* No side effects */
+#define IORESOURCE_MEM_64   0x00100000
 
 //#define DEVICE_ASSIGNMENT_DEBUG
 
@@ -442,9 +443,13 @@ static int assigned_dev_register_regions(PCIRegion *io_regions,
 
         /* handle memory io regions */
         if (cur_region->type & IORESOURCE_MEM) {
-            int t = cur_region->type & IORESOURCE_PREFETCH
-                ? PCI_BASE_ADDRESS_MEM_PREFETCH
-                : PCI_BASE_ADDRESS_SPACE_MEMORY;
+            int t = PCI_BASE_ADDRESS_SPACE_MEMORY;
+            if (cur_region->type & IORESOURCE_PREFETCH) {
+                t |= PCI_BASE_ADDRESS_MEM_PREFETCH;
+            }
+            if (cur_region->type & IORESOURCE_MEM_64) {
+                t |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+            }
 
             /* map physical memory */
             pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL, cur_region->size,
@@ -632,7 +637,8 @@ again:
         rp->valid = 0;
         rp->resource_fd = -1;
         size = end - start + 1;
-        flags &= IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH;
+        flags &= IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH
+                 | IORESOURCE_MEM_64;
         if (size == 0 || (flags & ~IORESOURCE_PREFETCH) == 0) {
             continue;
         }
@@ -930,8 +936,8 @@ retry:
             /* Retry with host-side MSI. There might be an IRQ conflict and
              * either the kernel or the device doesn't support sharing. */
             error_report("Host-side INTx sharing not supported, "
-                         "using MSI instead.\n"
-                         "Some devices do not to work properly in this mode.");
+                         "using MSI instead");
+            error_printf("Some devices do not work properly in this mode.\n");
             dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK;
             goto retry;
         }
@@ -1025,6 +1031,19 @@ static bool assigned_dev_msix_masked(MSIXTableEntry *entry)
     return (entry->ctrl & cpu_to_le32(0x1)) != 0;
 }
 
+/*
+ * When MSI-X is first enabled the vector table typically has all the
+ * vectors masked, so we can't use that as the obvious test to figure out
+ * how many vectors to initially enable.  Instead we look at the data field
+ * because this is what worked for pci-assign for a long time.  This makes
+ * sure the physical MSI-X state tracks the guest's view, which is important
+ * for some VF/PF and PF/fw communication channels.
+ */
+static bool assigned_dev_msix_skipped(MSIXTableEntry *entry)
+{
+    return !entry->data;
+}
+
 static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
 {
     AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
@@ -1035,7 +1054,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
 
     /* Get the usable entry number for allocating */
     for (i = 0; i < adev->msix_max; i++, entry++) {
-        if (assigned_dev_msix_masked(entry)) {
+        if (assigned_dev_msix_skipped(entry)) {
             continue;
         }
         entries_nr++;
@@ -1064,7 +1083,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
     for (i = 0; i < adev->msix_max; i++, entry++) {
         adev->msi_virq[i] = -1;
 
-        if (assigned_dev_msix_masked(entry)) {
+        if (assigned_dev_msix_skipped(entry)) {
             continue;
         }
 
@@ -1884,10 +1903,10 @@ static void assigned_dev_load_option_rom(AssignedDevice *dev)
     memset(ptr, 0xff, st.st_size);
 
     if (!fread(ptr, 1, st.st_size, fp)) {
-        error_report("pci-assign: Cannot read from host %s\n"
-                     "\tDevice option ROM contents are probably invalid "
-                     "(check dmesg).\n\tSkip option ROM probe with rombar=0, "
-                     "or load from file with romfile=", rom_file);
+        error_report("pci-assign: Cannot read from host %s", rom_file);
+        error_printf("Device option ROM contents are probably invalid "
+                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
+                     "or load from file with romfile=\n");
         memory_region_destroy(&dev->dev.rom);
         goto close_rom;
     }
diff --git a/hw/kvmvapic.c b/hw/kvmvapic.c
index e04c401..1b5f416 100644
--- a/hw/kvmvapic.c
+++ b/hw/kvmvapic.c
@@ -8,9 +8,9 @@
  * (at your option) any later version. See the COPYING file in the
  * top-level directory.
  */
-#include "sysemu.h"
-#include "cpus.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/cpus.h"
+#include "sysemu/kvm.h"
 #include "apic_internal.h"
 
 #define APIC_DEFAULT_ADDRESS    0xfee00000
@@ -387,7 +387,6 @@ static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong i
     VAPICHandlers *handlers;
     uint8_t opcode[2];
     uint32_t imm32;
-    TranslationBlock *current_tb;
     target_ulong current_pc = 0;
     target_ulong current_cs_base = 0;
     int current_flags = 0;
@@ -399,8 +398,7 @@ static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong i
     }
 
     if (!kvm_enabled()) {
-        current_tb = tb_find_pc(env->mem_io_pc);
-        cpu_restore_state(current_tb, env, env->mem_io_pc);
+        cpu_restore_state(env, env->mem_io_pc);
         cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                              &current_flags);
     }
@@ -806,7 +804,7 @@ static void vapic_class_init(ObjectClass *klass, void *data)
     sc->init    = vapic_init;
 }
 
-static TypeInfo vapic_type = {
+static const TypeInfo vapic_type = {
     .name          = "kvmvapic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(VAPICROMState),
diff --git a/hw/kzm.c b/hw/kzm.c
index 687daf3..fb33165 100644
--- a/hw/kzm.c
+++ b/hw/kzm.c
@@ -14,12 +14,12 @@
  */
 
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "hw.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "serial.h"
 #include "imx.h"
@@ -146,6 +146,7 @@ static QEMUMachine kzm_machine = {
     .name = "kzm",
     .desc = "ARM KZM Emulation Baseboard (ARM1136)",
     .init = kzm_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void kzm_machine_init(void)
diff --git a/hw/lan9118.c b/hw/lan9118.c
index f724e1c..0e844e5 100644
--- a/hw/lan9118.c
+++ b/hw/lan9118.c
@@ -11,9 +11,9 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "ptimer.h"
 /* For crc32 */
 #include <zlib.h>
@@ -341,7 +341,7 @@ static void lan9118_update(lan9118_state *s)
 
 static void lan9118_mac_changed(lan9118_state *s)
 {
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 }
 
 static void lan9118_reload_eeprom(lan9118_state *s)
@@ -373,7 +373,7 @@ static void phy_update_irq(lan9118_state *s)
 static void phy_update_link(lan9118_state *s)
 {
     /* Autonegotiation status mirrors link status.  */
-    if (s->nic->nc.link_down) {
+    if (qemu_get_queue(s->nic)->link_down) {
         s->phy_status &= ~0x0024;
         s->phy_int |= PHY_INT_DOWN;
     } else {
@@ -386,7 +386,7 @@ static void phy_update_link(lan9118_state *s)
 
 static void lan9118_set_link(NetClientState *nc)
 {
-    phy_update_link(DO_UPCAST(NICState, nc, nc)->opaque);
+    phy_update_link(qemu_get_nic_opaque(nc));
 }
 
 static void phy_reset(lan9118_state *s)
@@ -401,7 +401,7 @@ static void phy_reset(lan9118_state *s)
 
 static void lan9118_reset(DeviceState *d)
 {
-    lan9118_state *s = FROM_SYSBUS(lan9118_state, sysbus_from_qdev(d));
+    lan9118_state *s = FROM_SYSBUS(lan9118_state, SYS_BUS_DEVICE(d));
     s->irq_cfg &= (IRQ_TYPE | IRQ_POL);
     s->int_sts = 0;
     s->int_en = 0;
@@ -512,7 +512,7 @@ static int lan9118_filter(lan9118_state *s, const uint8_t *addr)
 static ssize_t lan9118_receive(NetClientState *nc, const uint8_t *buf,
                                size_t size)
 {
-    lan9118_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    lan9118_state *s = qemu_get_nic_opaque(nc);
     int fifo_len;
     int offset;
     int src_pos;
@@ -657,9 +657,9 @@ static void do_tx_packet(lan9118_state *s)
     /* FIXME: Honor TX disable, and allow queueing of packets.  */
     if (s->phy_control & 0x4000)  {
         /* This assumes the receive routine doesn't touch the VLANClient.  */
-        lan9118_receive(&s->nic->nc, s->txp->data, s->txp->len);
+        lan9118_receive(qemu_get_queue(s->nic), s->txp->data, s->txp->len);
     } else {
-        qemu_send_packet(&s->nic->nc, s->txp->data, s->txp->len);
+        qemu_send_packet(qemu_get_queue(s->nic), s->txp->data, s->txp->len);
     }
     s->txp->fifo_used = 0;
 
@@ -1306,7 +1306,7 @@ static const MemoryRegionOps lan9118_16bit_mem_ops = {
 
 static void lan9118_cleanup(NetClientState *nc)
 {
-    lan9118_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    lan9118_state *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -1335,7 +1335,7 @@ static int lan9118_init1(SysBusDevice *dev)
 
     s->nic = qemu_new_nic(&net_lan9118_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
     s->eeprom[0] = 0xa5;
     for (i = 0; i < 6; i++) {
         s->eeprom[i + 1] = s->conf.macaddr.a[i];
@@ -1368,7 +1368,7 @@ static void lan9118_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lan9118;
 }
 
-static TypeInfo lan9118_info = {
+static const TypeInfo lan9118_info = {
     .name          = "lan9118",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(lan9118_state),
@@ -1391,7 +1391,7 @@ void lan9118_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     dev = qdev_create(NULL, "lan9118");
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, base);
     sysbus_connect_irq(s, 0, irq);
 }
diff --git a/hw/lance.c b/hw/lance.c
index a3e6dd9..4b92425 100644
--- a/hw/lance.c
+++ b/hw/lance.c
@@ -36,9 +36,9 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
-#include "qemu-timer.h"
-#include "qemu_socket.h"
+#include "net/net.h"
+#include "qemu/timer.h"
+#include "qemu/sockets.h"
 #include "sun4m.h"
 #include "pcnet.h"
 #include "trace.h"
@@ -87,7 +87,7 @@ static const MemoryRegionOps lance_mem_ops = {
 
 static void lance_cleanup(NetClientState *nc)
 {
-    PCNetState *d = DO_UPCAST(NICState, nc, nc)->opaque;
+    PCNetState *d = qemu_get_nic_opaque(nc);
 
     pcnet_common_cleanup(d);
 }
@@ -155,7 +155,7 @@ static void lance_class_init(ObjectClass *klass, void *data)
     dc->props = lance_properties;
 }
 
-static TypeInfo lance_info = {
+static const TypeInfo lance_info = {
     .name          = "lance",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusPCNetState),
diff --git a/hw/leon3.c b/hw/leon3.c
index 7742738..f16a8bb 100644
--- a/hw/leon3.c
+++ b/hw/leon3.c
@@ -22,15 +22,15 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-char.h"
-#include "sysemu.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #include "grlib.h"
 
@@ -212,7 +212,7 @@ static QEMUMachine leon3_generic_machine = {
     .name     = "leon3_generic",
     .desc     = "Leon-3 generic",
     .init     = leon3_generic_hw_init,
-    .use_scsi = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void leon3_machine_init(void)
diff --git a/hw/lm32.h b/hw/lm32.h
index 0a67632..236686e 100644
--- a/hw/lm32.h
+++ b/hw/lm32.h
@@ -1,3 +1,6 @@
+#ifndef HW_LM32_H
+#define HW_LM32_H 1
+
 
 #include "qemu-common.h"
 
@@ -8,7 +11,7 @@ static inline DeviceState *lm32_pic_init(qemu_irq cpu_irq)
 
     dev = qdev_create(NULL, "lm32-pic");
     qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
+    d = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(d, 0, cpu_irq);
 
     return dev;
@@ -23,3 +26,5 @@ static inline DeviceState *lm32_juart_init(void)
 
     return dev;
 }
+
+#endif
diff --git a/hw/lm32_boards.c b/hw/lm32_boards.c
index 772cb8b..2bc06d7 100644
--- a/hw/lm32_boards.c
+++ b/hw/lm32_boards.c
@@ -19,16 +19,15 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
 #include "flash.h"
 #include "devices.h"
 #include "boards.h"
 #include "loader.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "elf.h"
 #include "lm32_hwsetup.h"
 #include "lm32.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     LM32CPU *cpu;
@@ -288,14 +287,16 @@ static QEMUMachine lm32_evr_machine = {
     .name = "lm32-evr",
     .desc = "LatticeMico32 EVR32 eval system",
     .init = lm32_evr_init,
-    .is_default = 1
+    .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine lm32_uclinux_machine = {
     .name = "lm32-uclinux",
     .desc = "lm32 platform for uClinux and u-boot by Theobroma Systems",
     .init = lm32_uclinux_init,
-    .is_default = 0
+    .is_default = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void lm32_machine_init(void)
diff --git a/hw/lm32_juart.c b/hw/lm32_juart.c
index f07ed39..8c82c85 100644
--- a/hw/lm32_juart.c
+++ b/hw/lm32_juart.c
@@ -20,7 +20,7 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 #include "lm32_juart.h"
 
@@ -144,7 +144,7 @@ static void lm32_juart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm32_juart;
 }
 
-static TypeInfo lm32_juart_info = {
+static const TypeInfo lm32_juart_info = {
     .name          = "lm32-juart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32JuartState),
diff --git a/hw/lm32_pic.c b/hw/lm32_pic.c
index 32f65db..42f298a 100644
--- a/hw/lm32_pic.c
+++ b/hw/lm32_pic.c
@@ -21,7 +21,7 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
 #include "trace.h"
 #include "lm32_pic.h"
@@ -39,7 +39,7 @@ struct LM32PicState {
 typedef struct LM32PicState LM32PicState;
 
 static LM32PicState *pic;
-void lm32_do_pic_info(Monitor *mon)
+void lm32_do_pic_info(Monitor *mon, const QDict *qdict)
 {
     if (pic == NULL) {
         return;
@@ -49,7 +49,7 @@ void lm32_do_pic_info(Monitor *mon)
             pic->im, pic->ip, pic->irq_state);
 }
 
-void lm32_irq_info(Monitor *mon)
+void lm32_irq_info(Monitor *mon, const QDict *qdict)
 {
     int i;
     uint32_t count;
@@ -184,7 +184,7 @@ static void lm32_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm32_pic;
 }
 
-static TypeInfo lm32_pic_info = {
+static const TypeInfo lm32_pic_info = {
     .name          = "lm32-pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32PicState),
diff --git a/hw/lm32_pic.h b/hw/lm32_pic.h
index 14456f3..5556803 100644
--- a/hw/lm32_pic.h
+++ b/hw/lm32_pic.h
@@ -8,7 +8,7 @@ uint32_t lm32_pic_get_im(DeviceState *d);
 void lm32_pic_set_ip(DeviceState *d, uint32_t ip);
 void lm32_pic_set_im(DeviceState *d, uint32_t im);
 
-void lm32_do_pic_info(Monitor *mon);
-void lm32_irq_info(Monitor *mon);
+void lm32_do_pic_info(Monitor *mon, const QDict *qdict);
+void lm32_irq_info(Monitor *mon, const QDict *qdict);
 
 #endif /* QEMU_HW_LM32_PIC_H */
diff --git a/hw/lm32_sys.c b/hw/lm32_sys.c
index a7887d1..187ef6d 100644
--- a/hw/lm32_sys.c
+++ b/hw/lm32_sys.c
@@ -31,10 +31,10 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-log.h"
-#include "qemu-error.h"
-#include "sysemu.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
 
 enum {
     R_CTRL = 0,
@@ -157,7 +157,7 @@ static void lm32_sys_class_init(ObjectClass *klass, void *data)
     dc->props = lm32_sys_properties;
 }
 
-static TypeInfo lm32_sys_info = {
+static const TypeInfo lm32_sys_info = {
     .name          = "lm32-sys",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32SysState),
diff --git a/hw/lm32_timer.c b/hw/lm32_timer.c
index a8be9cc..db527e9 100644
--- a/hw/lm32_timer.c
+++ b/hw/lm32_timer.c
@@ -24,9 +24,9 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #define DEFAULT_FREQUENCY (50*1000000)
 
@@ -215,7 +215,7 @@ static void lm32_timer_class_init(ObjectClass *klass, void *data)
     dc->props = lm32_timer_properties;
 }
 
-static TypeInfo lm32_timer_info = {
+static const TypeInfo lm32_timer_info = {
     .name          = "lm32-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32TimerState),
diff --git a/hw/lm32_uart.c b/hw/lm32_uart.c
index adb9287..9c89cca 100644
--- a/hw/lm32_uart.c
+++ b/hw/lm32_uart.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-char.h"
-#include "qemu-error.h"
+#include "char/char.h"
+#include "qemu/error-report.h"
 
 enum {
     R_RXTX = 0,
@@ -281,7 +281,7 @@ static void lm32_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm32_uart;
 }
 
-static TypeInfo lm32_uart_info = {
+static const TypeInfo lm32_uart_info = {
     .name          = "lm32-uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32UartState),
diff --git a/hw/lm832x.c b/hw/lm832x.c
index 8e09f9b..94b8ae0 100644
--- a/hw/lm832x.c
+++ b/hw/lm832x.c
@@ -20,8 +20,8 @@
 
 #include "hw.h"
 #include "i2c.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 
 typedef struct {
     I2CSlave i2c;
@@ -476,7 +476,7 @@ static int lm8323_init(I2CSlave *i2c)
 
 void lm832x_key_event(DeviceState *dev, int key, int state)
 {
-    LM823KbdState *s = FROM_I2C_SLAVE(LM823KbdState, I2C_SLAVE_FROM_QDEV(dev));
+    LM823KbdState *s = FROM_I2C_SLAVE(LM823KbdState, I2C_SLAVE(dev));
 
     if ((s->status & INT_ERROR) && (s->error & ERR_FIFOOVR))
         return;
@@ -506,7 +506,7 @@ static void lm8323_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lm_kbd;
 }
 
-static TypeInfo lm8323_info = {
+static const TypeInfo lm8323_info = {
     .name          = "lm8323",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(LM823KbdState),
diff --git a/hw/loader.c b/hw/loader.c
index ba01ca6..995edc3 100644
--- a/hw/loader.c
+++ b/hw/loader.c
@@ -43,14 +43,14 @@
  */
 
 #include "hw.h"
-#include "disas.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "disas/disas.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "uboot_image.h"
 #include "loader.h"
 #include "fw_cfg.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
 #include <zlib.h>
 
@@ -778,7 +778,7 @@ void *rom_ptr(hwaddr addr)
     return rom->data + (addr - rom->addr);
 }
 
-void do_info_roms(Monitor *mon)
+void do_info_roms(Monitor *mon, const QDict *qdict)
 {
     Rom *rom;
 
diff --git a/hw/loader.h b/hw/loader.h
index 26480ad..5e61c95 100644
--- a/hw/loader.h
+++ b/hw/loader.h
@@ -1,5 +1,6 @@
 #ifndef LOADER_H
 #define LOADER_H
+#include "qapi/qmp/qdict.h"
 
 /* loader.c */
 int get_image_size(const char *filename);
@@ -30,7 +31,7 @@ int rom_load_all(void);
 void rom_set_fw(void *f);
 int rom_copy(uint8_t *dest, hwaddr addr, size_t size);
 void *rom_ptr(hwaddr addr);
-void do_info_roms(Monitor *mon);
+void do_info_roms(Monitor *mon, const QDict *qdict);
 
 #define rom_add_file_fixed(_f, _a, _i)          \
     rom_add_file(_f, NULL, _a, _i)
diff --git a/hw/lpc_ich9.c b/hw/lpc_ich9.c
index 2fc83a4..e25689b 100644
--- a/hw/lpc_ich9.c
+++ b/hw/lpc_ich9.c
@@ -1,5 +1,13 @@
 /*
+ * QEMU ICH9 Emulation
+ *
  * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009, 2010, 2011
+ *               Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on piix_pci.c, but heavily modified.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -19,47 +27,24 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-/*
- * QEMU ICH9 Emulation
- *
- *  Copyright (c) 2009, 2010, 2011
- *                Isaku Yamahata <yamahata at valinux co jp>
- *                VA Linux Systems Japan K.K.
- *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
- *
- *  This is based on piix_pci.c, but heavily modified.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-
 #include "qemu-common.h"
 #include "hw.h"
-#include "range.h"
+#include "qemu/range.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "pc.h"
 #include "apm.h"
 #include "ioapic.h"
-#include "pci.h"
-#include "pcie_host.h"
-#include "pci_bridge.h"
+#include "pci/pci.h"
+#include "pci/pcie_host.h"
+#include "pci/pci_bridge.h"
 #include "ich9.h"
 #include "acpi.h"
 #include "acpi_ich9.h"
 #include "pam.h"
-#include "pci_internals.h"
-#include "exec-memory.h"
+#include "pci/pci_bus.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 
 static int ich9_lpc_sci_irq(ICH9LPCState *lpc);
 
@@ -173,6 +158,7 @@ static void ich9_cc_write(void *opaque, hwaddr addr,
 
     ich9_cc_addr_len(&addr, &len);
     memcpy(lpc->chip_config + addr, &val, len);
+    pci_bus_fire_intx_routing_notifier(lpc->d.bus);
     ich9_cc_update(lpc);
 }
 
@@ -301,6 +287,32 @@ int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx)
     return lpc->irr[PCI_SLOT(pci_dev->devfn)][intx];
 }
 
+PCIINTxRoute ich9_route_intx_pin_to_irq(void *opaque, int pirq_pin)
+{
+    ICH9LPCState *lpc = opaque;
+    PCIINTxRoute route;
+    int pic_irq;
+    int pic_dis;
+
+    assert(0 <= pirq_pin);
+    assert(pirq_pin < ICH9_LPC_NB_PIRQS);
+
+    route.mode = PCI_INTX_ENABLED;
+    ich9_lpc_pic_irq(lpc, pirq_pin, &pic_irq, &pic_dis);
+    if (!pic_dis) {
+        if (pic_irq < ICH9_LPC_PIC_NUM_PINS) {
+            route.irq = pic_irq;
+        } else {
+            route.mode = PCI_INTX_DISABLED;
+            route.irq = -1;
+        }
+    } else {
+        route.irq = ich9_pirq_to_gsi(pirq_pin);
+    }
+
+    return route;
+}
+
 static int ich9_lpc_sci_irq(ICH9LPCState *lpc)
 {
     switch (lpc->d.config[ICH9_LPC_ACPI_CTRL] &
@@ -351,7 +363,7 @@ void ich9_lpc_pm_init(PCIDevice *lpc_pci, qemu_irq cmos_s3)
     qemu_irq *sci_irq;
 
     sci_irq = qemu_allocate_irqs(ich9_set_sci, lpc, 1);
-    ich9_pm_init(&lpc->pm, sci_irq[0], cmos_s3);
+    ich9_pm_init(lpc_pci, &lpc->pm, sci_irq[0], cmos_s3);
 
     ich9_lpc_reset(&lpc->d.qdev);
 }
@@ -420,6 +432,12 @@ static void ich9_lpc_config_write(PCIDevice *d,
     if (ranges_overlap(addr, len, ICH9_LPC_RCBA, 4)) {
         ich9_lpc_rcba_update(lpc, rbca_old);
     }
+    if (ranges_overlap(addr, len, ICH9_LPC_PIRQA_ROUT, 4)) {
+        pci_bus_fire_intx_routing_notifier(lpc->d.bus);
+    }
+    if (ranges_overlap(addr, len, ICH9_LPC_PIRQE_ROUT, 4)) {
+        pci_bus_fire_intx_routing_notifier(lpc->d.bus);
+    }
 }
 
 static void ich9_lpc_reset(DeviceState *qdev)
@@ -456,6 +474,30 @@ static const MemoryRegionOps rbca_mmio_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+static void ich9_lpc_machine_ready(Notifier *n, void *opaque)
+{
+    ICH9LPCState *s = container_of(n, ICH9LPCState, machine_ready);
+    uint8_t *pci_conf;
+
+    pci_conf = s->d.config;
+    if (isa_is_ioport_assigned(0x3f8)) {
+        /* com1 */
+        pci_conf[0x82] |= 0x01;
+    }
+    if (isa_is_ioport_assigned(0x2f8)) {
+        /* com2 */
+        pci_conf[0x82] |= 0x02;
+    }
+    if (isa_is_ioport_assigned(0x378)) {
+        /* lpt */
+        pci_conf[0x82] |= 0x04;
+    }
+    if (isa_is_ioport_assigned(0x3f0)) {
+        /* floppy */
+        pci_conf[0x82] |= 0x08;
+    }
+}
+
 static int ich9_lpc_initfn(PCIDevice *d)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
@@ -472,7 +514,11 @@ static int ich9_lpc_initfn(PCIDevice *d)
     lpc->isa_bus = isa_bus;
 
     ich9_cc_init(lpc);
-    apm_init(&lpc->apm, ich9_apm_ctrl_changed, lpc);
+    apm_init(d, &lpc->apm, ich9_apm_ctrl_changed, lpc);
+
+    lpc->machine_ready.notify = ich9_lpc_machine_ready;
+    qemu_add_machine_init_done_notifier(&lpc->machine_ready);
+
     return 0;
 }
 
diff --git a/hw/lsi53c895a.c b/hw/lsi53c895a.c
index 04f2fae..860df32 100644
--- a/hw/lsi53c895a.c
+++ b/hw/lsi53c895a.c
@@ -13,9 +13,9 @@
 #include <assert.h>
 
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "scsi.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 //#define DEBUG_LSI
 //#define DEBUG_LSI_REG
@@ -1670,12 +1670,7 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
         }
         if (val & LSI_SCNTL1_RST) {
             if (!(s->sstat0 & LSI_SSTAT0_RST)) {
-                BusChild *kid;
-
-                QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-                    DeviceState *dev = kid->child;
-                    device_reset(dev);
-                }
+                qbus_reset_all(&s->bus.qbus);
                 s->sstat0 |= LSI_SSTAT0_RST;
                 lsi_script_scsi_interrupt(s, LSI_SIST0_RST, 0);
             }
@@ -2126,7 +2121,7 @@ static void lsi_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_lsi_scsi;
 }
 
-static TypeInfo lsi_info = {
+static const TypeInfo lsi_info = {
     .name          = "lsi53c895a",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(LSIState),
diff --git a/hw/m25p80.c b/hw/m25p80.c
index 3895e73..461b41c 100644
--- a/hw/m25p80.c
+++ b/hw/m25p80.c
@@ -22,7 +22,7 @@
  */
 
 #include "hw.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "ssi.h"
 #include "devices.h"
 
@@ -184,6 +184,7 @@ static const FlashPartInfo known_devices[] = {
 
 typedef enum {
     NOP = 0,
+    WRSR = 0x1,
     WRDI = 0x4,
     RDSR = 0x5,
     WREN = 0x6,
@@ -358,6 +359,8 @@ static void complete_collecting_data(Flash *s)
     s->cur_addr |= s->data[1] << 8;
     s->cur_addr |= s->data[2];
 
+    s->state = STATE_IDLE;
+
     switch (s->cmd_in_progress) {
     case DPP:
     case QPP:
@@ -377,6 +380,11 @@ static void complete_collecting_data(Flash *s)
     case ERASE_SECTOR:
         flash_erase(s, s->cur_addr, s->cmd_in_progress);
         break;
+    case WRSR:
+        if (s->write_enable) {
+            s->write_enable = false;
+        }
+        break;
     default:
         break;
     }
@@ -441,6 +449,15 @@ static void decode_new_cmd(Flash *s, uint32_t value)
         s->state = STATE_COLLECTING_DATA;
         break;
 
+    case WRSR:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+
     case WRDI:
         s->write_enable = false;
         break;
diff --git a/hw/m48t59.c b/hw/m48t59.c
index 7da7e7c..427d95b 100644
--- a/hw/m48t59.c
+++ b/hw/m48t59.c
@@ -23,11 +23,11 @@
  */
 #include "hw.h"
 #include "nvram.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 #include "isa.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_NVRAM
 
@@ -646,7 +646,7 @@ M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
     qdev_prop_set_uint32(dev, "size", size);
     qdev_prop_set_uint32(dev, "io_base", io_base);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     d = FROM_SYSBUS(M48t59SysBusState, s);
     state = &d->state;
     sysbus_connect_irq(s, 0, IRQ);
@@ -738,7 +738,7 @@ static void m48t59_init_class_isa1(ObjectClass *klass, void *data)
     dc->props = m48t59_isa_properties;
 }
 
-static TypeInfo m48t59_isa_info = {
+static const TypeInfo m48t59_isa_info = {
     .name          = "m48t59_isa",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(M48t59ISAState),
@@ -762,7 +762,7 @@ static void m48t59_class_init(ObjectClass *klass, void *data)
     dc->props = m48t59_properties;
 }
 
-static TypeInfo m48t59_info = {
+static const TypeInfo m48t59_info = {
     .name          = "m48t59",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(M48t59SysBusState),
diff --git a/hw/mac_dbdma.c b/hw/mac_dbdma.c
index e551156..b894ab2 100644
--- a/hw/mac_dbdma.c
+++ b/hw/mac_dbdma.c
@@ -39,6 +39,7 @@
 #include "hw.h"
 #include "isa.h"
 #include "mac_dbdma.h"
+#include "qemu/main-loop.h"
 
 /* debug DBDMA */
 //#define DEBUG_DBDMA
diff --git a/hw/mac_dbdma.h b/hw/mac_dbdma.h
index bfdb0dd..691263e 100644
--- a/hw/mac_dbdma.h
+++ b/hw/mac_dbdma.h
@@ -19,8 +19,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_MAC_DBDMA_H
+#define HW_MAC_DBDMA_H 1
 
-#include "memory.h"
+#include "exec/memory.h"
 
 typedef struct DBDMA_io DBDMA_io;
 
@@ -42,3 +44,5 @@ void DBDMA_register_channel(void *dbdma, int nchan, qemu_irq irq,
                             DBDMA_rw rw, DBDMA_flush flush,
                             void *opaque);
 void* DBDMA_init (MemoryRegion **dbdma_mem);
+
+#endif
diff --git a/hw/mac_nvram.c b/hw/mac_nvram.c
index a0d14dd..25121fa 100644
--- a/hw/mac_nvram.c
+++ b/hw/mac_nvram.c
@@ -24,8 +24,8 @@
  */
 #include "hw.h"
 #include "firmware_abi.h"
-#include "sysemu.h"
-#include "ppc_mac.h"
+#include "sysemu/sysemu.h"
+#include "ppc/mac.h"
 
 /* debug NVR */
 //#define DEBUG_NVR
@@ -37,37 +37,29 @@
 #define NVR_DPRINTF(fmt, ...)
 #endif
 
-struct MacIONVRAMState {
-    uint32_t size;
-    MemoryRegion mem;
-    unsigned int it_shift;
-    uint8_t *data;
-};
-
 #define DEF_SYSTEM_SIZE 0xc10
 
 /* Direct access to NVRAM */
-uint32_t macio_nvram_read (void *opaque, uint32_t addr)
+uint8_t macio_nvram_read(MacIONVRAMState *s, uint32_t addr)
 {
-    MacIONVRAMState *s = opaque;
     uint32_t ret;
 
-    if (addr < s->size)
+    if (addr < s->size) {
         ret = s->data[addr];
-    else
+    } else {
         ret = -1;
-    NVR_DPRINTF("read addr %04x val %x\n", addr, ret);
+    }
+    NVR_DPRINTF("read addr %04" PRIx32 " val %" PRIx8 "\n", addr, ret);
 
     return ret;
 }
 
-void macio_nvram_write (void *opaque, uint32_t addr, uint32_t val)
+void macio_nvram_write(MacIONVRAMState *s, uint32_t addr, uint8_t val)
 {
-    MacIONVRAMState *s = opaque;
-
-    NVR_DPRINTF("write addr %04x val %x\n", addr, val);
-    if (addr < s->size)
+    NVR_DPRINTF("write addr %04" PRIx32 " val %" PRIx8 "\n", addr, val);
+    if (addr < s->size) {
         s->data[addr] = val;
+    }
 }
 
 /* macio style NVRAM device */
@@ -78,7 +70,7 @@ static void macio_nvram_writeb(void *opaque, hwaddr addr,
 
     addr = (addr >> s->it_shift) & (s->size - 1);
     s->data[addr] = value;
-    NVR_DPRINTF("writeb addr %04x val %x\n", (int)addr, value);
+    NVR_DPRINTF("writeb addr %04" PHYS_PRIx " val %" PRIx64 "\n", addr, value);
 }
 
 static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
@@ -97,7 +89,7 @@ static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
 static const MemoryRegionOps macio_nvram_ops = {
     .read = macio_nvram_readb,
     .write = macio_nvram_writeb,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_BIG_ENDIAN,
 };
 
 static const VMStateDescription vmstate_macio_nvram = {
@@ -112,32 +104,56 @@ static const VMStateDescription vmstate_macio_nvram = {
 };
 
 
-static void macio_nvram_reset(void *opaque)
+static void macio_nvram_reset(DeviceState *dev)
 {
 }
 
-MacIONVRAMState *macio_nvram_init (hwaddr size,
-                                   unsigned int it_shift)
+static void macio_nvram_realizefn(DeviceState *dev, Error **errp)
 {
-    MacIONVRAMState *s;
+    SysBusDevice *d = SYS_BUS_DEVICE(dev);
+    MacIONVRAMState *s = MACIO_NVRAM(dev);
 
-    s = g_malloc0(sizeof(MacIONVRAMState));
-    s->data = g_malloc0(size);
-    s->size = size;
-    s->it_shift = it_shift;
+    s->data = g_malloc0(s->size);
 
     memory_region_init_io(&s->mem, &macio_nvram_ops, s, "macio-nvram",
-                          size << it_shift);
-    vmstate_register(NULL, -1, &vmstate_macio_nvram, s);
-    qemu_register_reset(macio_nvram_reset, s);
+                          s->size << s->it_shift);
+    sysbus_init_mmio(d, &s->mem);
+}
+
+static void macio_nvram_unrealizefn(DeviceState *dev, Error **errp)
+{
+    MacIONVRAMState *s = MACIO_NVRAM(dev);
 
-    return s;
+    g_free(s->data);
 }
 
-void macio_nvram_setup_bar(MacIONVRAMState *s, MemoryRegion *bar,
-                           hwaddr mem_base)
+static Property macio_nvram_properties[] = {
+    DEFINE_PROP_UINT32("size", MacIONVRAMState, size, 0),
+    DEFINE_PROP_UINT32("it_shift", MacIONVRAMState, it_shift, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void macio_nvram_class_init(ObjectClass *oc, void *data)
 {
-    memory_region_add_subregion(bar, mem_base, &s->mem);
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = macio_nvram_realizefn;
+    dc->unrealize = macio_nvram_unrealizefn;
+    dc->reset = macio_nvram_reset;
+    dc->vmsd = &vmstate_macio_nvram;
+    dc->props = macio_nvram_properties;
+}
+
+static const TypeInfo macio_nvram_type_info = {
+    .name = TYPE_MACIO_NVRAM,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MacIONVRAMState),
+    .class_init = macio_nvram_class_init,
+};
+
+static void macio_nvram_register_types(void)
+{
+    type_register_static(&macio_nvram_type_info);
 }
 
 /* Set up a system OpenBIOS NVRAM partition */
@@ -176,3 +192,5 @@ void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len)
     end = len;
     OpenBIOS_finish_partition(part_header, end - start);
 }
+
+type_init(macio_nvram_register_types)
diff --git a/hw/macio.c b/hw/macio.c
index eb15b89..74bdcd1 100644
--- a/hw/macio.c
+++ b/hw/macio.c
@@ -23,118 +23,283 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
-#include "pci.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
+#include "mac_dbdma.h"
 #include "escc.h"
 
+#define TYPE_MACIO "macio"
+#define MACIO(obj) OBJECT_CHECK(MacIOState, (obj), TYPE_MACIO)
+
 typedef struct MacIOState
 {
+    /*< private >*/
     PCIDevice parent;
-    int is_oldworld;
+    /*< public >*/
+
     MemoryRegion bar;
+    CUDAState cuda;
+    void *dbdma;
     MemoryRegion *pic_mem;
-    MemoryRegion *dbdma_mem;
-    MemoryRegion *cuda_mem;
     MemoryRegion *escc_mem;
-    void *nvram;
-    int nb_ide;
-    MemoryRegion *ide_mem[4];
 } MacIOState;
 
+#define OLDWORLD_MACIO(obj) \
+    OBJECT_CHECK(OldWorldMacIOState, (obj), TYPE_OLDWORLD_MACIO)
+
+typedef struct OldWorldMacIOState {
+    /*< private >*/
+    MacIOState parent_obj;
+    /*< public >*/
+
+    qemu_irq irqs[3];
+
+    MacIONVRAMState nvram;
+    MACIOIDEState ide;
+} OldWorldMacIOState;
+
+#define NEWWORLD_MACIO(obj) \
+    OBJECT_CHECK(NewWorldMacIOState, (obj), TYPE_NEWWORLD_MACIO)
+
+typedef struct NewWorldMacIOState {
+    /*< private >*/
+    MacIOState parent_obj;
+    /*< public >*/
+    qemu_irq irqs[5];
+    MACIOIDEState ide[2];
+} NewWorldMacIOState;
+
 static void macio_bar_setup(MacIOState *macio_state)
 {
-    int i;
     MemoryRegion *bar = &macio_state->bar;
 
-    memory_region_init(bar, "macio", 0x80000);
-    if (macio_state->pic_mem) {
-        if (macio_state->is_oldworld) {
-            /* Heathrow PIC */
-            memory_region_add_subregion(bar, 0x00000, macio_state->pic_mem);
-        } else {
-            /* OpenPIC */
-            memory_region_add_subregion(bar, 0x40000, macio_state->pic_mem);
-        }
-    }
-    if (macio_state->dbdma_mem) {
-        memory_region_add_subregion(bar, 0x08000, macio_state->dbdma_mem);
-    }
     if (macio_state->escc_mem) {
         memory_region_add_subregion(bar, 0x13000, macio_state->escc_mem);
     }
-    if (macio_state->cuda_mem) {
-        memory_region_add_subregion(bar, 0x16000, macio_state->cuda_mem);
+}
+
+static int macio_common_initfn(PCIDevice *d)
+{
+    MacIOState *s = MACIO(d);
+    SysBusDevice *sysbus_dev;
+    int ret;
+
+    d->config[0x3d] = 0x01; // interrupt on pin 1
+
+    ret = qdev_init(DEVICE(&s->cuda));
+    if (ret < 0) {
+        return ret;
     }
-    for (i = 0; i < macio_state->nb_ide; i++) {
-        if (macio_state->ide_mem[i]) {
-            memory_region_add_subregion(bar, 0x1f000 + (i * 0x1000),
-                                        macio_state->ide_mem[i]);
-        }
+    sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
+    memory_region_add_subregion(&s->bar, 0x16000,
+                                sysbus_mmio_get_region(sysbus_dev, 0));
+
+    macio_bar_setup(s);
+    pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+
+    return 0;
+}
+
+static int macio_oldworld_initfn(PCIDevice *d)
+{
+    MacIOState *s = MACIO(d);
+    OldWorldMacIOState *os = OLDWORLD_MACIO(d);
+    SysBusDevice *sysbus_dev;
+    int ret = macio_common_initfn(d);
+    if (ret < 0) {
+        return ret;
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
+    sysbus_connect_irq(sysbus_dev, 0, os->irqs[0]);
+
+    ret = qdev_init(DEVICE(&os->nvram));
+    if (ret < 0) {
+        return ret;
     }
-    if (macio_state->nvram != NULL)
-        macio_nvram_setup_bar(macio_state->nvram, bar, 0x60000);
+    sysbus_dev = SYS_BUS_DEVICE(&os->nvram);
+    memory_region_add_subregion(&s->bar, 0x60000,
+                                sysbus_mmio_get_region(sysbus_dev, 0));
+    pmac_format_nvram_partition(&os->nvram, os->nvram.size);
+
+    if (s->pic_mem) {
+        /* Heathrow PIC */
+        memory_region_add_subregion(&s->bar, 0x00000, s->pic_mem);
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&os->ide);
+    sysbus_connect_irq(sysbus_dev, 0, os->irqs[1]);
+    sysbus_connect_irq(sysbus_dev, 1, os->irqs[2]);
+    macio_ide_register_dma(&os->ide, s->dbdma, 0x16);
+    ret = qdev_init(DEVICE(&os->ide));
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
 }
 
-static int macio_initfn(PCIDevice *d)
+static void macio_oldworld_init(Object *obj)
 {
-    d->config[0x3d] = 0x01; // interrupt on pin 1
+    MacIOState *s = MACIO(obj);
+    OldWorldMacIOState *os = OLDWORLD_MACIO(obj);
+    DeviceState *dev;
+
+    qdev_init_gpio_out(DEVICE(obj), os->irqs, ARRAY_SIZE(os->irqs));
+
+    object_initialize(&os->nvram, TYPE_MACIO_NVRAM);
+    dev = DEVICE(&os->nvram);
+    qdev_prop_set_uint32(dev, "size", 0x2000);
+    qdev_prop_set_uint32(dev, "it_shift", 4);
+
+    object_initialize(&os->ide, TYPE_MACIO_IDE);
+    qdev_set_parent_bus(DEVICE(&os->ide), sysbus_get_default());
+    memory_region_add_subregion(&s->bar, 0x1f000 + (1 * 0x1000), &os->ide.mem);
+    object_property_add_child(obj, "ide", OBJECT(&os->ide), NULL);
+}
+
+static int macio_newworld_initfn(PCIDevice *d)
+{
+    MacIOState *s = MACIO(d);
+    NewWorldMacIOState *ns = NEWWORLD_MACIO(d);
+    SysBusDevice *sysbus_dev;
+    int ret = macio_common_initfn(d);
+    if (ret < 0) {
+        return ret;
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
+    sysbus_connect_irq(sysbus_dev, 0, ns->irqs[0]);
+
+    if (s->pic_mem) {
+        /* OpenPIC */
+        memory_region_add_subregion(&s->bar, 0x40000, s->pic_mem);
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&ns->ide[0]);
+    sysbus_connect_irq(sysbus_dev, 0, ns->irqs[1]);
+    sysbus_connect_irq(sysbus_dev, 1, ns->irqs[2]);
+    macio_ide_register_dma(&ns->ide[0], s->dbdma, 0x16);
+    ret = qdev_init(DEVICE(&ns->ide[0]));
+    if (ret < 0) {
+        return ret;
+    }
+
+    sysbus_dev = SYS_BUS_DEVICE(&ns->ide[1]);
+    sysbus_connect_irq(sysbus_dev, 0, ns->irqs[3]);
+    sysbus_connect_irq(sysbus_dev, 1, ns->irqs[4]);
+    macio_ide_register_dma(&ns->ide[0], s->dbdma, 0x1a);
+    ret = qdev_init(DEVICE(&ns->ide[1]));
+    if (ret < 0) {
+        return ret;
+    }
+
     return 0;
 }
 
+static void macio_newworld_init(Object *obj)
+{
+    MacIOState *s = MACIO(obj);
+    NewWorldMacIOState *ns = NEWWORLD_MACIO(obj);
+    int i;
+    gchar *name;
+
+    qdev_init_gpio_out(DEVICE(obj), ns->irqs, ARRAY_SIZE(ns->irqs));
+
+    for (i = 0; i < 2; i++) {
+        object_initialize(&ns->ide[i], TYPE_MACIO_IDE);
+        qdev_set_parent_bus(DEVICE(&ns->ide[i]), sysbus_get_default());
+        memory_region_add_subregion(&s->bar, 0x1f000 + ((i + 1) * 0x1000),
+                                    &ns->ide[i].mem);
+        name = g_strdup_printf("ide[%i]", i);
+        object_property_add_child(obj, name, OBJECT(&ns->ide[i]), NULL);
+        g_free(name);
+    }
+}
+
+static void macio_instance_init(Object *obj)
+{
+    MacIOState *s = MACIO(obj);
+    MemoryRegion *dbdma_mem;
+
+    memory_region_init(&s->bar, "macio", 0x80000);
+
+    object_initialize(&s->cuda, TYPE_CUDA);
+    qdev_set_parent_bus(DEVICE(&s->cuda), sysbus_get_default());
+    object_property_add_child(obj, "cuda", OBJECT(&s->cuda), NULL);
+
+    s->dbdma = DBDMA_init(&dbdma_mem);
+    memory_region_add_subregion(&s->bar, 0x08000, dbdma_mem);
+}
+
+static void macio_oldworld_class_init(ObjectClass *oc, void *data)
+{
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(oc);
+
+    pdc->init = macio_oldworld_initfn;
+    pdc->device_id = PCI_DEVICE_ID_APPLE_343S1201;
+}
+
+static void macio_newworld_class_init(ObjectClass *oc, void *data)
+{
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(oc);
+
+    pdc->init = macio_newworld_initfn;
+    pdc->device_id = PCI_DEVICE_ID_APPLE_UNI_N_KEYL;
+}
+
 static void macio_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = macio_initfn;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->class_id = PCI_CLASS_OTHERS << 8;
 }
 
-static TypeInfo macio_info = {
-    .name          = "macio",
+static const TypeInfo macio_oldworld_type_info = {
+    .name          = TYPE_OLDWORLD_MACIO,
+    .parent        = TYPE_MACIO,
+    .instance_size = sizeof(OldWorldMacIOState),
+    .instance_init = macio_oldworld_init,
+    .class_init    = macio_oldworld_class_init,
+};
+
+static const TypeInfo macio_newworld_type_info = {
+    .name          = TYPE_NEWWORLD_MACIO,
+    .parent        = TYPE_MACIO,
+    .instance_size = sizeof(NewWorldMacIOState),
+    .instance_init = macio_newworld_init,
+    .class_init    = macio_newworld_class_init,
+};
+
+static const TypeInfo macio_type_info = {
+    .name          = TYPE_MACIO,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(MacIOState),
+    .instance_init = macio_instance_init,
+    .abstract      = true,
     .class_init    = macio_class_init,
 };
 
 static void macio_register_types(void)
 {
-    type_register_static(&macio_info);
+    type_register_static(&macio_type_info);
+    type_register_static(&macio_oldworld_type_info);
+    type_register_static(&macio_newworld_type_info);
 }
 
 type_init(macio_register_types)
 
-void macio_init (PCIBus *bus, int device_id, int is_oldworld,
-                 MemoryRegion *pic_mem, MemoryRegion *dbdma_mem,
-                 MemoryRegion *cuda_mem, void *nvram,
-                 int nb_ide, MemoryRegion **ide_mem,
-                 MemoryRegion *escc_mem)
+void macio_init(PCIDevice *d,
+                MemoryRegion *pic_mem,
+                MemoryRegion *escc_mem)
 {
-    PCIDevice *d;
-    MacIOState *macio_state;
-    int i;
+    MacIOState *macio_state = MACIO(d);
 
-    d = pci_create_simple(bus, -1, "macio");
-
-    macio_state = DO_UPCAST(MacIOState, parent, d);
-    macio_state->is_oldworld = is_oldworld;
     macio_state->pic_mem = pic_mem;
-    macio_state->dbdma_mem = dbdma_mem;
-    macio_state->cuda_mem = cuda_mem;
     macio_state->escc_mem = escc_mem;
-    macio_state->nvram = nvram;
-    if (nb_ide > 4)
-        nb_ide = 4;
-    macio_state->nb_ide = nb_ide;
-    for (i = 0; i < nb_ide; i++)
-        macio_state->ide_mem[i] = ide_mem[i];
-    for (; i < 4; i++)
-        macio_state->ide_mem[i] = NULL;
     /* Note: this code is strongly inspirated from the corresponding code
        in PearPC */
 
-    pci_config_set_device_id(d->config, device_id);
-
-    macio_bar_setup(macio_state);
-    pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &macio_state->bar);
+    qdev_init_nofail(DEVICE(d));
 }
diff --git a/hw/mainstone.c b/hw/mainstone.c
index 5bbecb7..d1ff6e7 100644
--- a/hw/mainstone.c
+++ b/hw/mainstone.c
@@ -14,13 +14,13 @@
 #include "hw.h"
 #include "pxa.h"
 #include "arm-misc.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
 #include "boards.h"
 #include "flash.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 /* Device addresses */
 #define MST_FPGA_PHYS	0x08000000
@@ -179,6 +179,7 @@ static QEMUMachine mainstone2_machine = {
     .name = "mainstone",
     .desc = "Mainstone II (PXA27x)",
     .init = mainstone_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mainstone_machine_init(void)
diff --git a/hw/marvell_88w8618_audio.c b/hw/marvell_88w8618_audio.c
index de16cfa..c792caf 100644
--- a/hw/marvell_88w8618_audio.c
+++ b/hw/marvell_88w8618_audio.c
@@ -222,7 +222,7 @@ static void mv88w8618_audio_write(void *opaque, hwaddr offset,
 static void mv88w8618_audio_reset(DeviceState *d)
 {
     mv88w8618_audio_state *s = FROM_SYSBUS(mv88w8618_audio_state,
-                                           sysbus_from_qdev(d));
+                                           SYS_BUS_DEVICE(d));
 
     s->playback_mode = 0;
     s->status = 0;
@@ -288,7 +288,7 @@ static void mv88w8618_audio_class_init(ObjectClass *klass, void *data)
     dc->props = mv88w8618_audio_properties;
 }
 
-static TypeInfo mv88w8618_audio_info = {
+static const TypeInfo mv88w8618_audio_info = {
     .name          = "mv88w8618_audio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_audio_state),
diff --git a/hw/max111x.c b/hw/max111x.c
index 67640f1..de1be4d 100644
--- a/hw/max111x.c
+++ b/hw/max111x.c
@@ -162,7 +162,7 @@ static void max1110_class_init(ObjectClass *klass, void *data)
     k->transfer = max111x_transfer;
 }
 
-static TypeInfo max1110_info = {
+static const TypeInfo max1110_info = {
     .name          = "max1110",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(MAX111xState),
@@ -177,7 +177,7 @@ static void max1111_class_init(ObjectClass *klass, void *data)
     k->transfer = max111x_transfer;
 }
 
-static TypeInfo max1111_info = {
+static const TypeInfo max1111_info = {
     .name          = "max1111",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(MAX111xState),
diff --git a/hw/max7310.c b/hw/max7310.c
index 1ed18ba..c2df0b4 100644
--- a/hw/max7310.c
+++ b/hw/max7310.c
@@ -25,7 +25,7 @@ typedef struct {
 
 static void max7310_reset(DeviceState *dev)
 {
-    MAX7310State *s = FROM_I2C_SLAVE(MAX7310State, I2C_SLAVE_FROM_QDEV(dev));
+    MAX7310State *s = FROM_I2C_SLAVE(MAX7310State, I2C_SLAVE(dev));
     s->level &= s->direction;
     s->direction = 0xff;
     s->polarity = 0xf0;
@@ -198,7 +198,7 @@ static void max7310_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_max7310;
 }
 
-static TypeInfo max7310_info = {
+static const TypeInfo max7310_info = {
     .name          = "max7310",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(MAX7310State),
diff --git a/hw/mc146818rtc.c b/hw/mc146818rtc.c
index c79fca7..2fb11f6 100644
--- a/hw/mc146818rtc.c
+++ b/hw/mc146818rtc.c
@@ -22,10 +22,10 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "mc146818rtc.h"
-#include "qapi/qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 #ifdef TARGET_I386
 #include "apic.h"
@@ -898,7 +898,7 @@ static void rtc_class_initfn(ObjectClass *klass, void *data)
     dc->props = mc146818rtc_properties;
 }
 
-static TypeInfo mc146818rtc_info = {
+static const TypeInfo mc146818rtc_info = {
     .name          = "mc146818rtc",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(RTCState),
diff --git a/hw/mcf5206.c b/hw/mcf5206.c
index 510d770..d8c0059 100644
--- a/hw/mcf5206.c
+++ b/hw/mcf5206.c
@@ -7,10 +7,10 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "sysemu.h"
-#include "exec-memory.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
 
 /* General purpose timer module.  */
 typedef struct {
@@ -359,7 +359,7 @@ static void m5206_mbar_write(m5206_mbar_state *s, uint32_t offset,
 
 /* Internal peripherals use a variety of register widths.
    This lookup table allows a single routine to handle all of them.  */
-static const int m5206_mbar_width[] =
+static const uint8_t m5206_mbar_width[] =
 {
   /* 000-040 */ 1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  2, 2, 2, 2,
   /* 040-080 */ 1, 2, 2, 2,  4, 1, 2, 4,  1, 2, 4, 2,  2, 4, 2, 2,
diff --git a/hw/mcf5208.c b/hw/mcf5208.c
index b1db549..2c9a5dc 100644
--- a/hw/mcf5208.c
+++ b/hw/mcf5208.c
@@ -7,14 +7,14 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "sysemu.h"
-#include "net.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define SYS_FREQ 66000000
 
@@ -292,6 +292,7 @@ static QEMUMachine mcf5208evb_machine = {
     .desc = "MCF5206EVB",
     .init = mcf5208evb_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mcf5208evb_machine_init(void)
diff --git a/hw/mcf_fec.c b/hw/mcf_fec.c
index 1ed193c..8e60f09 100644
--- a/hw/mcf_fec.c
+++ b/hw/mcf_fec.c
@@ -6,11 +6,11 @@
  * This code is licensed under the GPL
  */
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "mcf.h"
 /* For crc32 */
 #include <zlib.h>
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_FEC 1
 
@@ -174,7 +174,7 @@ static void mcf_fec_do_tx(mcf_fec_state *s)
         if (bd.flags & FEC_BD_L) {
             /* Last buffer in frame.  */
             DPRINTF("Sending packet\n");
-            qemu_send_packet(&s->nic->nc, frame, len);
+            qemu_send_packet(qemu_get_queue(s->nic), frame, len);
             ptr = frame;
             frame_size = 0;
             s->eir |= FEC_INT_TXF;
@@ -353,13 +353,13 @@ static void mcf_fec_write(void *opaque, hwaddr addr,
 
 static int mcf_fec_can_receive(NetClientState *nc)
 {
-    mcf_fec_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    mcf_fec_state *s = qemu_get_nic_opaque(nc);
     return s->rx_enabled;
 }
 
 static ssize_t mcf_fec_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    mcf_fec_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    mcf_fec_state *s = qemu_get_nic_opaque(nc);
     mcf_fec_bd bd;
     uint32_t flags = 0;
     uint32_t addr;
@@ -441,7 +441,7 @@ static const MemoryRegionOps mcf_fec_ops = {
 
 static void mcf_fec_cleanup(NetClientState *nc)
 {
-    mcf_fec_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    mcf_fec_state *s = qemu_get_nic_opaque(nc);
 
     memory_region_del_subregion(s->sysmem, &s->iomem);
     memory_region_destroy(&s->iomem);
@@ -472,9 +472,9 @@ void mcf_fec_init(MemoryRegion *sysmem, NICInfo *nd,
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
     s->conf.macaddr = nd->macaddr;
-    s->conf.peer = nd->netdev;
+    s->conf.peers.ncs[0] = nd->netdev;
 
     s->nic = qemu_new_nic(&net_mcf_fec_info, &s->conf, nd->model, nd->name, s);
 
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 }
diff --git a/hw/mcf_intc.c b/hw/mcf_intc.c
index 6ef6dac..3bed3a2 100644
--- a/hw/mcf_intc.c
+++ b/hw/mcf_intc.c
@@ -7,7 +7,7 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     MemoryRegion iomem;
diff --git a/hw/mcf_uart.c b/hw/mcf_uart.c
index d1655f8..c443443 100644
--- a/hw/mcf_uart.c
+++ b/hw/mcf_uart.c
@@ -7,8 +7,8 @@
  */
 #include "hw.h"
 #include "mcf.h"
-#include "qemu-char.h"
-#include "exec-memory.h"
+#include "char/char.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     MemoryRegion iomem;
diff --git a/hw/megasas.c b/hw/megasas.c
index 61b6527..eb191f5 100644
--- a/hw/megasas.c
+++ b/hw/megasas.c
@@ -19,10 +19,10 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "dma.h"
-#include "msix.h"
-#include "iov.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
+#include "pci/msix.h"
+#include "qemu/iov.h"
 #include "scsi.h"
 #include "scsi-defs.h"
 #include "trace.h"
diff --git a/hw/microblaze_boot.c b/hw/microblaze_boot.c
index 02c349c..3ec5c0f 100644
--- a/hw/microblaze_boot.c
+++ b/hw/microblaze_boot.c
@@ -24,10 +24,10 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "qemu-common.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "loader.h"
 #include "elf.h"
 
diff --git a/hw/milkymist-ac97.c b/hw/milkymist-ac97.c
index d87656c..d51d1ac 100644
--- a/hw/milkymist-ac97.c
+++ b/hw/milkymist-ac97.c
@@ -25,7 +25,7 @@
 #include "sysbus.h"
 #include "trace.h"
 #include "audio/audio.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     R_AC97_CTRL = 0,
@@ -329,7 +329,7 @@ static void milkymist_ac97_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_ac97;
 }
 
-static TypeInfo milkymist_ac97_info = {
+static const TypeInfo milkymist_ac97_info = {
     .name          = "milkymist-ac97",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistAC97State),
diff --git a/hw/milkymist-hpdmc.c b/hw/milkymist-hpdmc.c
index 5d120a4..ea4d210 100644
--- a/hw/milkymist-hpdmc.c
+++ b/hw/milkymist-hpdmc.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     R_SYSTEM = 0,
@@ -155,7 +155,7 @@ static void milkymist_hpdmc_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_hpdmc;
 }
 
-static TypeInfo milkymist_hpdmc_info = {
+static const TypeInfo milkymist_hpdmc_info = {
     .name          = "milkymist-hpdmc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistHpdmcState),
diff --git a/hw/milkymist-hw.h b/hw/milkymist-hw.h
index 96b2a7f..c8bd7e9 100644
--- a/hw/milkymist-hw.h
+++ b/hw/milkymist-hw.h
@@ -3,6 +3,7 @@
 
 #include "qdev.h"
 #include "qdev-addr.h"
+#include "net/net.h"
 
 static inline DeviceState *milkymist_uart_create(hwaddr base,
         qemu_irq irq)
@@ -11,8 +12,8 @@ static inline DeviceState *milkymist_uart_create(hwaddr base,
 
     dev = qdev_create(NULL, "milkymist-uart");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
@@ -23,7 +24,7 @@ static inline DeviceState *milkymist_hpdmc_create(hwaddr base)
 
     dev = qdev_create(NULL, "milkymist-hpdmc");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     return dev;
 }
@@ -34,7 +35,7 @@ static inline DeviceState *milkymist_memcard_create(hwaddr base)
 
     dev = qdev_create(NULL, "milkymist-memcard");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     return dev;
 }
@@ -48,7 +49,7 @@ static inline DeviceState *milkymist_vgafb_create(hwaddr base,
     qdev_prop_set_uint32(dev, "fb_offset", fb_offset);
     qdev_prop_set_uint32(dev, "fb_mask", fb_mask);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     return dev;
 }
@@ -66,10 +67,10 @@ static inline DeviceState *milkymist_sysctl_create(hwaddr base,
     qdev_prop_set_uint32(dev, "capabilities", capabilities);
     qdev_prop_set_uint32(dev, "gpio_strappings", gpio_strappings);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, gpio_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, timer0_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 2, timer1_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, gpio_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, timer0_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, timer1_irq);
 
     return dev;
 }
@@ -81,8 +82,8 @@ static inline DeviceState *milkymist_pfpu_create(hwaddr base,
 
     dev = qdev_create(NULL, "milkymist-pfpu");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
@@ -143,8 +144,8 @@ static inline DeviceState *milkymist_tmu2_create(hwaddr base,
 
     dev = qdev_create(NULL, "milkymist-tmu2");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 #else
@@ -160,11 +161,11 @@ static inline DeviceState *milkymist_ac97_create(hwaddr base,
 
     dev = qdev_create(NULL, "milkymist-ac97");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, crrequest_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, crreply_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 2, dmar_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 3, dmaw_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, crrequest_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, crreply_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2, dmar_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 3, dmaw_irq);
 
     return dev;
 }
@@ -178,9 +179,9 @@ static inline DeviceState *milkymist_minimac_create(hwaddr base,
     dev = qdev_create(NULL, "milkymist-minimac");
     qdev_set_nic_properties(dev, &nd_table[0]);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, rx_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, tx_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, rx_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, tx_irq);
 
     return dev;
 }
@@ -195,9 +196,9 @@ static inline DeviceState *milkymist_minimac2_create(hwaddr base,
     qdev_prop_set_taddr(dev, "buffers_base", buffers_base);
     qdev_set_nic_properties(dev, &nd_table[0]);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, rx_irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, tx_irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, rx_irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, tx_irq);
 
     return dev;
 }
@@ -214,8 +215,8 @@ static inline DeviceState *milkymist_softusb_create(hwaddr base,
     qdev_prop_set_uint32(dev, "dmem_base", dmem_base);
     qdev_prop_set_uint32(dev, "dmem_size", dmem_size);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
diff --git a/hw/milkymist-memcard.c b/hw/milkymist-memcard.c
index 318fc6d..d428b16 100644
--- a/hw/milkymist-memcard.c
+++ b/hw/milkymist-memcard.c
@@ -23,10 +23,10 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
-#include "qemu-error.h"
-#include "blockdev.h"
+#include "qemu/error-report.h"
+#include "sysemu/blockdev.h"
 #include "sd.h"
 
 enum {
@@ -288,7 +288,7 @@ static void milkymist_memcard_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_memcard;
 }
 
-static TypeInfo milkymist_memcard_info = {
+static const TypeInfo milkymist_memcard_info = {
     .name          = "milkymist-memcard",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistMemcardState),
diff --git a/hw/milkymist-minimac2.c b/hw/milkymist-minimac2.c
index b204e5f..9992dcc 100644
--- a/hw/milkymist-minimac2.c
+++ b/hw/milkymist-minimac2.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "net.h"
-#include "qemu-error.h"
+#include "net/net.h"
+#include "qemu/error-report.h"
 #include "qdev-addr.h"
 
 #include <zlib.h>
@@ -257,7 +257,7 @@ static void minimac2_tx(MilkymistMinimac2State *s)
     trace_milkymist_minimac2_tx_frame(txcount - 12);
 
     /* send packet, skipping preamble and sfd */
-    qemu_send_packet_raw(&s->nic->nc, buf + 8, txcount - 12);
+    qemu_send_packet_raw(qemu_get_queue(s->nic), buf + 8, txcount - 12);
 
     s->regs[R_TXCOUNT] = 0;
 
@@ -280,7 +280,7 @@ static void update_rx_interrupt(MilkymistMinimac2State *s)
 
 static ssize_t minimac2_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    MilkymistMinimac2State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    MilkymistMinimac2State *s = qemu_get_nic_opaque(nc);
 
     uint32_t r_count;
     uint32_t r_state;
@@ -410,7 +410,7 @@ static const MemoryRegionOps minimac2_ops = {
 
 static int minimac2_can_rx(NetClientState *nc)
 {
-    MilkymistMinimac2State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    MilkymistMinimac2State *s = qemu_get_nic_opaque(nc);
 
     if (s->regs[R_STATE0] == STATE_LOADED) {
         return 1;
@@ -424,7 +424,7 @@ static int minimac2_can_rx(NetClientState *nc)
 
 static void minimac2_cleanup(NetClientState *nc)
 {
-    MilkymistMinimac2State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    MilkymistMinimac2State *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -480,7 +480,7 @@ static int milkymist_minimac2_init(SysBusDevice *dev)
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_milkymist_minimac2_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     return 0;
 }
@@ -535,7 +535,7 @@ static void milkymist_minimac2_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_minimac2_properties;
 }
 
-static TypeInfo milkymist_minimac2_info = {
+static const TypeInfo milkymist_minimac2_info = {
     .name          = "milkymist-minimac2",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistMinimac2State),
diff --git a/hw/milkymist-pfpu.c b/hw/milkymist-pfpu.c
index 450bab9..c347680 100644
--- a/hw/milkymist-pfpu.c
+++ b/hw/milkymist-pfpu.c
@@ -25,8 +25,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-log.h"
-#include "qemu-error.h"
+#include "qemu/log.h"
+#include "qemu/error-report.h"
 #include <math.h>
 
 /* #define TRACE_EXEC */
@@ -529,7 +529,7 @@ static void milkymist_pfpu_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_pfpu;
 }
 
-static TypeInfo milkymist_pfpu_info = {
+static const TypeInfo milkymist_pfpu_info = {
     .name          = "milkymist-pfpu",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistPFPUState),
diff --git a/hw/milkymist-softusb.c b/hw/milkymist-softusb.c
index b162b88..01660be 100644
--- a/hw/milkymist-softusb.c
+++ b/hw/milkymist-softusb.c
@@ -24,9 +24,9 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hid.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     R_CTRL = 0,
@@ -316,7 +316,7 @@ static void milkymist_softusb_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_softusb_properties;
 }
 
-static TypeInfo milkymist_softusb_info = {
+static const TypeInfo milkymist_softusb_info = {
     .name          = "milkymist-softusb",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistSoftUsbState),
diff --git a/hw/milkymist-sysctl.c b/hw/milkymist-sysctl.c
index f951ef9..e69ac6f 100644
--- a/hw/milkymist-sysctl.c
+++ b/hw/milkymist-sysctl.c
@@ -23,11 +23,11 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 enum {
     CTRL_ENABLE      = (1<<0),
@@ -323,7 +323,7 @@ static void milkymist_sysctl_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_sysctl_properties;
 }
 
-static TypeInfo milkymist_sysctl_info = {
+static const TypeInfo milkymist_sysctl_info = {
     .name          = "milkymist-sysctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistSysctlState),
diff --git a/hw/milkymist-tmu2.c b/hw/milkymist-tmu2.c
index 3f9a684..42de10a 100644
--- a/hw/milkymist-tmu2.c
+++ b/hw/milkymist-tmu2.c
@@ -27,7 +27,7 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #include <X11/Xlib.h>
 #include <GL/gl.h>
@@ -475,7 +475,7 @@ static void milkymist_tmu2_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_tmu2;
 }
 
-static TypeInfo milkymist_tmu2_info = {
+static const TypeInfo milkymist_tmu2_info = {
     .name          = "milkymist-tmu2",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistTMU2State),
diff --git a/hw/milkymist-uart.c b/hw/milkymist-uart.c
index aefa8c7..e73eb84 100644
--- a/hw/milkymist-uart.c
+++ b/hw/milkymist-uart.c
@@ -24,8 +24,8 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "qemu-char.h"
-#include "qemu-error.h"
+#include "char/char.h"
+#include "qemu/error-report.h"
 
 enum {
     R_RXTX = 0,
@@ -228,7 +228,7 @@ static void milkymist_uart_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_milkymist_uart;
 }
 
-static TypeInfo milkymist_uart_info = {
+static const TypeInfo milkymist_uart_info = {
     .name          = "milkymist-uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistUartState),
diff --git a/hw/milkymist-vgafb.c b/hw/milkymist-vgafb.c
index 833881c..4d0a5df 100644
--- a/hw/milkymist-vgafb.c
+++ b/hw/milkymist-vgafb.c
@@ -25,10 +25,10 @@
 #include "hw.h"
 #include "sysbus.h"
 #include "trace.h"
-#include "console.h"
+#include "ui/console.h"
 #include "framebuffer.h"
-#include "pixel_ops.h"
-#include "qemu-error.h"
+#include "ui/pixel_ops.h"
+#include "qemu/error-report.h"
 
 #define BITS 8
 #include "milkymist-vgafb_template.h"
@@ -319,7 +319,7 @@ static void milkymist_vgafb_class_init(ObjectClass *klass, void *data)
     dc->props = milkymist_vgafb_properties;
 }
 
-static TypeInfo milkymist_vgafb_info = {
+static const TypeInfo milkymist_vgafb_info = {
     .name          = "milkymist-vgafb",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistVgafbState),
diff --git a/hw/milkymist.c b/hw/milkymist.c
index 4c8111a..c04eb35 100644
--- a/hw/milkymist.c
+++ b/hw/milkymist.c
@@ -19,17 +19,16 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "milkymist-hw.h"
 #include "lm32.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define BIOS_FILENAME    "mmone-bios.bin"
 #define BIOS_OFFSET      0x00860000
@@ -207,7 +206,8 @@ static QEMUMachine milkymist_machine = {
     .name = "milkymist",
     .desc = "Milkymist One",
     .init = milkymist_init,
-    .is_default = 0
+    .is_default = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void milkymist_machine_init(void)
diff --git a/hw/mips.h b/hw/mips.h
index f7e9b7e..291e85f 100644
--- a/hw/mips.h
+++ b/hw/mips.h
@@ -2,7 +2,7 @@
 #define HW_MIPS_H
 /* Definitions for mips board emulation.  */
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* gt64xxx.c */
 PCIBus *gt64120_register(qemu_irq *pic);
diff --git a/hw/mips_fulong2e.c b/hw/mips_fulong2e.c
index 5fcf900..8b532e1 100644
--- a/hw/mips_fulong2e.c
+++ b/hw/mips_fulong2e.c
@@ -22,18 +22,18 @@
 #include "pc.h"
 #include "serial.h"
 #include "fdc.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
 #include "smbus.h"
-#include "block.h"
+#include "block/block.h"
 #include "flash.h"
 #include "mips.h"
 #include "mips_cpudevs.h"
-#include "pci.h"
-#include "qemu-char.h"
-#include "sysemu.h"
+#include "pci/pci.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
 #include "audio/audio.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
 #include "mips-bios.h"
 #include "ide.h"
@@ -41,8 +41,8 @@
 #include "vt82c686.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define DEBUG_FULONG2E_INIT
 
@@ -400,6 +400,7 @@ static QEMUMachine mips_fulong2e_machine = {
     .name = "fulong2e",
     .desc = "Fulong 2e mini pc",
     .init = mips_fulong2e_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_fulong2e_machine_init(void)
diff --git a/hw/mips_jazz.c b/hw/mips_jazz.c
index 0847427..17fbdde 100644
--- a/hw/mips_jazz.c
+++ b/hw/mips_jazz.c
@@ -29,19 +29,19 @@
 #include "serial.h"
 #include "isa.h"
 #include "fdc.h"
-#include "sysemu.h"
-#include "arch_init.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/arch_init.h"
 #include "boards.h"
-#include "net.h"
+#include "net/net.h"
 #include "esp.h"
 #include "mips-bios.h"
 #include "loader.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
 #include "pcspk.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 enum jazz_model_e
 {
@@ -209,7 +209,7 @@ static void mips_jazz_init(MemoryRegion *address_space,
     case JAZZ_MAGNUM:
         dev = qdev_create(NULL, "sysbus-g364");
         qdev_init_nofail(dev);
-        sysbus = sysbus_from_qdev(dev);
+        sysbus = SYS_BUS_DEVICE(dev);
         sysbus_mmio_map(sysbus, 0, 0x60080000);
         sysbus_mmio_map(sysbus, 1, 0x40000000);
         sysbus_connect_irq(sysbus, 0, rc4030[3]);
@@ -295,7 +295,7 @@ static void mips_jazz_init(MemoryRegion *address_space,
     /* NVRAM */
     dev = qdev_create(NULL, "ds1225y");
     qdev_init_nofail(dev);
-    sysbus = sysbus_from_qdev(dev);
+    sysbus = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(sysbus, 0, 0x80009000);
 
     /* LED indicator */
@@ -324,14 +324,16 @@ static QEMUMachine mips_magnum_machine = {
     .name = "magnum",
     .desc = "MIPS Magnum",
     .init = mips_magnum_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine mips_pica61_machine = {
     .name = "pica61",
     .desc = "Acer Pica 61",
     .init = mips_pica61_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_jazz_machine_init(void)
diff --git a/hw/mips_malta.c b/hw/mips_malta.c
index 4d2464a..2a150df 100644
--- a/hw/mips_malta.c
+++ b/hw/mips_malta.c
@@ -26,27 +26,27 @@
 #include "pc.h"
 #include "serial.h"
 #include "fdc.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
 #include "smbus.h"
-#include "block.h"
+#include "block/block.h"
 #include "flash.h"
 #include "mips.h"
 #include "mips_cpudevs.h"
-#include "pci.h"
-#include "qemu-char.h"
-#include "sysemu.h"
-#include "arch_init.h"
+#include "pci/pci.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/arch_init.h"
 #include "boards.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "mips-bios.h"
 #include "ide.h"
 #include "loader.h"
 #include "elf.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"             /* SysBusDevice */
 
 //#define DEBUG_BOARD_INIT
@@ -743,10 +743,13 @@ static int64_t load_kernel (void)
     return kernel_entry;
 }
 
-static void malta_mips_config(CPUMIPSState *env)
+static void malta_mips_config(MIPSCPU *cpu)
 {
+    CPUMIPSState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+
     env->mvp->CP0_MVPConf0 |= ((smp_cpus - 1) << CP0MVPC0_PVPE) |
-                         ((smp_cpus * env->nr_threads - 1) << CP0MVPC0_PTC);
+                         ((smp_cpus * cs->nr_threads - 1) << CP0MVPC0_PTC);
 }
 
 static void main_cpu_reset(void *opaque)
@@ -763,7 +766,7 @@ static void main_cpu_reset(void *opaque)
         env->CP0_Status &= ~((1 << CP0St_BEV) | (1 << CP0St_ERL));
     }
 
-    malta_mips_config(env);
+    malta_mips_config(cpu);
 }
 
 static void cpu_request_exit(void *opaque, int irq, int level)
@@ -1004,7 +1007,7 @@ static void mips_malta_class_init(ObjectClass *klass, void *data)
     k->init = mips_malta_sysbus_device_init;
 }
 
-static TypeInfo mips_malta_device = {
+static const TypeInfo mips_malta_device = {
     .name          = "mips-malta",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MaltaState),
@@ -1017,6 +1020,7 @@ static QEMUMachine mips_malta_machine = {
     .init = mips_malta_init,
     .max_cpus = 16,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_malta_register_types(void)
diff --git a/hw/mips_mipssim.c b/hw/mips_mipssim.c
index a95a3c1..b0ab8f6 100644
--- a/hw/mips_mipssim.c
+++ b/hw/mips_mipssim.c
@@ -29,14 +29,14 @@
 #include "mips_cpudevs.h"
 #include "serial.h"
 #include "isa.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "mips-bios.h"
 #include "loader.h"
 #include "elf.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static struct _loaderparams {
     int ram_size;
@@ -123,7 +123,7 @@ static void mipsnet_init(int base, qemu_irq irq, NICInfo *nd)
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     memory_region_add_subregion(get_system_io(),
                                 base,
@@ -217,7 +217,8 @@ mips_mipssim_init(QEMUMachineInitArgs *args)
     /* A single 16450 sits at offset 0x3f8. It is attached to
        MIPS CPU INT2, which is interrupt 4. */
     if (serial_hds[0])
-        serial_init(0x3f8, env->irq[4], 115200, serial_hds[0]);
+        serial_init(0x3f8, env->irq[4], 115200, serial_hds[0],
+                    get_system_io());
 
     if (nd_table[0].used)
         /* MIPSnet uses the MIPS CPU INT0, which is interrupt 2. */
@@ -228,6 +229,7 @@ static QEMUMachine mips_mipssim_machine = {
     .name = "mipssim",
     .desc = "MIPS MIPSsim platform",
     .init = mips_mipssim_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_mipssim_machine_init(void)
diff --git a/hw/mips_r4k.c b/hw/mips_r4k.c
index 325098a..5df7eb4 100644
--- a/hw/mips_r4k.c
+++ b/hw/mips_r4k.c
@@ -13,19 +13,19 @@
 #include "pc.h"
 #include "serial.h"
 #include "isa.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "flash.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "mips-bios.h"
 #include "ide.h"
 #include "loader.h"
 #include "elf.h"
 #include "mc146818rtc.h"
 #include "i8254.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define MAX_IDE_BUS 2
 
@@ -302,6 +302,7 @@ static QEMUMachine mips_machine = {
     .name = "mips",
     .desc = "mips r4k platform",
     .init = mips_r4k_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void mips_machine_init(void)
diff --git a/hw/mips_timer.c b/hw/mips_timer.c
index 7aa9004..83c400c 100644
--- a/hw/mips_timer.c
+++ b/hw/mips_timer.c
@@ -22,7 +22,7 @@
 
 #include "hw.h"
 #include "mips_cpudevs.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #define TIMER_FREQ	100 * 1000 * 1000
 
diff --git a/hw/mipsnet.c b/hw/mipsnet.c
index bece332..ff6bf7f 100644
--- a/hw/mipsnet.c
+++ b/hw/mipsnet.c
@@ -1,5 +1,5 @@
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "trace.h"
 #include "sysbus.h"
 
@@ -64,7 +64,7 @@ static int mipsnet_buffer_full(MIPSnetState *s)
 
 static int mipsnet_can_receive(NetClientState *nc)
 {
-    MIPSnetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    MIPSnetState *s = qemu_get_nic_opaque(nc);
 
     if (s->busy)
         return 0;
@@ -73,7 +73,7 @@ static int mipsnet_can_receive(NetClientState *nc)
 
 static ssize_t mipsnet_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    MIPSnetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    MIPSnetState *s = qemu_get_nic_opaque(nc);
 
     trace_mipsnet_receive(size);
     if (!mipsnet_can_receive(nc))
@@ -173,7 +173,7 @@ static void mipsnet_ioport_write(void *opaque, hwaddr addr,
         if (s->tx_written == s->tx_count) {
             /* Send buffer. */
             trace_mipsnet_send(s->tx_count);
-            qemu_send_packet(&s->nic->nc, s->tx_buffer, s->tx_count);
+            qemu_send_packet(qemu_get_queue(s->nic), s->tx_buffer, s->tx_count);
             s->tx_count = s->tx_written = 0;
             s->intctl |= MIPSNET_INTCTL_TXDONE;
             s->busy = 1;
@@ -211,7 +211,7 @@ static const VMStateDescription vmstate_mipsnet = {
 
 static void mipsnet_cleanup(NetClientState *nc)
 {
-    MIPSnetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    MIPSnetState *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -241,7 +241,7 @@ static int mipsnet_sysbus_init(SysBusDevice *dev)
 
     s->nic = qemu_new_nic(&net_mipsnet_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     return 0;
 }
@@ -269,7 +269,7 @@ static void mipsnet_class_init(ObjectClass *klass, void *data)
     dc->props = mipsnet_properties;
 }
 
-static TypeInfo mipsnet_info = {
+static const TypeInfo mipsnet_info = {
     .name          = "mipsnet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MIPSnetState),
diff --git a/hw/mpc8544_guts.c b/hw/mpc8544_guts.c
index 873cb8c..728723c 100644
--- a/hw/mpc8544_guts.c
+++ b/hw/mpc8544_guts.c
@@ -18,7 +18,7 @@
  */
 
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 
 #define MPC8544_GUTS_MMIO_SIZE        0x1000
@@ -112,7 +112,7 @@ static int mpc8544_guts_initfn(SysBusDevice *dev)
 {
     GutsState *s;
 
-    s = FROM_SYSBUS(GutsState, sysbus_from_qdev(dev));
+    s = FROM_SYSBUS(GutsState, SYS_BUS_DEVICE(dev));
 
     memory_region_init_io(&s->iomem, &mpc8544_guts_ops, s,
                           "mpc6544.guts", MPC8544_GUTS_MMIO_SIZE);
@@ -128,7 +128,7 @@ static void mpc8544_guts_class_init(ObjectClass *klass, void *data)
     k->init = mpc8544_guts_initfn;
 }
 
-static TypeInfo mpc8544_guts_info = {
+static const TypeInfo mpc8544_guts_info = {
     .name          = "mpc8544-guts",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GutsState),
diff --git a/hw/msmouse.c b/hw/msmouse.c
index 9c492a4..ef47aed 100644
--- a/hw/msmouse.c
+++ b/hw/msmouse.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 #include <stdlib.h>
-#include "../qemu-common.h"
-#include "../qemu-char.h"
-#include "../console.h"
+#include "qemu-common.h"
+#include "char/char.h"
+#include "ui/console.h"
 #include "msmouse.h"
 
 #define MSMOUSE_LO6(n) ((n) & 0x3f)
diff --git a/hw/msmouse.h b/hw/msmouse.h
index 456cb21..8cff3a7 100644
--- a/hw/msmouse.h
+++ b/hw/msmouse.h
@@ -1,2 +1,7 @@
+#ifndef HW_MSMOUSE_H
+#define HW_MSMOUSE_H 1
+
 /* msmouse.c */
 CharDriverState *qemu_chr_open_msmouse(QemuOpts *opts);
+
+#endif
diff --git a/hw/mst_fpga.c b/hw/mst_fpga.c
index fb4b739..7ae05e3 100644
--- a/hw/mst_fpga.c
+++ b/hw/mst_fpga.c
@@ -248,7 +248,7 @@ static void mst_fpga_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_mst_fpga_regs;
 }
 
-static TypeInfo mst_fpga_info = {
+static const TypeInfo mst_fpga_info = {
     .name          = "mainstone-fpga",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mst_irq_state),
diff --git a/hw/multiboot.c b/hw/multiboot.c
index 09ec5b2..c4ec2e3 100644
--- a/hw/multiboot.c
+++ b/hw/multiboot.c
@@ -27,7 +27,7 @@
 #include "multiboot.h"
 #include "loader.h"
 #include "elf.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* Show multiboot debug output */
 //#define DEBUG_MULTIBOOT
diff --git a/hw/musicpal.c b/hw/musicpal.c
index e0c57c8..272cb80 100644
--- a/hw/musicpal.c
+++ b/hw/musicpal.c
@@ -12,18 +12,19 @@
 #include "sysbus.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "serial.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "block.h"
+#include "block/block.h"
 #include "flash.h"
-#include "console.h"
+#include "ui/console.h"
 #include "i2c.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
+#include "ui/pixel_ops.h"
 
 #define MP_MISC_BASE            0x80002000
 #define MP_MISC_SIZE            0x00001000
@@ -189,7 +190,7 @@ static int eth_can_receive(NetClientState *nc)
 
 static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    mv88w8618_eth_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    mv88w8618_eth_state *s = qemu_get_nic_opaque(nc);
     uint32_t desc_addr;
     mv88w8618_rx_desc desc;
     int i;
@@ -256,7 +257,7 @@ static void eth_send(mv88w8618_eth_state *s, int queue_index)
             len = desc.bytes;
             if (len < 2048) {
                 cpu_physical_memory_read(desc.buffer, buf, len);
-                qemu_send_packet(&s->nic->nc, buf, len);
+                qemu_send_packet(qemu_get_queue(s->nic), buf, len);
             }
             desc.cmdstat &= ~MP_ETH_TX_OWN;
             s->icr |= 1 << (MP_ETH_IRQ_TXLO_BIT - queue_index);
@@ -368,7 +369,7 @@ static const MemoryRegionOps mv88w8618_eth_ops = {
 
 static void eth_cleanup(NetClientState *nc)
 {
-    mv88w8618_eth_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    mv88w8618_eth_state *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -427,7 +428,7 @@ static void mv88w8618_eth_class_init(ObjectClass *klass, void *data)
     dc->props = mv88w8618_eth_properties;
 }
 
-static TypeInfo mv88w8618_eth_info = {
+static const TypeInfo mv88w8618_eth_info = {
     .name          = "mv88w8618_eth",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_eth_state),
@@ -492,8 +493,6 @@ SET_LCD_PIXEL(8, uint8_t)
 SET_LCD_PIXEL(16, uint16_t)
 SET_LCD_PIXEL(32, uint32_t)
 
-#include "pixel_ops.h"
-
 static void lcd_refresh(void *opaque)
 {
     musicpal_lcd_state *s = opaque;
@@ -644,7 +643,7 @@ static void musicpal_lcd_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &musicpal_lcd_vmsd;
 }
 
-static TypeInfo musicpal_lcd_info = {
+static const TypeInfo musicpal_lcd_info = {
     .name          = "musicpal_lcd",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(musicpal_lcd_state),
@@ -717,7 +716,7 @@ static void mv88w8618_pic_write(void *opaque, hwaddr offset,
 static void mv88w8618_pic_reset(DeviceState *d)
 {
     mv88w8618_pic_state *s = FROM_SYSBUS(mv88w8618_pic_state,
-                                         sysbus_from_qdev(d));
+                                         SYS_BUS_DEVICE(d));
 
     s->level = 0;
     s->enabled = 0;
@@ -763,7 +762,7 @@ static void mv88w8618_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &mv88w8618_pic_vmsd;
 }
 
-static TypeInfo mv88w8618_pic_info = {
+static const TypeInfo mv88w8618_pic_info = {
     .name          = "mv88w8618_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_pic_state),
@@ -874,7 +873,7 @@ static void mv88w8618_pit_write(void *opaque, hwaddr offset,
 static void mv88w8618_pit_reset(DeviceState *d)
 {
     mv88w8618_pit_state *s = FROM_SYSBUS(mv88w8618_pit_state,
-                                         sysbus_from_qdev(d));
+                                         SYS_BUS_DEVICE(d));
     int i;
 
     for (i = 0; i < 4; i++) {
@@ -940,7 +939,7 @@ static void mv88w8618_pit_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &mv88w8618_pit_vmsd;
 }
 
-static TypeInfo mv88w8618_pit_info = {
+static const TypeInfo mv88w8618_pit_info = {
     .name          = "mv88w8618_pit",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_pit_state),
@@ -1020,7 +1019,7 @@ static void mv88w8618_flashcfg_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &mv88w8618_flashcfg_vmsd;
 }
 
-static TypeInfo mv88w8618_flashcfg_info = {
+static const TypeInfo mv88w8618_flashcfg_info = {
     .name          = "mv88w8618_flashcfg",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mv88w8618_flashcfg_state),
@@ -1289,7 +1288,7 @@ static const MemoryRegionOps musicpal_gpio_ops = {
 static void musicpal_gpio_reset(DeviceState *d)
 {
     musicpal_gpio_state *s = FROM_SYSBUS(musicpal_gpio_state,
-                                         sysbus_from_qdev(d));
+                                         SYS_BUS_DEVICE(d));
 
     s->lcd_brightness = 0;
     s->out_state = 0;
@@ -1342,7 +1341,7 @@ static void musicpal_gpio_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &musicpal_gpio_vmsd;
 }
 
-static TypeInfo musicpal_gpio_info = {
+static const TypeInfo musicpal_gpio_info = {
     .name          = "musicpal_gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(musicpal_gpio_state),
@@ -1496,7 +1495,7 @@ static void musicpal_key_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &musicpal_key_vmsd;
 }
 
-static TypeInfo musicpal_key_info = {
+static const TypeInfo musicpal_key_info = {
     .name          = "musicpal_key",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(musicpal_key_state),
@@ -1608,12 +1607,12 @@ static void musicpal_init(QEMUMachineInitArgs *args)
     dev = qdev_create(NULL, "mv88w8618_eth");
     qdev_set_nic_properties(dev, &nd_table[0]);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, MP_ETH_BASE);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, pic[MP_ETH_IRQ]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, MP_ETH_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[MP_ETH_IRQ]);
 
     sysbus_create_simple("mv88w8618_wlan", MP_WLAN_BASE, NULL);
 
-    musicpal_misc_init(sysbus_from_qdev(dev));
+    musicpal_misc_init(SYS_BUS_DEVICE(dev));
 
     dev = sysbus_create_simple("musicpal_gpio", MP_GPIO_BASE, pic[MP_GPIO_IRQ]);
     i2c_dev = sysbus_create_simple("gpio_i2c", -1, NULL);
@@ -1642,7 +1641,7 @@ static void musicpal_init(QEMUMachineInitArgs *args)
 
     wm8750_dev = i2c_create_slave(i2c, "wm8750", MP_WM_ADDR);
     dev = qdev_create(NULL, "mv88w8618_audio");
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     qdev_prop_set_ptr(dev, "wm8750", wm8750_dev);
     qdev_init_nofail(dev);
     sysbus_mmio_map(s, 0, MP_AUDIO_BASE);
@@ -1659,6 +1658,7 @@ static QEMUMachine musicpal_machine = {
     .name = "musicpal",
     .desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)",
     .init = musicpal_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void musicpal_machine_init(void)
@@ -1675,7 +1675,7 @@ static void mv88w8618_wlan_class_init(ObjectClass *klass, void *data)
     sdc->init = mv88w8618_wlan_init;
 }
 
-static TypeInfo mv88w8618_wlan_info = {
+static const TypeInfo mv88w8618_wlan_info = {
     .name          = "mv88w8618_wlan",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
diff --git a/hw/nand.c b/hw/nand.c
index 5ff1a91..6c99068 100644
--- a/hw/nand.c
+++ b/hw/nand.c
@@ -20,9 +20,9 @@
 
 # include "hw.h"
 # include "flash.h"
-# include "blockdev.h"
+# include "sysemu/blockdev.h"
 # include "sysbus.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 # define NAND_CMD_READ0		0x00
 # define NAND_CMD_READ1		0x01
@@ -227,7 +227,7 @@ static const struct {
 
 static void nand_reset(DeviceState *dev)
 {
-    NANDFlashState *s = FROM_SYSBUS(NANDFlashState, sysbus_from_qdev(dev));
+    NANDFlashState *s = FROM_SYSBUS(NANDFlashState, SYS_BUS_DEVICE(dev));
     s->cmd = NAND_CMD_READ0;
     s->addr = 0;
     s->addrlen = 0;
@@ -440,7 +440,7 @@ static void nand_class_init(ObjectClass *klass, void *data)
     dc->props = nand_properties;
 }
 
-static TypeInfo nand_info = {
+static const TypeInfo nand_info = {
     .name          = "nand",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(NANDFlashState),
diff --git a/hw/ne2000-isa.c b/hw/ne2000-isa.c
index 69982a9..342c6bd 100644
--- a/hw/ne2000-isa.c
+++ b/hw/ne2000-isa.c
@@ -25,9 +25,9 @@
 #include "pc.h"
 #include "isa.h"
 #include "qdev.h"
-#include "net.h"
+#include "net/net.h"
 #include "ne2000.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef struct ISANE2000State {
     ISADevice dev;
@@ -38,7 +38,7 @@ typedef struct ISANE2000State {
 
 static void isa_ne2000_cleanup(NetClientState *nc)
 {
-    NE2000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    NE2000State *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -77,7 +77,7 @@ static int isa_ne2000_initfn(ISADevice *dev)
 
     s->nic = qemu_new_nic(&net_ne2000_isa_info, &s->c,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->c.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->c.macaddr.a);
 
     return 0;
 }
@@ -97,7 +97,7 @@ static void isa_ne2000_class_initfn(ObjectClass *klass, void *data)
     dc->props = ne2000_isa_properties;
 }
 
-static TypeInfo ne2000_isa_info = {
+static const TypeInfo ne2000_isa_info = {
     .name          = "ne2k_isa",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISANE2000State),
diff --git a/hw/ne2000.c b/hw/ne2000.c
index d3dd9a6..3dd1c84 100644
--- a/hw/ne2000.c
+++ b/hw/ne2000.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "ne2000.h"
 #include "loader.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* debug NE2000 card */
 //#define DEBUG_NE2000
@@ -167,7 +167,7 @@ static int ne2000_buffer_full(NE2000State *s)
 
 int ne2000_can_receive(NetClientState *nc)
 {
-    NE2000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    NE2000State *s = qemu_get_nic_opaque(nc);
 
     if (s->cmd & E8390_STOP)
         return 1;
@@ -178,7 +178,7 @@ int ne2000_can_receive(NetClientState *nc)
 
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
 {
-    NE2000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    NE2000State *s = qemu_get_nic_opaque(nc);
     int size = size_;
     uint8_t *p;
     unsigned int total_len, next, avail, len, index, mcast_idx;
@@ -300,7 +300,8 @@ static void ne2000_ioport_write(void *opaque, uint32_t addr, uint32_t val)
                     index -= NE2000_PMEM_SIZE;
                 /* fail safe: check range on the transmitted length  */
                 if (index + s->tcnt <= NE2000_PMEM_END) {
-                    qemu_send_packet(&s->nic->nc, s->mem + index, s->tcnt);
+                    qemu_send_packet(qemu_get_queue(s->nic), s->mem + index,
+                                     s->tcnt);
                 }
                 /* signal end of transfer */
                 s->tsr = ENTSR_PTX;
@@ -705,7 +706,7 @@ void ne2000_setup_io(NE2000State *s, unsigned size)
 
 static void ne2000_cleanup(NetClientState *nc)
 {
-    NE2000State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    NE2000State *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -737,7 +738,7 @@ static int pci_ne2000_init(PCIDevice *pci_dev)
 
     s->nic = qemu_new_nic(&net_ne2000_info, &s->c,
                           object_get_typename(OBJECT(pci_dev)), pci_dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->c.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->c.macaddr.a);
 
     add_boot_device_path(s->c.bootindex, &pci_dev->qdev, "/ethernet-phy@0");
 
@@ -750,7 +751,7 @@ static void pci_ne2000_exit(PCIDevice *pci_dev)
     NE2000State *s = &d->ne2000;
 
     memory_region_destroy(&s->io);
-    qemu_del_net_client(&s->nic->nc);
+    qemu_del_nic(s->nic);
 }
 
 static Property ne2000_properties[] = {
@@ -773,7 +774,7 @@ static void ne2000_class_init(ObjectClass *klass, void *data)
     dc->props = ne2000_properties;
 }
 
-static TypeInfo ne2000_info = {
+static const TypeInfo ne2000_info = {
     .name          = "ne2k_pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCINE2000State),
diff --git a/hw/ne2000.h b/hw/ne2000.h
index 1e7ab07..b31ae03 100644
--- a/hw/ne2000.h
+++ b/hw/ne2000.h
@@ -1,3 +1,6 @@
+#ifndef HW_NE2000_H
+#define HW_NE2000_H 1
+
 #define NE2000_PMEM_SIZE    (32*1024)
 #define NE2000_PMEM_START   (16*1024)
 #define NE2000_PMEM_END     (NE2000_PMEM_SIZE+NE2000_PMEM_START)
@@ -33,3 +36,5 @@ extern const VMStateDescription vmstate_ne2000;
 void ne2000_reset(NE2000State *s);
 int ne2000_can_receive(NetClientState *nc);
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_);
+
+#endif
diff --git a/hw/nseries.c b/hw/nseries.c
index 83c33a9..f8e1c0a 100644
--- a/hw/nseries.c
+++ b/hw/nseries.c
@@ -19,11 +19,11 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "omap.h"
 #include "arm-misc.h"
 #include "irq.h"
-#include "console.h"
+#include "ui/console.h"
 #include "boards.h"
 #include "i2c.h"
 #include "spi.h"
@@ -31,11 +31,11 @@
 #include "flash.h"
 #include "hw.h"
 #include "bt.h"
-#include "net.h"
+#include "net/net.h"
 #include "loader.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 //#define MIPID_DEBUG
 
@@ -186,10 +186,10 @@ static void n8x0_nand_setup(struct n800_s *s)
         qdev_prop_set_drive_nofail(s->nand, "drive", dinfo->bdrv);
     }
     qdev_init_nofail(s->nand);
-    sysbus_connect_irq(sysbus_from_qdev(s->nand), 0,
+    sysbus_connect_irq(SYS_BUS_DEVICE(s->nand), 0,
                        qdev_get_gpio_in(s->mpu->gpio, N8X0_ONENAND_GPIO));
     omap_gpmc_attach(s->mpu->gpmc, N8X0_ONENAND_CS,
-                     sysbus_mmio_get_region(sysbus_from_qdev(s->nand), 0));
+                     sysbus_mmio_get_region(SYS_BUS_DEVICE(s->nand), 0));
     otp_region = onenand_raw_otp(s->nand);
 
     memcpy(otp_region + 0x000, n8x0_cal_wlan_mac, sizeof(n8x0_cal_wlan_mac));
@@ -932,7 +932,7 @@ static void n8x0_usb_setup(struct n800_s *s)
 {
     SysBusDevice *dev;
     s->usb = qdev_create(NULL, "tusb6010");
-    dev = sysbus_from_qdev(s->usb);
+    dev = SYS_BUS_DEVICE(s->usb);
     qdev_init_nofail(s->usb);
     sysbus_connect_irq(dev, 0,
                        qdev_get_gpio_in(s->mpu->gpio, N8X0_TUSB_INT_GPIO));
@@ -1560,12 +1560,14 @@ static QEMUMachine n800_machine = {
     .name = "n800",
     .desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)",
     .init = n800_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine n810_machine = {
     .name = "n810",
     .desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)",
     .init = n810_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define N900_SDRAM_SIZE (256 * 1024 * 1024)
@@ -1779,7 +1781,7 @@ static void lis302dl_step(void *opaque, int axis, int high, int activate)
 
 static void lis302dl_reset(DeviceState *ds)
 {
-    LIS302DLState *s = FROM_I2C_SLAVE(LIS302DLState, I2C_SLAVE_FROM_QDEV(ds));
+    LIS302DLState *s = FROM_I2C_SLAVE(LIS302DLState, I2C_SLAVE(ds));
     
     s->firstbyte = 0;
     s->reg = 0;
@@ -2104,7 +2106,7 @@ typedef struct BQ2415XState_s {
 
 static void bq2415x_reset(DeviceState *ds)
 {
-    BQ2415XState *s = FROM_I2C_SLAVE(BQ2415XState, I2C_SLAVE_FROM_QDEV(ds));
+    BQ2415XState *s = FROM_I2C_SLAVE(BQ2415XState, I2C_SLAVE(ds));
     
     s->firstbyte = 0;
     s->reg = 0;
@@ -2229,7 +2231,7 @@ typedef struct tpa6130_s {
 
 static void tpa6130_reset(DeviceState *ds)
 {
-    TPA6130State *s = FROM_I2C_SLAVE(TPA6130State, I2C_SLAVE_FROM_QDEV(ds));
+    TPA6130State *s = FROM_I2C_SLAVE(TPA6130State, I2C_SLAVE(ds));
     s->firstbyte = 0;
     s->reg = 0;
     memset(s->data, 0, sizeof(s->data));
@@ -2547,10 +2549,10 @@ static void n900_init(QEMUMachineInitArgs *args)
         qdev_prop_set_drive_nofail(s->nand, "drive", dmtd->bdrv);
     }
     qdev_init_nofail(s->nand);
-    sysbus_connect_irq(sysbus_from_qdev(s->nand), 0,
+    sysbus_connect_irq(SYS_BUS_DEVICE(s->nand), 0,
                        qdev_get_gpio_in(s->cpu->gpio, N900_ONENAND_GPIO));
     omap_gpmc_attach(s->cpu->gpmc, 0,
-                     sysbus_mmio_get_region(sysbus_from_qdev(s->nand), 0));
+                     sysbus_mmio_get_region(SYS_BUS_DEVICE(s->nand), 0));
 
     if (dsd) {
         omap3_mmc_attach(s->cpu->omap3_mmc[1], dsd->bdrv, 0, 1);
@@ -2592,10 +2594,10 @@ static void n900_init(QEMUMachineInitArgs *args)
         s->smc = qdev_create(NULL, "smc91c111");
         qdev_set_nic_properties(s->smc, &nd_table[i]);
         qdev_init_nofail(s->smc);
-        sysbus_connect_irq(sysbus_from_qdev(s->smc), 0,
+        sysbus_connect_irq(SYS_BUS_DEVICE(s->smc), 0,
                            qdev_get_gpio_in(s->cpu->gpio, 54));
         omap_gpmc_attach(s->cpu->gpmc, 1,
-                         sysbus_mmio_get_region(sysbus_from_qdev(s->smc), 0));
+                         sysbus_mmio_get_region(SYS_BUS_DEVICE(s->smc), 0));
     } else {
         hw_error("%s: no NIC for smc91c111\n", __FUNCTION__);
     }
diff --git a/hw/null-machine.c b/hw/null-machine.c
index d813c08..bdf109f 100644
--- a/hw/null-machine.c
+++ b/hw/null-machine.c
@@ -24,6 +24,7 @@ static QEMUMachine machine_none = {
     .desc = "empty machine",
     .init = machine_none_init,
     .max_cpus = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void register_machines(void)
diff --git a/hw/omap.h b/hw/omap.h
index eb85c71..eb2fc7a 100644
--- a/hw/omap.h
+++ b/hw/omap.h
@@ -17,11 +17,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef hw_omap_h
-#include "memory.h"
+#include "exec/memory.h"
 # define hw_omap_h		"omap.h"
 #include "hw/irq.h"
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "dsi.h"
 #include "spi.h"
 
diff --git a/hw/omap1.c b/hw/omap1.c
index c49c630..c4cce09 100644
--- a/hw/omap1.c
+++ b/hw/omap1.c
@@ -19,10 +19,10 @@
 #include "hw.h"
 #include "arm-misc.h"
 #include "omap.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "soc_dma.h"
-#include "blockdev.h"
-#include "range.h"
+#include "sysemu/blockdev.h"
+#include "qemu/range.h"
 #include "sysbus.h"
 
 /* Should signal the TCMI/GPMC */
@@ -529,6 +529,7 @@ static uint64_t omap_ulpd_pm_read(void *opaque, hwaddr addr,
     case 0x28:	/* Reserved */
     case 0x2c:	/* Reserved */
         OMAP_BAD_REG(addr);
+        /* fall through */
     case 0x00:	/* COUNTER_32_LSB */
     case 0x04:	/* COUNTER_32_MSB */
     case 0x08:	/* COUNTER_HIGH_FREQ_LSB */
@@ -633,6 +634,7 @@ static void omap_ulpd_pm_write(void *opaque, hwaddr addr,
     case 0x28:	/* Reserved */
     case 0x2c:	/* Reserved */
         OMAP_BAD_REG(addr);
+        /* fall through */
     case 0x24:	/* SETUP_ANALOG_CELL3_ULPD1 */
     case 0x38:	/* COUNTER_32_FIQ */
     case 0x48:	/* LOCL_TIME */
@@ -1089,6 +1091,7 @@ static void omap_mpui_write(void *opaque, hwaddr addr,
     /* Not in OMAP310 */
     case 0x14:	/* DSP_STATUS */
         OMAP_RO_REG(addr);
+        break;
     case 0x18:	/* DSP_BOOT_CONFIG */
     case 0x1c:	/* DSP_MPUI_CONFIG */
         break;
@@ -2830,7 +2833,7 @@ static void omap_rtc_tick(void *opaque)
         s->round = 0;
     }
 
-    memcpy(&s->current_tm, localtime(&s->ti), sizeof(s->current_tm));
+    localtime_r(&s->ti, &s->current_tm);
 
     if ((s->interrupts & 0x08) && s->ti == s->alarm_ti) {
         s->status |= 0x40;
@@ -3856,7 +3859,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_uint32(s->ih[0], "size", 0x100);
     qdev_prop_set_ptr(s->ih[0], "clk", omap_findclk(s, "arminth_ck"));
     qdev_init_nofail(s->ih[0]);
-    busdev = sysbus_from_qdev(s->ih[0]);
+    busdev = SYS_BUS_DEVICE(s->ih[0]);
     sysbus_connect_irq(busdev, 0, cpu_irq[ARM_PIC_CPU_IRQ]);
     sysbus_connect_irq(busdev, 1, cpu_irq[ARM_PIC_CPU_FIQ]);
     sysbus_mmio_map(busdev, 0, 0xfffecb00);
@@ -3864,7 +3867,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_uint32(s->ih[1], "size", 0x800);
     qdev_prop_set_ptr(s->ih[1], "clk", omap_findclk(s, "arminth_ck"));
     qdev_init_nofail(s->ih[1]);
-    busdev = sysbus_from_qdev(s->ih[1]);
+    busdev = SYS_BUS_DEVICE(s->ih[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_15XX_IH2_IRQ));
     /* The second interrupt controller's FIQ output is not wired up */
@@ -3936,7 +3939,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
                          omap_clk_getrate(omap_findclk(s, "uart1_ck")) / 16);
     qdev_prop_set_chr(s->uart[0], "chardev", serial_hds[0]);
     qdev_init_nofail(s->uart[0]);
-    busdev = sysbus_from_qdev(s->uart[0]);
+    busdev = SYS_BUS_DEVICE(s->uart[0]);
     sysbus_connect_irq(busdev, 0, qdev_get_gpio_in(s->ih[1], OMAP_INT_UART1));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP_DMA_UART1_TX]);
     sysbus_connect_irq(busdev, 2, s->drq[OMAP_DMA_UART1_RX]);
@@ -3950,7 +3953,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_chr(s->uart[1], "chardev",
                       serial_hds[0] ? serial_hds[1] : NULL);
     qdev_init_nofail(s->uart[1]);
-    busdev = sysbus_from_qdev(s->uart[1]);
+    busdev = SYS_BUS_DEVICE(s->uart[1]);
     sysbus_connect_irq(busdev, 0, qdev_get_gpio_in(s->ih[1], OMAP_INT_UART2));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP_DMA_UART2_TX]);
     sysbus_connect_irq(busdev, 2, s->drq[OMAP_DMA_UART2_RX]);
@@ -3964,7 +3967,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_chr(s->uart[2], "chardev",
                       serial_hds[0] && serial_hds[1] ? serial_hds[2] : NULL);
     qdev_init_nofail(s->uart[2]);
-    busdev = sysbus_from_qdev(s->uart[2]);
+    busdev = SYS_BUS_DEVICE(s->uart[2]);
     sysbus_connect_irq(busdev, 0, qdev_get_gpio_in(s->ih[0], OMAP_INT_UART3));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP_DMA_UART3_TX]);
     sysbus_connect_irq(busdev, 2, s->drq[OMAP_DMA_UART3_RX]);
@@ -3996,9 +3999,9 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_int32(s->gpio, "mpu_model", s->mpu_model);
     qdev_prop_set_ptr(s->gpio, "clk", omap_findclk(s, "arm_gpio_ck"));
     qdev_init_nofail(s->gpio);
-    sysbus_connect_irq(sysbus_from_qdev(s->gpio), 0,
+    sysbus_connect_irq(SYS_BUS_DEVICE(s->gpio), 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_GPIO_BANK1));
-    sysbus_mmio_map(sysbus_from_qdev(s->gpio), 0, 0xfffce000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(s->gpio), 0, 0xfffce000);
 
     s->microwire = omap_uwire_init(system_memory, 0xfffb3000,
                                    qdev_get_gpio_in(s->ih[1], OMAP_INT_uWireTX),
@@ -4014,7 +4017,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
     qdev_prop_set_uint8(s->i2c[0], "revision", 0x11);
     qdev_prop_set_ptr(s->i2c[0], "fclk", omap_findclk(s, "mpuper_ck"));
     qdev_init_nofail(s->i2c[0]);
-    busdev = sysbus_from_qdev(s->i2c[0]);
+    busdev = SYS_BUS_DEVICE(s->i2c[0]);
     sysbus_connect_irq(busdev, 0, qdev_get_gpio_in(s->ih[1], OMAP_INT_I2C));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP_DMA_I2C_TX]);
     sysbus_connect_irq(busdev, 2, s->drq[OMAP_DMA_I2C_RX]);
diff --git a/hw/omap2.c b/hw/omap2.c
index 471a75e..c222b64 100644
--- a/hw/omap2.c
+++ b/hw/omap2.c
@@ -18,13 +18,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw.h"
 #include "arm-misc.h"
 #include "omap.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "flash.h"
 #include "soc_dma.h"
 #include "sysbus.h"
@@ -2277,7 +2277,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->ih[0], "fclk", omap_findclk(s, "mpu_intc_fclk"));
     qdev_prop_set_ptr(s->ih[0], "iclk", omap_findclk(s, "mpu_intc_iclk"));
     qdev_init_nofail(s->ih[0]);
-    busdev = sysbus_from_qdev(s->ih[0]);
+    busdev = SYS_BUS_DEVICE(s->ih[0]);
     sysbus_connect_irq(busdev, 0, cpu_irq[ARM_PIC_CPU_IRQ]);
     sysbus_connect_irq(busdev, 1, cpu_irq[ARM_PIC_CPU_FIQ]);
     sysbus_mmio_map(busdev, 0, 0x480fe000);
@@ -2311,7 +2311,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                          omap_clk_getrate(omap_findclk(s, "uart1_fclk")) / 16);
     qdev_prop_set_chr(s->uart[0], "chardev", serial_hds[0]);
     qdev_init_nofail(s->uart[0]);
-    busdev = sysbus_from_qdev(s->uart[0]);
+    busdev = SYS_BUS_DEVICE(s->uart[0]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_UART1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_UART1_TX]);
@@ -2326,7 +2326,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_chr(s->uart[1], "chardev",
                       serial_hds[0] ? serial_hds[1] : NULL);
     qdev_init_nofail(s->uart[1]);
-    busdev = sysbus_from_qdev(s->uart[1]);
+    busdev = SYS_BUS_DEVICE(s->uart[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_UART2_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_UART2_TX]);
@@ -2341,7 +2341,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_chr(s->uart[2], "chardev",
                       serial_hds[0] && serial_hds[1] ? serial_hds[2] : NULL);
     qdev_init_nofail(s->uart[2]);
-    busdev = sysbus_from_qdev(s->uart[2]);
+    busdev = SYS_BUS_DEVICE(s->uart[2]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_UART3_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_UART3_TX]);
@@ -2408,7 +2408,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[0], "iclk", omap_findclk(s, "i2c1.iclk"));
     qdev_prop_set_ptr(s->i2c[0], "fclk", omap_findclk(s, "i2c1.fclk"));
     qdev_init_nofail(s->i2c[0]);
-    busdev = sysbus_from_qdev(s->i2c[0]);
+    busdev = SYS_BUS_DEVICE(s->i2c[0]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_I2C1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_I2C1_TX]);
@@ -2420,7 +2420,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[1], "iclk", omap_findclk(s, "i2c2.iclk"));
     qdev_prop_set_ptr(s->i2c[1], "fclk", omap_findclk(s, "i2c2.fclk"));
     qdev_init_nofail(s->i2c[1]);
-    busdev = sysbus_from_qdev(s->i2c[1]);
+    busdev = SYS_BUS_DEVICE(s->i2c[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_I2C2_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_I2C2_TX]);
@@ -2438,7 +2438,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
         qdev_prop_set_ptr(s->gpio, "fclk4", omap_findclk(s, "gpio5_dbclk"));
     }
     qdev_init_nofail(s->gpio);
-    busdev = sysbus_from_qdev(s->gpio);
+    busdev = SYS_BUS_DEVICE(s->gpio);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_GPIO_BANK1));
     sysbus_connect_irq(busdev, 3,
@@ -2477,7 +2477,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     s->mcspi = qdev_create(NULL, "omap_mcspi");
     qdev_prop_set_int32(s->mcspi, "mpu_model", s->mpu_model);
     qdev_init_nofail(s->mcspi);
-    busdev = sysbus_from_qdev(s->mcspi);
+    busdev = SYS_BUS_DEVICE(s->mcspi);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_MCSPI1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_SPI1_TX0]);
@@ -2500,7 +2500,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
     s->dss = qdev_create(NULL, "omap_dss");
     qdev_prop_set_int32(s->dss, "mpu_model", s->mpu_model);
     qdev_init_nofail(s->dss);
-    busdev = sysbus_from_qdev(s->dss);
+    busdev = SYS_BUS_DEVICE(s->dss);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_24XX_DSS_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP24XX_DMA_DSS]);
diff --git a/hw/omap3.c b/hw/omap3.c
index b93f0ac..aca4855 100644
--- a/hw/omap3.c
+++ b/hw/omap3.c
@@ -24,14 +24,14 @@
 #include "hw.h"
 #include "arm-misc.h"
 #include "omap.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "flash.h"
 #include "soc_dma.h"
 #include "sysbus.h"
 #include "audio/audio.h"
-#include "block.h"
+#include "block/block.h"
 #include "qdev-addr.h"
 
 //#define OMAP3_DEBUG
@@ -4092,7 +4092,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->ih[0], "fclk", omap_findclk(s, "omap3_mpu_intc_fclk"));
     qdev_prop_set_ptr(s->ih[0], "iclk", omap_findclk(s, "omap3_mpu_intc_iclk"));
     qdev_init_nofail(s->ih[0]);
-    busdev = sysbus_from_qdev(s->ih[0]);
+    busdev = SYS_BUS_DEVICE(s->ih[0]);
     sysbus_connect_irq(busdev, 0, cpu_irq[ARM_PIC_CPU_IRQ]);
     sysbus_connect_irq(busdev, 1, cpu_irq[ARM_PIC_CPU_FIQ]);
     sysbus_mmio_map(busdev, 0, 0x48200000);
@@ -4207,7 +4207,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_chr(s->uart[0], "chardev", chr_uart1);
     qdev_prop_set_uint32(s->uart[0], "revision", uart_revision);
     qdev_init_nofail(s->uart[0]);
-    busdev = sysbus_from_qdev(s->uart[0]);
+    busdev = SYS_BUS_DEVICE(s->uart[0]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_UART1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_UART1_TX]);
@@ -4224,7 +4224,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_chr(s->uart[1], "chardev", chr_uart2);
     qdev_prop_set_uint32(s->uart[1], "revision", uart_revision);
     qdev_init_nofail(s->uart[1]);
-    busdev = sysbus_from_qdev(s->uart[1]);
+    busdev = SYS_BUS_DEVICE(s->uart[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_UART2_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_UART2_TX]);
@@ -4241,7 +4241,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_chr(s->uart[2], "chardev", chr_uart3);
     qdev_prop_set_uint32(s->uart[2], "revision", uart_revision);
     qdev_init_nofail(s->uart[2]);
-    busdev = sysbus_from_qdev(s->uart[2]);
+    busdev = SYS_BUS_DEVICE(s->uart[2]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_UART3_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_UART3_TX]);
@@ -4260,7 +4260,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
         qdev_prop_set_chr(s->uart[3], "chardev", chr_uart4);
         qdev_prop_set_uint32(s->uart[3], "revision", uart_revision);
         qdev_init_nofail(s->uart[3]);
-        busdev = sysbus_from_qdev(s->uart[3]);
+        busdev = SYS_BUS_DEVICE(s->uart[3]);
         sysbus_connect_irq(busdev, 0,
                            qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_UART4_IRQ));
         sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_UART4_TX]);
@@ -4272,7 +4272,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     s->dss = qdev_create(NULL, "omap_dss");
     qdev_prop_set_int32(s->dss, "mpu_model", s->mpu_model);
     qdev_init_nofail(s->dss);
-    busdev = sysbus_from_qdev(s->dss);
+    busdev = SYS_BUS_DEVICE(s->dss);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_DSS_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_DSS_LINETRIGGER]);
@@ -4297,7 +4297,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->gpio, "fclk4", omap_findclk(s, "omap3_per_32k_fclk"));
     qdev_prop_set_ptr(s->gpio, "fclk5", omap_findclk(s, "omap3_per_32k_fclk"));
     qdev_init_nofail(s->gpio);
-    busdev = sysbus_from_qdev(s->gpio);
+    busdev = SYS_BUS_DEVICE(s->gpio);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_GPIO1_MPU_IRQ));
     sysbus_connect_irq(busdev, 3,
@@ -4328,7 +4328,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     s->omap3_mmc[0] = qdev_create(NULL, "omap3_mmc");
     s->omap3_mmc[0]->id = "mmc1";
     qdev_init_nofail(s->omap3_mmc[0]);
-    busdev = sysbus_from_qdev(s->omap3_mmc[0]);
+    busdev = SYS_BUS_DEVICE(s->omap3_mmc[0]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_MMC1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_MMC1_TX]);
@@ -4338,7 +4338,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     s->omap3_mmc[1] = qdev_create(NULL, "omap3_mmc");
     s->omap3_mmc[1]->id = "mmc2";
     qdev_init_nofail(s->omap3_mmc[1]);
-    busdev = sysbus_from_qdev(s->omap3_mmc[1]);
+    busdev = SYS_BUS_DEVICE(s->omap3_mmc[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_MMC2_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_MMC2_TX]);
@@ -4348,7 +4348,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     s->omap3_mmc[2] = qdev_create(NULL, "omap3_mmc");
     s->omap3_mmc[2]->id = "mmc3";
     qdev_init_nofail(s->omap3_mmc[2]);
-    busdev = sysbus_from_qdev(s->omap3_mmc[2]);
+    busdev = SYS_BUS_DEVICE(s->omap3_mmc[2]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_MMC3_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_MMC3_TX]);
@@ -4365,7 +4365,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[0], "iclk", omap_findclk(s, "omap3_i2c1_iclk"));
     qdev_prop_set_ptr(s->i2c[0], "fclk", omap_findclk(s, "omap3_i2c1_fclk"));
     qdev_init_nofail(s->i2c[0]);
-    busdev = sysbus_from_qdev(s->i2c[0]);
+    busdev = SYS_BUS_DEVICE(s->i2c[0]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_I2C1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_I2C1_TX]);
@@ -4379,7 +4379,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[1], "iclk", omap_findclk(s, "omap3_i2c2_iclk"));
     qdev_prop_set_ptr(s->i2c[1], "fclk", omap_findclk(s, "omap3_i2c2_fclk"));
     qdev_init_nofail(s->i2c[1]);
-    busdev = sysbus_from_qdev(s->i2c[1]);
+    busdev = SYS_BUS_DEVICE(s->i2c[1]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_I2C2_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_I2C2_TX]);
@@ -4393,7 +4393,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_ptr(s->i2c[2], "iclk", omap_findclk(s, "omap3_i2c3_iclk"));
     qdev_prop_set_ptr(s->i2c[2], "fclk", omap_findclk(s, "omap3_i2c3_fclk"));
     qdev_init_nofail(s->i2c[2]);
-    busdev = sysbus_from_qdev(s->i2c[2]);
+    busdev = SYS_BUS_DEVICE(s->i2c[2]);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_I2C3_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_I2C3_TX]);
@@ -4403,7 +4403,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
 
     s->omap3_usb_otg = qdev_create(NULL, "omap3_hsusb_otg");
     qdev_init_nofail(s->omap3_usb_otg);
-    busdev = sysbus_from_qdev(s->omap3_usb_otg);
+    busdev = SYS_BUS_DEVICE(s->omap3_usb_otg);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_HSUSB_MC));
     sysbus_connect_irq(busdev, 1,
@@ -4417,7 +4417,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
 
     s->omap3_usb_host = qdev_create(NULL, "omap3_hsusb_host");
     qdev_init_nofail(s->omap3_usb_host);
-    busdev =  sysbus_from_qdev(s->omap3_usb_host);
+    busdev =  SYS_BUS_DEVICE(s->omap3_usb_host);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_EHCI_IRQ));
     sysbus_connect_irq(busdev, 1,
@@ -4434,7 +4434,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     qdev_prop_set_uint32(s->omap3_usb_ohci, "num-ports", 3);
     qdev_prop_set_taddr(s->omap3_usb_ohci, "dma-offset", 0);
     qdev_init_nofail(s->omap3_usb_ohci);
-    busdev = sysbus_from_qdev(s->omap3_usb_ohci);
+    busdev = SYS_BUS_DEVICE(s->omap3_usb_ohci);
     sysbus_mmio_map(busdev, 0, omap_l4_region_base(usbhost_ta, 1));
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_OHCI_IRQ));
@@ -4442,7 +4442,7 @@ struct omap_mpu_state_s *omap3_mpu_init(MemoryRegion *sysmem,
     s->mcspi = qdev_create(NULL, "omap_mcspi");
     qdev_prop_set_int32(s->mcspi, "mpu_model", s->mpu_model);
     qdev_init_nofail(s->mcspi);
-    busdev = sysbus_from_qdev(s->mcspi);
+    busdev = SYS_BUS_DEVICE(s->mcspi);
     sysbus_connect_irq(busdev, 0,
                        qdev_get_gpio_in(s->ih[0], OMAP_INT_3XXX_MCSPI1_IRQ));
     sysbus_connect_irq(busdev, 1, s->drq[OMAP3XXX_DMA_SPI1_TX0]);
diff --git a/hw/omap3_boot.c b/hw/omap3_boot.c
index ee05afd..f9ce253 100644
--- a/hw/omap3_boot.c
+++ b/hw/omap3_boot.c
@@ -26,11 +26,11 @@
 #include "hw.h"
 #include "arm-misc.h"
 #include "omap.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 //#define OMAP3_BOOT_DEBUG
 
diff --git a/hw/omap3_mmc.c b/hw/omap3_mmc.c
index ca4dbd2..99adf58 100644
--- a/hw/omap3_mmc.c
+++ b/hw/omap3_mmc.c
@@ -129,7 +129,7 @@ struct omap3_mmc_s
 static void omap3_mmc_reset(DeviceState *dev)
 {
     struct omap3_mmc_s *s = FROM_SYSBUS(struct omap3_mmc_s,
-                                        sysbus_from_qdev(dev));
+                                        SYS_BUS_DEVICE(dev));
     s->sysconfig = 0x00000015;
     s->sysstatus = 0;
     s->csre      = 0;
@@ -787,7 +787,7 @@ void omap3_mmc_attach(DeviceState *dev, BlockDriverState *bs,
                       int is_spi, int is_mmc)
 {
     struct omap3_mmc_s *s = FROM_SYSBUS(struct omap3_mmc_s,
-                                        sysbus_from_qdev(dev));
+                                        SYS_BUS_DEVICE(dev));
     if (s->card) {
         hw_error("%s: card already attached!", __FUNCTION__);
     }
diff --git a/hw/omap_dma.c b/hw/omap_dma.c
index bab9ed0..5bf40c7 100644
--- a/hw/omap_dma.c
+++ b/hw/omap_dma.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "omap.h"
 #include "irq.h"
 #include "soc_dma.h"
@@ -1635,6 +1635,7 @@ static void omap_dma_write(void *opaque, hwaddr addr,
     case 0x404 ... 0x4fe:
         if (s->model <= omap_dma_3_1)
             break;
+        /* fall through */
     case 0x400:
         /* Fall through. */
         if (omap_dma_sys_write(s, addr, value))
@@ -1837,19 +1838,25 @@ static uint64_t omap_dma4_read(void *opaque, hwaddr addr,
 
     case 0x14:	/* DMA4_IRQSTATUS_L3 */
         irqn ++;
+        /* fall through */
     case 0x10:	/* DMA4_IRQSTATUS_L2 */
         irqn ++;
+        /* fall through */
     case 0x0c:	/* DMA4_IRQSTATUS_L1 */
         irqn ++;
+        /* fall through */
     case 0x08:	/* DMA4_IRQSTATUS_L0 */
         return s->irqstat[irqn];
 
     case 0x24:	/* DMA4_IRQENABLE_L3 */
         irqn ++;
+        /* fall through */
     case 0x20:	/* DMA4_IRQENABLE_L2 */
         irqn ++;
+        /* fall through */
     case 0x1c:	/* DMA4_IRQENABLE_L1 */
         irqn ++;
+        /* fall through */
     case 0x18:	/* DMA4_IRQENABLE_L0 */
         return s->irqen[irqn];
 
@@ -2159,10 +2166,13 @@ static void omap_dma4_write(void *opaque, hwaddr addr,
     switch (addr) {
     case 0x14: /* DMA4_IRQSTATUS_L3 */
         irqn++;
+        /* fall through */
     case 0x10: /* DMA4_IRQSTATUS_L2 */
         irqn++;
+        /* fall through */
     case 0x0c: /* DMA4_IRQSTATUS_L1 */
         irqn++;
+        /* fall through */
     case 0x08: /* DMA4_IRQSTATUS_L0 */
         s->irqstat[irqn] &= ~value;
         if (!s->irqstat[irqn]) {
diff --git a/hw/omap_dss.c b/hw/omap_dss.c
index 8ecc31d..b91adfc 100644
--- a/hw/omap_dss.c
+++ b/hw/omap_dss.c
@@ -18,9 +18,9 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "console.h"
 #include "omap.h"
 #include "sysbus.h"
+#include "ui/console.h"
 #include "dsi.h"
 
 //#define OMAP_DSS_DEBUG
@@ -234,7 +234,7 @@ struct omap_dss_s {
     } dsi;
 };
 
-#include "pixel_ops.h"
+#include "ui/pixel_ops.h"
 #include "framebuffer.h"
 #define DEPTH 8
 #include "omap_dss_drawfn.h"
@@ -324,7 +324,7 @@ static void omap_dss_framedone(void *opaque)
 static void omap_dsi_te_trigger(DeviceState *dev, int vc)
 {
     struct omap_dss_s *s = FROM_SYSBUS(struct omap_dss_s,
-                                       sysbus_from_qdev(dev));
+                                       SYS_BUS_DEVICE(dev));
     if ((s->dsi.ctrl & 1) &&        /* IF_EN */
         (s->dsi.vc[vc].ctrl & 1)) { /* VC_EN */
         s->dsi.irqst |= 1 << 16;    /* TE_TRIGGER_IRQ */
@@ -713,7 +713,7 @@ static void omap_dss_reset(DeviceState *dev)
     int i, j;
 
     struct omap_dss_s *s = FROM_SYSBUS(struct omap_dss_s,
-                                       sysbus_from_qdev(dev));
+                                       SYS_BUS_DEVICE(dev));
     s->autoidle = 0x10; /* was 0 for OMAP2 but bit4 must be set for OMAP3 */
     s->control = 0;
     if (s->mpu_model == omap3430) {
@@ -2555,7 +2555,7 @@ void omap_rfbi_attach(DeviceState *dev, int cs,
                       const struct rfbi_chip_s *chip)
 {
     struct omap_dss_s *s = FROM_SYSBUS(struct omap_dss_s,
-                                       sysbus_from_qdev(dev));
+                                       SYS_BUS_DEVICE(dev));
     if (cs < 0 || cs > 1) {
         hw_error("%s: wrong CS %i\n", __FUNCTION__, cs);
     }
@@ -2569,13 +2569,13 @@ void omap_rfbi_attach(DeviceState *dev, int cs,
 DSIHost *omap_dsi_host(DeviceState *dev)
 {
     return FROM_SYSBUS(struct omap_dss_s,
-                       sysbus_from_qdev(dev))->dsi.host;
+                       SYS_BUS_DEVICE(dev))->dsi.host;
 }
 
 void omap_lcd_panel_attach(DeviceState *dev)
 {
     struct omap_dss_s *s = FROM_SYSBUS(struct omap_dss_s,
-                                       sysbus_from_qdev(dev));
+                                       SYS_BUS_DEVICE(dev));
     if (!s->lcd.attached) {
         s->lcd.attached = 1;
         s->lcd.invalidate = 1;
@@ -2588,7 +2588,7 @@ void omap_lcd_panel_attach(DeviceState *dev)
 void omap_digital_panel_attach(DeviceState *dev)
 {
     struct omap_dss_s *s = FROM_SYSBUS(struct omap_dss_s,
-                                       sysbus_from_qdev(dev));
+                                       SYS_BUS_DEVICE(dev));
     if (!s->dig.attached) {
         s->dig.attached = 1;
         s->dig.invalidate = 1;
diff --git a/hw/omap_gpio.c b/hw/omap_gpio.c
index add57cf..49a81c2 100644
--- a/hw/omap_gpio.c
+++ b/hw/omap_gpio.c
@@ -588,7 +588,7 @@ static const MemoryRegionOps omap2_gpio_module_ops = {
 static void omap_gpif_reset(DeviceState *dev)
 {
     struct omap_gpif_s *s = FROM_SYSBUS(struct omap_gpif_s,
-                    sysbus_from_qdev(dev));
+                    SYS_BUS_DEVICE(dev));
     omap_gpio_reset(&s->omap1);
 }
 
@@ -596,7 +596,7 @@ static void omap2_gpif_reset(DeviceState *dev)
 {
     int i;
     struct omap2_gpif_s *s = FROM_SYSBUS(struct omap2_gpif_s,
-                    sysbus_from_qdev(dev));
+                    SYS_BUS_DEVICE(dev));
     for (i = 0; i < s->modulecount; i++) {
         omap2_gpio_module_reset(&s->modules[i]);
     }
@@ -747,7 +747,7 @@ static void omap_gpio_class_init(ObjectClass *klass, void *data)
     dc->props = omap_gpio_properties;
 }
 
-static TypeInfo omap_gpio_info = {
+static const TypeInfo omap_gpio_info = {
     .name          = "omap-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_gpif_s),
@@ -776,7 +776,7 @@ static void omap2_gpio_class_init(ObjectClass *klass, void *data)
     dc->props = omap2_gpio_properties;
 }
 
-static TypeInfo omap2_gpio_info = {
+static const TypeInfo omap2_gpio_info = {
     .name          = "omap2-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap2_gpif_s),
diff --git a/hw/omap_gpmc.c b/hw/omap_gpmc.c
index 1f7c5bc..02ab0ab 100644
--- a/hw/omap_gpmc.c
+++ b/hw/omap_gpmc.c
@@ -21,8 +21,8 @@
 #include "hw.h"
 #include "flash.h"
 #include "omap.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
 /* General-Purpose Memory Controller */
 struct omap_gpmc_s {
diff --git a/hw/omap_gptimer.c b/hw/omap_gptimer.c
index 1f7c648..b019295 100644
--- a/hw/omap_gptimer.c
+++ b/hw/omap_gptimer.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "omap.h"
 
 /* GP timers */
diff --git a/hw/omap_i2c.c b/hw/omap_i2c.c
index b28e14e..2103ee8 100644
--- a/hw/omap_i2c.c
+++ b/hw/omap_i2c.c
@@ -157,7 +157,7 @@ static void omap_i2c_fifo_run(OMAPI2CState *s)
 static void omap_i2c_reset(DeviceState *dev)
 {
     OMAPI2CState *s = FROM_SYSBUS(OMAPI2CState,
-                                  sysbus_from_qdev(dev));
+                                  SYS_BUS_DEVICE(dev));
     s->mask = 0;
     s->stat = 0;
     s->dma = 0;
@@ -727,7 +727,7 @@ static void omap_i2c_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_omap_i2c;
 }
 
-static TypeInfo omap_i2c_info = {
+static const TypeInfo omap_i2c_info = {
     .name = "omap_i2c",
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OMAPI2CState),
@@ -741,7 +741,7 @@ static void omap_i2c_register_types(void)
 
 i2c_bus *omap_i2c_bus(DeviceState *omap_i2c)
 {
-    OMAPI2CState *s = FROM_SYSBUS(OMAPI2CState, sysbus_from_qdev(omap_i2c));
+    OMAPI2CState *s = FROM_SYSBUS(OMAPI2CState, SYS_BUS_DEVICE(omap_i2c));
     return s->bus;
 }
 
diff --git a/hw/omap_intc.c b/hw/omap_intc.c
index dd55c47..6191e19 100644
--- a/hw/omap_intc.c
+++ b/hw/omap_intc.c
@@ -329,7 +329,7 @@ static const MemoryRegionOps omap_inth_mem_ops = {
 static void omap_inth_reset(DeviceState *dev)
 {
     struct omap_intr_handler_s *s = FROM_SYSBUS(struct omap_intr_handler_s,
-                                                sysbus_from_qdev(dev));
+                                                SYS_BUS_DEVICE(dev));
     int i;
 
     for (i = 0; i < s->nbanks; ++i){
@@ -389,7 +389,7 @@ static void omap_intc_class_init(ObjectClass *klass, void *data)
     dc->props = omap_intc_properties;
 }
 
-static TypeInfo omap_intc_info = {
+static const TypeInfo omap_intc_info = {
     .name          = "omap-intc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_intr_handler_s),
@@ -637,7 +637,7 @@ static void omap2_intc_class_init(ObjectClass *klass, void *data)
     dc->props = omap2_intc_properties;
 }
 
-static TypeInfo omap2_intc_info = {
+static const TypeInfo omap2_intc_info = {
     .name          = "omap2-intc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_intr_handler_s),
diff --git a/hw/omap_lcdc.c b/hw/omap_lcdc.c
index d7ae303..936850a 100644
--- a/hw/omap_lcdc.c
+++ b/hw/omap_lcdc.c
@@ -17,9 +17,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "omap.h"
 #include "framebuffer.h"
+#include "ui/pixel_ops.h"
 
 struct omap_lcd_panel_s {
     MemoryRegion *sysmem;
@@ -66,8 +67,6 @@ static void omap_lcd_interrupts(struct omap_lcd_panel_s *s)
     qemu_irq_lower(s->irq);
 }
 
-#include "pixel_ops.h"
-
 #define draw_line_func drawfn
 
 #define DEPTH 8
diff --git a/hw/omap_spi.c b/hw/omap_spi.c
index 08d2462..6d173a1 100644
--- a/hw/omap_spi.c
+++ b/hw/omap_spi.c
@@ -321,40 +321,55 @@ static uint64_t omap_mcspi_read(void *opaque, hwaddr addr,
         return s->control;
 
     case 0x68: ch ++;
+        /* fall through */
     case 0x54: ch ++;
+        /* fall through */
     case 0x40: ch ++;
+        /* fall through */
     case 0x2c:	/* MCSPI_CHCONF */
         TRACE("CHCONF%d = 0x%08x", ch,
               (ch < s->chnum) ? s->ch[ch].config : 0);
         return (ch < s->chnum) ? s->ch[ch].config : 0;
 
     case 0x6c: ch ++;
+        /* fall through */
     case 0x58: ch ++;
+        /* fall through */
     case 0x44: ch ++;
+        /* fall through */
     case 0x30:	/* MCSPI_CHSTAT */
         TRACE("CHSTAT%d = 0x%08x", ch,
               (ch < s->chnum) ? s->ch[ch].status : 0);
         return (ch < s->chnum) ? s->ch[ch].status : 0;
 
     case 0x70: ch ++;
+        /* fall through */
     case 0x5c: ch ++;
+        /* fall through */
     case 0x48: ch ++;
+        /* fall through */
     case 0x34:	/* MCSPI_CHCTRL */
         TRACE("CHCTRL%d = 0x%08x", ch,
               (ch < s->chnum) ? s->ch[ch].control : 0);
         return (ch < s->chnum) ? s->ch[ch].control : 0;
 
     case 0x74: ch ++;
+        /* fall through */
     case 0x60: ch ++;
+        /* fall through */
     case 0x4c: ch ++;
+        /* fall through */
     case 0x38:	/* MCSPI_TX */
         TRACE("TX%d = 0x%08x", ch,
               (ch < s->chnum) ? s->ch[ch].tx : 0);
         return (ch < s->chnum) ? s->ch[ch].tx : 0;
 
     case 0x78: ch ++;
+        /* fall through */
     case 0x64: ch ++;
+        /* fall through */
     case 0x50: ch ++;
+        /* fall through */
     case 0x3c:	/* MCSPI_RX */
         if (ch < s->chnum) {
             if (!IS_OMAP3_SPI(s) || ch != s->fifo_ch ||
@@ -479,8 +494,11 @@ static void omap_mcspi_write(void *opaque, hwaddr addr,
         break;
 
     case 0x68: ch ++;
+        /* fall through */
     case 0x54: ch ++;
+        /* fall through */
     case 0x40: ch ++;
+        /* fall through */
     case 0x2c:	/* MCSPI_CHCONF */
         TRACE("CHCONF%d = 0x%08x", ch, value);
         if (ch < s->chnum) {
@@ -509,8 +527,11 @@ static void omap_mcspi_write(void *opaque, hwaddr addr,
         break;
 
     case 0x70: ch ++;
+        /* fall through */
     case 0x5c: ch ++;
+        /* fall through */
     case 0x48: ch ++;
+        /* fall through */
     case 0x34:	/* MCSPI_CHCTRL */
         TRACE("CHCTRL%d = 0x%08x", ch, value);
         if (ch < s->chnum) {
@@ -525,8 +546,11 @@ static void omap_mcspi_write(void *opaque, hwaddr addr,
         break;
 
     case 0x74: ch ++;
+        /* fall through */
     case 0x60: ch ++;
+        /* fall through */
     case 0x4c: ch ++;
+        /* fall through */
     case 0x38:	/* MCSPI_TX */
         TRACE("TX%d = 0x%08x", ch, value);
         if (ch < s->chnum) {
@@ -586,7 +610,7 @@ static const MemoryRegionOps omap_mcspi_ops = {
 static void omap_mcspi_reset(DeviceState *qdev)
 {
     int i;
-    OMAPSPIState *s = FROM_SYSBUS(OMAPSPIState, sysbus_from_qdev(qdev));
+    OMAPSPIState *s = FROM_SYSBUS(OMAPSPIState, SYS_BUS_DEVICE(qdev));
     for (i = 0; i < s->buscount; i++) {
         omap_mcspi_bus_reset(&s->bus[i]);
     }
@@ -625,7 +649,7 @@ static int omap_mcspi_init(SysBusDevice *busdev)
 
 SPIBus *omap_mcspi_bus(DeviceState *qdev, int bus_number)
 {
-    OMAPSPIState *s = FROM_SYSBUS(OMAPSPIState, sysbus_from_qdev(qdev));
+    OMAPSPIState *s = FROM_SYSBUS(OMAPSPIState, SYS_BUS_DEVICE(qdev));
     if (bus_number < s->buscount) {
         return s->bus[bus_number].bus;
     }
diff --git a/hw/omap_sx1.c b/hw/omap_sx1.c
index 21a5bbb..30998c5 100644
--- a/hw/omap_sx1.c
+++ b/hw/omap_sx1.c
@@ -26,13 +26,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "omap.h"
 #include "boards.h"
 #include "arm-misc.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 /*****************************************************************************/
 /* Siemens SX1 Cellphone V1 */
@@ -219,12 +219,14 @@ static QEMUMachine sx1_machine_v2 = {
     .name = "sx1",
     .desc = "Siemens SX1 (OMAP310) V2",
     .init = sx1_init_v2,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine sx1_machine_v1 = {
     .name = "sx1-v1",
     .desc = "Siemens SX1 (OMAP310) V1",
     .init = sx1_init_v1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void sx1_machine_init(void)
diff --git a/hw/omap_synctimer.c b/hw/omap_synctimer.c
index 308b0fa..122cb4f 100644
--- a/hw/omap_synctimer.c
+++ b/hw/omap_synctimer.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "omap.h"
 struct omap_synctimer_s {
     MemoryRegion iomem;
diff --git a/hw/omap_uart.c b/hw/omap_uart.c
index e50f766..e94d655 100644
--- a/hw/omap_uart.c
+++ b/hw/omap_uart.c
@@ -17,11 +17,11 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "qemu-char.h"
+#include "char/char.h"
 #include "hw.h"
 #include "omap.h"
 #include "serial.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"
 
 /* The OMAP UART functionality is similar to the TI16C752; it is
@@ -84,7 +84,7 @@ static int tcr_tlr_mode(struct omap_uart_s *s)
 static void omap_uart_reset(DeviceState *qdev)
 {
     struct omap_uart_s *s = FROM_SYSBUS(struct omap_uart_s,
-                                        sysbus_from_qdev(qdev));
+                                        SYS_BUS_DEVICE(qdev));
     s->eblr = 0x00;
     s->syscontrol = 0;
     s->wkup = 0x3f;
@@ -368,7 +368,7 @@ void omap_uart_attach(DeviceState *qdev, CharDriverState *chr,
                       const char *label)
 {
     struct omap_uart_s *s = FROM_SYSBUS(struct omap_uart_s,
-                                        sysbus_from_qdev(qdev));
+                                        SYS_BUS_DEVICE(qdev));
     s->chr = chr ?: qemu_chr_new(label, "null", NULL);
     serial_change_char_driver(s->serial, s->chr);
 }
diff --git a/hw/omap_usb.c b/hw/omap_usb.c
index 856a704..92b1e0b 100644
--- a/hw/omap_usb.c
+++ b/hw/omap_usb.c
@@ -18,7 +18,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "usb.h"
 #include "omap.h"
 #include "irq.h"
@@ -77,7 +77,7 @@ static void omap3_hsusb_otg_stdby_update(OMAP3HSUSBOTGState *s)
 static void omap3_hsusb_otg_reset(DeviceState *dev)
 {
     OMAP3HSUSBOTGState *s = FROM_SYSBUS(OMAP3HSUSBOTGState,
-                                        sysbus_from_qdev(dev));
+                                        SYS_BUS_DEVICE(dev));
     s->rev = 0x33;
     s->sysconfig = 0;
     s->interfsel = 0x1;
@@ -285,7 +285,7 @@ static const VMStateDescription vmstate_omap3_hsusb_host = {
 static void omap3_hsusb_host_reset(DeviceState *dev)
 {
     OMAP3HSUSBHostState *s = FROM_SYSBUS(OMAP3HSUSBHostState,
-                                         sysbus_from_qdev(dev));
+                                         SYS_BUS_DEVICE(dev));
     s->uhh_sysconfig = 1;
     s->uhh_hostconfig = 0x700;
     s->uhh_debug_csr = 0x20;
diff --git a/hw/onenand.c b/hw/onenand.c
index f1befca..b63e64f 100644
--- a/hw/onenand.c
+++ b/hw/onenand.c
@@ -22,11 +22,11 @@
 #include "hw.h"
 #include "flash.h"
 #include "irq.h"
-#include "blockdev.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 /* 11 for 2kB-page OneNAND ("2nd generation") and 10 for 1kB-page chips */
 #define PAGE_SHIFT	11
@@ -224,7 +224,7 @@ static void onenand_reset(OneNANDState *s, int cold)
 
 static void onenand_system_reset(DeviceState *dev)
 {
-    onenand_reset(FROM_SYSBUS(OneNANDState, sysbus_from_qdev(dev)), 1);
+    onenand_reset(FROM_SYSBUS(OneNANDState, SYS_BUS_DEVICE(dev)), 1);
 }
 
 static inline int onenand_load_main(OneNANDState *s, int sec, int secn,
@@ -828,7 +828,7 @@ static void onenand_class_init(ObjectClass *klass, void *data)
     dc->props = onenand_properties;
 }
 
-static TypeInfo onenand_info = {
+static const TypeInfo onenand_info = {
     .name          = "onenand",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OneNANDState),
@@ -842,7 +842,7 @@ static void onenand_register_types(void)
 
 void *onenand_raw_otp(DeviceState *onenand_device)
 {
-    return FROM_SYSBUS(OneNANDState, sysbus_from_qdev(onenand_device))->otp;
+    return FROM_SYSBUS(OneNANDState, SYS_BUS_DEVICE(onenand_device))->otp;
 }
 
 type_init(onenand_register_types)
diff --git a/hw/opencores_eth.c b/hw/opencores_eth.c
index b2780b9..f9ba5ee 100644
--- a/hw/opencores_eth.c
+++ b/hw/opencores_eth.c
@@ -33,8 +33,8 @@
 
 #include "hw.h"
 #include "sysbus.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
 /* RECSMALL is not used because it breaks tap networking in linux:
@@ -313,7 +313,7 @@ static void open_eth_int_source_write(OpenEthState *s,
 
 static void open_eth_set_link_status(NetClientState *nc)
 {
-    OpenEthState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    OpenEthState *s = qemu_get_nic_opaque(nc);
 
     if (GET_REGBIT(s, MIICOMMAND, SCANSTAT)) {
         SET_REGFIELD(s, MIISTATUS, LINKFAIL, nc->link_down);
@@ -339,12 +339,12 @@ static void open_eth_reset(void *opaque)
     s->rx_desc = 0x40;
 
     mii_reset(&s->mii);
-    open_eth_set_link_status(&s->nic->nc);
+    open_eth_set_link_status(qemu_get_queue(s->nic));
 }
 
 static int open_eth_can_receive(NetClientState *nc)
 {
-    OpenEthState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    OpenEthState *s = qemu_get_nic_opaque(nc);
 
     return GET_REGBIT(s, MODER, RXEN) &&
         (s->regs[TX_BD_NUM] < 0x80) &&
@@ -354,7 +354,7 @@ static int open_eth_can_receive(NetClientState *nc)
 static ssize_t open_eth_receive(NetClientState *nc,
         const uint8_t *buf, size_t size)
 {
-    OpenEthState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    OpenEthState *s = qemu_get_nic_opaque(nc);
     size_t maxfl = GET_REGFIELD(s, PACKETLEN, MAXFL);
     size_t minfl = GET_REGFIELD(s, PACKETLEN, MINFL);
     size_t fcsl = 4;
@@ -499,7 +499,7 @@ static void open_eth_start_xmit(OpenEthState *s, desc *tx)
     if (tx_len > len) {
         memset(buf + len, 0, tx_len - len);
     }
-    qemu_send_packet(&s->nic->nc, buf, tx_len);
+    qemu_send_packet(qemu_get_queue(s->nic), buf, tx_len);
 
     if (tx->len_flags & TXD_WR) {
         s->tx_desc = 0;
@@ -606,7 +606,7 @@ static void open_eth_mii_command_host_write(OpenEthState *s, uint32_t val)
         } else {
             s->regs[MIIRX_DATA] = 0xffff;
         }
-        SET_REGFIELD(s, MIISTATUS, LINKFAIL, s->nic->nc.link_down);
+        SET_REGFIELD(s, MIISTATUS, LINKFAIL, qemu_get_queue(s->nic)->link_down);
     }
 }
 
@@ -718,7 +718,7 @@ static void open_eth_class_init(ObjectClass *klass, void *data)
     dc->props = open_eth_properties;
 }
 
-static TypeInfo open_eth_info = {
+static const TypeInfo open_eth_info = {
     .name          = "open_eth",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OpenEthState),
diff --git a/hw/openpic.c b/hw/openpic.c
index 8b3784a..20a479c 100644
--- a/hw/openpic.c
+++ b/hw/openpic.c
@@ -34,148 +34,164 @@
  *
  */
 #include "hw.h"
-#include "ppc_mac.h"
-#include "pci.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
 #include "openpic.h"
+#include "sysbus.h"
+#include "pci/msi.h"
+#include "qemu/bitops.h"
+#include "ppc.h"
 
 //#define DEBUG_OPENPIC
 
 #ifdef DEBUG_OPENPIC
-#define DPRINTF(fmt, ...) do { printf(fmt , ## __VA_ARGS__); } while (0)
+static const int debug_openpic = 1;
 #else
-#define DPRINTF(fmt, ...) do { } while (0)
+static const int debug_openpic = 0;
 #endif
 
-#define USE_MPCxxx /* Intel model is broken, for now */
+#define DPRINTF(fmt, ...) do { \
+        if (debug_openpic) { \
+            printf(fmt , ## __VA_ARGS__); \
+        } \
+    } while (0)
 
-#if defined (USE_INTEL_GW80314)
-/* Intel GW80314 I/O Companion chip */
-
-#define MAX_CPU     4
-#define MAX_IRQ    32
-#define MAX_DBL     4
-#define MAX_MBX     4
+#define MAX_CPU     32
+#define MAX_SRC     256
 #define MAX_TMR     4
-#define VECTOR_BITS 8
 #define MAX_IPI     4
+#define MAX_MSI     8
+#define MAX_IRQ     (MAX_SRC + MAX_IPI + MAX_TMR)
+#define VID         0x03 /* MPIC version ID */
 
-#define VID (0x00000000)
-
-#elif defined(USE_MPCxxx)
+/* OpenPIC capability flags */
+#define OPENPIC_FLAG_IDR_CRIT     (1 << 0)
+#define OPENPIC_FLAG_ILR          (2 << 0)
+
+/* OpenPIC address map */
+#define OPENPIC_GLB_REG_START        0x0
+#define OPENPIC_GLB_REG_SIZE         0x10F0
+#define OPENPIC_TMR_REG_START        0x10F0
+#define OPENPIC_TMR_REG_SIZE         0x220
+#define OPENPIC_MSI_REG_START        0x1600
+#define OPENPIC_MSI_REG_SIZE         0x200
+#define OPENPIC_SUMMARY_REG_START   0x3800
+#define OPENPIC_SUMMARY_REG_SIZE    0x800
+#define OPENPIC_SRC_REG_START        0x10000
+#define OPENPIC_SRC_REG_SIZE         (MAX_SRC * 0x20)
+#define OPENPIC_CPU_REG_START        0x20000
+#define OPENPIC_CPU_REG_SIZE         0x100 + ((MAX_CPU - 1) * 0x1000)
+
+/* Raven */
+#define RAVEN_MAX_CPU      2
+#define RAVEN_MAX_EXT     48
+#define RAVEN_MAX_IRQ     64
+#define RAVEN_MAX_TMR      MAX_TMR
+#define RAVEN_MAX_IPI      MAX_IPI
 
-#define MAX_CPU    15
-#define MAX_IRQ   128
-#define MAX_DBL     0
-#define MAX_MBX     0
-#define MAX_TMR     4
-#define VECTOR_BITS 8
-#define MAX_IPI     4
-#define VID         0x03 /* MPIC version ID */
-#define VENI        0x00000000 /* Vendor ID */
+/* Interrupt definitions */
+#define RAVEN_FE_IRQ     (RAVEN_MAX_EXT)     /* Internal functional IRQ */
+#define RAVEN_ERR_IRQ    (RAVEN_MAX_EXT + 1) /* Error IRQ */
+#define RAVEN_TMR_IRQ    (RAVEN_MAX_EXT + 2) /* First timer IRQ */
+#define RAVEN_IPI_IRQ    (RAVEN_TMR_IRQ + RAVEN_MAX_TMR) /* First IPI IRQ */
+/* First doorbell IRQ */
+#define RAVEN_DBL_IRQ    (RAVEN_IPI_IRQ + (RAVEN_MAX_CPU * RAVEN_MAX_IPI))
+
+typedef struct FslMpicInfo {
+    int max_ext;
+} FslMpicInfo;
+
+static FslMpicInfo fsl_mpic_20 = {
+    .max_ext = 12,
+};
 
-enum {
-    IRQ_IPVP = 0,
-    IRQ_IDE,
+static FslMpicInfo fsl_mpic_42 = {
+    .max_ext = 12,
 };
 
-/* OpenPIC */
-#define OPENPIC_MAX_CPU      2
-#define OPENPIC_MAX_IRQ     64
-#define OPENPIC_EXT_IRQ     48
-#define OPENPIC_MAX_TMR      MAX_TMR
-#define OPENPIC_MAX_IPI      MAX_IPI
+#define FRR_NIRQ_SHIFT    16
+#define FRR_NCPU_SHIFT     8
+#define FRR_VID_SHIFT      0
 
-/* Interrupt definitions */
-#define OPENPIC_IRQ_FE     (OPENPIC_EXT_IRQ)     /* Internal functional IRQ */
-#define OPENPIC_IRQ_ERR    (OPENPIC_EXT_IRQ + 1) /* Error IRQ */
-#define OPENPIC_IRQ_TIM0   (OPENPIC_EXT_IRQ + 2) /* First timer IRQ */
-#if OPENPIC_MAX_IPI > 0
-#define OPENPIC_IRQ_IPI0   (OPENPIC_IRQ_TIM0 + OPENPIC_MAX_TMR) /* First IPI IRQ */
-#define OPENPIC_IRQ_DBL0   (OPENPIC_IRQ_IPI0 + (OPENPIC_MAX_CPU * OPENPIC_MAX_IPI)) /* First doorbell IRQ */
-#else
-#define OPENPIC_IRQ_DBL0   (OPENPIC_IRQ_TIM0 + OPENPIC_MAX_TMR) /* First doorbell IRQ */
-#define OPENPIC_IRQ_MBX0   (OPENPIC_IRQ_DBL0 + OPENPIC_MAX_DBL) /* First mailbox IRQ */
-#endif
+#define VID_REVISION_1_2   2
+#define VID_REVISION_1_3   3
 
-/* MPIC */
-#define MPIC_MAX_CPU      1
-#define MPIC_MAX_EXT     12
-#define MPIC_MAX_INT     64
-#define MPIC_MAX_MSG      4
-#define MPIC_MAX_MSI      8
-#define MPIC_MAX_TMR      MAX_TMR
-#define MPIC_MAX_IPI      MAX_IPI
-#define MPIC_MAX_IRQ     (MPIC_MAX_EXT + MPIC_MAX_INT + MPIC_MAX_TMR + MPIC_MAX_MSG + MPIC_MAX_MSI + (MPIC_MAX_IPI * MPIC_MAX_CPU))
+#define VIR_GENERIC      0x00000000 /* Generic Vendor ID */
 
-/* Interrupt definitions */
-#define MPIC_EXT_IRQ      0
-#define MPIC_INT_IRQ      (MPIC_EXT_IRQ + MPIC_MAX_EXT)
-#define MPIC_TMR_IRQ      (MPIC_INT_IRQ + MPIC_MAX_INT)
-#define MPIC_MSG_IRQ      (MPIC_TMR_IRQ + MPIC_MAX_TMR)
-#define MPIC_MSI_IRQ      (MPIC_MSG_IRQ + MPIC_MAX_MSG)
-#define MPIC_IPI_IRQ      (MPIC_MSI_IRQ + MPIC_MAX_MSI)
-
-#define MPIC_GLB_REG_START        0x0
-#define MPIC_GLB_REG_SIZE         0x10F0
-#define MPIC_TMR_REG_START        0x10F0
-#define MPIC_TMR_REG_SIZE         0x220
-#define MPIC_EXT_REG_START        0x10000
-#define MPIC_EXT_REG_SIZE         0x180
-#define MPIC_INT_REG_START        0x10200
-#define MPIC_INT_REG_SIZE         0x800
-#define MPIC_MSG_REG_START        0x11600
-#define MPIC_MSG_REG_SIZE         0x100
-#define MPIC_MSI_REG_START        0x11C00
-#define MPIC_MSI_REG_SIZE         0x100
-#define MPIC_CPU_REG_START        0x20000
-#define MPIC_CPU_REG_SIZE         0x100 + ((MAX_CPU - 1) * 0x1000)
+#define GCR_RESET        0x80000000
+#define GCR_MODE_PASS    0x00000000
+#define GCR_MODE_MIXED   0x20000000
+#define GCR_MODE_PROXY   0x60000000
 
-/*
- * Block Revision Register1 (BRR1): QEMU does not fully emulate
- * any version on MPIC. So to start with, set the IP version to 0.
- *
- * NOTE: This is Freescale MPIC specific register. Keep it here till
- * this code is refactored for different variants of OPENPIC and MPIC.
- */
-#define FSL_BRR1_IPID (0x0040 << 16) /* 16 bit IP-block ID */
-#define FSL_BRR1_IPMJ (0x00 << 8) /* 8 bit IP major number */
-#define FSL_BRR1_IPMN 0x00 /* 8 bit IP minor number */
-
-enum mpic_ide_bits {
-    IDR_EP     = 31,
-    IDR_CI0     = 30,
-    IDR_CI1     = 29,
-    IDR_P1     = 1,
-    IDR_P0     = 0,
-};
+#define TBCR_CI           0x80000000 /* count inhibit */
+#define TCCR_TOG          0x80000000 /* toggles when decrement to zero */
 
-#else
-#error "Please select which OpenPic implementation is to be emulated"
-#endif
+#define IDR_EP_SHIFT      31
+#define IDR_EP_MASK       (1 << IDR_EP_SHIFT)
+#define IDR_CI0_SHIFT     30
+#define IDR_CI1_SHIFT     29
+#define IDR_P1_SHIFT      1
+#define IDR_P0_SHIFT      0
 
-#define OPENPIC_PAGE_SIZE 4096
+#define ILR_INTTGT_MASK   0x000000ff
+#define ILR_INTTGT_INT    0x00
+#define ILR_INTTGT_CINT   0x01 /* critical */
+#define ILR_INTTGT_MCP    0x02 /* machine check */
 
-#define BF_WIDTH(_bits_) \
-(((_bits_) + (sizeof(uint32_t) * 8) - 1) / (sizeof(uint32_t) * 8))
+/* The currently supported INTTGT values happen to be the same as QEMU's
+ * openpic output codes, but don't depend on this.  The output codes
+ * could change (unlikely, but...) or support could be added for
+ * more INTTGT values.
+ */
+static const int inttgt_output[][2] = {
+    { ILR_INTTGT_INT, OPENPIC_OUTPUT_INT },
+    { ILR_INTTGT_CINT, OPENPIC_OUTPUT_CINT },
+    { ILR_INTTGT_MCP, OPENPIC_OUTPUT_MCK },
+};
 
-static inline void set_bit (uint32_t *field, int bit)
+static int inttgt_to_output(int inttgt)
 {
-    field[bit >> 5] |= 1 << (bit & 0x1F);
-}
+    int i;
 
-static inline void reset_bit (uint32_t *field, int bit)
-{
-    field[bit >> 5] &= ~(1 << (bit & 0x1F));
+    for (i = 0; i < ARRAY_SIZE(inttgt_output); i++) {
+        if (inttgt_output[i][0] == inttgt) {
+            return inttgt_output[i][1];
+        }
+    }
+
+    fprintf(stderr, "%s: unsupported inttgt %d\n", __func__, inttgt);
+    return OPENPIC_OUTPUT_INT;
 }
 
-static inline int test_bit (uint32_t *field, int bit)
+static int output_to_inttgt(int output)
 {
-    return (field[bit >> 5] & 1 << (bit & 0x1F)) != 0;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(inttgt_output); i++) {
+        if (inttgt_output[i][1] == output) {
+            return inttgt_output[i][0];
+        }
+    }
+
+    abort();
 }
 
+#define MSIIR_OFFSET       0x140
+#define MSIIR_SRS_SHIFT    29
+#define MSIIR_SRS_MASK     (0x7 << MSIIR_SRS_SHIFT)
+#define MSIIR_IBS_SHIFT    24
+#define MSIIR_IBS_MASK     (0x1f << MSIIR_IBS_SHIFT)
+
 static int get_current_cpu(void)
 {
-  return cpu_single_env->cpu_index;
+    CPUState *cpu_single_cpu;
+
+    if (!cpu_single_env) {
+        return -1;
+    }
+
+    cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
+    return cpu_single_cpu->cpu_index;
 }
 
 static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
@@ -183,232 +199,298 @@ static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
 static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
                                        uint32_t val, int idx);
 
-enum {
-    IRQ_EXTERNAL = 0x01,
-    IRQ_INTERNAL = 0x02,
-    IRQ_TIMER    = 0x04,
-    IRQ_SPECIAL  = 0x08,
-};
-
-typedef struct IRQ_queue_t {
-    uint32_t queue[BF_WIDTH(MAX_IRQ)];
+typedef enum IRQType {
+    IRQ_TYPE_NORMAL = 0,
+    IRQ_TYPE_FSLINT,        /* FSL internal interrupt -- level only */
+    IRQ_TYPE_FSLSPECIAL,    /* FSL timer/IPI interrupt, edge, no polarity */
+} IRQType;
+
+typedef struct IRQQueue {
+    /* Round up to the nearest 64 IRQs so that the queue length
+     * won't change when moving between 32 and 64 bit hosts.
+     */
+    unsigned long queue[BITS_TO_LONGS((MAX_IRQ + 63) & ~63)];
     int next;
     int priority;
-} IRQ_queue_t;
+} IRQQueue;
 
-typedef struct IRQ_src_t {
-    uint32_t ipvp;  /* IRQ vector/priority register */
-    uint32_t ide;   /* IRQ destination register */
-    int type;
+typedef struct IRQSource {
+    uint32_t ivpr;  /* IRQ vector/priority register */
+    uint32_t idr;   /* IRQ destination register */
+    uint32_t destmask; /* bitmap of CPU destinations */
     int last_cpu;
+    int output;     /* IRQ level, e.g. OPENPIC_OUTPUT_INT */
     int pending;    /* TRUE if IRQ is pending */
-} IRQ_src_t;
-
-enum IPVP_bits {
-    IPVP_MASK     = 31,
-    IPVP_ACTIVITY = 30,
-    IPVP_MODE     = 29,
-    IPVP_POLARITY = 23,
-    IPVP_SENSE    = 22,
-};
-#define IPVP_PRIORITY_MASK     (0x1F << 16)
-#define IPVP_PRIORITY(_ipvpr_) ((int)(((_ipvpr_) & IPVP_PRIORITY_MASK) >> 16))
-#define IPVP_VECTOR_MASK       ((1 << VECTOR_BITS) - 1)
-#define IPVP_VECTOR(_ipvpr_)   ((_ipvpr_) & IPVP_VECTOR_MASK)
-
-typedef struct IRQ_dst_t {
-    uint32_t tfrr;
-    uint32_t pctp; /* CPU current task priority */
-    uint32_t pcsr; /* CPU sensitivity register */
-    IRQ_queue_t raised;
-    IRQ_queue_t servicing;
+    IRQType type;
+    bool level:1;   /* level-triggered */
+    bool nomask:1;  /* critical interrupts ignore mask on some FSL MPICs */
+} IRQSource;
+
+#define IVPR_MASK_SHIFT       31
+#define IVPR_MASK_MASK        (1 << IVPR_MASK_SHIFT)
+#define IVPR_ACTIVITY_SHIFT   30
+#define IVPR_ACTIVITY_MASK    (1 << IVPR_ACTIVITY_SHIFT)
+#define IVPR_MODE_SHIFT       29
+#define IVPR_MODE_MASK        (1 << IVPR_MODE_SHIFT)
+#define IVPR_POLARITY_SHIFT   23
+#define IVPR_POLARITY_MASK    (1 << IVPR_POLARITY_SHIFT)
+#define IVPR_SENSE_SHIFT      22
+#define IVPR_SENSE_MASK       (1 << IVPR_SENSE_SHIFT)
+
+#define IVPR_PRIORITY_MASK     (0xF << 16)
+#define IVPR_PRIORITY(_ivprr_) ((int)(((_ivprr_) & IVPR_PRIORITY_MASK) >> 16))
+#define IVPR_VECTOR(opp, _ivprr_) ((_ivprr_) & (opp)->vector_mask)
+
+/* IDR[EP/CI] are only for FSL MPIC prior to v4.0 */
+#define IDR_EP      0x80000000  /* external pin */
+#define IDR_CI      0x40000000  /* critical interrupt */
+
+typedef struct IRQDest {
+    int32_t ctpr; /* CPU current task priority */
+    IRQQueue raised;
+    IRQQueue servicing;
     qemu_irq *irqs;
-} IRQ_dst_t;
 
-typedef struct openpic_t {
-    PCIDevice pci_dev;
+    /* Count of IRQ sources asserting on non-INT outputs */
+    uint32_t outputs_active[OPENPIC_OUTPUT_NB];
+} IRQDest;
+
+typedef struct OpenPICState {
+    SysBusDevice busdev;
     MemoryRegion mem;
 
+    /* Behavior control */
+    FslMpicInfo *fsl;
+    uint32_t model;
+    uint32_t flags;
+    uint32_t nb_irqs;
+    uint32_t vid;
+    uint32_t vir; /* Vendor identification register */
+    uint32_t vector_mask;
+    uint32_t tfrr_reset;
+    uint32_t ivpr_reset;
+    uint32_t idr_reset;
+    uint32_t brr1;
+    uint32_t mpic_mode_mask;
+
     /* Sub-regions */
-    MemoryRegion sub_io_mem[7];
+    MemoryRegion sub_io_mem[6];
 
     /* Global registers */
-    uint32_t frep; /* Feature reporting register */
-    uint32_t glbc; /* Global configuration register  */
-    uint32_t micr; /* MPIC interrupt configuration register */
-    uint32_t veni; /* Vendor identification register */
-    uint32_t pint; /* Processor initialization register */
+    uint32_t frr; /* Feature reporting register */
+    uint32_t gcr; /* Global configuration register  */
+    uint32_t pir; /* Processor initialization register */
     uint32_t spve; /* Spurious vector register */
-    uint32_t tifr; /* Timer frequency reporting register */
+    uint32_t tfrr; /* Timer frequency reporting register */
     /* Source registers */
-    IRQ_src_t src[MAX_IRQ];
+    IRQSource src[MAX_IRQ];
     /* Local registers per output pin */
-    IRQ_dst_t dst[MAX_CPU];
-    int nb_cpus;
+    IRQDest dst[MAX_CPU];
+    uint32_t nb_cpus;
     /* Timer registers */
     struct {
-        uint32_t ticc;  /* Global timer current count register */
-        uint32_t tibc;  /* Global timer base count register */
+        uint32_t tccr;  /* Global timer current count register */
+        uint32_t tbcr;  /* Global timer base count register */
     } timers[MAX_TMR];
-#if MAX_DBL > 0
-    /* Doorbell registers */
-    uint32_t dar;        /* Doorbell activate register */
+    /* Shared MSI registers */
     struct {
-        uint32_t dmr;    /* Doorbell messaging register */
-    } doorbells[MAX_DBL];
-#endif
-#if MAX_MBX > 0
-    /* Mailbox registers */
-    struct {
-        uint32_t mbr;    /* Mailbox register */
-    } mailboxes[MAX_MAILBOXES];
-#endif
-    /* IRQ out is used when in bypass mode (not implemented) */
-    qemu_irq irq_out;
-    int max_irq;
-    int irq_ipi0;
-    int irq_tim0;
-    void (*reset) (void *);
-    void (*irq_raise) (struct openpic_t *, int, IRQ_src_t *);
-} openpic_t;
-
-static inline void IRQ_setbit (IRQ_queue_t *q, int n_IRQ)
+        uint32_t msir;   /* Shared Message Signaled Interrupt Register */
+    } msi[MAX_MSI];
+    uint32_t max_irq;
+    uint32_t irq_ipi0;
+    uint32_t irq_tim0;
+    uint32_t irq_msi;
+} OpenPICState;
+
+static inline void IRQ_setbit(IRQQueue *q, int n_IRQ)
 {
-    set_bit(q->queue, n_IRQ);
+    set_bit(n_IRQ, q->queue);
 }
 
-static inline void IRQ_resetbit (IRQ_queue_t *q, int n_IRQ)
+static inline void IRQ_resetbit(IRQQueue *q, int n_IRQ)
 {
-    reset_bit(q->queue, n_IRQ);
+    clear_bit(n_IRQ, q->queue);
 }
 
-static inline int IRQ_testbit (IRQ_queue_t *q, int n_IRQ)
+static inline int IRQ_testbit(IRQQueue *q, int n_IRQ)
 {
-    return test_bit(q->queue, n_IRQ);
+    return test_bit(n_IRQ, q->queue);
 }
 
-static void IRQ_check (openpic_t *opp, IRQ_queue_t *q)
+static void IRQ_check(OpenPICState *opp, IRQQueue *q)
 {
-    int next, i;
-    int priority;
+    int irq = -1;
+    int next = -1;
+    int priority = -1;
+
+    for (;;) {
+        irq = find_next_bit(q->queue, opp->max_irq, irq + 1);
+        if (irq == opp->max_irq) {
+            break;
+        }
 
-    next = -1;
-    priority = -1;
-    for (i = 0; i < opp->max_irq; i++) {
-        if (IRQ_testbit(q, i)) {
-            DPRINTF("IRQ_check: irq %d set ipvp_pr=%d pr=%d\n",
-                    i, IPVP_PRIORITY(opp->src[i].ipvp), priority);
-            if (IPVP_PRIORITY(opp->src[i].ipvp) > priority) {
-                next = i;
-                priority = IPVP_PRIORITY(opp->src[i].ipvp);
-            }
+        DPRINTF("IRQ_check: irq %d set ivpr_pr=%d pr=%d\n",
+                irq, IVPR_PRIORITY(opp->src[irq].ivpr), priority);
+
+        if (IVPR_PRIORITY(opp->src[irq].ivpr) > priority) {
+            next = irq;
+            priority = IVPR_PRIORITY(opp->src[irq].ivpr);
         }
     }
+
     q->next = next;
     q->priority = priority;
 }
 
-static int IRQ_get_next (openpic_t *opp, IRQ_queue_t *q)
+static int IRQ_get_next(OpenPICState *opp, IRQQueue *q)
 {
-    if (q->next == -1) {
-        /* XXX: optimize */
-        IRQ_check(opp, q);
-    }
+    /* XXX: optimize */
+    IRQ_check(opp, q);
 
     return q->next;
 }
 
-static void IRQ_local_pipe (openpic_t *opp, int n_CPU, int n_IRQ)
+static void IRQ_local_pipe(OpenPICState *opp, int n_CPU, int n_IRQ,
+                           bool active, bool was_active)
 {
-    IRQ_dst_t *dst;
-    IRQ_src_t *src;
+    IRQDest *dst;
+    IRQSource *src;
     int priority;
 
     dst = &opp->dst[n_CPU];
     src = &opp->src[n_IRQ];
-    priority = IPVP_PRIORITY(src->ipvp);
-    if (priority <= dst->pctp) {
-        /* Too low priority */
-        DPRINTF("%s: IRQ %d has too low priority on CPU %d\n",
-                __func__, n_IRQ, n_CPU);
+
+    DPRINTF("%s: IRQ %d active %d was %d\n",
+            __func__, n_IRQ, active, was_active);
+
+    if (src->output != OPENPIC_OUTPUT_INT) {
+        DPRINTF("%s: output %d irq %d active %d was %d count %d\n",
+                __func__, src->output, n_IRQ, active, was_active,
+                dst->outputs_active[src->output]);
+
+        /* On Freescale MPIC, critical interrupts ignore priority,
+         * IACK, EOI, etc.  Before MPIC v4.1 they also ignore
+         * masking.
+         */
+        if (active) {
+            if (!was_active && dst->outputs_active[src->output]++ == 0) {
+                DPRINTF("%s: Raise OpenPIC output %d cpu %d irq %d\n",
+                        __func__, src->output, n_CPU, n_IRQ);
+                qemu_irq_raise(dst->irqs[src->output]);
+            }
+        } else {
+            if (was_active && --dst->outputs_active[src->output] == 0) {
+                DPRINTF("%s: Lower OpenPIC output %d cpu %d irq %d\n",
+                        __func__, src->output, n_CPU, n_IRQ);
+                qemu_irq_lower(dst->irqs[src->output]);
+            }
+        }
+
         return;
     }
-    if (IRQ_testbit(&dst->raised, n_IRQ)) {
-        /* Interrupt miss */
-        DPRINTF("%s: IRQ %d was missed on CPU %d\n",
-                __func__, n_IRQ, n_CPU);
-        return;
+
+    priority = IVPR_PRIORITY(src->ivpr);
+
+    /* Even if the interrupt doesn't have enough priority,
+     * it is still raised, in case ctpr is lowered later.
+     */
+    if (active) {
+        IRQ_setbit(&dst->raised, n_IRQ);
+    } else {
+        IRQ_resetbit(&dst->raised, n_IRQ);
     }
-    set_bit(&src->ipvp, IPVP_ACTIVITY);
-    IRQ_setbit(&dst->raised, n_IRQ);
-    if (priority < dst->raised.priority) {
-        /* An higher priority IRQ is already raised */
-        DPRINTF("%s: IRQ %d is hidden by raised IRQ %d on CPU %d\n",
-                __func__, n_IRQ, dst->raised.next, n_CPU);
-        return;
+
+    IRQ_check(opp, &dst->raised);
+
+    if (active && priority <= dst->ctpr) {
+        DPRINTF("%s: IRQ %d priority %d too low for ctpr %d on CPU %d\n",
+                __func__, n_IRQ, priority, dst->ctpr, n_CPU);
+        active = 0;
     }
-    IRQ_get_next(opp, &dst->raised);
-    if (IRQ_get_next(opp, &dst->servicing) != -1 &&
-        priority <= dst->servicing.priority) {
-        DPRINTF("%s: IRQ %d is hidden by servicing IRQ %d on CPU %d\n",
-                __func__, n_IRQ, dst->servicing.next, n_CPU);
-        /* Already servicing a higher priority IRQ */
-        return;
+
+    if (active) {
+        if (IRQ_get_next(opp, &dst->servicing) >= 0 &&
+                priority <= dst->servicing.priority) {
+            DPRINTF("%s: IRQ %d is hidden by servicing IRQ %d on CPU %d\n",
+                    __func__, n_IRQ, dst->servicing.next, n_CPU);
+        } else {
+            DPRINTF("%s: Raise OpenPIC INT output cpu %d irq %d/%d\n",
+                    __func__, n_CPU, n_IRQ, dst->raised.next);
+            qemu_irq_raise(opp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
+        }
+    } else {
+        IRQ_get_next(opp, &dst->servicing);
+        if (dst->raised.priority > dst->ctpr &&
+                dst->raised.priority > dst->servicing.priority) {
+            DPRINTF("%s: IRQ %d inactive, IRQ %d prio %d above %d/%d, CPU %d\n",
+                    __func__, n_IRQ, dst->raised.next, dst->raised.priority,
+                    dst->ctpr, dst->servicing.priority, n_CPU);
+            /* IRQ line stays asserted */
+        } else {
+            DPRINTF("%s: IRQ %d inactive, current prio %d/%d, CPU %d\n",
+                    __func__, n_IRQ, dst->ctpr, dst->servicing.priority, n_CPU);
+            qemu_irq_lower(opp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
+        }
     }
-    DPRINTF("Raise OpenPIC INT output cpu %d irq %d\n", n_CPU, n_IRQ);
-    opp->irq_raise(opp, n_CPU, src);
 }
 
 /* update pic state because registers for n_IRQ have changed value */
-static void openpic_update_irq(openpic_t *opp, int n_IRQ)
+static void openpic_update_irq(OpenPICState *opp, int n_IRQ)
 {
-    IRQ_src_t *src;
+    IRQSource *src;
+    bool active, was_active;
     int i;
 
     src = &opp->src[n_IRQ];
+    active = src->pending;
 
-    if (!src->pending) {
-        /* no irq pending */
-        DPRINTF("%s: IRQ %d is not pending\n", __func__, n_IRQ);
-        return;
-    }
-    if (test_bit(&src->ipvp, IPVP_MASK)) {
+    if ((src->ivpr & IVPR_MASK_MASK) && !src->nomask) {
         /* Interrupt source is disabled */
         DPRINTF("%s: IRQ %d is disabled\n", __func__, n_IRQ);
-        return;
+        active = false;
     }
-    if (IPVP_PRIORITY(src->ipvp) == 0) {
-        /* Priority set to zero */
-        DPRINTF("%s: IRQ %d has 0 priority\n", __func__, n_IRQ);
+
+    was_active = !!(src->ivpr & IVPR_ACTIVITY_MASK);
+
+    /*
+     * We don't have a similar check for already-active because
+     * ctpr may have changed and we need to withdraw the interrupt.
+     */
+    if (!active && !was_active) {
+        DPRINTF("%s: IRQ %d is already inactive\n", __func__, n_IRQ);
         return;
     }
-    if (test_bit(&src->ipvp, IPVP_ACTIVITY)) {
-        /* IRQ already active */
-        DPRINTF("%s: IRQ %d is already active\n", __func__, n_IRQ);
-        return;
+
+    if (active) {
+        src->ivpr |= IVPR_ACTIVITY_MASK;
+    } else {
+        src->ivpr &= ~IVPR_ACTIVITY_MASK;
     }
-    if (src->ide == 0x00000000) {
+
+    if (src->destmask == 0) {
         /* No target */
         DPRINTF("%s: IRQ %d has no target\n", __func__, n_IRQ);
         return;
     }
 
-    if (src->ide == (1 << src->last_cpu)) {
+    if (src->destmask == (1 << src->last_cpu)) {
         /* Only one CPU is allowed to receive this IRQ */
-        IRQ_local_pipe(opp, src->last_cpu, n_IRQ);
-    } else if (!test_bit(&src->ipvp, IPVP_MODE)) {
+        IRQ_local_pipe(opp, src->last_cpu, n_IRQ, active, was_active);
+    } else if (!(src->ivpr & IVPR_MODE_MASK)) {
         /* Directed delivery mode */
         for (i = 0; i < opp->nb_cpus; i++) {
-            if (test_bit(&src->ide, i))
-                IRQ_local_pipe(opp, i, n_IRQ);
+            if (src->destmask & (1 << i)) {
+                IRQ_local_pipe(opp, i, n_IRQ, active, was_active);
+            }
         }
     } else {
         /* Distributed delivery mode */
         for (i = src->last_cpu + 1; i != src->last_cpu; i++) {
-            if (i == opp->nb_cpus)
+            if (i == opp->nb_cpus) {
                 i = 0;
-            if (test_bit(&src->ide, i)) {
-                IRQ_local_pipe(opp, i, n_IRQ);
+            }
+            if (src->destmask & (1 << i)) {
+                IRQ_local_pipe(opp, i, n_IRQ, active, was_active);
                 src->last_cpu = i;
                 break;
             }
@@ -418,193 +500,236 @@ static void openpic_update_irq(openpic_t *opp, int n_IRQ)
 
 static void openpic_set_irq(void *opaque, int n_IRQ, int level)
 {
-    openpic_t *opp = opaque;
-    IRQ_src_t *src;
+    OpenPICState *opp = opaque;
+    IRQSource *src;
+
+    if (n_IRQ >= MAX_IRQ) {
+        fprintf(stderr, "%s: IRQ %d out of range\n", __func__, n_IRQ);
+        abort();
+    }
 
     src = &opp->src[n_IRQ];
-    DPRINTF("openpic: set irq %d = %d ipvp=%08x\n",
-            n_IRQ, level, src->ipvp);
-    if (test_bit(&src->ipvp, IPVP_SENSE)) {
+    DPRINTF("openpic: set irq %d = %d ivpr=0x%08x\n",
+            n_IRQ, level, src->ivpr);
+    if (src->level) {
         /* level-sensitive irq */
         src->pending = level;
-        if (!level)
-            reset_bit(&src->ipvp, IPVP_ACTIVITY);
+        openpic_update_irq(opp, n_IRQ);
     } else {
         /* edge-sensitive irq */
-        if (level)
+        if (level) {
             src->pending = 1;
+            openpic_update_irq(opp, n_IRQ);
+        }
+
+        if (src->output != OPENPIC_OUTPUT_INT) {
+            /* Edge-triggered interrupts shouldn't be used
+             * with non-INT delivery, but just in case,
+             * try to make it do something sane rather than
+             * cause an interrupt storm.  This is close to
+             * what you'd probably see happen in real hardware.
+             */
+            src->pending = 0;
+            openpic_update_irq(opp, n_IRQ);
+        }
     }
-    openpic_update_irq(opp, n_IRQ);
 }
 
-static void openpic_reset (void *opaque)
+static void openpic_reset(DeviceState *d)
 {
-    openpic_t *opp = (openpic_t *)opaque;
+    OpenPICState *opp = FROM_SYSBUS(typeof(*opp), SYS_BUS_DEVICE(d));
     int i;
 
-    opp->glbc = 0x80000000;
+    opp->gcr = GCR_RESET;
     /* Initialise controller registers */
-    opp->frep = ((OPENPIC_EXT_IRQ - 1) << 16) | ((MAX_CPU - 1) << 8) | VID;
-    opp->veni = VENI;
-    opp->pint = 0x00000000;
-    opp->spve = 0x000000FF;
-    opp->tifr = 0x003F7A00;
-    /* ? */
-    opp->micr = 0x00000000;
+    opp->frr = ((opp->nb_irqs - 1) << FRR_NIRQ_SHIFT) |
+               ((opp->nb_cpus - 1) << FRR_NCPU_SHIFT) |
+               (opp->vid << FRR_VID_SHIFT);
+
+    opp->pir = 0;
+    opp->spve = -1 & opp->vector_mask;
+    opp->tfrr = opp->tfrr_reset;
     /* Initialise IRQ sources */
     for (i = 0; i < opp->max_irq; i++) {
-        opp->src[i].ipvp = 0xA0000000;
-        opp->src[i].ide  = 0x00000000;
+        opp->src[i].ivpr = opp->ivpr_reset;
+        opp->src[i].idr  = opp->idr_reset;
+
+        switch (opp->src[i].type) {
+        case IRQ_TYPE_NORMAL:
+            opp->src[i].level = !!(opp->ivpr_reset & IVPR_SENSE_MASK);
+            break;
+
+        case IRQ_TYPE_FSLINT:
+            opp->src[i].ivpr |= IVPR_POLARITY_MASK;
+            break;
+
+        case IRQ_TYPE_FSLSPECIAL:
+            break;
+        }
     }
     /* Initialise IRQ destinations */
     for (i = 0; i < MAX_CPU; i++) {
-        opp->dst[i].pctp      = 0x0000000F;
-        opp->dst[i].pcsr      = 0x00000000;
-        memset(&opp->dst[i].raised, 0, sizeof(IRQ_queue_t));
+        opp->dst[i].ctpr      = 15;
+        memset(&opp->dst[i].raised, 0, sizeof(IRQQueue));
         opp->dst[i].raised.next = -1;
-        memset(&opp->dst[i].servicing, 0, sizeof(IRQ_queue_t));
+        memset(&opp->dst[i].servicing, 0, sizeof(IRQQueue));
         opp->dst[i].servicing.next = -1;
     }
     /* Initialise timers */
     for (i = 0; i < MAX_TMR; i++) {
-        opp->timers[i].ticc = 0x00000000;
-        opp->timers[i].tibc = 0x80000000;
+        opp->timers[i].tccr = 0;
+        opp->timers[i].tbcr = TBCR_CI;
     }
-    /* Initialise doorbells */
-#if MAX_DBL > 0
-    opp->dar = 0x00000000;
-    for (i = 0; i < MAX_DBL; i++) {
-        opp->doorbells[i].dmr  = 0x00000000;
-    }
-#endif
-    /* Initialise mailboxes */
-#if MAX_MBX > 0
-    for (i = 0; i < MAX_MBX; i++) { /* ? */
-        opp->mailboxes[i].mbr   = 0x00000000;
-    }
-#endif
     /* Go out of RESET state */
-    opp->glbc = 0x00000000;
+    opp->gcr = 0;
 }
 
-static inline uint32_t read_IRQreg_ide(openpic_t *opp, int n_IRQ)
+static inline uint32_t read_IRQreg_idr(OpenPICState *opp, int n_IRQ)
 {
-    return opp->src[n_IRQ].ide;
+    return opp->src[n_IRQ].idr;
 }
 
-static inline uint32_t read_IRQreg_ipvp(openpic_t *opp, int n_IRQ)
+static inline uint32_t read_IRQreg_ilr(OpenPICState *opp, int n_IRQ)
 {
-    return opp->src[n_IRQ].ipvp;
-}
-
-static inline void write_IRQreg_ide(openpic_t *opp, int n_IRQ, uint32_t val)
-{
-    uint32_t tmp;
+    if (opp->flags & OPENPIC_FLAG_ILR) {
+        return output_to_inttgt(opp->src[n_IRQ].output);
+    }
 
-    tmp = val & 0xC0000000;
-    tmp |= val & ((1ULL << MAX_CPU) - 1);
-    opp->src[n_IRQ].ide = tmp;
-    DPRINTF("Set IDE %d to 0x%08x\n", n_IRQ, opp->src[n_IRQ].ide);
+    return 0xffffffff;
 }
 
-static inline void write_IRQreg_ipvp(openpic_t *opp, int n_IRQ, uint32_t val)
+static inline uint32_t read_IRQreg_ivpr(OpenPICState *opp, int n_IRQ)
 {
-    /* NOTE: not fully accurate for special IRQs, but simple and sufficient */
-    /* ACTIVITY bit is read-only */
-    opp->src[n_IRQ].ipvp = (opp->src[n_IRQ].ipvp & 0x40000000)
-                         | (val & 0x800F00FF);
-    openpic_update_irq(opp, n_IRQ);
-    DPRINTF("Set IPVP %d to 0x%08x -> 0x%08x\n", n_IRQ, val,
-            opp->src[n_IRQ].ipvp);
+    return opp->src[n_IRQ].ivpr;
 }
 
-#if 0 // Code provision for Intel model
-#if MAX_DBL > 0
-static uint32_t read_doorbell_register (openpic_t *opp,
-                                        int n_dbl, uint32_t offset)
+static inline void write_IRQreg_idr(OpenPICState *opp, int n_IRQ, uint32_t val)
 {
-    uint32_t retval;
+    IRQSource *src = &opp->src[n_IRQ];
+    uint32_t normal_mask = (1UL << opp->nb_cpus) - 1;
+    uint32_t crit_mask = 0;
+    uint32_t mask = normal_mask;
+    int crit_shift = IDR_EP_SHIFT - opp->nb_cpus;
+    int i;
 
-    switch (offset) {
-    case DBL_IPVP_OFFSET:
-        retval = read_IRQreg_ipvp(opp, IRQ_DBL0 + n_dbl);
-        break;
-    case DBL_IDE_OFFSET:
-        retval = read_IRQreg_ide(opp, IRQ_DBL0 + n_dbl);
-        break;
-    case DBL_DMR_OFFSET:
-        retval = opp->doorbells[n_dbl].dmr;
-        break;
+    if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
+        crit_mask = mask << crit_shift;
+        mask |= crit_mask | IDR_EP;
     }
 
-    return retval;
+    src->idr = val & mask;
+    DPRINTF("Set IDR %d to 0x%08x\n", n_IRQ, src->idr);
+
+    if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
+        if (src->idr & crit_mask) {
+            if (src->idr & normal_mask) {
+                DPRINTF("%s: IRQ configured for multiple output types, using "
+                        "critical\n", __func__);
+            }
+
+            src->output = OPENPIC_OUTPUT_CINT;
+            src->nomask = true;
+            src->destmask = 0;
+
+            for (i = 0; i < opp->nb_cpus; i++) {
+                int n_ci = IDR_CI0_SHIFT - i;
+
+                if (src->idr & (1UL << n_ci)) {
+                    src->destmask |= 1UL << i;
+                }
+            }
+        } else {
+            src->output = OPENPIC_OUTPUT_INT;
+            src->nomask = false;
+            src->destmask = src->idr & normal_mask;
+        }
+    } else {
+        src->destmask = src->idr;
+    }
 }
 
-static void write_doorbell_register (penpic_t *opp, int n_dbl,
-                                     uint32_t offset, uint32_t value)
+static inline void write_IRQreg_ilr(OpenPICState *opp, int n_IRQ, uint32_t val)
 {
-    switch (offset) {
-    case DBL_IVPR_OFFSET:
-        write_IRQreg_ipvp(opp, IRQ_DBL0 + n_dbl, value);
-        break;
-    case DBL_IDE_OFFSET:
-        write_IRQreg_ide(opp, IRQ_DBL0 + n_dbl, value);
-        break;
-    case DBL_DMR_OFFSET:
-        opp->doorbells[n_dbl].dmr = value;
-        break;
+    if (opp->flags & OPENPIC_FLAG_ILR) {
+        IRQSource *src = &opp->src[n_IRQ];
+
+        src->output = inttgt_to_output(val & ILR_INTTGT_MASK);
+        DPRINTF("Set ILR %d to 0x%08x, output %d\n", n_IRQ, src->idr,
+                src->output);
+
+        /* TODO: on MPIC v4.0 only, set nomask for non-INT */
     }
 }
-#endif
 
-#if MAX_MBX > 0
-static uint32_t read_mailbox_register (openpic_t *opp,
-                                       int n_mbx, uint32_t offset)
+static inline void write_IRQreg_ivpr(OpenPICState *opp, int n_IRQ, uint32_t val)
 {
-    uint32_t retval;
+    uint32_t mask;
+
+    /* NOTE when implementing newer FSL MPIC models: starting with v4.0,
+     * the polarity bit is read-only on internal interrupts.
+     */
+    mask = IVPR_MASK_MASK | IVPR_PRIORITY_MASK | IVPR_SENSE_MASK |
+           IVPR_POLARITY_MASK | opp->vector_mask;
 
-    switch (offset) {
-    case MBX_MBR_OFFSET:
-        retval = opp->mailboxes[n_mbx].mbr;
+    /* ACTIVITY bit is read-only */
+    opp->src[n_IRQ].ivpr =
+        (opp->src[n_IRQ].ivpr & IVPR_ACTIVITY_MASK) | (val & mask);
+
+    /* For FSL internal interrupts, The sense bit is reserved and zero,
+     * and the interrupt is always level-triggered.  Timers and IPIs
+     * have no sense or polarity bits, and are edge-triggered.
+     */
+    switch (opp->src[n_IRQ].type) {
+    case IRQ_TYPE_NORMAL:
+        opp->src[n_IRQ].level = !!(opp->src[n_IRQ].ivpr & IVPR_SENSE_MASK);
         break;
-    case MBX_IVPR_OFFSET:
-        retval = read_IRQreg_ipvp(opp, IRQ_MBX0 + n_mbx);
+
+    case IRQ_TYPE_FSLINT:
+        opp->src[n_IRQ].ivpr &= ~IVPR_SENSE_MASK;
         break;
-    case MBX_DMR_OFFSET:
-        retval = read_IRQreg_ide(opp, IRQ_MBX0 + n_mbx);
+
+    case IRQ_TYPE_FSLSPECIAL:
+        opp->src[n_IRQ].ivpr &= ~(IVPR_POLARITY_MASK | IVPR_SENSE_MASK);
         break;
     }
 
-    return retval;
+    openpic_update_irq(opp, n_IRQ);
+    DPRINTF("Set IVPR %d to 0x%08x -> 0x%08x\n", n_IRQ, val,
+            opp->src[n_IRQ].ivpr);
 }
 
-static void write_mailbox_register (openpic_t *opp, int n_mbx,
-                                    uint32_t address, uint32_t value)
+static void openpic_gcr_write(OpenPICState *opp, uint64_t val)
 {
-    switch (offset) {
-    case MBX_MBR_OFFSET:
-        opp->mailboxes[n_mbx].mbr = value;
-        break;
-    case MBX_IVPR_OFFSET:
-        write_IRQreg_ipvp(opp, IRQ_MBX0 + n_mbx, value);
-        break;
-    case MBX_DMR_OFFSET:
-        write_IRQreg_ide(opp, IRQ_MBX0 + n_mbx, value);
-        break;
+    bool mpic_proxy = false;
+
+    if (val & GCR_RESET) {
+        openpic_reset(&opp->busdev.qdev);
+        return;
+    }
+
+    opp->gcr &= ~opp->mpic_mode_mask;
+    opp->gcr |= val & opp->mpic_mode_mask;
+
+    /* Set external proxy mode */
+    if ((val & opp->mpic_mode_mask) == GCR_MODE_PROXY) {
+        mpic_proxy = true;
     }
+
+    ppce500_set_mpic_proxy(mpic_proxy);
 }
-#endif
-#endif /* 0 : Code provision for Intel model */
 
-static void openpic_gbl_write (void *opaque, hwaddr addr, uint32_t val)
+static void openpic_gbl_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned len)
 {
-    openpic_t *opp = opaque;
-    IRQ_dst_t *dst;
+    OpenPICState *opp = opaque;
+    IRQDest *dst;
     int idx;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= %08" PRIx64 "\n",
+            __func__, addr, val);
+    if (addr & 0xF) {
         return;
+    }
     switch (addr) {
     case 0x00: /* Block Revision Register1 (BRR1) is Readonly */
         break;
@@ -618,73 +743,71 @@ static void openpic_gbl_write (void *opaque, hwaddr addr, uint32_t val)
     case 0xB0:
         openpic_cpu_write_internal(opp, addr, val, get_current_cpu());
         break;
-    case 0x1000: /* FREP */
+    case 0x1000: /* FRR */
         break;
-    case 0x1020: /* GLBC */
-        if (val & 0x80000000 && opp->reset)
-            opp->reset(opp);
-        opp->glbc = val & ~0x80000000;
+    case 0x1020: /* GCR */
+        openpic_gcr_write(opp, val);
         break;
-    case 0x1080: /* VENI */
+    case 0x1080: /* VIR */
         break;
-    case 0x1090: /* PINT */
+    case 0x1090: /* PIR */
         for (idx = 0; idx < opp->nb_cpus; idx++) {
-            if ((val & (1 << idx)) && !(opp->pint & (1 << idx))) {
+            if ((val & (1 << idx)) && !(opp->pir & (1 << idx))) {
                 DPRINTF("Raise OpenPIC RESET output for CPU %d\n", idx);
                 dst = &opp->dst[idx];
                 qemu_irq_raise(dst->irqs[OPENPIC_OUTPUT_RESET]);
-            } else if (!(val & (1 << idx)) && (opp->pint & (1 << idx))) {
+            } else if (!(val & (1 << idx)) && (opp->pir & (1 << idx))) {
                 DPRINTF("Lower OpenPIC RESET output for CPU %d\n", idx);
                 dst = &opp->dst[idx];
                 qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_RESET]);
             }
         }
-        opp->pint = val;
+        opp->pir = val;
         break;
-    case 0x10A0: /* IPI_IPVP */
+    case 0x10A0: /* IPI_IVPR */
     case 0x10B0:
     case 0x10C0:
     case 0x10D0:
         {
             int idx;
             idx = (addr - 0x10A0) >> 4;
-            write_IRQreg_ipvp(opp, opp->irq_ipi0 + idx, val);
+            write_IRQreg_ivpr(opp, opp->irq_ipi0 + idx, val);
         }
         break;
     case 0x10E0: /* SPVE */
-        opp->spve = val & 0x000000FF;
-        break;
-    case 0x10F0: /* TIFR */
-        opp->tifr = val;
+        opp->spve = val & opp->vector_mask;
         break;
     default:
         break;
     }
 }
 
-static uint32_t openpic_gbl_read (void *opaque, hwaddr addr)
+static uint64_t openpic_gbl_read(void *opaque, hwaddr addr, unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     uint32_t retval;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
     retval = 0xFFFFFFFF;
-    if (addr & 0xF)
+    if (addr & 0xF) {
         return retval;
+    }
     switch (addr) {
-    case 0x1000: /* FREP */
-        retval = opp->frep;
+    case 0x1000: /* FRR */
+        retval = opp->frr;
         break;
-    case 0x1020: /* GLBC */
-        retval = opp->glbc;
+    case 0x1020: /* GCR */
+        retval = opp->gcr;
         break;
-    case 0x1080: /* VENI */
-        retval = opp->veni;
+    case 0x1080: /* VIR */
+        retval = opp->vir;
         break;
-    case 0x1090: /* PINT */
+    case 0x1090: /* PIR */
         retval = 0x00000000;
         break;
     case 0x00: /* Block Revision Register1 (BRR1) */
+        retval = opp->brr1;
+        break;
     case 0x40:
     case 0x50:
     case 0x60:
@@ -695,179 +818,309 @@ static uint32_t openpic_gbl_read (void *opaque, hwaddr addr)
     case 0xB0:
         retval = openpic_cpu_read_internal(opp, addr, get_current_cpu());
         break;
-    case 0x10A0: /* IPI_IPVP */
+    case 0x10A0: /* IPI_IVPR */
     case 0x10B0:
     case 0x10C0:
     case 0x10D0:
         {
             int idx;
             idx = (addr - 0x10A0) >> 4;
-            retval = read_IRQreg_ipvp(opp, opp->irq_ipi0 + idx);
+            retval = read_IRQreg_ivpr(opp, opp->irq_ipi0 + idx);
         }
         break;
     case 0x10E0: /* SPVE */
         retval = opp->spve;
         break;
-    case 0x10F0: /* TIFR */
-        retval = opp->tifr;
-        break;
     default:
         break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
 
     return retval;
 }
 
-static void openpic_timer_write (void *opaque, uint32_t addr, uint32_t val)
+static void openpic_tmr_write(void *opaque, hwaddr addr, uint64_t val,
+                                unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     int idx;
 
-    DPRINTF("%s: addr %08x <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
+    addr += 0x10f0;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= %08" PRIx64 "\n",
+            __func__, addr, val);
+    if (addr & 0xF) {
+        return;
+    }
+
+    if (addr == 0x10f0) {
+        /* TFRR */
+        opp->tfrr = val;
         return;
-    addr -= 0x10;
-    addr &= 0xFFFF;
-    idx = (addr & 0xFFF0) >> 6;
+    }
+
+    idx = (addr >> 6) & 0x3;
     addr = addr & 0x30;
-    switch (addr) {
-    case 0x00: /* TICC */
+
+    switch (addr & 0x30) {
+    case 0x00: /* TCCR */
         break;
-    case 0x10: /* TIBC */
-        if ((opp->timers[idx].ticc & 0x80000000) != 0 &&
-            (val & 0x80000000) == 0 &&
-            (opp->timers[idx].tibc & 0x80000000) != 0)
-            opp->timers[idx].ticc &= ~0x80000000;
-        opp->timers[idx].tibc = val;
+    case 0x10: /* TBCR */
+        if ((opp->timers[idx].tccr & TCCR_TOG) != 0 &&
+            (val & TBCR_CI) == 0 &&
+            (opp->timers[idx].tbcr & TBCR_CI) != 0) {
+            opp->timers[idx].tccr &= ~TCCR_TOG;
+        }
+        opp->timers[idx].tbcr = val;
         break;
-    case 0x20: /* TIVP */
-        write_IRQreg_ipvp(opp, opp->irq_tim0 + idx, val);
+    case 0x20: /* TVPR */
+        write_IRQreg_ivpr(opp, opp->irq_tim0 + idx, val);
         break;
-    case 0x30: /* TIDE */
-        write_IRQreg_ide(opp, opp->irq_tim0 + idx, val);
+    case 0x30: /* TDR */
+        write_IRQreg_idr(opp, opp->irq_tim0 + idx, val);
         break;
     }
 }
 
-static uint32_t openpic_timer_read (void *opaque, uint32_t addr)
+static uint64_t openpic_tmr_read(void *opaque, hwaddr addr, unsigned len)
 {
-    openpic_t *opp = opaque;
-    uint32_t retval;
+    OpenPICState *opp = opaque;
+    uint32_t retval = -1;
     int idx;
 
-    DPRINTF("%s: addr %08x\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-    addr -= 0x10;
-    addr &= 0xFFFF;
-    idx = (addr & 0xFFF0) >> 6;
-    addr = addr & 0x30;
-    switch (addr) {
-    case 0x00: /* TICC */
-        retval = opp->timers[idx].ticc;
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
+    if (addr & 0xF) {
+        goto out;
+    }
+    idx = (addr >> 6) & 0x3;
+    if (addr == 0x0) {
+        /* TFRR */
+        retval = opp->tfrr;
+        goto out;
+    }
+    switch (addr & 0x30) {
+    case 0x00: /* TCCR */
+        retval = opp->timers[idx].tccr;
         break;
-    case 0x10: /* TIBC */
-        retval = opp->timers[idx].tibc;
+    case 0x10: /* TBCR */
+        retval = opp->timers[idx].tbcr;
         break;
     case 0x20: /* TIPV */
-        retval = read_IRQreg_ipvp(opp, opp->irq_tim0 + idx);
+        retval = read_IRQreg_ivpr(opp, opp->irq_tim0 + idx);
         break;
-    case 0x30: /* TIDE */
-        retval = read_IRQreg_ide(opp, opp->irq_tim0 + idx);
+    case 0x30: /* TIDE (TIDR) */
+        retval = read_IRQreg_idr(opp, opp->irq_tim0 + idx);
         break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
+
+out:
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
 
     return retval;
 }
 
-static void openpic_src_write (void *opaque, uint32_t addr, uint32_t val)
+static void openpic_src_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     int idx;
 
-    DPRINTF("%s: addr %08x <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-    addr = addr & 0xFFF0;
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= %08" PRIx64 "\n",
+            __func__, addr, val);
+
+    addr = addr & 0xffff;
     idx = addr >> 5;
-    if (addr & 0x10) {
-        /* EXDE / IFEDE / IEEDE */
-        write_IRQreg_ide(opp, idx, val);
-    } else {
-        /* EXVP / IFEVP / IEEVP */
-        write_IRQreg_ipvp(opp, idx, val);
+
+    switch (addr & 0x1f) {
+    case 0x00:
+        write_IRQreg_ivpr(opp, idx, val);
+        break;
+    case 0x10:
+        write_IRQreg_idr(opp, idx, val);
+        break;
+    case 0x18:
+        write_IRQreg_ilr(opp, idx, val);
+        break;
     }
 }
 
-static uint32_t openpic_src_read (void *opaque, uint32_t addr)
+static uint64_t openpic_src_read(void *opaque, uint64_t addr, unsigned len)
 {
-    openpic_t *opp = opaque;
+    OpenPICState *opp = opaque;
     uint32_t retval;
     int idx;
 
-    DPRINTF("%s: addr %08x\n", __func__, addr);
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
     retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-    addr = addr & 0xFFF0;
+
+    addr = addr & 0xffff;
     idx = addr >> 5;
-    if (addr & 0x10) {
-        /* EXDE / IFEDE / IEEDE */
-        retval = read_IRQreg_ide(opp, idx);
-    } else {
-        /* EXVP / IFEVP / IEEVP */
-        retval = read_IRQreg_ipvp(opp, idx);
+
+    switch (addr & 0x1f) {
+    case 0x00:
+        retval = read_IRQreg_ivpr(opp, idx);
+        break;
+    case 0x10:
+        retval = read_IRQreg_idr(opp, idx);
+        break;
+    case 0x18:
+        retval = read_IRQreg_ilr(opp, idx);
+        break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
 
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
     return retval;
 }
 
+static void openpic_msi_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
+{
+    OpenPICState *opp = opaque;
+    int idx = opp->irq_msi;
+    int srs, ibs;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= 0x%08" PRIx64 "\n",
+            __func__, addr, val);
+    if (addr & 0xF) {
+        return;
+    }
+
+    switch (addr) {
+    case MSIIR_OFFSET:
+        srs = val >> MSIIR_SRS_SHIFT;
+        idx += srs;
+        ibs = (val & MSIIR_IBS_MASK) >> MSIIR_IBS_SHIFT;
+        opp->msi[srs].msir |= 1 << ibs;
+        openpic_set_irq(opp, idx, 1);
+        break;
+    default:
+        /* most registers are read-only, thus ignored */
+        break;
+    }
+}
+
+static uint64_t openpic_msi_read(void *opaque, hwaddr addr, unsigned size)
+{
+    OpenPICState *opp = opaque;
+    uint64_t r = 0;
+    int i, srs;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
+    if (addr & 0xF) {
+        return -1;
+    }
+
+    srs = addr >> 4;
+
+    switch (addr) {
+    case 0x00:
+    case 0x10:
+    case 0x20:
+    case 0x30:
+    case 0x40:
+    case 0x50:
+    case 0x60:
+    case 0x70: /* MSIRs */
+        r = opp->msi[srs].msir;
+        /* Clear on read */
+        opp->msi[srs].msir = 0;
+        openpic_set_irq(opp, opp->irq_msi + srs, 0);
+        break;
+    case 0x120: /* MSISR */
+        for (i = 0; i < MAX_MSI; i++) {
+            r |= (opp->msi[i].msir ? 1 : 0) << i;
+        }
+        break;
+    }
+
+    return r;
+}
+
+static uint64_t openpic_summary_read(void *opaque, hwaddr addr, unsigned size)
+{
+    uint64_t r = 0;
+
+    DPRINTF("%s: addr %#" HWADDR_PRIx "\n", __func__, addr);
+
+    /* TODO: EISR/EIMR */
+
+    return r;
+}
+
+static void openpic_summary_write(void *opaque, hwaddr addr, uint64_t val,
+                                  unsigned size)
+{
+    DPRINTF("%s: addr %#" HWADDR_PRIx " <= 0x%08" PRIx64 "\n",
+            __func__, addr, val);
+
+    /* TODO: EISR/EIMR */
+}
+
 static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
                                        uint32_t val, int idx)
 {
-    openpic_t *opp = opaque;
-    IRQ_src_t *src;
-    IRQ_dst_t *dst;
+    OpenPICState *opp = opaque;
+    IRQSource *src;
+    IRQDest *dst;
     int s_IRQ, n_IRQ;
 
-    DPRINTF("%s: cpu %d addr " TARGET_FMT_plx " <= %08x\n", __func__, idx,
+    DPRINTF("%s: cpu %d addr %#" HWADDR_PRIx " <= 0x%08x\n", __func__, idx,
             addr, val);
-    if (addr & 0xF)
+
+    if (idx < 0) {
         return;
+    }
+
+    if (addr & 0xF) {
+        return;
+    }
     dst = &opp->dst[idx];
     addr &= 0xFF0;
     switch (addr) {
-#if MAX_IPI > 0
     case 0x40: /* IPIDR */
     case 0x50:
     case 0x60:
     case 0x70:
         idx = (addr - 0x40) >> 4;
         /* we use IDE as mask which CPUs to deliver the IPI to still. */
-        write_IRQreg_ide(opp, opp->irq_ipi0 + idx,
-                         opp->src[opp->irq_ipi0 + idx].ide | val);
+        opp->src[opp->irq_ipi0 + idx].destmask |= val;
         openpic_set_irq(opp, opp->irq_ipi0 + idx, 1);
         openpic_set_irq(opp, opp->irq_ipi0 + idx, 0);
         break;
-#endif
-    case 0x80: /* PCTP */
-        dst->pctp = val & 0x0000000F;
+    case 0x80: /* CTPR */
+        dst->ctpr = val & 0x0000000F;
+
+        DPRINTF("%s: set CPU %d ctpr to %d, raised %d servicing %d\n",
+                __func__, idx, dst->ctpr, dst->raised.priority,
+                dst->servicing.priority);
+
+        if (dst->raised.priority <= dst->ctpr) {
+            DPRINTF("%s: Lower OpenPIC INT output cpu %d due to ctpr\n",
+                    __func__, idx);
+            qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_INT]);
+        } else if (dst->raised.priority > dst->servicing.priority) {
+            DPRINTF("%s: Raise OpenPIC INT output cpu %d irq %d\n",
+                    __func__, idx, dst->raised.next);
+            qemu_irq_raise(dst->irqs[OPENPIC_OUTPUT_INT]);
+        }
+
         break;
     case 0x90: /* WHOAMI */
         /* Read-only register */
         break;
-    case 0xA0: /* PIAC */
+    case 0xA0: /* IACK */
         /* Read-only register */
         break;
-    case 0xB0: /* PEOI */
-        DPRINTF("PEOI\n");
+    case 0xB0: /* EOI */
+        DPRINTF("EOI\n");
         s_IRQ = IRQ_get_next(opp, &dst->servicing);
+
+        if (s_IRQ < 0) {
+            DPRINTF("%s: EOI with no interrupt in service\n", __func__);
+            break;
+        }
+
         IRQ_resetbit(&dst->servicing, s_IRQ);
-        dst->servicing.next = -1;
         /* Set up next servicing IRQ */
         s_IRQ = IRQ_get_next(opp, &dst->servicing);
         /* Check queued interrupts. */
@@ -875,10 +1128,10 @@ static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
         src = &opp->src[n_IRQ];
         if (n_IRQ != -1 &&
             (s_IRQ == -1 ||
-             IPVP_PRIORITY(src->ipvp) > dst->servicing.priority)) {
+             IVPR_PRIORITY(src->ivpr) > dst->servicing.priority)) {
             DPRINTF("Raise OpenPIC INT output cpu %d irq %d\n",
                     idx, n_IRQ);
-            opp->irq_raise(opp, idx, src);
+            qemu_irq_raise(opp->dst[idx].irqs[OPENPIC_OUTPUT_INT]);
         }
         break;
     default:
@@ -886,188 +1139,222 @@ static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
     }
 }
 
-static void openpic_cpu_write(void *opaque, hwaddr addr, uint32_t val)
+static void openpic_cpu_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned len)
 {
     openpic_cpu_write_internal(opaque, addr, val, (addr & 0x1f000) >> 12);
 }
 
+
+static uint32_t openpic_iack(OpenPICState *opp, IRQDest *dst, int cpu)
+{
+    IRQSource *src;
+    int retval, irq;
+
+    DPRINTF("Lower OpenPIC INT output\n");
+    qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_INT]);
+
+    irq = IRQ_get_next(opp, &dst->raised);
+    DPRINTF("IACK: irq=%d\n", irq);
+
+    if (irq == -1) {
+        /* No more interrupt pending */
+        return opp->spve;
+    }
+
+    src = &opp->src[irq];
+    if (!(src->ivpr & IVPR_ACTIVITY_MASK) ||
+            !(IVPR_PRIORITY(src->ivpr) > dst->ctpr)) {
+        fprintf(stderr, "%s: bad raised IRQ %d ctpr %d ivpr 0x%08x\n",
+                __func__, irq, dst->ctpr, src->ivpr);
+        openpic_update_irq(opp, irq);
+        retval = opp->spve;
+    } else {
+        /* IRQ enter servicing state */
+        IRQ_setbit(&dst->servicing, irq);
+        retval = IVPR_VECTOR(opp, src->ivpr);
+    }
+
+    if (!src->level) {
+        /* edge-sensitive IRQ */
+        src->ivpr &= ~IVPR_ACTIVITY_MASK;
+        src->pending = 0;
+        IRQ_resetbit(&dst->raised, irq);
+    }
+
+    if ((irq >= opp->irq_ipi0) &&  (irq < (opp->irq_ipi0 + MAX_IPI))) {
+        src->destmask &= ~(1 << cpu);
+        if (src->destmask && !src->level) {
+            /* trigger on CPUs that didn't know about it yet */
+            openpic_set_irq(opp, irq, 1);
+            openpic_set_irq(opp, irq, 0);
+            /* if all CPUs knew about it, set active bit again */
+            src->ivpr |= IVPR_ACTIVITY_MASK;
+        }
+    }
+
+    return retval;
+}
+
 static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
                                           int idx)
 {
-    openpic_t *opp = opaque;
-    IRQ_src_t *src;
-    IRQ_dst_t *dst;
+    OpenPICState *opp = opaque;
+    IRQDest *dst;
     uint32_t retval;
-    int n_IRQ;
 
-    DPRINTF("%s: cpu %d addr " TARGET_FMT_plx "\n", __func__, idx, addr);
+    DPRINTF("%s: cpu %d addr %#" HWADDR_PRIx "\n", __func__, idx, addr);
     retval = 0xFFFFFFFF;
-    if (addr & 0xF)
+
+    if (idx < 0) {
+        return retval;
+    }
+
+    if (addr & 0xF) {
         return retval;
+    }
     dst = &opp->dst[idx];
     addr &= 0xFF0;
     switch (addr) {
-    case 0x00: /* Block Revision Register1 (BRR1) */
-        retval = FSL_BRR1_IPID | FSL_BRR1_IPMJ | FSL_BRR1_IPMN;
-        break;
-    case 0x80: /* PCTP */
-        retval = dst->pctp;
+    case 0x80: /* CTPR */
+        retval = dst->ctpr;
         break;
     case 0x90: /* WHOAMI */
         retval = idx;
         break;
-    case 0xA0: /* PIAC */
-        DPRINTF("Lower OpenPIC INT output\n");
-        qemu_irq_lower(dst->irqs[OPENPIC_OUTPUT_INT]);
-        n_IRQ = IRQ_get_next(opp, &dst->raised);
-        DPRINTF("PIAC: irq=%d\n", n_IRQ);
-        if (n_IRQ == -1) {
-            /* No more interrupt pending */
-            retval = IPVP_VECTOR(opp->spve);
-        } else {
-            src = &opp->src[n_IRQ];
-            if (!test_bit(&src->ipvp, IPVP_ACTIVITY) ||
-                !(IPVP_PRIORITY(src->ipvp) > dst->pctp)) {
-                /* - Spurious level-sensitive IRQ
-                 * - Priorities has been changed
-                 *   and the pending IRQ isn't allowed anymore
-                 */
-                reset_bit(&src->ipvp, IPVP_ACTIVITY);
-                retval = IPVP_VECTOR(opp->spve);
-            } else {
-                /* IRQ enter servicing state */
-                IRQ_setbit(&dst->servicing, n_IRQ);
-                retval = IPVP_VECTOR(src->ipvp);
-            }
-            IRQ_resetbit(&dst->raised, n_IRQ);
-            dst->raised.next = -1;
-            if (!test_bit(&src->ipvp, IPVP_SENSE)) {
-                /* edge-sensitive IRQ */
-                reset_bit(&src->ipvp, IPVP_ACTIVITY);
-                src->pending = 0;
-            }
-
-            if ((n_IRQ >= opp->irq_ipi0) &&  (n_IRQ < (opp->irq_ipi0 + MAX_IPI))) {
-                src->ide &= ~(1 << idx);
-                if (src->ide && !test_bit(&src->ipvp, IPVP_SENSE)) {
-                    /* trigger on CPUs that didn't know about it yet */
-                    openpic_set_irq(opp, n_IRQ, 1);
-                    openpic_set_irq(opp, n_IRQ, 0);
-                    /* if all CPUs knew about it, set active bit again */
-                    set_bit(&src->ipvp, IPVP_ACTIVITY);
-                }
-            }
-        }
+    case 0xA0: /* IACK */
+        retval = openpic_iack(opp, dst, idx);
         break;
-    case 0xB0: /* PEOI */
+    case 0xB0: /* EOI */
         retval = 0;
         break;
     default:
         break;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
+    DPRINTF("%s: => 0x%08x\n", __func__, retval);
 
     return retval;
 }
 
-static uint32_t openpic_cpu_read(void *opaque, hwaddr addr)
+static uint64_t openpic_cpu_read(void *opaque, hwaddr addr, unsigned len)
 {
     return openpic_cpu_read_internal(opaque, addr, (addr & 0x1f000) >> 12);
 }
 
-static void openpic_buggy_write (void *opaque,
-                                 hwaddr addr, uint32_t val)
-{
-    printf("Invalid OPENPIC write access !\n");
-}
-
-static uint32_t openpic_buggy_read (void *opaque, hwaddr addr)
-{
-    printf("Invalid OPENPIC read access !\n");
-
-    return -1;
-}
+static const MemoryRegionOps openpic_glb_ops_le = {
+    .write = openpic_gbl_write,
+    .read  = openpic_gbl_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static void openpic_writel (void *opaque,
-                            hwaddr addr, uint32_t val)
-{
-    openpic_t *opp = opaque;
-
-    addr &= 0x3FFFF;
-    DPRINTF("%s: offset %08x val: %08x\n", __func__, (int)addr, val);
-    if (addr < 0x1100) {
-        /* Global registers */
-        openpic_gbl_write(opp, addr, val);
-    } else if (addr < 0x10000) {
-        /* Timers registers */
-        openpic_timer_write(opp, addr, val);
-    } else if (addr < 0x20000) {
-        /* Source registers */
-        openpic_src_write(opp, addr, val);
-    } else {
-        /* CPU registers */
-        openpic_cpu_write(opp, addr, val);
-    }
-}
+static const MemoryRegionOps openpic_glb_ops_be = {
+    .write = openpic_gbl_write,
+    .read  = openpic_gbl_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static uint32_t openpic_readl (void *opaque,hwaddr addr)
-{
-    openpic_t *opp = opaque;
-    uint32_t retval;
+static const MemoryRegionOps openpic_tmr_ops_le = {
+    .write = openpic_tmr_write,
+    .read  = openpic_tmr_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    addr &= 0x3FFFF;
-    DPRINTF("%s: offset %08x\n", __func__, (int)addr);
-    if (addr < 0x1100) {
-        /* Global registers */
-        retval = openpic_gbl_read(opp, addr);
-    } else if (addr < 0x10000) {
-        /* Timers registers */
-        retval = openpic_timer_read(opp, addr);
-    } else if (addr < 0x20000) {
-        /* Source registers */
-        retval = openpic_src_read(opp, addr);
-    } else {
-        /* CPU registers */
-        retval = openpic_cpu_read(opp, addr);
-    }
+static const MemoryRegionOps openpic_tmr_ops_be = {
+    .write = openpic_tmr_write,
+    .read  = openpic_tmr_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    return retval;
-}
+static const MemoryRegionOps openpic_cpu_ops_le = {
+    .write = openpic_cpu_write,
+    .read  = openpic_cpu_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static uint64_t openpic_read(void *opaque, hwaddr addr,
-                             unsigned size)
-{
-    openpic_t *opp = opaque;
+static const MemoryRegionOps openpic_cpu_ops_be = {
+    .write = openpic_cpu_write,
+    .read  = openpic_cpu_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    switch (size) {
-    case 4: return openpic_readl(opp, addr);
-    default: return openpic_buggy_read(opp, addr);
-    }
-}
+static const MemoryRegionOps openpic_src_ops_le = {
+    .write = openpic_src_write,
+    .read  = openpic_src_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static void openpic_write(void *opaque, hwaddr addr,
-                          uint64_t data, unsigned size)
-{
-    openpic_t *opp = opaque;
+static const MemoryRegionOps openpic_src_ops_be = {
+    .write = openpic_src_write,
+    .read  = openpic_src_read,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-    switch (size) {
-    case 4: return openpic_writel(opp, addr, data);
-    default: return openpic_buggy_write(opp, addr, data);
-    }
-}
+static const MemoryRegionOps openpic_msi_ops_be = {
+    .read = openpic_msi_read,
+    .write = openpic_msi_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
 
-static const MemoryRegionOps openpic_ops = {
-    .read = openpic_read,
-    .write = openpic_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
+static const MemoryRegionOps openpic_summary_ops_be = {
+    .read = openpic_summary_read,
+    .write = openpic_summary_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
 };
 
-static void openpic_save_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
+static void openpic_save_IRQ_queue(QEMUFile* f, IRQQueue *q)
 {
     unsigned int i;
 
-    for (i = 0; i < BF_WIDTH(MAX_IRQ); i++)
-        qemu_put_be32s(f, &q->queue[i]);
+    for (i = 0; i < ARRAY_SIZE(q->queue); i++) {
+        /* Always put the lower half of a 64-bit long first, in case we
+         * restore on a 32-bit host.  The least significant bits correspond
+         * to lower IRQ numbers in the bitmap.
+         */
+        qemu_put_be32(f, (uint32_t)q->queue[i]);
+#if LONG_MAX > 0x7FFFFFFF
+        qemu_put_be32(f, (uint32_t)(q->queue[i] >> 32));
+#endif
+    }
 
     qemu_put_sbe32s(f, &q->next);
     qemu_put_sbe32s(f, &q->priority);
@@ -1075,63 +1362,54 @@ static void openpic_save_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
 
 static void openpic_save(QEMUFile* f, void *opaque)
 {
-    openpic_t *opp = (openpic_t *)opaque;
+    OpenPICState *opp = (OpenPICState *)opaque;
     unsigned int i;
 
-    qemu_put_be32s(f, &opp->frep);
-    qemu_put_be32s(f, &opp->glbc);
-    qemu_put_be32s(f, &opp->micr);
-    qemu_put_be32s(f, &opp->veni);
-    qemu_put_be32s(f, &opp->pint);
+    qemu_put_be32s(f, &opp->gcr);
+    qemu_put_be32s(f, &opp->vir);
+    qemu_put_be32s(f, &opp->pir);
     qemu_put_be32s(f, &opp->spve);
-    qemu_put_be32s(f, &opp->tifr);
-
-    for (i = 0; i < opp->max_irq; i++) {
-        qemu_put_be32s(f, &opp->src[i].ipvp);
-        qemu_put_be32s(f, &opp->src[i].ide);
-        qemu_put_sbe32s(f, &opp->src[i].type);
-        qemu_put_sbe32s(f, &opp->src[i].last_cpu);
-        qemu_put_sbe32s(f, &opp->src[i].pending);
-    }
+    qemu_put_be32s(f, &opp->tfrr);
 
-    qemu_put_sbe32s(f, &opp->nb_cpus);
+    qemu_put_be32s(f, &opp->nb_cpus);
 
     for (i = 0; i < opp->nb_cpus; i++) {
-        qemu_put_be32s(f, &opp->dst[i].tfrr);
-        qemu_put_be32s(f, &opp->dst[i].pctp);
-        qemu_put_be32s(f, &opp->dst[i].pcsr);
+        qemu_put_sbe32s(f, &opp->dst[i].ctpr);
         openpic_save_IRQ_queue(f, &opp->dst[i].raised);
         openpic_save_IRQ_queue(f, &opp->dst[i].servicing);
+        qemu_put_buffer(f, (uint8_t *)&opp->dst[i].outputs_active,
+                        sizeof(opp->dst[i].outputs_active));
     }
 
     for (i = 0; i < MAX_TMR; i++) {
-        qemu_put_be32s(f, &opp->timers[i].ticc);
-        qemu_put_be32s(f, &opp->timers[i].tibc);
+        qemu_put_be32s(f, &opp->timers[i].tccr);
+        qemu_put_be32s(f, &opp->timers[i].tbcr);
     }
 
-#if MAX_DBL > 0
-    qemu_put_be32s(f, &opp->dar);
-
-    for (i = 0; i < MAX_DBL; i++) {
-        qemu_put_be32s(f, &opp->doorbells[i].dmr);
-    }
-#endif
-
-#if MAX_MBX > 0
-    for (i = 0; i < MAX_MAILBOXES; i++) {
-        qemu_put_be32s(f, &opp->mailboxes[i].mbr);
+    for (i = 0; i < opp->max_irq; i++) {
+        qemu_put_be32s(f, &opp->src[i].ivpr);
+        qemu_put_be32s(f, &opp->src[i].idr);
+        qemu_get_be32s(f, &opp->src[i].destmask);
+        qemu_put_sbe32s(f, &opp->src[i].last_cpu);
+        qemu_put_sbe32s(f, &opp->src[i].pending);
     }
-#endif
-
-    pci_device_save(&opp->pci_dev, f);
 }
 
-static void openpic_load_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
+static void openpic_load_IRQ_queue(QEMUFile* f, IRQQueue *q)
 {
     unsigned int i;
 
-    for (i = 0; i < BF_WIDTH(MAX_IRQ); i++)
-        qemu_get_be32s(f, &q->queue[i]);
+    for (i = 0; i < ARRAY_SIZE(q->queue); i++) {
+        unsigned long val;
+
+        val = qemu_get_be32(f);
+#if LONG_MAX > 0x7FFFFFFF
+        val <<= 32;
+        val |= qemu_get_be32(f);
+#endif
+
+        q->queue[i] = val;
+    }
 
     qemu_get_sbe32s(f, &q->next);
     qemu_get_sbe32s(f, &q->priority);
@@ -1139,572 +1417,245 @@ static void openpic_load_IRQ_queue(QEMUFile* f, IRQ_queue_t *q)
 
 static int openpic_load(QEMUFile* f, void *opaque, int version_id)
 {
-    openpic_t *opp = (openpic_t *)opaque;
+    OpenPICState *opp = (OpenPICState *)opaque;
     unsigned int i;
 
-    if (version_id != 1)
+    if (version_id != 1) {
         return -EINVAL;
+    }
 
-    qemu_get_be32s(f, &opp->frep);
-    qemu_get_be32s(f, &opp->glbc);
-    qemu_get_be32s(f, &opp->micr);
-    qemu_get_be32s(f, &opp->veni);
-    qemu_get_be32s(f, &opp->pint);
+    qemu_get_be32s(f, &opp->gcr);
+    qemu_get_be32s(f, &opp->vir);
+    qemu_get_be32s(f, &opp->pir);
     qemu_get_be32s(f, &opp->spve);
-    qemu_get_be32s(f, &opp->tifr);
-
-    for (i = 0; i < opp->max_irq; i++) {
-        qemu_get_be32s(f, &opp->src[i].ipvp);
-        qemu_get_be32s(f, &opp->src[i].ide);
-        qemu_get_sbe32s(f, &opp->src[i].type);
-        qemu_get_sbe32s(f, &opp->src[i].last_cpu);
-        qemu_get_sbe32s(f, &opp->src[i].pending);
-    }
+    qemu_get_be32s(f, &opp->tfrr);
 
-    qemu_get_sbe32s(f, &opp->nb_cpus);
+    qemu_get_be32s(f, &opp->nb_cpus);
 
     for (i = 0; i < opp->nb_cpus; i++) {
-        qemu_get_be32s(f, &opp->dst[i].tfrr);
-        qemu_get_be32s(f, &opp->dst[i].pctp);
-        qemu_get_be32s(f, &opp->dst[i].pcsr);
+        qemu_get_sbe32s(f, &opp->dst[i].ctpr);
         openpic_load_IRQ_queue(f, &opp->dst[i].raised);
         openpic_load_IRQ_queue(f, &opp->dst[i].servicing);
+        qemu_get_buffer(f, (uint8_t *)&opp->dst[i].outputs_active,
+                        sizeof(opp->dst[i].outputs_active));
     }
 
     for (i = 0; i < MAX_TMR; i++) {
-        qemu_get_be32s(f, &opp->timers[i].ticc);
-        qemu_get_be32s(f, &opp->timers[i].tibc);
+        qemu_get_be32s(f, &opp->timers[i].tccr);
+        qemu_get_be32s(f, &opp->timers[i].tbcr);
     }
 
-#if MAX_DBL > 0
-    qemu_get_be32s(f, &opp->dar);
+    for (i = 0; i < opp->max_irq; i++) {
+        uint32_t val;
 
-    for (i = 0; i < MAX_DBL; i++) {
-        qemu_get_be32s(f, &opp->doorbells[i].dmr);
-    }
-#endif
+        val = qemu_get_be32(f);
+        write_IRQreg_idr(opp, i, val);
+        val = qemu_get_be32(f);
+        write_IRQreg_ivpr(opp, i, val);
 
-#if MAX_MBX > 0
-    for (i = 0; i < MAX_MAILBOXES; i++) {
-        qemu_get_be32s(f, &opp->mailboxes[i].mbr);
+        qemu_get_be32s(f, &opp->src[i].ivpr);
+        qemu_get_be32s(f, &opp->src[i].idr);
+        qemu_get_be32s(f, &opp->src[i].destmask);
+        qemu_get_sbe32s(f, &opp->src[i].last_cpu);
+        qemu_get_sbe32s(f, &opp->src[i].pending);
     }
-#endif
 
-    return pci_device_load(&opp->pci_dev, f);
+    return 0;
 }
 
-static void openpic_irq_raise(openpic_t *opp, int n_CPU, IRQ_src_t *src)
-{
-    qemu_irq_raise(opp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
-}
+typedef struct MemReg {
+    const char             *name;
+    MemoryRegionOps const  *ops;
+    hwaddr      start_addr;
+    ram_addr_t              size;
+} MemReg;
 
-qemu_irq *openpic_init (MemoryRegion **pmem, int nb_cpus,
-                        qemu_irq **irqs, qemu_irq irq_out)
+static void fsl_common_init(OpenPICState *opp)
 {
-    openpic_t *opp;
-    int i, m;
-
-    /* XXX: for now, only one CPU is supported */
-    if (nb_cpus != 1)
-        return NULL;
-    opp = g_malloc0(sizeof(openpic_t));
-    memory_region_init_io(&opp->mem, &openpic_ops, opp, "openpic", 0x40000);
-
-    //    isu_base &= 0xFFFC0000;
-    opp->nb_cpus = nb_cpus;
-    opp->max_irq = OPENPIC_MAX_IRQ;
-    opp->irq_ipi0 = OPENPIC_IRQ_IPI0;
-    opp->irq_tim0 = OPENPIC_IRQ_TIM0;
-    /* Set IRQ types */
-    for (i = 0; i < OPENPIC_EXT_IRQ; i++) {
-        opp->src[i].type = IRQ_EXTERNAL;
-    }
-    for (; i < OPENPIC_IRQ_TIM0; i++) {
-        opp->src[i].type = IRQ_SPECIAL;
-    }
-#if MAX_IPI > 0
-    m = OPENPIC_IRQ_IPI0;
-#else
-    m = OPENPIC_IRQ_DBL0;
-#endif
-    for (; i < m; i++) {
-        opp->src[i].type = IRQ_TIMER;
-    }
-    for (; i < OPENPIC_MAX_IRQ; i++) {
-        opp->src[i].type = IRQ_INTERNAL;
-    }
-    for (i = 0; i < nb_cpus; i++)
-        opp->dst[i].irqs = irqs[i];
-    opp->irq_out = irq_out;
-
-    register_savevm(&opp->pci_dev.qdev, "openpic", 0, 2,
-                    openpic_save, openpic_load, opp);
-    qemu_register_reset(openpic_reset, opp);
+    int i;
+    int virq = MAX_SRC;
 
-    opp->irq_raise = openpic_irq_raise;
-    opp->reset = openpic_reset;
+    opp->vid = VID_REVISION_1_2;
+    opp->vir = VIR_GENERIC;
+    opp->vector_mask = 0xFFFF;
+    opp->tfrr_reset = 0;
+    opp->ivpr_reset = IVPR_MASK_MASK;
+    opp->idr_reset = 1 << 0;
+    opp->max_irq = MAX_IRQ;
 
-    if (pmem)
-        *pmem = &opp->mem;
+    opp->irq_ipi0 = virq;
+    virq += MAX_IPI;
+    opp->irq_tim0 = virq;
+    virq += MAX_TMR;
 
-    return qemu_allocate_irqs(openpic_set_irq, opp, opp->max_irq);
-}
+    assert(virq <= MAX_IRQ);
 
-static void mpic_irq_raise(openpic_t *mpp, int n_CPU, IRQ_src_t *src)
-{
-    int n_ci = IDR_CI0 - n_CPU;
+    opp->irq_msi = 224;
 
-    if(test_bit(&src->ide, n_ci)) {
-        qemu_irq_raise(mpp->dst[n_CPU].irqs[OPENPIC_OUTPUT_CINT]);
+    msi_supported = true;
+    for (i = 0; i < opp->fsl->max_ext; i++) {
+        opp->src[i].level = false;
     }
-    else {
-        qemu_irq_raise(mpp->dst[n_CPU].irqs[OPENPIC_OUTPUT_INT]);
-    }
-}
-
-static void mpic_reset (void *opaque)
-{
-    openpic_t *mpp = (openpic_t *)opaque;
-    int i;
 
-    mpp->glbc = 0x80000000;
-    /* Initialise controller registers */
-    mpp->frep = 0x004f0002 | ((mpp->nb_cpus - 1) << 8);
-    mpp->veni = VENI;
-    mpp->pint = 0x00000000;
-    mpp->spve = 0x0000FFFF;
-    /* Initialise IRQ sources */
-    for (i = 0; i < mpp->max_irq; i++) {
-        mpp->src[i].ipvp = 0x80800000;
-        mpp->src[i].ide  = 0x00000001;
-    }
-    /* Set IDE for IPIs to 0 so we don't get spurious interrupts */
-    for (i = mpp->irq_ipi0; i < (mpp->irq_ipi0 + MAX_IPI); i++) {
-        mpp->src[i].ide = 0;
-    }
-    /* Initialise IRQ destinations */
-    for (i = 0; i < MAX_CPU; i++) {
-        mpp->dst[i].pctp      = 0x0000000F;
-        mpp->dst[i].tfrr      = 0x00000000;
-        memset(&mpp->dst[i].raised, 0, sizeof(IRQ_queue_t));
-        mpp->dst[i].raised.next = -1;
-        memset(&mpp->dst[i].servicing, 0, sizeof(IRQ_queue_t));
-        mpp->dst[i].servicing.next = -1;
-    }
-    /* Initialise timers */
-    for (i = 0; i < MAX_TMR; i++) {
-        mpp->timers[i].ticc = 0x00000000;
-        mpp->timers[i].tibc = 0x80000000;
-    }
-    /* Go out of RESET state */
-    mpp->glbc = 0x00000000;
-}
-
-static void mpic_timer_write (void *opaque, hwaddr addr, uint32_t val)
-{
-    openpic_t *mpp = opaque;
-    int idx, cpu;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-    addr &= 0xFFFF;
-    cpu = addr >> 12;
-    idx = (addr >> 6) & 0x3;
-    switch (addr & 0x30) {
-    case 0x00: /* gtccr */
-        break;
-    case 0x10: /* gtbcr */
-        if ((mpp->timers[idx].ticc & 0x80000000) != 0 &&
-            (val & 0x80000000) == 0 &&
-            (mpp->timers[idx].tibc & 0x80000000) != 0)
-            mpp->timers[idx].ticc &= ~0x80000000;
-        mpp->timers[idx].tibc = val;
-        break;
-    case 0x20: /* GTIVPR */
-        write_IRQreg_ipvp(mpp, MPIC_TMR_IRQ + idx, val);
-        break;
-    case 0x30: /* GTIDR & TFRR */
-        if ((addr & 0xF0) == 0xF0)
-            mpp->dst[cpu].tfrr = val;
-        else
-            write_IRQreg_ide(mpp, MPIC_TMR_IRQ + idx, val);
-        break;
+    /* Internal interrupts, including message and MSI */
+    for (i = 16; i < MAX_SRC; i++) {
+        opp->src[i].type = IRQ_TYPE_FSLINT;
+        opp->src[i].level = true;
     }
-}
-
-static uint32_t mpic_timer_read (void *opaque, hwaddr addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx, cpu;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-    addr &= 0xFFFF;
-    cpu = addr >> 12;
-    idx = (addr >> 6) & 0x3;
-    switch (addr & 0x30) {
-    case 0x00: /* gtccr */
-        retval = mpp->timers[idx].ticc;
-        break;
-    case 0x10: /* gtbcr */
-        retval = mpp->timers[idx].tibc;
-        break;
-    case 0x20: /* TIPV */
-        retval = read_IRQreg_ipvp(mpp, MPIC_TMR_IRQ + idx);
-        break;
-    case 0x30: /* TIDR */
-        if ((addr &0xF0) == 0XF0)
-            retval = mpp->dst[cpu].tfrr;
-        else
-            retval = read_IRQreg_ide(mpp, MPIC_TMR_IRQ + idx);
-        break;
+    /* timers and IPIs */
+    for (i = MAX_SRC; i < virq; i++) {
+        opp->src[i].type = IRQ_TYPE_FSLSPECIAL;
+        opp->src[i].level = false;
     }
-    DPRINTF("%s: => %08x\n", __func__, retval);
-
-    return retval;
 }
 
-static void mpic_src_ext_write (void *opaque, hwaddr addr,
-                                uint32_t val)
+static void map_list(OpenPICState *opp, const MemReg *list, int *count)
 {
-    openpic_t *mpp = opaque;
-    int idx = MPIC_EXT_IRQ;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-
-    if (addr < MPIC_EXT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
-        }
-    }
-}
+    while (list->name) {
+        assert(*count < ARRAY_SIZE(opp->sub_io_mem));
 
-static uint32_t mpic_src_ext_read (void *opaque, hwaddr addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_EXT_IRQ;
+        memory_region_init_io(&opp->sub_io_mem[*count], list->ops, opp,
+                              list->name, list->size);
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
+        memory_region_add_subregion(&opp->mem, list->start_addr,
+                                    &opp->sub_io_mem[*count]);
 
-    if (addr < MPIC_EXT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
+        (*count)++;
+        list++;
     }
-
-    return retval;
 }
 
-static void mpic_src_int_write (void *opaque, hwaddr addr,
-                                uint32_t val)
+static int openpic_init(SysBusDevice *dev)
 {
-    openpic_t *mpp = opaque;
-    int idx = MPIC_INT_IRQ;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
+    OpenPICState *opp = FROM_SYSBUS(typeof (*opp), dev);
+    int i, j;
+    int list_count = 0;
+    static const MemReg list_le[] = {
+        {"glb", &openpic_glb_ops_le,
+                OPENPIC_GLB_REG_START, OPENPIC_GLB_REG_SIZE},
+        {"tmr", &openpic_tmr_ops_le,
+                OPENPIC_TMR_REG_START, OPENPIC_TMR_REG_SIZE},
+        {"src", &openpic_src_ops_le,
+                OPENPIC_SRC_REG_START, OPENPIC_SRC_REG_SIZE},
+        {"cpu", &openpic_cpu_ops_le,
+                OPENPIC_CPU_REG_START, OPENPIC_CPU_REG_SIZE},
+        {NULL}
+    };
+    static const MemReg list_be[] = {
+        {"glb", &openpic_glb_ops_be,
+                OPENPIC_GLB_REG_START, OPENPIC_GLB_REG_SIZE},
+        {"tmr", &openpic_tmr_ops_be,
+                OPENPIC_TMR_REG_START, OPENPIC_TMR_REG_SIZE},
+        {"src", &openpic_src_ops_be,
+                OPENPIC_SRC_REG_START, OPENPIC_SRC_REG_SIZE},
+        {"cpu", &openpic_cpu_ops_be,
+                OPENPIC_CPU_REG_START, OPENPIC_CPU_REG_SIZE},
+        {NULL}
+    };
+    static const MemReg list_fsl[] = {
+        {"msi", &openpic_msi_ops_be,
+                OPENPIC_MSI_REG_START, OPENPIC_MSI_REG_SIZE},
+        {"summary", &openpic_summary_ops_be,
+                OPENPIC_SUMMARY_REG_START, OPENPIC_SUMMARY_REG_SIZE},
+        {NULL}
+    };
 
-    if (addr < MPIC_INT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
-        }
-    }
-}
+    memory_region_init(&opp->mem, "openpic", 0x40000);
 
-static uint32_t mpic_src_int_read (void *opaque, hwaddr addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_INT_IRQ;
+    switch (opp->model) {
+    case OPENPIC_MODEL_FSL_MPIC_20:
+    default:
+        opp->fsl = &fsl_mpic_20;
+        opp->brr1 = 0x00400200;
+        opp->flags |= OPENPIC_FLAG_IDR_CRIT;
+        opp->nb_irqs = 80;
+        opp->mpic_mode_mask = GCR_MODE_MIXED;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
+        fsl_common_init(opp);
+        map_list(opp, list_be, &list_count);
+        map_list(opp, list_fsl, &list_count);
 
-    if (addr < MPIC_INT_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
-    }
+        break;
 
-    return retval;
-}
+    case OPENPIC_MODEL_FSL_MPIC_42:
+        opp->fsl = &fsl_mpic_42;
+        opp->brr1 = 0x00400402;
+        opp->flags |= OPENPIC_FLAG_ILR;
+        opp->nb_irqs = 196;
+        opp->mpic_mode_mask = GCR_MODE_PROXY;
 
-static void mpic_src_msg_write (void *opaque, hwaddr addr,
-                                uint32_t val)
-{
-    openpic_t *mpp = opaque;
-    int idx = MPIC_MSG_IRQ;
+        fsl_common_init(opp);
+        map_list(opp, list_be, &list_count);
+        map_list(opp, list_fsl, &list_count);
 
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
+        break;
 
-    if (addr < MPIC_MSG_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
+    case OPENPIC_MODEL_RAVEN:
+        opp->nb_irqs = RAVEN_MAX_EXT;
+        opp->vid = VID_REVISION_1_3;
+        opp->vir = VIR_GENERIC;
+        opp->vector_mask = 0xFF;
+        opp->tfrr_reset = 4160000;
+        opp->ivpr_reset = IVPR_MASK_MASK | IVPR_MODE_MASK;
+        opp->idr_reset = 0;
+        opp->max_irq = RAVEN_MAX_IRQ;
+        opp->irq_ipi0 = RAVEN_IPI_IRQ;
+        opp->irq_tim0 = RAVEN_TMR_IRQ;
+        opp->brr1 = -1;
+        opp->mpic_mode_mask = GCR_MODE_MIXED;
+
+        /* Only UP supported today */
+        if (opp->nb_cpus != 1) {
+            return -EINVAL;
         }
-    }
-}
-
-static uint32_t mpic_src_msg_read (void *opaque, hwaddr addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_MSG_IRQ;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
-
-    if (addr < MPIC_MSG_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
+        map_list(opp, list_le, &list_count);
+        break;
     }
 
-    return retval;
-}
-
-static void mpic_src_msi_write (void *opaque, hwaddr addr,
-                                uint32_t val)
-{
-    openpic_t *mpp = opaque;
-    int idx = MPIC_MSI_IRQ;
-
-    DPRINTF("%s: addr " TARGET_FMT_plx " <= %08x\n", __func__, addr, val);
-    if (addr & 0xF)
-        return;
-
-    if (addr < MPIC_MSI_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            write_IRQreg_ide(mpp, idx, val);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            write_IRQreg_ipvp(mpp, idx, val);
+    for (i = 0; i < opp->nb_cpus; i++) {
+        opp->dst[i].irqs = g_new(qemu_irq, OPENPIC_OUTPUT_NB);
+        for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
+            sysbus_init_irq(dev, &opp->dst[i].irqs[j]);
         }
     }
-}
-static uint32_t mpic_src_msi_read (void *opaque, hwaddr addr)
-{
-    openpic_t *mpp = opaque;
-    uint32_t retval;
-    int idx = MPIC_MSI_IRQ;
 
-    DPRINTF("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
-    retval = 0xFFFFFFFF;
-    if (addr & 0xF)
-        return retval;
+    register_savevm(&opp->busdev.qdev, "openpic", 0, 2,
+                    openpic_save, openpic_load, opp);
 
-    if (addr < MPIC_MSI_REG_SIZE) {
-        idx += (addr & 0xFFF0) >> 5;
-        if (addr & 0x10) {
-            /* EXDE / IFEDE / IEEDE */
-            retval = read_IRQreg_ide(mpp, idx);
-        } else {
-            /* EXVP / IFEVP / IEEVP */
-            retval = read_IRQreg_ipvp(mpp, idx);
-        }
-        DPRINTF("%s: => %08x\n", __func__, retval);
-    }
+    sysbus_init_mmio(dev, &opp->mem);
+    qdev_init_gpio_in(&dev->qdev, openpic_set_irq, opp->max_irq);
 
-    return retval;
+    return 0;
 }
 
-static const MemoryRegionOps mpic_glb_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   openpic_gbl_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   openpic_gbl_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
-
-static const MemoryRegionOps mpic_tmr_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_timer_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_timer_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
+static Property openpic_properties[] = {
+    DEFINE_PROP_UINT32("model", OpenPICState, model, OPENPIC_MODEL_FSL_MPIC_20),
+    DEFINE_PROP_UINT32("nb_cpus", OpenPICState, nb_cpus, 1),
+    DEFINE_PROP_END_OF_LIST(),
 };
 
-static const MemoryRegionOps mpic_cpu_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   openpic_cpu_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   openpic_cpu_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
-
-static const MemoryRegionOps mpic_ext_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_ext_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_ext_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
+static void openpic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-static const MemoryRegionOps mpic_int_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_int_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_int_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
+    k->init = openpic_init;
+    dc->props = openpic_properties;
+    dc->reset = openpic_reset;
+}
 
-static const MemoryRegionOps mpic_msg_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_msg_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_msg_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
+static const TypeInfo openpic_info = {
+    .name          = "openpic",
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(OpenPICState),
+    .class_init    = openpic_class_init,
 };
 
-static const MemoryRegionOps mpic_msi_ops = {
-    .old_mmio = {
-        .write = { openpic_buggy_write,
-                   openpic_buggy_write,
-                   mpic_src_msi_write,
-        },
-        .read  = { openpic_buggy_read,
-                   openpic_buggy_read,
-                   mpic_src_msi_read,
-        },
-    },
-    .endianness = DEVICE_BIG_ENDIAN,
-};
-
-qemu_irq *mpic_init (MemoryRegion *address_space, hwaddr base,
-                     int nb_cpus, qemu_irq **irqs, qemu_irq irq_out)
+static void openpic_register_types(void)
 {
-    openpic_t    *mpp;
-    int           i;
-    struct {
-        const char             *name;
-        MemoryRegionOps const  *ops;
-        hwaddr      start_addr;
-        ram_addr_t              size;
-    } const list[] = {
-        {"glb", &mpic_glb_ops, MPIC_GLB_REG_START, MPIC_GLB_REG_SIZE},
-        {"tmr", &mpic_tmr_ops, MPIC_TMR_REG_START, MPIC_TMR_REG_SIZE},
-        {"ext", &mpic_ext_ops, MPIC_EXT_REG_START, MPIC_EXT_REG_SIZE},
-        {"int", &mpic_int_ops, MPIC_INT_REG_START, MPIC_INT_REG_SIZE},
-        {"msg", &mpic_msg_ops, MPIC_MSG_REG_START, MPIC_MSG_REG_SIZE},
-        {"msi", &mpic_msi_ops, MPIC_MSI_REG_START, MPIC_MSI_REG_SIZE},
-        {"cpu", &mpic_cpu_ops, MPIC_CPU_REG_START, MPIC_CPU_REG_SIZE},
-    };
-
-    mpp = g_malloc0(sizeof(openpic_t));
-
-    memory_region_init(&mpp->mem, "mpic", 0x40000);
-    memory_region_add_subregion(address_space, base, &mpp->mem);
-
-    for (i = 0; i < sizeof(list)/sizeof(list[0]); i++) {
-
-        memory_region_init_io(&mpp->sub_io_mem[i], list[i].ops, mpp,
-                              list[i].name, list[i].size);
-
-        memory_region_add_subregion(&mpp->mem, list[i].start_addr,
-                                    &mpp->sub_io_mem[i]);
-    }
-
-    mpp->nb_cpus = nb_cpus;
-    mpp->max_irq = MPIC_MAX_IRQ;
-    mpp->irq_ipi0 = MPIC_IPI_IRQ;
-    mpp->irq_tim0 = MPIC_TMR_IRQ;
-
-    for (i = 0; i < nb_cpus; i++)
-        mpp->dst[i].irqs = irqs[i];
-    mpp->irq_out = irq_out;
-
-    mpp->irq_raise = mpic_irq_raise;
-    mpp->reset = mpic_reset;
-
-    register_savevm(NULL, "mpic", 0, 2, openpic_save, openpic_load, mpp);
-    qemu_register_reset(mpic_reset, mpp);
-
-    return qemu_allocate_irqs(openpic_set_irq, mpp, mpp->max_irq);
+    type_register_static(&openpic_info);
 }
+
+type_init(openpic_register_types)
diff --git a/hw/openpic.h b/hw/openpic.h
index f50a1e4..9dcaf0e 100644
--- a/hw/openpic.h
+++ b/hw/openpic.h
@@ -11,8 +11,8 @@ enum {
     OPENPIC_OUTPUT_NB,
 };
 
-qemu_irq *openpic_init (MemoryRegion **pmem, int nb_cpus,
-                        qemu_irq **irqs, qemu_irq irq_out);
-qemu_irq *mpic_init (MemoryRegion *address_space, hwaddr base,
-                     int nb_cpus, qemu_irq **irqs, qemu_irq irq_out);
+#define OPENPIC_MODEL_RAVEN       0
+#define OPENPIC_MODEL_FSL_MPIC_20 1
+#define OPENPIC_MODEL_FSL_MPIC_42 2
+
 #endif /* __OPENPIC_H__ */
diff --git a/hw/openrisc_sim.c b/hw/openrisc_sim.c
index 23c66df..30947de 100644
--- a/hw/openrisc_sim.c
+++ b/hw/openrisc_sim.c
@@ -22,12 +22,12 @@
 #include "boards.h"
 #include "elf.h"
 #include "serial.h"
-#include "net.h"
+#include "net/net.h"
 #include "loader.h"
-#include "exec-memory.h"
-#include "sysemu.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "qtest.h"
+#include "sysemu/qtest.h"
 
 #define KERNEL_LOAD_ADDR 0x100
 
@@ -50,7 +50,7 @@ static void openrisc_sim_net_init(MemoryRegion *address_space,
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     memory_region_add_subregion(address_space, base,
                                 sysbus_mmio_get_region(s, 0));
@@ -107,7 +107,7 @@ static void openrisc_sim_init(QEMUMachineInitArgs *args)
     for (n = 0; n < smp_cpus; n++) {
         cpu = cpu_openrisc_init(cpu_model);
         if (cpu == NULL) {
-            qemu_log("Unable to find CPU defineition!\n");
+            qemu_log("Unable to find CPU definition!\n");
             exit(1);
         }
         qemu_register_reset(main_cpu_reset, cpu);
@@ -139,6 +139,7 @@ static QEMUMachine openrisc_sim_machine = {
     .init = openrisc_sim_init,
     .max_cpus = 1,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void openrisc_sim_machine_init(void)
diff --git a/hw/openrisc_timer.c b/hw/openrisc_timer.c
index 7916e61..d965be7 100644
--- a/hw/openrisc_timer.c
+++ b/hw/openrisc_timer.c
@@ -20,7 +20,7 @@
 
 #include "cpu.h"
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #define TIMER_FREQ    (20 * 1000 * 1000)    /* 20MHz */
 
diff --git a/hw/overo.c b/hw/overo.c
index f4fa692..846a6c2 100644
--- a/hw/overo.c
+++ b/hw/overo.c
@@ -19,17 +19,17 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "omap.h"
 #include "arm-misc.h"
 #include "boards.h"
 #include "i2c.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
 #include "flash.h"
 #include "sysbus.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define OVERO_NAND_CS       0
 #define OVERO_NET_CS        5
@@ -91,8 +91,8 @@ static void overo_init(QEMUMachineInitArgs *args)
         qdev_set_nic_properties(s->eth, nd);
         qdev_init_nofail(s->eth);
         omap_gpmc_attach(s->cpu->gpmc, OVERO_NET_CS,
-                         sysbus_mmio_get_region(sysbus_from_qdev(s->eth), 0));
-        sysbus_connect_irq(sysbus_from_qdev(s->eth), 0,
+                         sysbus_mmio_get_region(SYS_BUS_DEVICE(s->eth), 0));
+        sysbus_connect_irq(SYS_BUS_DEVICE(s->eth), 0,
                            qdev_get_gpio_in(s->cpu->gpio, 176));
     }
 }
diff --git a/hw/palm.c b/hw/palm.c
index 6f6f414..a633dfc 100644
--- a/hw/palm.c
+++ b/hw/palm.c
@@ -18,14 +18,14 @@
  */
 #include "hw.h"
 #include "audio/audio.h"
-#include "sysemu.h"
-#include "console.h"
+#include "sysemu/sysemu.h"
+#include "ui/console.h"
 #include "omap.h"
 #include "boards.h"
 #include "arm-misc.h"
 #include "devices.h"
 #include "loader.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static uint32_t static_readb(void *opaque, hwaddr offset)
 {
@@ -280,6 +280,7 @@ static QEMUMachine palmte_machine = {
     .name = "cheetah",
     .desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)",
     .init = palmte_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void palmte_machine_init(void)
diff --git a/hw/pam.c b/hw/pam.c
index a95e2cf..1d72e88 100644
--- a/hw/pam.c
+++ b/hw/pam.c
@@ -26,7 +26,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "pam.h"
 
 void smram_update(MemoryRegion *smram_region, uint8_t smram,
diff --git a/hw/pam.h b/hw/pam.h
index 2d77ebe..8e9e349 100644
--- a/hw/pam.h
+++ b/hw/pam.h
@@ -51,7 +51,7 @@
  */
 
 #include "qemu-common.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 #define SMRAM_C_BASE    0xa0000
 #define SMRAM_C_END     0xc0000
diff --git a/hw/parallel.c b/hw/parallel.c
index c4705bc..3a4e06b 100644
--- a/hw/parallel.c
+++ b/hw/parallel.c
@@ -23,10 +23,10 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "isa.h"
 #include "pc.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG_PARALLEL
 
@@ -599,7 +599,7 @@ static void parallel_isa_class_initfn(ObjectClass *klass, void *data)
     dc->props = parallel_isa_properties;
 }
 
-static TypeInfo parallel_isa_info = {
+static const TypeInfo parallel_isa_info = {
     .name          = "isa-parallel",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAParallelState),
diff --git a/hw/pc-testdev.c b/hw/pc-testdev.c
new file mode 100644
index 0000000..cf64a1f
--- /dev/null
+++ b/hw/pc-testdev.c
@@ -0,0 +1,187 @@
+/*
+ * QEMU x86 ISA testdev
+ *
+ * Copyright (c) 2012 Avi Kivity, Gerd Hoffmann, Marcelo Tosatti
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * This device is used to test KVM features specific to the x86 port, such
+ * as emulation, power management, interrupt routing, among others. It's meant
+ * to be used like:
+ *
+ * qemu-system-x86_64 -device pc-testdev -serial stdio \
+ * -device isa-debug-exit,iobase=0xf4,iosize=0x4 \
+ * -kernel /home/lmr/Code/virt-test.git/kvm/unittests/msr.flat
+ *
+ * Where msr.flat is one of the KVM unittests, present on a separate repo,
+ * git://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git
+*/
+
+#include "config-host.h"
+#if defined(CONFIG_POSIX)
+#include <sys/mman.h>
+#endif
+#include "hw.h"
+#include "qdev.h"
+#include "isa.h"
+
+#define IOMEM_LEN    0x10000
+
+typedef struct PCTestdev {
+    ISADevice parent_obj;
+
+    MemoryRegion ioport;
+    MemoryRegion flush;
+    MemoryRegion irq;
+    MemoryRegion iomem;
+    uint32_t ioport_data;
+    char iomem_buf[IOMEM_LEN];
+} PCTestdev;
+
+#define TYPE_TESTDEV "pc-testdev"
+#define TESTDEV(obj) \
+     OBJECT_CHECK(PCTestdev, (obj), TYPE_TESTDEV)
+
+static void test_irq_line(void *opaque, hwaddr addr, uint64_t data,
+                          unsigned len)
+{
+    PCTestdev *dev = opaque;
+    ISADevice *isa = ISA_DEVICE(dev);
+
+    qemu_set_irq(isa_get_irq(isa, addr), !!data);
+}
+
+static const MemoryRegionOps test_irq_ops = {
+    .write = test_irq_line,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void test_ioport_write(void *opaque, hwaddr addr, uint64_t data,
+                              unsigned len)
+{
+    PCTestdev *dev = opaque;
+    dev->ioport_data = data;
+}
+
+static uint64_t test_ioport_read(void *opaque, hwaddr addr, unsigned len)
+{
+    PCTestdev *dev = opaque;
+    return dev->ioport_data;
+}
+
+static const MemoryRegionOps test_ioport_ops = {
+    .read = test_ioport_read,
+    .write = test_ioport_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void test_flush_page(void *opaque, hwaddr addr, uint64_t data,
+                            unsigned len)
+{
+    hwaddr page = 4096;
+    void *a = cpu_physical_memory_map(data & ~0xffful, &page, 0);
+
+    /* We might not be able to get the full page, only mprotect what we actually
+       have mapped */
+#if defined(CONFIG_POSIX)
+    mprotect(a, page, PROT_NONE);
+    mprotect(a, page, PROT_READ|PROT_WRITE);
+#endif
+    cpu_physical_memory_unmap(a, page, 0, 0);
+}
+
+static const MemoryRegionOps test_flush_ops = {
+    .write = test_flush_page,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t test_iomem_read(void *opaque, hwaddr addr, unsigned len)
+{
+    PCTestdev *dev = opaque;
+    uint64_t ret = 0;
+    memcpy(&ret, &dev->iomem_buf[addr], len);
+    ret = le64_to_cpu(ret);
+
+    return ret;
+}
+
+static void test_iomem_write(void *opaque, hwaddr addr, uint64_t val,
+                             unsigned len)
+{
+    PCTestdev *dev = opaque;
+    val = cpu_to_le64(val);
+    memcpy(&dev->iomem_buf[addr], &val, len);
+    dev->iomem_buf[addr] = val;
+}
+
+static const MemoryRegionOps test_iomem_ops = {
+    .read = test_iomem_read,
+    .write = test_iomem_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int init_test_device(ISADevice *isa)
+{
+    PCTestdev *dev = TESTDEV(isa);
+    MemoryRegion *mem = isa_address_space(isa);
+    MemoryRegion *io = isa_address_space_io(isa);
+
+    memory_region_init_io(&dev->ioport, &test_ioport_ops, dev,
+                          "pc-testdev-ioport", 4);
+    memory_region_init_io(&dev->flush, &test_flush_ops, dev,
+                          "pc-testdev-flush-page", 4);
+    memory_region_init_io(&dev->irq, &test_irq_ops, dev,
+                          "pc-testdev-irq-line", 24);
+    memory_region_init_io(&dev->iomem, &test_iomem_ops, dev,
+                          "pc-testdev-iomem", IOMEM_LEN);
+
+    memory_region_add_subregion(io,  0xe0,       &dev->ioport);
+    memory_region_add_subregion(io,  0xe4,       &dev->flush);
+    memory_region_add_subregion(io,  0x2000,     &dev->irq);
+    memory_region_add_subregion(mem, 0xff000000, &dev->iomem);
+
+    return 0;
+}
+
+static void testdev_class_init(ObjectClass *klass, void *data)
+{
+    ISADeviceClass *k = ISA_DEVICE_CLASS(klass);
+
+    k->init = init_test_device;
+}
+
+static const TypeInfo testdev_info = {
+    .name           = TYPE_TESTDEV,
+    .parent         = TYPE_ISA_DEVICE,
+    .instance_size  = sizeof(PCTestdev),
+    .class_init     = testdev_class_init,
+};
+
+static void testdev_register_types(void)
+{
+    type_register_static(&testdev_info);
+}
+
+type_init(testdev_register_types)
diff --git a/hw/pc.c b/hw/pc.c
index 2b5bbbf..53cc173 100644
--- a/hw/pc.c
+++ b/hw/pc.c
@@ -27,8 +27,8 @@
 #include "apic.h"
 #include "fdc.h"
 #include "ide.h"
-#include "pci.h"
-#include "monitor.h"
+#include "pci/pci.h"
+#include "monitor/monitor.h"
 #include "fw_cfg.h"
 #include "hpet_emul.h"
 #include "smbios.h"
@@ -38,19 +38,19 @@
 #include "mc146818rtc.h"
 #include "i8254.h"
 #include "pcspk.h"
-#include "msi.h"
+#include "pci/msi.h"
 #include "sysbus.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #include "xen.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
 #include "ui/qemu-spice.h"
-#include "memory.h"
-#include "exec-memory.h"
-#include "arch_init.h"
-#include "bitmap.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "sysemu/arch_init.h"
+#include "qemu/bitmap.h"
 
 /* debug PC/ISA interrupts */
 //#define DEBUG_IRQ
@@ -98,10 +98,16 @@ void gsi_handler(void *opaque, int n, int level)
     qemu_set_irq(s->ioapic_irq[n], level);
 }
 
-static void ioport80_write(void *opaque, uint32_t addr, uint32_t data)
+static void ioport80_write(void *opaque, hwaddr addr, uint64_t data,
+                           unsigned size)
 {
 }
 
+static uint64_t ioport80_read(void *opaque, hwaddr addr, unsigned size)
+{
+    return 0xffffffffffffffffULL;
+}
+
 /* MSDOS compatibility mode FPU exception support */
 static qemu_irq ferr_irq;
 
@@ -116,11 +122,17 @@ void cpu_set_ferr(CPUX86State *s)
     qemu_irq_raise(ferr_irq);
 }
 
-static void ioportF0_write(void *opaque, uint32_t addr, uint32_t data)
+static void ioportF0_write(void *opaque, hwaddr addr, uint64_t data,
+                           unsigned size)
 {
     qemu_irq_lower(ferr_irq);
 }
 
+static uint64_t ioportF0_read(void *opaque, hwaddr addr, unsigned size)
+{
+    return 0xffffffffffffffffULL;
+}
+
 /* TSC handling */
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
@@ -499,7 +511,7 @@ static void port92_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_port92_isa;
 }
 
-static TypeInfo port92_info = {
+static const TypeInfo port92_info = {
     .name          = "port92",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(Port92State),
@@ -515,39 +527,11 @@ type_init(port92_register_types)
 
 static void handle_a20_line_change(void *opaque, int irq, int level)
 {
-    CPUX86State *cpu = opaque;
+    X86CPU *cpu = opaque;
 
     /* XXX: send to all CPUs ? */
     /* XXX: add logic to handle multiple A20 line sources */
-    cpu_x86_set_a20(cpu, level);
-}
-
-/***********************************************************/
-/* Bochs BIOS debug ports */
-
-static void bochs_bios_write(void *opaque, uint32_t addr, uint32_t val)
-{
-    static const char shutdown_str[8] = "Shutdown";
-    static int shutdown_index = 0;
-
-    switch(addr) {
-    case 0x8900:
-        /* same as Bochs power off */
-        if (val == shutdown_str[shutdown_index]) {
-            shutdown_index++;
-            if (shutdown_index == 8) {
-                shutdown_index = 0;
-                qemu_system_shutdown_request();
-            }
-        } else {
-            shutdown_index = 0;
-        }
-        break;
-
-    case 0x501:
-    case 0x502:
-        exit((val << 1) | 1);
-    }
+    x86_cpu_set_a20(cpu, level);
 }
 
 int e820_add_entry(uint64_t address, uint64_t length, uint32_t type)
@@ -567,6 +551,18 @@ int e820_add_entry(uint64_t address, uint64_t length, uint32_t type)
     return index;
 }
 
+/* Calculates the limit to CPU APIC ID values
+ *
+ * This function returns the limit for the APIC ID value, so that all
+ * CPU APIC IDs are < pc_apic_id_limit().
+ *
+ * This is used for FW_CFG_MAX_CPUS. See comments on bochs_bios_init().
+ */
+static unsigned int pc_apic_id_limit(unsigned int max_cpus)
+{
+    return x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
+}
+
 static void *bochs_bios_init(void)
 {
     void *fw_cfg;
@@ -574,49 +570,60 @@ static void *bochs_bios_init(void)
     size_t smbios_len;
     uint64_t *numa_fw_cfg;
     int i, j;
-
-    register_ioport_write(0x8900, 1, 1, bochs_bios_write, NULL);
-
-    register_ioport_write(0x501, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x501, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x502, 1, 2, bochs_bios_write, NULL);
+    unsigned int apic_id_limit = pc_apic_id_limit(max_cpus);
 
     fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
-
+    /* FW_CFG_MAX_CPUS is a bit confusing/problematic on x86:
+     *
+     * SeaBIOS needs FW_CFG_MAX_CPUS for CPU hotplug, but the CPU hotplug
+     * QEMU<->SeaBIOS interface is not based on the "CPU index", but on the APIC
+     * ID of hotplugged CPUs[1]. This means that FW_CFG_MAX_CPUS is not the
+     * "maximum number of CPUs", but the "limit to the APIC ID values SeaBIOS
+     * may see".
+     *
+     * So, this means we must not use max_cpus, here, but the maximum possible
+     * APIC ID value, plus one.
+     *
+     * [1] The only kind of "CPU identifier" used between SeaBIOS and QEMU is
+     *     the APIC ID, not the "CPU index"
+     */
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)apic_id_limit);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_ACPI_TABLES, (uint8_t *)acpi_tables,
-                     acpi_tables_len);
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_ACPI_TABLES,
+                     acpi_tables, acpi_tables_len);
     fw_cfg_add_i32(fw_cfg, FW_CFG_IRQ0_OVERRIDE, kvm_allows_irq0_override());
 
     smbios_table = smbios_get_table(&smbios_len);
     if (smbios_table)
         fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
                          smbios_table, smbios_len);
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE, (uint8_t *)&e820_table,
-                     sizeof(struct e820_table));
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE,
+                     &e820_table, sizeof(e820_table));
 
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_HPET, (uint8_t *)&hpet_cfg,
-                     sizeof(struct hpet_fw_config));
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_HPET, &hpet_cfg, sizeof(hpet_cfg));
     /* allocate memory for the NUMA channel: one (64bit) word for the number
      * of nodes, one word for each VCPU->node and one word for each node to
      * hold the amount of memory.
      */
-    numa_fw_cfg = g_malloc0((1 + max_cpus + nb_numa_nodes) * 8);
+    numa_fw_cfg = g_new0(uint64_t, 1 + apic_id_limit + nb_numa_nodes);
     numa_fw_cfg[0] = cpu_to_le64(nb_numa_nodes);
     for (i = 0; i < max_cpus; i++) {
+        unsigned int apic_id = x86_cpu_apic_id_from_index(i);
+        assert(apic_id < apic_id_limit);
         for (j = 0; j < nb_numa_nodes; j++) {
             if (test_bit(i, node_cpumask[j])) {
-                numa_fw_cfg[i + 1] = cpu_to_le64(j);
+                numa_fw_cfg[apic_id + 1] = cpu_to_le64(j);
                 break;
             }
         }
     }
     for (i = 0; i < nb_numa_nodes; i++) {
-        numa_fw_cfg[max_cpus + 1 + i] = cpu_to_le64(node_mem[i]);
+        numa_fw_cfg[apic_id_limit + 1 + i] = cpu_to_le64(node_mem[i]);
     }
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_NUMA, (uint8_t *)numa_fw_cfg,
-                     (1 + max_cpus + nb_numa_nodes) * 8);
+    fw_cfg_add_bytes(fw_cfg, FW_CFG_NUMA, numa_fw_cfg,
+                     (1 + apic_id_limit + nb_numa_nodes) *
+                     sizeof(*numa_fw_cfg));
 
     return fw_cfg;
 }
@@ -715,9 +722,7 @@ static void load_linux(void *fw_cfg,
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_ADDR, cmdline_addr);
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, strlen(kernel_cmdline)+1);
-    fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                     (uint8_t*)strdup(kernel_cmdline),
-                     strlen(kernel_cmdline)+1);
+    fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
 
     if (protocol >= 0x202) {
 	stl_p(header+0x228, cmdline_addr);
@@ -877,6 +882,29 @@ void pc_cpus_init(const char *cpu_model)
     }
 }
 
+void pc_acpi_init(const char *default_dsdt)
+{
+    char *filename = NULL, *arg = NULL;
+
+    if (acpi_tables != NULL) {
+        /* manually set via -acpitable, leave it alone */
+        return;
+    }
+
+    filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, default_dsdt);
+    if (filename == NULL) {
+        fprintf(stderr, "WARNING: failed to find %s\n", default_dsdt);
+        return;
+    }
+
+    arg = g_strdup_printf("file=%s", filename);
+    if (acpi_table_add(arg) != 0) {
+        fprintf(stderr, "WARNING: failed to load %s\n", filename);
+    }
+    g_free(arg);
+    g_free(filename);
+}
+
 void *pc_memory_init(MemoryRegion *system_memory,
                     const char *kernel_filename,
                     const char *kernel_cmdline,
@@ -967,6 +995,26 @@ static void cpu_request_exit(void *opaque, int irq, int level)
     }
 }
 
+static const MemoryRegionOps ioport80_io_ops = {
+    .write = ioport80_write,
+    .read = ioport80_read,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+static const MemoryRegionOps ioportF0_io_ops = {
+    .write = ioportF0_write,
+    .read = ioportF0_read,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
 void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           ISADevice **rtc_state,
                           ISADevice **floppy,
@@ -981,10 +1029,14 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     qemu_irq *a20_line;
     ISADevice *i8042, *port92, *vmmouse, *pit = NULL;
     qemu_irq *cpu_exit_irq;
+    MemoryRegion *ioport80_io = g_new(MemoryRegion, 1);
+    MemoryRegion *ioportF0_io = g_new(MemoryRegion, 1);
 
-    register_ioport_write(0x80, 1, 1, ioport80_write, NULL);
+    memory_region_init_io(ioport80_io, &ioport80_io_ops, NULL, "ioport80", 1);
+    memory_region_add_subregion(isa_bus->address_space_io, 0x80, ioport80_io);
 
-    register_ioport_write(0xf0, 1, 1, ioportF0_write, NULL);
+    memory_region_init_io(ioportF0_io, &ioportF0_io_ops, NULL, "ioportF0", 1);
+    memory_region_add_subregion(isa_bus->address_space_io, 0xf0, ioportF0_io);
 
     /*
      * Check if an HPET shall be created.
@@ -997,7 +1049,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
 
         if (hpet) {
             for (i = 0; i < GSI_NUM_PINS; i++) {
-                sysbus_connect_irq(sysbus_from_qdev(hpet), i, gsi[i]);
+                sysbus_connect_irq(SYS_BUS_DEVICE(hpet), i, gsi[i]);
             }
             pit_isa_irq = -1;
             pit_alt_irq = qdev_get_gpio_in(hpet, HPET_LEGACY_PIT_INT);
@@ -1033,7 +1085,8 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
         }
     }
 
-    a20_line = qemu_allocate_irqs(handle_a20_line_change, first_cpu, 2);
+    a20_line = qemu_allocate_irqs(handle_a20_line_change,
+                                  x86_env_get_cpu(first_cpu), 2);
     i8042 = isa_create_simple(isa_bus, "i8042");
     i8042_setup_a20_line(i8042, &a20_line[0]);
     if (!no_vmport) {
@@ -1100,7 +1153,7 @@ void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name)
                                   "ioapic", OBJECT(dev), NULL);
     }
     qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
+    d = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(d, 0, 0xfec00000);
 
     for (i = 0; i < IOAPIC_NUM_PINS; i++) {
diff --git a/hw/pc.h b/hw/pc.h
index 2237e86..fbcf43d 100644
--- a/hw/pc.h
+++ b/hw/pc.h
@@ -2,12 +2,12 @@
 #define HW_PC_H
 
 #include "qemu-common.h"
-#include "memory.h"
-#include "ioport.h"
+#include "exec/memory.h"
+#include "exec/ioport.h"
 #include "isa.h"
 #include "fdc.h"
-#include "net.h"
-#include "memory.h"
+#include "net/net.h"
+#include "exec/memory.h"
 #include "ioapic.h"
 
 /* PC-style peripherals (also used by other machines).  */
@@ -40,8 +40,8 @@ qemu_irq *i8259_init(ISABus *bus, qemu_irq parent_irq);
 qemu_irq *kvm_i8259_init(ISABus *bus);
 int pic_read_irq(DeviceState *d);
 int pic_get_output(DeviceState *d);
-void pic_info(Monitor *mon);
-void irq_info(Monitor *mon);
+void pic_info(Monitor *mon, const QDict *qdict);
+void irq_info(Monitor *mon, const QDict *qdict);
 
 /* Global System Interrupts */
 
@@ -79,6 +79,7 @@ void pc_register_ferr_irq(qemu_irq irq);
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level);
 
 void pc_cpus_init(const char *cpu_model);
+void pc_acpi_init(const char *default_dsdt);
 void *pc_memory_init(MemoryRegion *system_memory,
                     const char *kernel_filename,
                     const char *kernel_cmdline,
diff --git a/hw/pc87312.c b/hw/pc87312.c
new file mode 100644
index 0000000..38af4c1
--- /dev/null
+++ b/hw/pc87312.c
@@ -0,0 +1,401 @@
+/*
+ * QEMU National Semiconductor PC87312 (Super I/O)
+ *
+ * Copyright (c) 2010-2012 Herve Poussineau
+ * Copyright (c) 2011-2012 Andreas Färber
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "pc87312.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
+#include "trace.h"
+
+
+#define REG_FER 0
+#define REG_FAR 1
+#define REG_PTR 2
+
+#define FER_PARALLEL_EN   0x01
+#define FER_UART1_EN      0x02
+#define FER_UART2_EN      0x04
+#define FER_FDC_EN        0x08
+#define FER_FDC_4         0x10
+#define FER_FDC_ADDR      0x20
+#define FER_IDE_EN        0x40
+#define FER_IDE_ADDR      0x80
+
+#define FAR_PARALLEL_ADDR 0x03
+#define FAR_UART1_ADDR    0x0C
+#define FAR_UART2_ADDR    0x30
+#define FAR_UART_3_4      0xC0
+
+#define PTR_POWER_DOWN    0x01
+#define PTR_CLOCK_DOWN    0x02
+#define PTR_PWDN          0x04
+#define PTR_IRQ_5_7       0x08
+#define PTR_UART1_TEST    0x10
+#define PTR_UART2_TEST    0x20
+#define PTR_LOCK_CONF     0x40
+#define PTR_EPP_MODE      0x80
+
+
+/* Parallel port */
+
+static inline bool is_parallel_enabled(PC87312State *s)
+{
+    return s->regs[REG_FER] & FER_PARALLEL_EN;
+}
+
+static const uint32_t parallel_base[] = { 0x378, 0x3bc, 0x278, 0x00 };
+
+static inline uint32_t get_parallel_iobase(PC87312State *s)
+{
+    return parallel_base[s->regs[REG_FAR] & FAR_PARALLEL_ADDR];
+}
+
+static const uint32_t parallel_irq[] = { 5, 7, 5, 0 };
+
+static inline uint32_t get_parallel_irq(PC87312State *s)
+{
+    int idx;
+    idx = (s->regs[REG_FAR] & FAR_PARALLEL_ADDR);
+    if (idx == 0) {
+        return (s->regs[REG_PTR] & PTR_IRQ_5_7) ? 7 : 5;
+    } else {
+        return parallel_irq[idx];
+    }
+}
+
+static inline bool is_parallel_epp(PC87312State *s)
+{
+    return s->regs[REG_PTR] & PTR_EPP_MODE;
+}
+
+
+/* UARTs */
+
+static const uint32_t uart_base[2][4] = {
+    { 0x3e8, 0x338, 0x2e8, 0x220 },
+    { 0x2e8, 0x238, 0x2e0, 0x228 }
+};
+
+static inline uint32_t get_uart_iobase(PC87312State *s, int i)
+{
+    int idx;
+    idx = (s->regs[REG_FAR] >> (2 * i + 2)) & 0x3;
+    if (idx == 0) {
+        return 0x3f8;
+    } else if (idx == 1) {
+        return 0x2f8;
+    } else {
+        return uart_base[idx & 1][(s->regs[REG_FAR] & FAR_UART_3_4) >> 6];
+    }
+}
+
+static inline uint32_t get_uart_irq(PC87312State *s, int i)
+{
+    int idx;
+    idx = (s->regs[REG_FAR] >> (2 * i + 2)) & 0x3;
+    return (idx & 1) ? 3 : 4;
+}
+
+static inline bool is_uart_enabled(PC87312State *s, int i)
+{
+    return s->regs[REG_FER] & (FER_UART1_EN << i);
+}
+
+
+/* Floppy controller */
+
+static inline bool is_fdc_enabled(PC87312State *s)
+{
+    return s->regs[REG_FER] & FER_FDC_EN;
+}
+
+static inline uint32_t get_fdc_iobase(PC87312State *s)
+{
+    return (s->regs[REG_FER] & FER_FDC_ADDR) ? 0x370 : 0x3f0;
+}
+
+
+/* IDE controller */
+
+static inline bool is_ide_enabled(PC87312State *s)
+{
+    return s->regs[REG_FER] & FER_IDE_EN;
+}
+
+static inline uint32_t get_ide_iobase(PC87312State *s)
+{
+    return (s->regs[REG_FER] & FER_IDE_ADDR) ? 0x170 : 0x1f0;
+}
+
+
+static void reconfigure_devices(PC87312State *s)
+{
+    error_report("pc87312: unsupported device reconfiguration (%02x %02x %02x)",
+                 s->regs[REG_FER], s->regs[REG_FAR], s->regs[REG_PTR]);
+}
+
+static void pc87312_soft_reset(PC87312State *s)
+{
+    static const uint8_t fer_init[] = {
+        0x4f, 0x4f, 0x4f, 0x4f, 0x4f, 0x4f, 0x4b, 0x4b,
+        0x4b, 0x4b, 0x4b, 0x4b, 0x0f, 0x0f, 0x0f, 0x0f,
+        0x49, 0x49, 0x49, 0x49, 0x07, 0x07, 0x07, 0x07,
+        0x47, 0x47, 0x47, 0x47, 0x47, 0x47, 0x08, 0x00,
+    };
+    static const uint8_t far_init[] = {
+        0x10, 0x11, 0x11, 0x39, 0x24, 0x38, 0x00, 0x01,
+        0x01, 0x09, 0x08, 0x08, 0x10, 0x11, 0x39, 0x24,
+        0x00, 0x01, 0x01, 0x00, 0x10, 0x11, 0x39, 0x24,
+        0x10, 0x11, 0x11, 0x39, 0x24, 0x38, 0x10, 0x10,
+    };
+    static const uint8_t ptr_init[] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+    };
+
+    s->read_id_step = 0;
+    s->selected_index = REG_FER;
+
+    s->regs[REG_FER] = fer_init[s->config & 0x1f];
+    s->regs[REG_FAR] = far_init[s->config & 0x1f];
+    s->regs[REG_PTR] = ptr_init[s->config & 0x1f];
+}
+
+static void pc87312_hard_reset(PC87312State *s)
+{
+    pc87312_soft_reset(s);
+}
+
+static void pc87312_io_write(void *opaque, hwaddr addr, uint64_t val,
+                             unsigned int size)
+{
+    PC87312State *s = opaque;
+
+    trace_pc87312_io_write(addr, val);
+
+    if ((addr & 1) == 0) {
+        /* Index register */
+        s->read_id_step = 2;
+        s->selected_index = val;
+    } else {
+        /* Data register */
+        if (s->selected_index < 3) {
+            s->regs[s->selected_index] = val;
+            reconfigure_devices(s);
+        }
+    }
+}
+
+static uint64_t pc87312_io_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    PC87312State *s = opaque;
+    uint32_t val;
+
+    if ((addr & 1) == 0) {
+        /* Index register */
+        if (s->read_id_step++ == 0) {
+            val = 0x88;
+        } else if (s->read_id_step++ == 1) {
+            val = 0;
+        } else {
+            val = s->selected_index;
+        }
+    } else {
+        /* Data register */
+        if (s->selected_index < 3) {
+            val = s->regs[s->selected_index];
+        } else {
+            /* Invalid selected index */
+            val = 0;
+        }
+    }
+
+    trace_pc87312_io_read(addr, val);
+    return val;
+}
+
+static const MemoryRegionOps pc87312_io_ops = {
+    .read  = pc87312_io_read,
+    .write = pc87312_io_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+static int pc87312_post_load(void *opaque, int version_id)
+{
+    PC87312State *s = opaque;
+
+    reconfigure_devices(s);
+    return 0;
+}
+
+static void pc87312_reset(DeviceState *d)
+{
+    PC87312State *s = PC87312(d);
+
+    pc87312_soft_reset(s);
+}
+
+static int pc87312_init(ISADevice *dev)
+{
+    PC87312State *s;
+    DeviceState *d;
+    ISADevice *isa;
+    ISABus *bus;
+    CharDriverState *chr;
+    DriveInfo *drive;
+    char name[5];
+    int i;
+
+    s = PC87312(dev);
+    bus = isa_bus_from_device(dev);
+    pc87312_hard_reset(s);
+    isa_register_ioport(dev, &s->io, s->iobase);
+
+    if (is_parallel_enabled(s)) {
+        chr = parallel_hds[0];
+        if (chr == NULL) {
+            chr = qemu_chr_new("par0", "null", NULL);
+        }
+        isa = isa_create(bus, "isa-parallel");
+        d = DEVICE(isa);
+        qdev_prop_set_uint32(d, "index", 0);
+        qdev_prop_set_uint32(d, "iobase", get_parallel_iobase(s));
+        qdev_prop_set_uint32(d, "irq", get_parallel_irq(s));
+        qdev_prop_set_chr(d, "chardev", chr);
+        qdev_init_nofail(d);
+        s->parallel.dev = isa;
+        trace_pc87312_info_parallel(get_parallel_iobase(s),
+                                    get_parallel_irq(s));
+    }
+
+    for (i = 0; i < 2; i++) {
+        if (is_uart_enabled(s, i)) {
+            chr = serial_hds[i];
+            if (chr == NULL) {
+                snprintf(name, sizeof(name), "ser%d", i);
+                chr = qemu_chr_new(name, "null", NULL);
+            }
+            isa = isa_create(bus, "isa-serial");
+            d = DEVICE(isa);
+            qdev_prop_set_uint32(d, "index", i);
+            qdev_prop_set_uint32(d, "iobase", get_uart_iobase(s, i));
+            qdev_prop_set_uint32(d, "irq", get_uart_irq(s, i));
+            qdev_prop_set_chr(d, "chardev", chr);
+            qdev_init_nofail(d);
+            s->uart[i].dev = isa;
+            trace_pc87312_info_serial(i, get_uart_iobase(s, i),
+                                      get_uart_irq(s, i));
+        }
+    }
+
+    if (is_fdc_enabled(s)) {
+        isa = isa_create(bus, "isa-fdc");
+        d = DEVICE(isa);
+        qdev_prop_set_uint32(d, "iobase", get_fdc_iobase(s));
+        qdev_prop_set_uint32(d, "irq", 6);
+        drive = drive_get(IF_FLOPPY, 0, 0);
+        if (drive != NULL) {
+            qdev_prop_set_drive_nofail(d, "driveA", drive->bdrv);
+        }
+        drive = drive_get(IF_FLOPPY, 0, 1);
+        if (drive != NULL) {
+            qdev_prop_set_drive_nofail(d, "driveB", drive->bdrv);
+        }
+        qdev_init_nofail(d);
+        s->fdc.dev = isa;
+        trace_pc87312_info_floppy(get_fdc_iobase(s));
+    }
+
+    if (is_ide_enabled(s)) {
+        isa = isa_create(bus, "isa-ide");
+        d = DEVICE(isa);
+        qdev_prop_set_uint32(d, "iobase", get_ide_iobase(s));
+        qdev_prop_set_uint32(d, "iobase2", get_ide_iobase(s) + 0x206);
+        qdev_prop_set_uint32(d, "irq", 14);
+        qdev_init_nofail(d);
+        s->ide.dev = isa;
+        trace_pc87312_info_ide(get_ide_iobase(s));
+    }
+
+    return 0;
+}
+
+static void pc87312_initfn(Object *obj)
+{
+    PC87312State *s = PC87312(obj);
+
+    memory_region_init_io(&s->io, &pc87312_io_ops, s, "pc87312", 2);
+}
+
+static const VMStateDescription vmstate_pc87312 = {
+    .name = "pc87312",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = pc87312_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(read_id_step, PC87312State),
+        VMSTATE_UINT8(selected_index, PC87312State),
+        VMSTATE_UINT8_ARRAY(regs, PC87312State, 3),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property pc87312_properties[] = {
+    DEFINE_PROP_HEX32("iobase", PC87312State, iobase, 0x398),
+    DEFINE_PROP_UINT8("config", PC87312State, config, 1),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void pc87312_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
+
+    ic->init = pc87312_init;
+    dc->reset = pc87312_reset;
+    dc->vmsd = &vmstate_pc87312;
+    dc->props = pc87312_properties;
+}
+
+static const TypeInfo pc87312_type_info = {
+    .name          = TYPE_PC87312,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(PC87312State),
+    .instance_init = pc87312_initfn,
+    .class_init    = pc87312_class_init,
+};
+
+static void pc87312_register_types(void)
+{
+    type_register_static(&pc87312_type_info);
+}
+
+type_init(pc87312_register_types)
diff --git a/hw/pc87312.h b/hw/pc87312.h
new file mode 100644
index 0000000..7b9e6f6
--- /dev/null
+++ b/hw/pc87312.h
@@ -0,0 +1,68 @@
+/*
+ * QEMU National Semiconductor PC87312 (Super I/O)
+ *
+ * Copyright (c) 2010-2012 Herve Poussineau
+ * Copyright (c) 2011-2012 Andreas Färber
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef QEMU_PC87312_H
+#define QEMU_PC87312_H
+
+#include "isa.h"
+
+
+#define TYPE_PC87312 "pc87312"
+#define PC87312(obj) OBJECT_CHECK(PC87312State, (obj), TYPE_PC87312)
+
+typedef struct PC87312State {
+    ISADevice dev;
+
+    uint32_t iobase;
+    uint8_t config; /* initial configuration */
+
+    struct {
+        ISADevice *dev;
+    } parallel;
+
+    struct {
+        ISADevice *dev;
+    } uart[2];
+
+    struct {
+        ISADevice *dev;
+        BlockDriverState *drive[2];
+        uint32_t base;
+    } fdc;
+
+    struct {
+        ISADevice *dev;
+        uint32_t base;
+    } ide;
+
+    MemoryRegion io;
+
+    uint8_t read_id_step;
+    uint8_t selected_index;
+
+    uint8_t regs[3];
+} PC87312State;
+
+
+#endif
diff --git a/hw/pc_piix.c b/hw/pc_piix.c
index aa3e7f4..0af436c 100644
--- a/hw/pc_piix.c
+++ b/hw/pc_piix.c
@@ -27,22 +27,22 @@
 #include "hw.h"
 #include "pc.h"
 #include "apic.h"
-#include "pci.h"
-#include "pci_ids.h"
+#include "pci/pci.h"
+#include "pci/pci_ids.h"
 #include "usb.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
 #include "ide.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm/clock.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "arch_init.h"
-#include "blockdev.h"
+#include "sysemu/arch_init.h"
+#include "sysemu/blockdev.h"
 #include "smbus.h"
 #include "xen.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 #include "cpu.h"
 #ifdef CONFIG_XEN
 #  include <xen/hvm/hvm_info_table.h>
@@ -87,6 +87,7 @@ static void pc_init1(MemoryRegion *system_memory,
     void *fw_cfg = NULL;
 
     pc_cpus_init(cpu_model);
+    pc_acpi_init("acpi-dsdt.aml");
 
     if (kvmclock_enabled) {
         kvmclock_create();
@@ -234,10 +235,18 @@ static void pc_init_pci(QEMUMachineInitArgs *args)
 
 static void pc_init_pci_1_3(QEMUMachineInitArgs *args)
 {
-    enable_kvm_pv_eoi();
+    enable_compat_apic_id_mode();
     pc_init_pci(args);
 }
 
+/* PC machine init function for pc-0.14 to pc-1.2 */
+static void pc_init_pci_1_2(QEMUMachineInitArgs *args)
+{
+    disable_kvm_pv_eoi();
+    pc_init_pci_1_3(args);
+}
+
+/* PC init function for pc-0.10 to pc-0.13, and reused by xenfv */
 static void pc_init_pci_no_kvmclock(QEMUMachineInitArgs *args)
 {
     ram_addr_t ram_size = args->ram_size;
@@ -246,6 +255,8 @@ static void pc_init_pci_no_kvmclock(QEMUMachineInitArgs *args)
     const char *kernel_cmdline = args->kernel_cmdline;
     const char *initrd_filename = args->initrd_filename;
     const char *boot_device = args->boot_device;
+    disable_kvm_pv_eoi();
+    enable_compat_apic_id_mode();
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -263,6 +274,8 @@ static void pc_init_isa(QEMUMachineInitArgs *args)
     const char *boot_device = args->boot_device;
     if (cpu_model == NULL)
         cpu_model = "486";
+    disable_kvm_pv_eoi();
+    enable_compat_apic_id_mode();
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -281,16 +294,45 @@ static void pc_xen_hvm_init(QEMUMachineInitArgs *args)
 }
 #endif
 
+static QEMUMachine pc_i440fx_machine_v1_4 = {
+    .name = "pc-i440fx-1.4",
+    .alias = "pc",
+    .desc = "Standard PC (i440FX + PIIX, 1996)",
+    .init = pc_init_pci,
+    .max_cpus = 255,
+    .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
+};
+
+#define PC_COMPAT_1_3 \
+        {\
+            .driver   = "usb-tablet",\
+            .property = "usb_version",\
+            .value    = stringify(1),\
+        },{\
+            .driver   = "virtio-net-pci",\
+            .property = "ctrl_mac_addr",\
+            .value    = "off",      \
+        },{ \
+            .driver   = "virtio-net-pci", \
+            .property = "mq", \
+            .value    = "off", \
+        }
+
 static QEMUMachine pc_machine_v1_3 = {
     .name = "pc-1.3",
-    .alias = "pc",
     .desc = "Standard PC",
     .init = pc_init_pci_1_3,
     .max_cpus = 255,
-    .is_default = 1,
+    .compat_props = (GlobalProperty[]) {
+        PC_COMPAT_1_3,
+        { /* end of list */ }
+    },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_1_2 \
+        PC_COMPAT_1_3,\
         {\
             .driver   = "nec-usb-xhci",\
             .property = "msi",\
@@ -320,12 +362,13 @@ static QEMUMachine pc_machine_v1_3 = {
 static QEMUMachine pc_machine_v1_2 = {
     .name = "pc-1.2",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_1_2,
         { /* end of list */ }
     },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_1_1 \
@@ -363,12 +406,13 @@ static QEMUMachine pc_machine_v1_2 = {
 static QEMUMachine pc_machine_v1_1 = {
     .name = "pc-1.1",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_1_1,
         { /* end of list */ }
     },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_1_0 \
@@ -398,13 +442,14 @@ static QEMUMachine pc_machine_v1_1 = {
 static QEMUMachine pc_machine_v1_0 = {
     .name = "pc-1.0",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_1_0,
         { /* end of list */ }
     },
     .hw_version = "1.0",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_15 \
@@ -413,13 +458,14 @@ static QEMUMachine pc_machine_v1_0 = {
 static QEMUMachine pc_machine_v0_15 = {
     .name = "pc-0.15",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_0_15,
         { /* end of list */ }
     },
     .hw_version = "0.15",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_14 \
@@ -445,7 +491,7 @@ static QEMUMachine pc_machine_v0_15 = {
 static QEMUMachine pc_machine_v0_14 = {
     .name = "pc-0.14",
     .desc = "Standard PC",
-    .init = pc_init_pci,
+    .init = pc_init_pci_1_2,
     .max_cpus = 255,
     .compat_props = (GlobalProperty[]) {
         PC_COMPAT_0_14, 
@@ -461,6 +507,7 @@ static QEMUMachine pc_machine_v0_14 = {
         { /* end of list */ }
     },
     .hw_version = "0.14",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_13 \
@@ -498,6 +545,7 @@ static QEMUMachine pc_machine_v0_13 = {
         { /* end of list */ }
     },
     .hw_version = "0.13",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_12 \
@@ -531,6 +579,7 @@ static QEMUMachine pc_machine_v0_12 = {
         { /* end of list */ }
     },
     .hw_version = "0.12",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #define PC_COMPAT_0_11 \
@@ -564,6 +613,7 @@ static QEMUMachine pc_machine_v0_11 = {
         { /* end of list */ }
     },
     .hw_version = "0.11",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine pc_machine_v0_10 = {
@@ -597,6 +647,7 @@ static QEMUMachine pc_machine_v0_10 = {
         { /* end of list */ }
     },
     .hw_version = "0.10",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine isapc_machine = {
@@ -612,6 +663,7 @@ static QEMUMachine isapc_machine = {
         },
         { /* end of list */ }
     },
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 #ifdef CONFIG_XEN
@@ -621,11 +673,13 @@ static QEMUMachine xenfv_machine = {
     .init = pc_xen_hvm_init,
     .max_cpus = HVM_MAX_VCPUS,
     .default_machine_opts = "accel=xen",
+    DEFAULT_MACHINE_OPTIONS,
 };
 #endif
 
 static void pc_machine_init(void)
 {
+    qemu_register_machine(&pc_i440fx_machine_v1_4);
     qemu_register_machine(&pc_machine_v1_3);
     qemu_register_machine(&pc_machine_v1_2);
     qemu_register_machine(&pc_machine_v1_1);
diff --git a/hw/pc_q35.c b/hw/pc_q35.c
index 3429a9a..6f5ff8d 100644
--- a/hw/pc_q35.c
+++ b/hw/pc_q35.c
@@ -28,15 +28,15 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
 #include "smbus.h"
 #include "boards.h"
 #include "mc146818rtc.h"
 #include "xen.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm/clock.h"
 #include "q35.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "ich9.h"
 #include "hw/ide/pci.h"
 #include "hw/ide/ahci.h"
@@ -87,6 +87,7 @@ static void pc_q35_init(QEMUMachineInitArgs *args)
     qemu_irq *cmos_s3;
 
     pc_cpus_init(cpu_model);
+    pc_acpi_init("q35-acpi-dsdt.aml");
 
     kvmclock_create();
 
@@ -146,6 +147,7 @@ static void pc_q35_init(QEMUMachineInitArgs *args)
     ich9_lpc->ioapic = gsi_state->ioapic_irq;
     pci_bus_irqs(host_bus, ich9_lpc_set_irq, ich9_lpc_map_irq, ich9_lpc,
                  ICH9_LPC_NB_PIRQS);
+    pci_bus_set_route_irq_fn(host_bus, ich9_route_intx_pin_to_irq);
     isa_bus = ich9_lpc->isa_bus;
 
     /*end early*/
@@ -208,11 +210,12 @@ static void pc_q35_init(QEMUMachineInitArgs *args)
 }
 
 static QEMUMachine pc_q35_machine = {
-    .name = "q35-next",
+    .name = "pc-q35-1.4",
     .alias = "q35",
-    .desc = "Q35 chipset PC",
+    .desc = "Standard PC (Q35 + ICH9, 2009)",
     .init = pc_q35_init,
     .max_cpus = 255,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void pc_q35_machine_init(void)
diff --git a/hw/pc_sysfw.c b/hw/pc_sysfw.c
index 9d7c5f4..7f6c12c 100644
--- a/hw/pc_sysfw.c
+++ b/hw/pc_sysfw.c
@@ -23,15 +23,15 @@
  * THE SOFTWARE.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
 #include "hw.h"
 #include "pc.h"
 #include "hw/boards.h"
 #include "loader.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "flash.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #define BIOS_FILENAME "bios.bin"
 
@@ -84,6 +84,10 @@ static void pc_fw_add_pflash_drv(void)
         bios_name = BIOS_FILENAME;
     }
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+    if (!filename) {
+        error_report("Can't open BIOS image %s", bios_name);
+        exit(1);
+    }
 
     opts = drive_add(IF_PFLASH, -1, filename, "readonly=on");
 
@@ -98,7 +102,9 @@ static void pc_fw_add_pflash_drv(void)
       return;
     }
 
-    drive_init(opts, machine->use_scsi);
+    if (!drive_init(opts, machine->block_default_type)) {
+        qemu_opts_del(opts);
+    }
 }
 
 static void pc_system_flash_init(MemoryRegion *rom_memory,
@@ -250,7 +256,7 @@ static void pcsysfw_class_init (ObjectClass *klass, void *data)
     dc->props = pcsysfw_properties;
 }
 
-static TypeInfo pcsysfw_info = {
+static const TypeInfo pcsysfw_info = {
     .name          = "pc-sysfw",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof (PcSysFwDevice),
diff --git a/hw/pci/Makefile.objs b/hw/pci/Makefile.objs
new file mode 100644
index 0000000..1cd6cde
--- /dev/null
+++ b/hw/pci/Makefile.objs
@@ -0,0 +1,9 @@
+common-obj-$(CONFIG_PCI) += pci.o pci_bridge.o
+common-obj-$(CONFIG_PCI) += msix.o msi.o
+common-obj-$(CONFIG_PCI) += shpc.o
+common-obj-$(CONFIG_PCI) += slotid_cap.o
+common-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
+common-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
+common-obj-$(CONFIG_NO_PCI) += pci-stub.o
+
+common-obj-$(CONFIG_ALL) += pci-stub.o
diff --git a/hw/msi.c b/hw/pci/msi.c
index 33037a8..2a04d18 100644
--- a/hw/msi.c
+++ b/hw/pci/msi.c
@@ -18,8 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "msi.h"
-#include "range.h"
+#include "hw/pci/msi.h"
+#include "qemu/range.h"
 
 /* Eventually those constants should go to Linux pci_regs.h */
 #define PCI_MSI_PENDING_32      0x10
diff --git a/hw/msi.h b/hw/pci/msi.h
index 150b09a..81a3848 100644
--- a/hw/msi.h
+++ b/hw/pci/msi.h
@@ -22,7 +22,7 @@
 #define QEMU_MSI_H
 
 #include "qemu-common.h"
-#include "pci.h"
+#include "hw/pci/pci.h"
 
 struct MSIMessage {
     uint64_t address;
diff --git a/hw/msix.c b/hw/pci/msix.c
index 136ef09..e231a0d 100644
--- a/hw/msix.c
+++ b/hw/pci/msix.c
@@ -14,11 +14,11 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw.h"
-#include "msi.h"
-#include "msix.h"
-#include "pci.h"
-#include "range.h"
+#include "hw/hw.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/pci.h"
+#include "qemu/range.h"
 
 #define MSIX_CAP_LENGTH 12
 
@@ -27,7 +27,7 @@
 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
 
-static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
+MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
 {
     uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
     MSIMessage msg;
@@ -65,7 +65,7 @@ static int msix_is_pending(PCIDevice *dev, int vector)
     return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
 }
 
-static void msix_set_pending(PCIDevice *dev, int vector)
+void msix_set_pending(PCIDevice *dev, unsigned int vector)
 {
     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
 }
@@ -75,13 +75,13 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 }
 
-static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
+static bool msix_vector_masked(PCIDevice *dev, unsigned int vector, bool fmask)
 {
     unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
     return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
 }
 
-static bool msix_is_masked(PCIDevice *dev, int vector)
+bool msix_is_masked(PCIDevice *dev, unsigned int vector)
 {
     return msix_vector_masked(dev, vector, dev->msix_function_masked);
 }
@@ -180,8 +180,7 @@ static void msix_table_mmio_write(void *opaque, hwaddr addr,
 static const MemoryRegionOps msix_table_mmio_ops = {
     .read = msix_table_mmio_read,
     .write = msix_table_mmio_write,
-    /* TODO: MSIX should be LITTLE_ENDIAN. */
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
         .max_access_size = 4,
@@ -192,14 +191,18 @@ static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     PCIDevice *dev = opaque;
+    if (dev->msix_vector_poll_notifier) {
+        unsigned vector_start = addr * 8;
+        unsigned vector_end = MIN(addr + size * 8, dev->msix_entries_nr);
+        dev->msix_vector_poll_notifier(dev, vector_start, vector_end);
+    }
 
     return pci_get_long(dev->msix_pba + addr);
 }
 
 static const MemoryRegionOps msix_pba_mmio_ops = {
     .read = msix_pba_mmio_read,
-    /* TODO: MSIX should be LITTLE_ENDIAN. */
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
         .max_access_size = 4,
@@ -515,7 +518,8 @@ static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
 
 int msix_set_vector_notifiers(PCIDevice *dev,
                               MSIVectorUseNotifier use_notifier,
-                              MSIVectorReleaseNotifier release_notifier)
+                              MSIVectorReleaseNotifier release_notifier,
+                              MSIVectorPollNotifier poll_notifier)
 {
     int vector, ret;
 
@@ -523,6 +527,7 @@ int msix_set_vector_notifiers(PCIDevice *dev,
 
     dev->msix_vector_use_notifier = use_notifier;
     dev->msix_vector_release_notifier = release_notifier;
+    dev->msix_vector_poll_notifier = poll_notifier;
 
     if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
         (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
@@ -533,6 +538,9 @@ int msix_set_vector_notifiers(PCIDevice *dev,
             }
         }
     }
+    if (dev->msix_vector_poll_notifier) {
+        dev->msix_vector_poll_notifier(dev, 0, dev->msix_entries_nr);
+    }
     return 0;
 
 undo:
@@ -559,4 +567,5 @@ void msix_unset_vector_notifiers(PCIDevice *dev)
     }
     dev->msix_vector_use_notifier = NULL;
     dev->msix_vector_release_notifier = NULL;
+    dev->msix_vector_poll_notifier = NULL;
 }
diff --git a/hw/msix.h b/hw/pci/msix.h
index 15211cb..e648410 100644
--- a/hw/msix.h
+++ b/hw/pci/msix.h
@@ -2,9 +2,10 @@
 #define QEMU_MSIX_H
 
 #include "qemu-common.h"
-#include "pci.h"
+#include "hw/pci/pci.h"
 
 void msix_set_message(PCIDevice *dev, int vector, MSIMessage msg);
+MSIMessage msix_get_message(PCIDevice *dev, unsigned int vector);
 int msix_init(PCIDevice *dev, unsigned short nentries,
               MemoryRegion *table_bar, uint8_t table_bar_nr,
               unsigned table_offset, MemoryRegion *pba_bar,
@@ -26,6 +27,9 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
 int msix_enabled(PCIDevice *dev);
 int msix_present(PCIDevice *dev);
 
+bool msix_is_masked(PCIDevice *dev, unsigned vector);
+void msix_set_pending(PCIDevice *dev, unsigned vector);
+
 int msix_vector_use(PCIDevice *dev, unsigned vector);
 void msix_vector_unuse(PCIDevice *dev, unsigned vector);
 void msix_unuse_all_vectors(PCIDevice *dev);
@@ -36,6 +40,7 @@ void msix_reset(PCIDevice *dev);
 
 int msix_set_vector_notifiers(PCIDevice *dev,
                               MSIVectorUseNotifier use_notifier,
-                              MSIVectorReleaseNotifier release_notifier);
+                              MSIVectorReleaseNotifier release_notifier,
+                              MSIVectorPollNotifier poll_notifier);
 void msix_unset_vector_notifiers(PCIDevice *dev);
 #endif
diff --git a/hw/pci-hotplug.c b/hw/pci/pci-hotplug.c
index 0ca5546..f38df30 100644
--- a/hw/pci-hotplug.c
+++ b/hw/pci/pci-hotplug.c
@@ -22,17 +22,17 @@
  * THE SOFTWARE.
  */
 
-#include "hw.h"
-#include "boards.h"
-#include "pci.h"
-#include "net.h"
-#include "pc.h"
-#include "monitor.h"
-#include "scsi.h"
-#include "virtio-blk.h"
-#include "qemu-config.h"
-#include "blockdev.h"
-#include "error.h"
+#include "hw/hw.h"
+#include "hw/boards.h"
+#include "hw/pci/pci.h"
+#include "net/net.h"
+#include "hw/pc.h"
+#include "monitor/monitor.h"
+#include "hw/scsi.h"
+#include "hw/virtio-blk.h"
+#include "qemu/config-file.h"
+#include "sysemu/blockdev.h"
+#include "qapi/error.h"
 
 #if defined(TARGET_I386)
 static PCIDevice *qemu_pci_hot_add_nic(Monitor *mon,
@@ -111,15 +111,14 @@ static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
     return 0;
 }
 
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
-                      DriveInfo *dinfo, int type)
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo)
 {
     int dom, pci_bus;
     unsigned slot;
     PCIDevice *dev;
     const char *pci_addr = qdict_get_str(qdict, "pci_addr");
 
-    switch (type) {
+    switch (dinfo->type) {
     case IF_SCSI:
         if (pci_read_devaddr(mon, pci_addr, &dom, &pci_bus, &slot)) {
             goto err;
@@ -135,7 +134,7 @@ int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
         }
         break;
     default:
-        monitor_printf(mon, "Can't hot-add drive to type %d\n", type);
+        monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
         goto err;
     }
 
diff --git a/hw/pci-stub.c b/hw/pci/pci-stub.c
index 134c448..1dda89b 100644
--- a/hw/pci-stub.c
+++ b/hw/pci/pci-stub.c
@@ -18,9 +18,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sysemu.h"
-#include "monitor.h"
-#include "pci.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "hw/pci/pci.h"
 #include "qmp-commands.h"
 
 PciInfoList *qmp_query_pci(Error **errp)
diff --git a/hw/pci.c b/hw/pci/pci.c
index 97a0cd7..2f45c8f 100644
--- a/hw/pci.c
+++ b/hw/pci/pci.c
@@ -21,19 +21,19 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
-#include "pci.h"
-#include "pci_bridge.h"
-#include "pci_internals.h"
-#include "monitor.h"
-#include "net.h"
-#include "sysemu.h"
-#include "loader.h"
-#include "range.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pci_bus.h"
+#include "monitor/monitor.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "hw/loader.h"
+#include "qemu/range.h"
 #include "qmp-commands.h"
-#include "msi.h"
-#include "msix.h"
-#include "exec-memory.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_PCI
 #ifdef DEBUG_PCI
@@ -274,13 +274,12 @@ int pci_find_domain(const PCIBus *bus)
     return -1;
 }
 
-void pci_bus_new_inplace(PCIBus *bus, DeviceState *parent,
+static void pci_bus_init(PCIBus *bus, DeviceState *parent,
                          const char *name,
                          MemoryRegion *address_space_mem,
                          MemoryRegion *address_space_io,
                          uint8_t devfn_min)
 {
-    qbus_create_inplace(&bus->qbus, TYPE_PCI_BUS, parent, name);
     assert(PCI_FUNC(devfn_min) == 0);
     bus->devfn_min = devfn_min;
     bus->address_space_mem = address_space_mem;
@@ -293,6 +292,17 @@ void pci_bus_new_inplace(PCIBus *bus, DeviceState *parent,
     vmstate_register(NULL, -1, &vmstate_pcibus, bus);
 }
 
+void pci_bus_new_inplace(PCIBus *bus, DeviceState *parent,
+                         const char *name,
+                         MemoryRegion *address_space_mem,
+                         MemoryRegion *address_space_io,
+                         uint8_t devfn_min)
+{
+    qbus_create_inplace(bus, TYPE_PCI_BUS, parent, name);
+    pci_bus_init(bus, parent, name, address_space_mem,
+                 address_space_io, devfn_min);
+}
+
 PCIBus *pci_bus_new(DeviceState *parent, const char *name,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
@@ -300,10 +310,9 @@ PCIBus *pci_bus_new(DeviceState *parent, const char *name,
 {
     PCIBus *bus;
 
-    bus = g_malloc0(sizeof(*bus));
-    pci_bus_new_inplace(bus, parent, name, address_space_mem,
-                        address_space_io, devfn_min);
-    OBJECT(bus)->free = g_free;
+    bus = PCI_BUS(qbus_create(TYPE_PCI_BUS, parent, name));
+    pci_bus_init(bus, parent, name, address_space_mem,
+                 address_space_io, devfn_min);
     return bus;
 }
 
@@ -1123,7 +1132,7 @@ PCIINTxRoute pci_device_route_intx_to_irq(PCIDevice *dev, int pin)
     } while (dev);
 
     if (!bus->route_intx_to_irq) {
-        error_report("PCI: Bug - unimplemented PCI INTx routing (%s)\n",
+        error_report("PCI: Bug - unimplemented PCI INTx routing (%s)",
                      object_get_typename(OBJECT(bus->qbus.parent)));
         return (PCIINTxRoute) { PCI_INTX_DISABLED, -1 };
     }
@@ -2150,7 +2159,7 @@ void pci_setup_iommu(PCIBus *bus, PCIDMAContextFunc fn, void *opaque)
     bus->dma_context_opaque = opaque;
 }
 
-static TypeInfo pci_device_type_info = {
+static const TypeInfo pci_device_type_info = {
     .name = TYPE_PCI_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(PCIDevice),
diff --git a/hw/pci.h b/hw/pci/pci.h
index 4da0c2a..f340fe5 100644
--- a/hw/pci.h
+++ b/hw/pci/pci.h
@@ -3,14 +3,14 @@
 
 #include "qemu-common.h"
 
-#include "qdev.h"
-#include "memory.h"
-#include "dma.h"
+#include "hw/qdev.h"
+#include "exec/memory.h"
+#include "sysemu/dma.h"
 
 /* PCI includes legacy ISA access.  */
-#include "isa.h"
+#include "hw/isa.h"
 
-#include "pcie.h"
+#include "hw/pci/pcie.h"
 
 /* PCI bus */
 
@@ -21,7 +21,7 @@
 #define PCI_FUNC_MAX            8
 
 /* Class, Vendor and Device IDs from Linux's pci_ids.h */
-#include "pci_ids.h"
+#include "hw/pci/pci_ids.h"
 
 /* QEMU-specific Vendor and Device ID definitions */
 
@@ -77,6 +77,14 @@
 #define PCI_DEVICE_ID_VIRTIO_CONSOLE     0x1003
 #define PCI_DEVICE_ID_VIRTIO_SCSI        0x1004
 #define PCI_DEVICE_ID_VIRTIO_RNG         0x1005
+#define PCI_DEVICE_ID_VIRTIO_9P          0x1009
+
+#define PCI_VENDOR_ID_REDHAT             0x1b36
+#define PCI_DEVICE_ID_REDHAT_BRIDGE      0x0001
+#define PCI_DEVICE_ID_REDHAT_SERIAL      0x0002
+#define PCI_DEVICE_ID_REDHAT_SERIAL2     0x0003
+#define PCI_DEVICE_ID_REDHAT_SERIAL4     0x0004
+#define PCI_DEVICE_ID_REDHAT_QXL         0x0100
 
 #define FMT_PCIBUS                      PRIx64
 
@@ -100,7 +108,7 @@ typedef struct PCIIORegion {
 #define PCI_ROM_SLOT 6
 #define PCI_NUM_REGIONS 7
 
-#include "pci_regs.h"
+#include "hw/pci/pci_regs.h"
 
 /* PCI HEADER_TYPE */
 #define  PCI_HEADER_TYPE_MULTI_FUNCTION 0x80
@@ -187,6 +195,9 @@ typedef void (*PCIINTxRoutingNotifier)(PCIDevice *dev);
 typedef int (*MSIVectorUseNotifier)(PCIDevice *dev, unsigned int vector,
                                       MSIMessage msg);
 typedef void (*MSIVectorReleaseNotifier)(PCIDevice *dev, unsigned int vector);
+typedef void (*MSIVectorPollNotifier)(PCIDevice *dev,
+                                      unsigned int vector_start,
+                                      unsigned int vector_end);
 
 struct PCIDevice {
     DeviceState qdev;
@@ -271,6 +282,7 @@ struct PCIDevice {
     /* MSI-X notifiers */
     MSIVectorUseNotifier msix_vector_use_notifier;
     MSIVectorReleaseNotifier msix_vector_release_notifier;
+    MSIVectorPollNotifier msix_vector_poll_notifier;
 };
 
 void pci_register_bar(PCIDevice *pci_dev, int region_num,
diff --git a/hw/pci_bridge.c b/hw/pci/pci_bridge.c
index 4680501..995842a 100644
--- a/hw/pci_bridge.c
+++ b/hw/pci/pci_bridge.c
@@ -29,9 +29,9 @@
  *                    VA Linux Systems Japan K.K.
  */
 
-#include "pci_bridge.h"
-#include "pci_internals.h"
-#include "range.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pci_bus.h"
+#include "qemu/range.h"
 
 /* PCI bridge subsystem vendor ID helper functions */
 #define PCI_SSVID_SIZEOF        8
diff --git a/hw/pci_bridge.h b/hw/pci/pci_bridge.h
index a00accc..455cb66 100644
--- a/hw/pci_bridge.h
+++ b/hw/pci/pci_bridge.h
@@ -26,7 +26,7 @@
 #ifndef QEMU_PCI_BRIDGE_H
 #define QEMU_PCI_BRIDGE_H
 
-#include "pci.h"
+#include "hw/pci/pci.h"
 
 int pci_bridge_ssvid_init(PCIDevice *dev, uint8_t offset,
                           uint16_t svid, uint16_t ssid);
diff --git a/hw/pci_internals.h b/hw/pci/pci_bus.h
index 21d0ce6..f905b9e 100644
--- a/hw/pci_internals.h
+++ b/hw/pci/pci_bus.h
@@ -1,15 +1,11 @@
-#ifndef QEMU_PCI_INTERNALS_H
-#define QEMU_PCI_INTERNALS_H
+#ifndef QEMU_PCI_BUS_H
+#define QEMU_PCI_BUS_H
 
 /*
- * This header files is private to pci.c and pci_bridge.c
- * So following structures are opaque to others and shouldn't be
- * accessed.
+ * PCI Bus and Bridge datastructures.
  *
- * For pci-to-pci bridge needs to include this header file to embed
- * PCIBridge in its structure or to get sizeof(PCIBridge),
- * However, they shouldn't access those following members directly.
- * Use accessor function in pci.h, pci_bridge.h
+ * Do not access the following members directly;
+ * use accessor functions in pci.h, pci_bridge.h
  */
 
 #define TYPE_PCI_BUS "PCI"
@@ -75,4 +71,4 @@ struct PCIBridge {
     const char *bus_name;
 };
 
-#endif /* QEMU_PCI_INTERNALS_H */
+#endif /* QEMU_PCI_BUS_H */
diff --git a/hw/pci_host.c b/hw/pci/pci_host.c
index 68e328c..daca1c1 100644
--- a/hw/pci_host.c
+++ b/hw/pci/pci_host.c
@@ -18,8 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci.h"
-#include "pci_host.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_host.h"
 
 /* debug PCI */
 //#define DEBUG_PCI
diff --git a/hw/pci_host.h b/hw/pci/pci_host.h
index 4b9c300..1845d4d 100644
--- a/hw/pci_host.h
+++ b/hw/pci/pci_host.h
@@ -28,7 +28,7 @@
 #ifndef PCI_HOST_H
 #define PCI_HOST_H
 
-#include "sysbus.h"
+#include "hw/sysbus.h"
 
 #define TYPE_PCI_HOST_BRIDGE "pci-host-bridge"
 #define PCI_HOST_BRIDGE(obj) \
diff --git a/hw/pci_ids.h b/hw/pci/pci_ids.h
index 5df7245..d8dc2f1 100644
--- a/hw/pci_ids.h
+++ b/hw/pci/pci_ids.h
@@ -7,6 +7,8 @@
  *
  *      QEMU-specific definitions belong in pci.h
  */
+#ifndef HW_PCI_IDS_H
+#define HW_PCI_IDS_H 1
 
 /* Device classes and subclasses */
 
@@ -145,3 +147,8 @@
 
 #define PCI_VENDOR_ID_NEC                0x1033
 #define PCI_DEVICE_ID_NEC_UPD720200      0x0194
+
+#define PCI_VENDOR_ID_TEWS               0x1498
+#define PCI_DEVICE_ID_TEWS_TPCI200       0x30C8
+
+#endif
diff --git a/hw/pci_regs.h b/hw/pci/pci_regs.h
index 56a404b..56a404b 100644
--- a/hw/pci_regs.h
+++ b/hw/pci/pci_regs.h
diff --git a/hw/pcie.c b/hw/pci/pcie.c
index 7c92f19..485c94c 100644
--- a/hw/pcie.c
+++ b/hw/pci/pcie.c
@@ -19,13 +19,13 @@
  */
 
 #include "qemu-common.h"
-#include "pci_bridge.h"
-#include "pcie.h"
-#include "msix.h"
-#include "msi.h"
-#include "pci_internals.h"
-#include "pcie_regs.h"
-#include "range.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pcie.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/pcie_regs.h"
+#include "qemu/range.h"
 
 //#define DEBUG_PCIE
 #ifdef DEBUG_PCIE
@@ -494,7 +494,7 @@ uint16_t pcie_find_capability(PCIDevice *dev, uint16_t cap_id)
 
 static void pcie_ext_cap_set_next(PCIDevice *dev, uint16_t pos, uint16_t next)
 {
-    uint16_t header = pci_get_long(dev->config + pos);
+    uint32_t header = pci_get_long(dev->config + pos);
     assert(!(next & (PCI_EXT_CAP_ALIGN - 1)));
     header = (header & ~PCI_EXT_CAP_NEXT_MASK) |
         ((next << PCI_EXT_CAP_NEXT_SHIFT) & PCI_EXT_CAP_NEXT_MASK);
diff --git a/hw/pcie.h b/hw/pci/pcie.h
index 4889194..31604e2 100644
--- a/hw/pcie.h
+++ b/hw/pci/pcie.h
@@ -21,10 +21,10 @@
 #ifndef QEMU_PCIE_H
 #define QEMU_PCIE_H
 
-#include "hw.h"
-#include "pci_regs.h"
-#include "pcie_regs.h"
-#include "pcie_aer.h"
+#include "hw/hw.h"
+#include "hw/pci/pci_regs.h"
+#include "hw/pci/pcie_regs.h"
+#include "hw/pci/pcie_aer.h"
 
 typedef enum {
     /* for attention and power indicator */
diff --git a/hw/pcie_aer.c b/hw/pci/pcie_aer.c
index b04c164..1ce72ce 100644
--- a/hw/pcie_aer.c
+++ b/hw/pci/pcie_aer.c
@@ -18,15 +18,15 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sysemu.h"
-#include "qemu-objects.h"
-#include "monitor.h"
-#include "pci_bridge.h"
-#include "pcie.h"
-#include "msix.h"
-#include "msi.h"
-#include "pci_internals.h"
-#include "pcie_regs.h"
+#include "sysemu/sysemu.h"
+#include "qapi/qmp/types.h"
+#include "monitor/monitor.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pcie.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/pcie_regs.h"
 
 //#define DEBUG_PCIE
 #ifdef DEBUG_PCIE
diff --git a/hw/pcie_aer.h b/hw/pci/pcie_aer.h
index 7539500..bcac80a 100644
--- a/hw/pcie_aer.h
+++ b/hw/pci/pcie_aer.h
@@ -21,7 +21,7 @@
 #ifndef QEMU_PCIE_AER_H
 #define QEMU_PCIE_AER_H
 
-#include "hw.h"
+#include "hw/hw.h"
 
 /* definitions which PCIExpressDevice uses */
 
diff --git a/hw/pcie_host.c b/hw/pci/pcie_host.c
index c257fb4..b2d942b 100644
--- a/hw/pcie_host.c
+++ b/hw/pci/pcie_host.c
@@ -19,10 +19,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "hw.h"
-#include "pci.h"
-#include "pcie_host.h"
-#include "exec-memory.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pcie_host.h"
+#include "exec/address-spaces.h"
 
 /*
  * PCI express mmcfig address
diff --git a/hw/pcie_host.h b/hw/pci/pcie_host.h
index 3921935..1228e36 100644
--- a/hw/pcie_host.h
+++ b/hw/pci/pcie_host.h
@@ -21,8 +21,8 @@
 #ifndef PCIE_HOST_H
 #define PCIE_HOST_H
 
-#include "pci_host.h"
-#include "memory.h"
+#include "hw/pci/pci_host.h"
+#include "exec/memory.h"
 
 #define TYPE_PCIE_HOST_BRIDGE "pcie-host-bridge"
 #define PCIE_HOST_BRIDGE(obj) \
diff --git a/hw/pcie_port.c b/hw/pci/pcie_port.c
index d6350e5..33a6b0a 100644
--- a/hw/pcie_port.c
+++ b/hw/pci/pcie_port.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pcie_port.h"
+#include "hw/pci/pcie_port.h"
 
 void pcie_port_init_reg(PCIDevice *d)
 {
diff --git a/hw/pcie_port.h b/hw/pci/pcie_port.h
index 3709583..d89aa61 100644
--- a/hw/pcie_port.h
+++ b/hw/pci/pcie_port.h
@@ -21,8 +21,8 @@
 #ifndef QEMU_PCIE_PORT_H
 #define QEMU_PCIE_PORT_H
 
-#include "pci_bridge.h"
-#include "pci_internals.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/pci/pci_bus.h"
 
 struct PCIEPort {
     PCIBridge   br;
diff --git a/hw/pcie_regs.h b/hw/pci/pcie_regs.h
index 4d123d9..4d123d9 100644
--- a/hw/pcie_regs.h
+++ b/hw/pci/pcie_regs.h
diff --git a/hw/shpc.c b/hw/pci/shpc.c
index 4597bbd..f07266d 100644
--- a/hw/shpc.c
+++ b/hw/pci/shpc.c
@@ -1,11 +1,11 @@
 #include <strings.h>
 #include <stdint.h>
-#include "range.h"
-#include "range.h"
-#include "shpc.h"
-#include "pci.h"
-#include "pci_internals.h"
-#include "msi.h"
+#include "qemu/range.h"
+#include "qemu/range.h"
+#include "hw/pci/shpc.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/msi.h"
 
 /* TODO: model power only and disabled slot states. */
 /* TODO: handle SERR and wakeups */
diff --git a/hw/shpc.h b/hw/pci/shpc.h
index 130b71d..467911a 100644
--- a/hw/shpc.h
+++ b/hw/pci/shpc.h
@@ -2,8 +2,8 @@
 #define SHPC_H
 
 #include "qemu-common.h"
-#include "memory.h"
-#include "vmstate.h"
+#include "exec/memory.h"
+#include "migration/vmstate.h"
 
 struct SHPCDevice {
     /* Capability offset in device's config space */
diff --git a/hw/slotid_cap.c b/hw/pci/slotid_cap.c
index 0106452..99a30f4 100644
--- a/hw/slotid_cap.c
+++ b/hw/pci/slotid_cap.c
@@ -1,5 +1,5 @@
-#include "slotid_cap.h"
-#include "pci.h"
+#include "hw/pci/slotid_cap.h"
+#include "hw/pci/pci.h"
 
 #define SLOTID_CAP_LENGTH 4
 #define SLOTID_NSLOTS_SHIFT (ffs(PCI_SID_ESR_NSLOTS) - 1)
diff --git a/hw/slotid_cap.h b/hw/pci/slotid_cap.h
index 70db047..70db047 100644
--- a/hw/slotid_cap.h
+++ b/hw/pci/slotid_cap.h
diff --git a/hw/pci_bridge_dev.c b/hw/pci_bridge_dev.c
index f706396..1124c53 100644
--- a/hw/pci_bridge_dev.c
+++ b/hw/pci_bridge_dev.c
@@ -19,17 +19,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_bridge.h"
-#include "pci_ids.h"
-#include "msi.h"
-#include "shpc.h"
-#include "slotid_cap.h"
-#include "memory.h"
-#include "pci_internals.h"
-
-#define REDHAT_PCI_VENDOR_ID 0x1b36
-#define PCI_BRIDGE_DEV_VENDOR_ID REDHAT_PCI_VENDOR_ID
-#define PCI_BRIDGE_DEV_DEVICE_ID 0x1
+#include "pci/pci_bridge.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/shpc.h"
+#include "pci/slotid_cap.h"
+#include "exec/memory.h"
+#include "pci/pci_bus.h"
 
 struct PCIBridgeDev {
     PCIBridge bridge;
@@ -146,8 +142,8 @@ static void pci_bridge_dev_class_init(ObjectClass *klass, void *data)
     k->init = pci_bridge_dev_initfn;
     k->exit = pci_bridge_dev_exitfn;
     k->config_write = pci_bridge_dev_write_config;
-    k->vendor_id = PCI_BRIDGE_DEV_VENDOR_ID;
-    k->device_id = PCI_BRIDGE_DEV_DEVICE_ID;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_BRIDGE;
     k->class_id = PCI_CLASS_BRIDGE_PCI;
     k->is_bridge = 1,
     dc->desc = "Standard PCI Bridge";
@@ -156,7 +152,7 @@ static void pci_bridge_dev_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &pci_bridge_dev_vmstate;
 }
 
-static TypeInfo pci_bridge_dev_info = {
+static const TypeInfo pci_bridge_dev_info = {
     .name = "pci-bridge",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIBridgeDev),
diff --git a/hw/pckbd.c b/hw/pckbd.c
index 5bb3e0a..3bad09b 100644
--- a/hw/pckbd.c
+++ b/hw/pckbd.c
@@ -25,7 +25,7 @@
 #include "isa.h"
 #include "pc.h"
 #include "ps2.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 /* debug PC keyboard */
 //#define DEBUG_KBD
@@ -512,7 +512,7 @@ static void i8042_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_kbd_isa;
 }
 
-static TypeInfo i8042_info = {
+static const TypeInfo i8042_info = {
     .name          = "i8042",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAKBDState),
diff --git a/hw/pcmcia.h b/hw/pcmcia.h
index 50648c9..f916693 100644
--- a/hw/pcmcia.h
+++ b/hw/pcmcia.h
@@ -1,3 +1,6 @@
+#ifndef HW_PCMCIA_H
+#define HW_PCMCIA_H 1
+
 /* PCMCIA/Cardbus */
 
 #include "qemu-common.h"
@@ -11,7 +14,7 @@ typedef struct {
 
 void pcmcia_socket_register(PCMCIASocket *socket);
 void pcmcia_socket_unregister(PCMCIASocket *socket);
-void pcmcia_info(Monitor *mon);
+void pcmcia_info(Monitor *mon, const QDict *qdict);
 
 struct PCMCIACardState {
     void *state;
@@ -49,3 +52,5 @@ struct PCMCIACardState {
 
 /* dscm1xxxx.c */
 PCMCIACardState *dscm1xxxx_init(DriveInfo *bdrv);
+
+#endif
diff --git a/hw/pcnet-pci.c b/hw/pcnet-pci.c
index 0bf438f..df63b22 100644
--- a/hw/pcnet-pci.c
+++ b/hw/pcnet-pci.c
@@ -27,11 +27,11 @@
  * AMD Publication# 19436  Rev:E  Amendment/0  Issue Date: June 2000
  */
 
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "loader.h"
-#include "qemu-timer.h"
-#include "dma.h"
+#include "qemu/timer.h"
+#include "sysemu/dma.h"
 
 #include "pcnet.h"
 
@@ -266,7 +266,7 @@ static void pci_physical_memory_read(void *dma_opaque, hwaddr addr,
 
 static void pci_pcnet_cleanup(NetClientState *nc)
 {
-    PCNetState *d = DO_UPCAST(NICState, nc, nc)->opaque;
+    PCNetState *d = qemu_get_nic_opaque(nc);
 
     pcnet_common_cleanup(d);
 }
@@ -279,7 +279,7 @@ static void pci_pcnet_uninit(PCIDevice *dev)
     memory_region_destroy(&d->io_bar);
     qemu_del_timer(d->state.poll_timer);
     qemu_free_timer(d->state.poll_timer);
-    qemu_del_net_client(&d->state.nic->nc);
+    qemu_del_nic(d->state.nic);
 }
 
 static NetClientInfo net_pci_pcnet_info = {
@@ -361,7 +361,7 @@ static void pcnet_class_init(ObjectClass *klass, void *data)
     dc->props = pcnet_properties;
 }
 
-static TypeInfo pcnet_info = {
+static const TypeInfo pcnet_info = {
     .name          = "pcnet",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIPCNetState),
diff --git a/hw/pcnet.c b/hw/pcnet.c
index 54eecd0..e0de1e3 100644
--- a/hw/pcnet.c
+++ b/hw/pcnet.c
@@ -36,10 +36,10 @@
  */
 
 #include "qdev.h"
-#include "net.h"
-#include "qemu-timer.h"
-#include "qemu_socket.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "qemu/timer.h"
+#include "qemu/sockets.h"
+#include "sysemu/sysemu.h"
 
 #include "pcnet.h"
 
@@ -1006,7 +1006,7 @@ static int pcnet_tdte_poll(PCNetState *s)
 
 int pcnet_can_receive(NetClientState *nc)
 {
-    PCNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    PCNetState *s = qemu_get_nic_opaque(nc);
     if (CSR_STOP(s) || CSR_SPND(s))
         return 0;
 
@@ -1017,7 +1017,7 @@ int pcnet_can_receive(NetClientState *nc)
 
 ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
 {
-    PCNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    PCNetState *s = qemu_get_nic_opaque(nc);
     int is_padr = 0, is_bcast = 0, is_ladr = 0;
     uint8_t buf1[60];
     int remaining;
@@ -1199,7 +1199,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
 
 void pcnet_set_link_status(NetClientState *nc)
 {
-    PCNetState *d = DO_UPCAST(NICState, nc, nc)->opaque;
+    PCNetState *d = qemu_get_nic_opaque(nc);
 
     d->lnkst = nc->link_down ? 0 : 0x40;
 }
@@ -1261,11 +1261,12 @@ static void pcnet_transmit(PCNetState *s)
                 if (BCR_SWSTYLE(s) == 1)
                     add_crc = !GET_FIELD(tmd.status, TMDS, NOFCS);
                 s->looptest = add_crc ? PCNET_LOOPTEST_CRC : PCNET_LOOPTEST_NOCRC;
-                pcnet_receive(&s->nic->nc, s->buffer, s->xmit_pos);
+                pcnet_receive(qemu_get_queue(s->nic), s->buffer, s->xmit_pos);
                 s->looptest = 0;
             } else
                 if (s->nic)
-                    qemu_send_packet(&s->nic->nc, s->buffer, s->xmit_pos);
+                    qemu_send_packet(qemu_get_queue(s->nic), s->buffer,
+                                     s->xmit_pos);
 
             s->csr[0] &= ~0x0008;   /* clear TDMD */
             s->csr[4] |= 0x0004;    /* set TXSTRT */
@@ -1730,7 +1731,7 @@ int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info)
 
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(info, &s->conf, object_get_typename(OBJECT(dev)), dev->id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     add_boot_device_path(s->conf.bootindex, dev, "/ethernet-phy@0");
 
diff --git a/hw/pcnet.h b/hw/pcnet.h
index da8c3bd..9dee6f3 100644
--- a/hw/pcnet.h
+++ b/hw/pcnet.h
@@ -1,10 +1,13 @@
+#ifndef HW_PCNET_H
+#define HW_PCNET_H 1
+
 #define PCNET_IOPORT_SIZE       0x20
 #define PCNET_PNPMMIO_SIZE      0x20
 
 #define PCNET_LOOPTEST_CRC	1
 #define PCNET_LOOPTEST_NOCRC	2
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* BUS CONFIGURATION REGISTERS */
 #define BCR_MSRDA    0
@@ -63,3 +66,5 @@ void pcnet_set_link_status(NetClientState *nc);
 void pcnet_common_cleanup(PCNetState *d);
 int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info);
 extern const VMStateDescription vmstate_pcnet;
+
+#endif
diff --git a/hw/pcspk.c b/hw/pcspk.c
index ad6491b..dfab955 100644
--- a/hw/pcspk.c
+++ b/hw/pcspk.c
@@ -26,7 +26,7 @@
 #include "pc.h"
 #include "isa.h"
 #include "audio/audio.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i8254.h"
 #include "pcspk.h"
 
@@ -187,7 +187,7 @@ static void pcspk_class_initfn(ObjectClass *klass, void *data)
     dc->props = pcspk_properties;
 }
 
-static TypeInfo pcspk_info = {
+static const TypeInfo pcspk_info = {
     .name           = "isa-pcspk",
     .parent         = TYPE_ISA_DEVICE,
     .instance_size  = sizeof(PCSpkState),
diff --git a/hw/petalogix_ml605_mmu.c b/hw/petalogix_ml605_mmu.c
index 3589a4b..82d7183 100644
--- a/hw/petalogix_ml605_mmu.c
+++ b/hw/petalogix_ml605_mmu.c
@@ -27,15 +27,15 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
 #include "xilinx.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "serial.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "ssi.h"
 
 #include "microblaze_boot.h"
@@ -129,17 +129,19 @@ petalogix_ml605_init(QEMUMachineInitArgs *args)
     xilinx_timer_create(TIMER_BASEADDR, irq[2], 0, 100 * 1000000);
 
     /* axi ethernet and dma initialization. */
+    qemu_check_nic_model(&nd_table[0], "xlnx.axi-ethernet");
+    eth0 = qdev_create(NULL, "xlnx.axi-ethernet");
     dma = qdev_create(NULL, "xlnx.axi-dma");
 
     /* FIXME: attach to the sysbus instead */
     object_property_add_child(container_get(qdev_get_machine(), "/unattached"),
                                   "xilinx-dma", OBJECT(dma), NULL);
 
-    eth0 = xilinx_axiethernet_create(&nd_table[0], STREAM_SLAVE(dma),
-                                     0x82780000, irq[3], 0x1000, 0x1000);
+    xilinx_axiethernet_init(eth0, &nd_table[0], STREAM_SLAVE(dma),
+                                   0x82780000, irq[3], 0x1000, 0x1000);
 
-    xilinx_axiethernetdma_init(dma, STREAM_SLAVE(eth0),
-                               0x84600000, irq[1], irq[0], 100 * 1000000);
+    xilinx_axidma_init(dma, STREAM_SLAVE(eth0), 0x84600000, irq[1], irq[0],
+                       100 * 1000000);
 
     {
         SSIBus *spi;
@@ -147,7 +149,7 @@ petalogix_ml605_init(QEMUMachineInitArgs *args)
         dev = qdev_create(NULL, "xlnx.xps-spi");
         qdev_prop_set_uint8(dev, "num-ss-bits", NUM_SPI_FLASHES);
         qdev_init_nofail(dev);
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         sysbus_mmio_map(busdev, 0, 0x40a00000);
         sysbus_connect_irq(busdev, 0, irq[4]);
 
@@ -173,7 +175,8 @@ static QEMUMachine petalogix_ml605_machine = {
     .name = "petalogix-ml605",
     .desc = "PetaLogix linux refdesign for xilinx ml605 little endian",
     .init = petalogix_ml605_init,
-    .is_default = 0
+    .is_default = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void petalogix_ml605_machine_init(void)
diff --git a/hw/petalogix_s3adsp1800_mmu.c b/hw/petalogix_s3adsp1800_mmu.c
index c5fd5e7..8605fb8 100644
--- a/hw/petalogix_s3adsp1800_mmu.c
+++ b/hw/petalogix_s3adsp1800_mmu.c
@@ -25,14 +25,14 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
 #include "xilinx.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #include "microblaze_boot.h"
 #include "microblaze_pic_cpu.h"
@@ -115,7 +115,8 @@ static QEMUMachine petalogix_s3adsp1800_machine = {
     .name = "petalogix-s3adsp1800",
     .desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800",
     .init = petalogix_s3adsp1800_init,
-    .is_default = 1
+    .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void petalogix_s3adsp1800_machine_init(void)
diff --git a/hw/pflash_cfi01.c b/hw/pflash_cfi01.c
index 7d040b5..9e6ff52 100644
--- a/hw/pflash_cfi01.c
+++ b/hw/pflash_cfi01.c
@@ -38,23 +38,23 @@
 
 #include "hw.h"
 #include "flash.h"
-#include "block.h"
-#include "qemu-timer.h"
-#include "exec-memory.h"
-#include "host-utils.h"
+#include "block/block.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
+#include "qemu/host-utils.h"
 #include "sysbus.h"
 
 #define PFLASH_BUG(fmt, ...) \
 do { \
-    printf("PFLASH: Possible BUG - " fmt, ## __VA_ARGS__); \
+    fprintf(stderr, "PFLASH: Possible BUG - " fmt, ## __VA_ARGS__); \
     exit(1); \
 } while(0)
 
 /* #define PFLASH_DEBUG */
 #ifdef PFLASH_DEBUG
-#define DPRINTF(fmt, ...)                          \
-do {                                               \
-    printf("PFLASH: " fmt , ## __VA_ARGS__);       \
+#define DPRINTF(fmt, ...)                                   \
+do {                                                        \
+    fprintf(stderr, "PFLASH: " fmt , ## __VA_ARGS__);       \
 } while (0)
 #else
 #define DPRINTF(fmt, ...) do { } while (0)
@@ -319,6 +319,9 @@ static void pflash_write(pflash_t *pfl, hwaddr offset,
             DPRINTF("%s: Write to buffer\n", __func__);
             pfl->status |= 0x80; /* Ready! */
             break;
+        case 0xf0: /* Probe for AMD flash */
+            DPRINTF("%s: Probe for AMD flash\n", __func__);
+            goto reset_flash;
         case 0xff: /* Read array mode */
             DPRINTF("%s: Read array mode\n", __func__);
             goto reset_flash;
@@ -438,9 +441,9 @@ static void pflash_write(pflash_t *pfl, hwaddr offset,
     return;
 
  error_flash:
-    printf("%s: Unimplemented flash cmd sequence "
-           "(offset " TARGET_FMT_plx ", wcycle 0x%x cmd 0x%x value 0x%x)\n",
-           __func__, offset, pfl->wcycle, pfl->cmd, value);
+    qemu_log_mask(LOG_UNIMP, "%s: Unimplemented flash cmd sequence "
+                  "(offset " TARGET_FMT_plx ", wcycle 0x%x cmd 0x%x value 0x%x)"
+                  "\n", __func__, offset, pfl->wcycle, pfl->cmd, value);
 
  reset_flash:
     memory_region_rom_device_set_readable(&pfl->mem, true);
@@ -726,7 +729,7 @@ pflash_t *pflash_cfi01_register(hwaddr base,
                                 uint16_t id2, uint16_t id3, int be)
 {
     DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
-    SysBusDevice *busdev = sysbus_from_qdev(dev);
+    SysBusDevice *busdev = SYS_BUS_DEVICE(dev);
     pflash_t *pfl = (pflash_t *)object_dynamic_cast(OBJECT(dev),
                                                     "cfi.pflash01");
 
diff --git a/hw/pflash_cfi02.c b/hw/pflash_cfi02.c
index f918e36..44bd465 100644
--- a/hw/pflash_cfi02.c
+++ b/hw/pflash_cfi02.c
@@ -37,17 +37,17 @@
 
 #include "hw.h"
 #include "flash.h"
-#include "qemu-timer.h"
-#include "block.h"
-#include "exec-memory.h"
-#include "host-utils.h"
+#include "qemu/timer.h"
+#include "block/block.h"
+#include "exec/address-spaces.h"
+#include "qemu/host-utils.h"
 #include "sysbus.h"
 
 //#define PFLASH_DEBUG
 #ifdef PFLASH_DEBUG
-#define DPRINTF(fmt, ...)                          \
-do {                                               \
-    printf("PFLASH: " fmt , ## __VA_ARGS__);       \
+#define DPRINTF(fmt, ...)                                  \
+do {                                                       \
+    fprintf(stderr "PFLASH: " fmt , ## __VA_ARGS__);       \
 } while (0)
 #else
 #define DPRINTF(fmt, ...) do { } while (0)
@@ -157,6 +157,7 @@ static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
         DPRINTF("%s: unknown command state: %x\n", __func__, pfl->cmd);
         pfl->wcycle = 0;
         pfl->cmd = 0;
+        /* fall through to the read code */
     case 0x80:
         /* We accept reads during second unlock sequence... */
     case 0x00:
@@ -760,7 +761,7 @@ pflash_t *pflash_cfi02_register(hwaddr base,
                                 int be)
 {
     DeviceState *dev = qdev_create(NULL, "cfi.pflash02");
-    SysBusDevice *busdev = sysbus_from_qdev(dev);
+    SysBusDevice *busdev = SYS_BUS_DEVICE(dev);
     pflash_t *pfl = (pflash_t *)object_dynamic_cast(OBJECT(dev),
                                                     "cfi.pflash02");
 
diff --git a/hw/piix4.c b/hw/piix4.c
index ce4eb0d..c1cb94d 100644
--- a/hw/piix4.c
+++ b/hw/piix4.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "isa.h"
 #include "sysbus.h"
 
@@ -117,7 +117,7 @@ static void piix4_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_piix4;
 }
 
-static TypeInfo piix4_info = {
+static const TypeInfo piix4_info = {
     .name          = "PIIX4",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PIIX4State),
diff --git a/hw/piix_pci.c b/hw/piix_pci.c
index ba1b3de..6c77e49 100644
--- a/hw/piix_pci.c
+++ b/hw/piix_pci.c
@@ -24,13 +24,14 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 #include "isa.h"
 #include "sysbus.h"
-#include "range.h"
+#include "qemu/range.h"
 #include "xen.h"
 #include "pam.h"
+#include "sysemu/sysemu.h"
 
 /*
  * I440FX chipset data sheet.
@@ -46,6 +47,12 @@ typedef struct I440FXState {
 #define XEN_PIIX_NUM_PIRQS      128ULL
 #define PIIX_PIRQC              0x60
 
+/*
+ * Reset Control Register: PCI-accessible ISA-Compatible Register at address
+ * 0xcf9, provided by the PCI/ISA bridge (PIIX3 PCI function 0, 8086:7000).
+ */
+#define RCR_IOPORT 0xcf9
+
 typedef struct PIIX3State {
     PCIDevice dev;
 
@@ -67,6 +74,12 @@ typedef struct PIIX3State {
 
     /* This member isn't used. Just for save/load compatibility */
     int32_t pci_irq_levels_vmstate[PIIX_NUM_PIRQS];
+
+    /* Reset Control Register contents */
+    uint8_t rcr;
+
+    /* IO memory region for Reset Control Register (RCR_IOPORT) */
+    MemoryRegion rcr_mem;
 } PIIX3State;
 
 struct PCII440FXState {
@@ -442,6 +455,7 @@ static void piix3_reset(void *opaque)
     pci_conf[0xae] = 0x00;
 
     d->pic_levels = 0;
+    d->rcr = 0;
 }
 
 static int piix3_post_load(void *opaque, int version_id)
@@ -462,6 +476,23 @@ static void piix3_pre_save(void *opaque)
     }
 }
 
+static bool piix3_rcr_needed(void *opaque)
+{
+    PIIX3State *piix3 = opaque;
+
+    return (piix3->rcr != 0);
+}
+
+static const VMStateDescription vmstate_piix3_rcr = {
+    .name = "PIIX3/rcr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField []) {
+        VMSTATE_UINT8(rcr, PIIX3State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_piix3 = {
     .name = "PIIX3",
     .version_id = 3,
@@ -469,12 +500,44 @@ static const VMStateDescription vmstate_piix3 = {
     .minimum_version_id_old = 2,
     .post_load = piix3_post_load,
     .pre_save = piix3_pre_save,
-    .fields      = (VMStateField []) {
+    .fields      = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(dev, PIIX3State),
         VMSTATE_INT32_ARRAY_V(pci_irq_levels_vmstate, PIIX3State,
                               PIIX_NUM_PIRQS, 3),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_piix3_rcr,
+            .needed = piix3_rcr_needed,
+        },
+        { 0 }
+    }
+};
+
+
+static void rcr_write(void *opaque, hwaddr addr, uint64_t val, unsigned len)
+{
+    PIIX3State *d = opaque;
+
+    if (val & 4) {
+        qemu_system_reset_request();
+        return;
     }
+    d->rcr = val & 2; /* keep System Reset type only */
+}
+
+static uint64_t rcr_read(void *opaque, hwaddr addr, unsigned len)
+{
+    PIIX3State *d = opaque;
+
+    return d->rcr;
+}
+
+static const MemoryRegionOps rcr_ops = {
+    .read = rcr_read,
+    .write = rcr_write,
+    .endianness = DEVICE_LITTLE_ENDIAN
 };
 
 static int piix3_initfn(PCIDevice *dev)
@@ -482,6 +545,11 @@ static int piix3_initfn(PCIDevice *dev)
     PIIX3State *d = DO_UPCAST(PIIX3State, dev, dev);
 
     isa_bus_new(&d->dev.qdev, pci_address_space_io(dev));
+
+    memory_region_init_io(&d->rcr_mem, &rcr_ops, d, "piix3-reset-control", 1);
+    memory_region_add_subregion_overlap(pci_address_space_io(dev), RCR_IOPORT,
+                                        &d->rcr_mem, 1);
+
     qemu_register_reset(piix3_reset, d);
     return 0;
 }
diff --git a/hw/pl011.c b/hw/pl011.c
index 1f7ce2f..002a50e 100644
--- a/hw/pl011.c
+++ b/hw/pl011.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 typedef struct {
     SysBusDevice busdev;
@@ -300,7 +300,7 @@ static void pl011_arm_class_init(ObjectClass *klass, void *data)
     sdc->init = pl011_arm_init;
 }
 
-static TypeInfo pl011_arm_info = {
+static const TypeInfo pl011_arm_info = {
     .name          = "pl011",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl011_state),
@@ -314,7 +314,7 @@ static void pl011_luminary_class_init(ObjectClass *klass, void *data)
     sdc->init = pl011_luminary_init;
 }
 
-static TypeInfo pl011_luminary_info = {
+static const TypeInfo pl011_luminary_info = {
     .name          = "pl011_luminary",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl011_state),
diff --git a/hw/pl022.c b/hw/pl022.c
index fbd7ded..c160e90 100644
--- a/hw/pl022.c
+++ b/hw/pl022.c
@@ -293,7 +293,7 @@ static void pl022_class_init(ObjectClass *klass, void *data)
     sdc->init = pl022_init;
 }
 
-static TypeInfo pl022_info = {
+static const TypeInfo pl022_info = {
     .name          = "pl022",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl022_state),
diff --git a/hw/pl031.c b/hw/pl031.c
index 8bf0183..757867f 100644
--- a/hw/pl031.c
+++ b/hw/pl031.c
@@ -12,8 +12,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 
 //#define DEBUG_PL031
 
@@ -250,7 +250,7 @@ static void pl031_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl031;
 }
 
-static TypeInfo pl031_info = {
+static const TypeInfo pl031_info = {
     .name          = "pl031",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl031_state),
diff --git a/hw/pl041.c b/hw/pl041.c
index 4436d97..0b71c45 100644
--- a/hw/pl041.c
+++ b/hw/pl041.c
@@ -632,7 +632,7 @@ static void pl041_device_class_init(ObjectClass *klass, void *data)
     dc->props = pl041_device_properties;
 }
 
-static TypeInfo pl041_device_info = {
+static const TypeInfo pl041_device_info = {
     .name          = "pl041",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl041_state),
diff --git a/hw/pl050.c b/hw/pl050.c
index 47032f1..5d06bc9 100644
--- a/hw/pl050.c
+++ b/hw/pl050.c
@@ -168,7 +168,7 @@ static void pl050_kbd_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl050;
 }
 
-static TypeInfo pl050_kbd_info = {
+static const TypeInfo pl050_kbd_info = {
     .name          = "pl050_keyboard",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl050_state),
@@ -184,7 +184,7 @@ static void pl050_mouse_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl050;
 }
 
-static TypeInfo pl050_mouse_info = {
+static const TypeInfo pl050_mouse_info = {
     .name          = "pl050_mouse",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl050_state),
diff --git a/hw/pl061.c b/hw/pl061.c
index f1ed5ce..a78e819 100644
--- a/hw/pl061.c
+++ b/hw/pl061.c
@@ -304,7 +304,7 @@ static void pl061_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl061;
 }
 
-static TypeInfo pl061_info = {
+static const TypeInfo pl061_info = {
     .name          = "pl061",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl061_state),
@@ -320,7 +320,7 @@ static void pl061_luminary_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl061;
 }
 
-static TypeInfo pl061_luminary_info = {
+static const TypeInfo pl061_luminary_info = {
     .name          = "pl061_luminary",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl061_state),
diff --git a/hw/pl080.c b/hw/pl080.c
index 26150af..f6bbf98 100644
--- a/hw/pl080.c
+++ b/hw/pl080.c
@@ -386,7 +386,7 @@ static void pl080_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl080;
 }
 
-static TypeInfo pl080_info = {
+static const TypeInfo pl080_info = {
     .name          = "pl080",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl080_state),
@@ -403,7 +403,7 @@ static void pl081_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl080;
 }
 
-static TypeInfo pl081_info = {
+static const TypeInfo pl081_info = {
     .name          = "pl081",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl080_state),
diff --git a/hw/pl110.c b/hw/pl110.c
index f869ba6..3d0ac00 100644
--- a/hw/pl110.c
+++ b/hw/pl110.c
@@ -8,8 +8,9 @@
  */
 
 #include "sysbus.h"
-#include "console.h"
+#include "ui/console.h"
 #include "framebuffer.h"
+#include "ui/pixel_ops.h"
 
 #define PL110_CR_EN   0x001
 #define PL110_CR_BGR  0x100
@@ -109,8 +110,6 @@ static const unsigned char *idregs[] = {
     pl111_id
 };
 
-#include "pixel_ops.h"
-
 #define BITS 8
 #include "pl110_template.h"
 #define BITS 15
@@ -481,7 +480,7 @@ static void pl110_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl110;
 }
 
-static TypeInfo pl110_info = {
+static const TypeInfo pl110_info = {
     .name          = "pl110",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl110_state),
@@ -498,7 +497,7 @@ static void pl110_versatile_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl110;
 }
 
-static TypeInfo pl110_versatile_info = {
+static const TypeInfo pl110_versatile_info = {
     .name          = "pl110_versatile",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl110_state),
@@ -515,7 +514,7 @@ static void pl111_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl110;
 }
 
-static TypeInfo pl111_info = {
+static const TypeInfo pl111_info = {
     .name          = "pl111",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl110_state),
diff --git a/hw/pl181.c b/hw/pl181.c
index ac18477..4996cba 100644
--- a/hw/pl181.c
+++ b/hw/pl181.c
@@ -7,7 +7,7 @@
  * This code is licensed under the GPL.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
 #include "sd.h"
 
@@ -500,7 +500,7 @@ static void pl181_class_init(ObjectClass *klass, void *data)
     k->no_user = 1;
 }
 
-static TypeInfo pl181_info = {
+static const TypeInfo pl181_info = {
     .name          = "pl181",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl181_state),
diff --git a/hw/pl190.c b/hw/pl190.c
index 4019930..76ac159 100644
--- a/hw/pl190.c
+++ b/hw/pl190.c
@@ -274,7 +274,7 @@ static void pl190_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pl190;
 }
 
-static TypeInfo pl190_info = {
+static const TypeInfo pl190_info = {
     .name          = "pl190",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(pl190_state),
diff --git a/hw/pm_smbus.c b/hw/pm_smbus.c
index 5d6046d..ea1380c 100644
--- a/hw/pm_smbus.c
+++ b/hw/pm_smbus.c
@@ -94,10 +94,11 @@ static void smb_transaction(PMSMBus *s)
     s->smb_stat |= 0x04;
 }
 
-void smb_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void smb_ioport_writeb(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned width)
 {
     PMSMBus *s = opaque;
-    addr &= 0x3f;
+
     SMBUS_DPRINTF("SMB writeb port=0x%04x val=0x%02x\n", addr, val);
     switch(addr) {
     case SMBHSTSTS:
@@ -131,12 +132,11 @@ void smb_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-uint32_t smb_ioport_readb(void *opaque, uint32_t addr)
+static uint64_t smb_ioport_readb(void *opaque, hwaddr addr, unsigned width)
 {
     PMSMBus *s = opaque;
     uint32_t val;
 
-    addr &= 0x3f;
     switch(addr) {
     case SMBHSTSTS:
         val = s->smb_stat;
@@ -170,7 +170,16 @@ uint32_t smb_ioport_readb(void *opaque, uint32_t addr)
     return val;
 }
 
+static const MemoryRegionOps pm_smbus_ops = {
+    .read = smb_ioport_readb,
+    .write = smb_ioport_writeb,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
 void pm_smbus_init(DeviceState *parent, PMSMBus *smb)
 {
     smb->smbus = i2c_init_bus(parent, "i2c");
+    memory_region_init_io(&smb->io, &pm_smbus_ops, smb, "pm-smbus", 64);
 }
diff --git a/hw/pm_smbus.h b/hw/pm_smbus.h
index 4750a40..e3069bf 100644
--- a/hw/pm_smbus.h
+++ b/hw/pm_smbus.h
@@ -3,6 +3,7 @@
 
 typedef struct PMSMBus {
     i2c_bus *smbus;
+    MemoryRegion io;
 
     uint8_t smb_stat;
     uint8_t smb_ctl;
@@ -15,7 +16,5 @@ typedef struct PMSMBus {
 } PMSMBus;
 
 void pm_smbus_init(DeviceState *parent, PMSMBus *smb);
-void smb_ioport_writeb(void *opaque, uint32_t addr, uint32_t val);
-uint32_t smb_ioport_readb(void *opaque, uint32_t addr);
 
 #endif /* !PM_SMBUS_H */
diff --git a/hw/ppc.c b/hw/ppc.c
index 11fd199..c52e22f 100644
--- a/hw/ppc.c
+++ b/hw/ppc.c
@@ -23,12 +23,12 @@
  */
 #include "hw.h"
 #include "ppc.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "nvram.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 //#define PPC_DEBUG_IRQ
@@ -50,8 +50,9 @@
 static void cpu_ppc_tb_stop (CPUPPCState *env);
 static void cpu_ppc_tb_start (CPUPPCState *env);
 
-void ppc_set_irq(CPUPPCState *env, int n_IRQ, int level)
+void ppc_set_irq(PowerPCCPU *cpu, int n_IRQ, int level)
 {
+    CPUPPCState *env = &cpu->env;
     unsigned int old_pending = env->pending_interrupts;
 
     if (level) {
@@ -65,7 +66,7 @@ void ppc_set_irq(CPUPPCState *env, int n_IRQ, int level)
 
     if (old_pending != env->pending_interrupts) {
 #ifdef CONFIG_KVM
-        kvmppc_set_interrupt(env, n_IRQ, level);
+        kvmppc_set_interrupt(cpu, n_IRQ, level);
 #endif
     }
 
@@ -100,13 +101,13 @@ static void ppc6xx_set_irq(void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the external IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPC6xx_INPUT_SMI:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the SMI IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_SMI, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_SMI, level);
             break;
         case PPC6xx_INPUT_MCP:
             /* Negative edge sensitive */
@@ -116,7 +117,7 @@ static void ppc6xx_set_irq(void *opaque, int pin, int level)
             if (cur_level == 1 && level == 0) {
                 LOG_IRQ("%s: raise machine check state\n",
                             __func__);
-                ppc_set_irq(env, PPC_INTERRUPT_MCK, 1);
+                ppc_set_irq(cpu, PPC_INTERRUPT_MCK, 1);
             }
             break;
         case PPC6xx_INPUT_CKSTP_IN:
@@ -138,7 +139,7 @@ static void ppc6xx_set_irq(void *opaque, int pin, int level)
         case PPC6xx_INPUT_SRESET:
             LOG_IRQ("%s: set the RESET IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_RESET, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_RESET, level);
             break;
         default:
             /* Unknown pin - do nothing */
@@ -178,13 +179,13 @@ static void ppc970_set_irq(void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the external IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPC970_INPUT_THINT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the SMI IRQ state to %d\n", __func__,
                         level);
-            ppc_set_irq(env, PPC_INTERRUPT_THERM, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_THERM, level);
             break;
         case PPC970_INPUT_MCP:
             /* Negative edge sensitive */
@@ -194,7 +195,7 @@ static void ppc970_set_irq(void *opaque, int pin, int level)
             if (cur_level == 1 && level == 0) {
                 LOG_IRQ("%s: raise machine check state\n",
                             __func__);
-                ppc_set_irq(env, PPC_INTERRUPT_MCK, 1);
+                ppc_set_irq(cpu, PPC_INTERRUPT_MCK, 1);
             }
             break;
         case PPC970_INPUT_CKSTP:
@@ -218,7 +219,7 @@ static void ppc970_set_irq(void *opaque, int pin, int level)
         case PPC970_INPUT_SRESET:
             LOG_IRQ("%s: set the RESET IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_RESET, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_RESET, level);
             break;
         case PPC970_INPUT_TBEN:
             LOG_IRQ("%s: set the TBEN state to %d\n", __func__,
@@ -259,7 +260,7 @@ static void power7_set_irq(void *opaque, int pin, int level)
         /* Level sensitive - active high */
         LOG_IRQ("%s: set the external IRQ state to %d\n",
                 __func__, level);
-        ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+        ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
         break;
     default:
         /* Unknown pin - do nothing */
@@ -319,13 +320,13 @@ static void ppc40x_set_irq(void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the critical IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_CEXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_CEXT, level);
             break;
         case PPC40x_INPUT_INT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the external IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPC40x_INPUT_HALT:
             /* Level sensitive - active low */
@@ -342,7 +343,7 @@ static void ppc40x_set_irq(void *opaque, int pin, int level)
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the debug pin state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_DEBUG, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_DEBUG, level);
             break;
         default:
             /* Unknown pin - do nothing */
@@ -387,26 +388,26 @@ static void ppce500_set_irq(void *opaque, int pin, int level)
         case PPCE500_INPUT_RESET_CORE:
             if (level) {
                 LOG_IRQ("%s: reset the PowerPC core\n", __func__);
-                ppc_set_irq(env, PPC_INTERRUPT_MCK, level);
+                ppc_set_irq(cpu, PPC_INTERRUPT_MCK, level);
             }
             break;
         case PPCE500_INPUT_CINT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the critical IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_CEXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_CEXT, level);
             break;
         case PPCE500_INPUT_INT:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the core IRQ state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_EXT, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_EXT, level);
             break;
         case PPCE500_INPUT_DEBUG:
             /* Level sensitive - active high */
             LOG_IRQ("%s: set the debug pin state to %d\n",
                         __func__, level);
-            ppc_set_irq(env, PPC_INTERRUPT_DEBUG, level);
+            ppc_set_irq(cpu, PPC_INTERRUPT_DEBUG, level);
             break;
         default:
             /* Unknown pin - do nothing */
@@ -427,6 +428,23 @@ void ppce500_irq_init(CPUPPCState *env)
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppce500_set_irq,
                                                   cpu, PPCE500_INPUT_NB);
 }
+
+/* Enable or Disable the E500 EPR capability */
+void ppce500_set_mpic_proxy(bool enabled)
+{
+    CPUPPCState *env;
+
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        PowerPCCPU *cpu = ppc_env_get_cpu(env);
+        CPUState *cs = CPU(cpu);
+
+        env->mpic_proxy = enabled;
+        if (kvm_enabled()) {
+            kvmppc_set_mpic_proxy(POWERPC_CPU(cs), enabled);
+        }
+    }
+}
+
 /*****************************************************************************/
 /* PowerPC time base and decrementer emulation */
 
@@ -643,26 +661,27 @@ uint64_t cpu_ppc_load_purr (CPUPPCState *env)
 /* When decrementer expires,
  * all we need to do is generate or queue a CPU exception
  */
-static inline void cpu_ppc_decr_excp(CPUPPCState *env)
+static inline void cpu_ppc_decr_excp(PowerPCCPU *cpu)
 {
     /* Raise it */
     LOG_TB("raise decrementer exception\n");
-    ppc_set_irq(env, PPC_INTERRUPT_DECR, 1);
+    ppc_set_irq(cpu, PPC_INTERRUPT_DECR, 1);
 }
 
-static inline void cpu_ppc_hdecr_excp(CPUPPCState *env)
+static inline void cpu_ppc_hdecr_excp(PowerPCCPU *cpu)
 {
     /* Raise it */
     LOG_TB("raise decrementer exception\n");
-    ppc_set_irq(env, PPC_INTERRUPT_HDECR, 1);
+    ppc_set_irq(cpu, PPC_INTERRUPT_HDECR, 1);
 }
 
-static void __cpu_ppc_store_decr (CPUPPCState *env, uint64_t *nextp,
-                                  struct QEMUTimer *timer,
-                                  void (*raise_excp)(CPUPPCState *),
-                                  uint32_t decr, uint32_t value,
-                                  int is_excp)
+static void __cpu_ppc_store_decr(PowerPCCPU *cpu, uint64_t *nextp,
+                                 struct QEMUTimer *timer,
+                                 void (*raise_excp)(PowerPCCPU *),
+                                 uint32_t decr, uint32_t value,
+                                 int is_excp)
 {
+    CPUPPCState *env = &cpu->env;
     ppc_tb_t *tb_env = env->tb_env;
     uint64_t now, next;
 
@@ -692,53 +711,61 @@ static void __cpu_ppc_store_decr (CPUPPCState *env, uint64_t *nextp,
     if ((tb_env->flags & PPC_DECR_UNDERFLOW_TRIGGERED)
         && (value & 0x80000000)
         && !(decr & 0x80000000)) {
-        (*raise_excp)(env);
+        (*raise_excp)(cpu);
     }
 }
 
-static inline void _cpu_ppc_store_decr(CPUPPCState *env, uint32_t decr,
+static inline void _cpu_ppc_store_decr(PowerPCCPU *cpu, uint32_t decr,
                                        uint32_t value, int is_excp)
 {
-    ppc_tb_t *tb_env = env->tb_env;
+    ppc_tb_t *tb_env = cpu->env.tb_env;
 
-    __cpu_ppc_store_decr(env, &tb_env->decr_next, tb_env->decr_timer,
+    __cpu_ppc_store_decr(cpu, &tb_env->decr_next, tb_env->decr_timer,
                          &cpu_ppc_decr_excp, decr, value, is_excp);
 }
 
 void cpu_ppc_store_decr (CPUPPCState *env, uint32_t value)
 {
-    _cpu_ppc_store_decr(env, cpu_ppc_load_decr(env), value, 0);
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    _cpu_ppc_store_decr(cpu, cpu_ppc_load_decr(env), value, 0);
 }
 
-static void cpu_ppc_decr_cb (void *opaque)
+static void cpu_ppc_decr_cb(void *opaque)
 {
-    _cpu_ppc_store_decr(opaque, 0x00000000, 0xFFFFFFFF, 1);
+    PowerPCCPU *cpu = opaque;
+
+    _cpu_ppc_store_decr(cpu, 0x00000000, 0xFFFFFFFF, 1);
 }
 
-static inline void _cpu_ppc_store_hdecr(CPUPPCState *env, uint32_t hdecr,
+static inline void _cpu_ppc_store_hdecr(PowerPCCPU *cpu, uint32_t hdecr,
                                         uint32_t value, int is_excp)
 {
-    ppc_tb_t *tb_env = env->tb_env;
+    ppc_tb_t *tb_env = cpu->env.tb_env;
 
     if (tb_env->hdecr_timer != NULL) {
-        __cpu_ppc_store_decr(env, &tb_env->hdecr_next, tb_env->hdecr_timer,
+        __cpu_ppc_store_decr(cpu, &tb_env->hdecr_next, tb_env->hdecr_timer,
                              &cpu_ppc_hdecr_excp, hdecr, value, is_excp);
     }
 }
 
 void cpu_ppc_store_hdecr (CPUPPCState *env, uint32_t value)
 {
-    _cpu_ppc_store_hdecr(env, cpu_ppc_load_hdecr(env), value, 0);
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    _cpu_ppc_store_hdecr(cpu, cpu_ppc_load_hdecr(env), value, 0);
 }
 
-static void cpu_ppc_hdecr_cb (void *opaque)
+static void cpu_ppc_hdecr_cb(void *opaque)
 {
-    _cpu_ppc_store_hdecr(opaque, 0x00000000, 0xFFFFFFFF, 1);
+    PowerPCCPU *cpu = opaque;
+
+    _cpu_ppc_store_hdecr(cpu, 0x00000000, 0xFFFFFFFF, 1);
 }
 
-static void cpu_ppc_store_purr(CPUPPCState *env, uint64_t value)
+static void cpu_ppc_store_purr(PowerPCCPU *cpu, uint64_t value)
 {
-    ppc_tb_t *tb_env = env->tb_env;
+    ppc_tb_t *tb_env = cpu->env.tb_env;
 
     tb_env->purr_load = value;
     tb_env->purr_start = qemu_get_clock_ns(vm_clock);
@@ -747,6 +774,7 @@ static void cpu_ppc_store_purr(CPUPPCState *env, uint64_t value)
 static void cpu_ppc_set_tb_clk (void *opaque, uint32_t freq)
 {
     CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_tb_t *tb_env = env->tb_env;
 
     tb_env->tb_freq = freq;
@@ -755,25 +783,27 @@ static void cpu_ppc_set_tb_clk (void *opaque, uint32_t freq)
      * if a decrementer exception is pending when it enables msr_ee at startup,
      * it's not ready to handle it...
      */
-    _cpu_ppc_store_decr(env, 0xFFFFFFFF, 0xFFFFFFFF, 0);
-    _cpu_ppc_store_hdecr(env, 0xFFFFFFFF, 0xFFFFFFFF, 0);
-    cpu_ppc_store_purr(env, 0x0000000000000000ULL);
+    _cpu_ppc_store_decr(cpu, 0xFFFFFFFF, 0xFFFFFFFF, 0);
+    _cpu_ppc_store_hdecr(cpu, 0xFFFFFFFF, 0xFFFFFFFF, 0);
+    cpu_ppc_store_purr(cpu, 0x0000000000000000ULL);
 }
 
 /* Set up (once) timebase frequency (in Hz) */
 clk_setup_cb cpu_ppc_tb_init (CPUPPCState *env, uint32_t freq)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_tb_t *tb_env;
 
     tb_env = g_malloc0(sizeof(ppc_tb_t));
     env->tb_env = tb_env;
     tb_env->flags = PPC_DECR_UNDERFLOW_TRIGGERED;
     /* Create new timer */
-    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_decr_cb, env);
+    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_decr_cb, cpu);
     if (0) {
         /* XXX: find a suitable condition to enable the hypervisor decrementer
          */
-        tb_env->hdecr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_hdecr_cb, env);
+        tb_env->hdecr_timer = qemu_new_timer_ns(vm_clock, &cpu_ppc_hdecr_cb,
+                                                cpu);
     } else {
         tb_env->hdecr_timer = NULL;
     }
@@ -829,12 +859,14 @@ struct ppc40x_timer_t {
 /* Fixed interval timer */
 static void cpu_4xx_fit_cb (void *opaque)
 {
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     ppc_tb_t *tb_env;
     ppc40x_timer_t *ppc40x_timer;
     uint64_t now, next;
 
     env = opaque;
+    cpu = ppc_env_get_cpu(env);
     tb_env = env->tb_env;
     ppc40x_timer = tb_env->opaque;
     now = qemu_get_clock_ns(vm_clock);
@@ -860,8 +892,9 @@ static void cpu_4xx_fit_cb (void *opaque)
         next++;
     qemu_mod_timer(ppc40x_timer->fit_timer, next);
     env->spr[SPR_40x_TSR] |= 1 << 26;
-    if ((env->spr[SPR_40x_TCR] >> 23) & 0x1)
-        ppc_set_irq(env, PPC_INTERRUPT_FIT, 1);
+    if ((env->spr[SPR_40x_TCR] >> 23) & 0x1) {
+        ppc_set_irq(cpu, PPC_INTERRUPT_FIT, 1);
+    }
     LOG_TB("%s: ir %d TCR " TARGET_FMT_lx " TSR " TARGET_FMT_lx "\n", __func__,
            (int)((env->spr[SPR_40x_TCR] >> 23) & 0x1),
            env->spr[SPR_40x_TCR], env->spr[SPR_40x_TSR]);
@@ -897,16 +930,19 @@ static void start_stop_pit (CPUPPCState *env, ppc_tb_t *tb_env, int is_excp)
 
 static void cpu_4xx_pit_cb (void *opaque)
 {
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     ppc_tb_t *tb_env;
     ppc40x_timer_t *ppc40x_timer;
 
     env = opaque;
+    cpu = ppc_env_get_cpu(env);
     tb_env = env->tb_env;
     ppc40x_timer = tb_env->opaque;
     env->spr[SPR_40x_TSR] |= 1 << 27;
-    if ((env->spr[SPR_40x_TCR] >> 26) & 0x1)
-        ppc_set_irq(env, ppc40x_timer->decr_excp, 1);
+    if ((env->spr[SPR_40x_TCR] >> 26) & 0x1) {
+        ppc_set_irq(cpu, ppc40x_timer->decr_excp, 1);
+    }
     start_stop_pit(env, tb_env, 1);
     LOG_TB("%s: ar %d ir %d TCR " TARGET_FMT_lx " TSR " TARGET_FMT_lx " "
            "%016" PRIx64 "\n", __func__,
@@ -919,12 +955,14 @@ static void cpu_4xx_pit_cb (void *opaque)
 /* Watchdog timer */
 static void cpu_4xx_wdt_cb (void *opaque)
 {
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     ppc_tb_t *tb_env;
     ppc40x_timer_t *ppc40x_timer;
     uint64_t now, next;
 
     env = opaque;
+    cpu = ppc_env_get_cpu(env);
     tb_env = env->tb_env;
     ppc40x_timer = tb_env->opaque;
     now = qemu_get_clock_ns(vm_clock);
@@ -961,8 +999,9 @@ static void cpu_4xx_wdt_cb (void *opaque)
         qemu_mod_timer(ppc40x_timer->wdt_timer, next);
         ppc40x_timer->wdt_next = next;
         env->spr[SPR_40x_TSR] |= 1 << 30;
-        if ((env->spr[SPR_40x_TCR] >> 27) & 0x1)
-            ppc_set_irq(env, PPC_INTERRUPT_WDT, 1);
+        if ((env->spr[SPR_40x_TCR] >> 27) & 0x1) {
+            ppc_set_irq(cpu, PPC_INTERRUPT_WDT, 1);
+        }
         break;
     case 0x3:
         env->spr[SPR_40x_TSR] &= ~0x30000000;
diff --git a/hw/ppc.h b/hw/ppc.h
index 2f3ea27..ee0cd16 100644
--- a/hw/ppc.h
+++ b/hw/ppc.h
@@ -1,4 +1,7 @@
-void ppc_set_irq (CPUPPCState *env, int n_IRQ, int level);
+#ifndef HW_PPC_H
+#define HW_PPC_H 1
+
+void ppc_set_irq(PowerPCCPU *cpu, int n_IRQ, int level);
 
 /* PowerPC hardware exceptions management helpers */
 typedef void (*clk_setup_cb)(void *opaque, uint32_t freq);
@@ -70,6 +73,8 @@ void ppc6xx_irq_init (CPUPPCState *env);
 void ppc970_irq_init (CPUPPCState *env);
 void ppcPOWER7_irq_init (CPUPPCState *env);
 
+void ppce500_set_mpic_proxy(bool enabled);
+
 /* PPC machines for OpenBIOS */
 enum {
     ARCH_PREP = 0,
@@ -89,4 +94,6 @@ enum {
 #define PPC_SERIAL_MM_BAUDBASE 399193
 
 /* ppc_booke.c */
-void ppc_booke_timers_init(CPUPPCState *env, uint32_t freq, uint32_t flags);
+void ppc_booke_timers_init(PowerPCCPU *cpu, uint32_t freq, uint32_t flags);
+
+#endif
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index 8fe2123..f762050 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -2,16 +2,11 @@
 obj-y = ppc.o ppc_booke.o
 # PREP target
 obj-y += mc146818rtc.o
-obj-y += ppc_prep.o
-# OldWorld PowerMac
-obj-y += ppc_oldworld.o
-# NewWorld PowerMac
-obj-y += ppc_newworld.o
 # IBM pSeries (sPAPR)
 obj-$(CONFIG_PSERIES) += spapr.o spapr_hcall.o spapr_rtas.o spapr_vio.o
 obj-$(CONFIG_PSERIES) += xics.o spapr_vty.o spapr_llan.o spapr_vscsi.o
-obj-$(CONFIG_PSERIES) += spapr_pci.o pci-hotplug.o spapr_iommu.o
-obj-$(CONFIG_PSERIES) += spapr_events.o
+obj-$(CONFIG_PSERIES) += spapr_pci.o pci/pci-hotplug.o spapr_iommu.o
+obj-$(CONFIG_PSERIES) += spapr_events.o spapr_nvram.o
 # PowerPC 4xx boards
 obj-y += ppc4xx_devs.o ppc4xx_pci.o ppc405_uc.o ppc405_boards.o
 obj-y += ppc440_bamboo.o
@@ -28,4 +23,11 @@ obj-y += xilinx_ethlite.o
 
 obj-y := $(addprefix ../,$(obj-y))
 
+# PReP
+obj-y += prep.o
+# OldWorld PowerMac
+obj-y += mac_oldworld.o
+# NewWorld PowerMac
+obj-y += mac_newworld.o
+# e500
 obj-$(CONFIG_FDT) += e500.o mpc8544ds.o e500plat.o
diff --git a/hw/ppc/e500-ccsr.h b/hw/ppc/e500-ccsr.h
new file mode 100644
index 0000000..f20f51b
--- /dev/null
+++ b/hw/ppc/e500-ccsr.h
@@ -0,0 +1,17 @@
+#ifndef E500_CCSR_H
+#define E500_CCSR_H
+
+#include "../sysbus.h"
+
+typedef struct PPCE500CCSRState {
+    /*< private >*/
+    SysBusDevice parent;
+    /*< public >*/
+
+    MemoryRegion ccsr_space;
+} PPCE500CCSRState;
+
+#define TYPE_CCSR "e500-ccsr"
+#define CCSR(obj) OBJECT_CHECK(PPCE500CCSRState, (obj), TYPE_CCSR)
+
+#endif /* E500_CCSR_H */
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index 6749fff..b7474c0 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -17,27 +17,31 @@
 #include "config.h"
 #include "qemu-common.h"
 #include "e500.h"
-#include "net.h"
+#include "e500-ccsr.h"
+#include "net/net.h"
+#include "qemu/config-file.h"
 #include "hw/hw.h"
 #include "hw/serial.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/boards.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "hw/openpic.h"
 #include "hw/ppc.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "hw/sysbus.h"
-#include "exec-memory.h"
-#include "host-utils.h"
+#include "exec/address-spaces.h"
+#include "qemu/host-utils.h"
+#include "hw/ppce500_pci.h"
 
 #define BINARY_DEVICE_TREE_FILE    "mpc8544ds.dtb"
 #define UIMAGE_LOAD_BASE           0
 #define DTC_LOAD_PAD               0x1800000
 #define DTC_PAD_MASK               0xFFFFF
+#define DTB_MAX_SIZE               (8 * 1024 * 1024)
 #define INITRD_LOAD_PAD            0x2000000
 #define INITRD_PAD_MASK            0xFFFFFF
 
@@ -46,13 +50,16 @@
 /* TODO: parameterize */
 #define MPC8544_CCSRBAR_BASE       0xE0000000ULL
 #define MPC8544_CCSRBAR_SIZE       0x00100000ULL
-#define MPC8544_MPIC_REGS_BASE     (MPC8544_CCSRBAR_BASE + 0x40000ULL)
-#define MPC8544_SERIAL0_REGS_BASE  (MPC8544_CCSRBAR_BASE + 0x4500ULL)
-#define MPC8544_SERIAL1_REGS_BASE  (MPC8544_CCSRBAR_BASE + 0x4600ULL)
-#define MPC8544_PCI_REGS_BASE      (MPC8544_CCSRBAR_BASE + 0x8000ULL)
+#define MPC8544_MPIC_REGS_OFFSET   0x40000ULL
+#define MPC8544_MSI_REGS_OFFSET   0x41600ULL
+#define MPC8544_SERIAL0_REGS_OFFSET 0x4500ULL
+#define MPC8544_SERIAL1_REGS_OFFSET 0x4600ULL
+#define MPC8544_PCI_REGS_OFFSET    0x8000ULL
+#define MPC8544_PCI_REGS_BASE      (MPC8544_CCSRBAR_BASE + \
+                                    MPC8544_PCI_REGS_OFFSET)
 #define MPC8544_PCI_REGS_SIZE      0x1000ULL
 #define MPC8544_PCI_IO             0xE1000000ULL
-#define MPC8544_UTIL_BASE          (MPC8544_CCSRBAR_BASE + 0xe0000ULL)
+#define MPC8544_UTIL_OFFSET        0xe0000ULL
 #define MPC8544_SPIN_BASE          0xEF000000ULL
 
 struct boot_info
@@ -62,25 +69,35 @@ struct boot_info
     uint32_t entry;
 };
 
-static void pci_map_create(void *fdt, uint32_t *pci_map, uint32_t mpic)
+static uint32_t *pci_map_create(void *fdt, uint32_t mpic, int first_slot,
+                                int nr_slots, int *len)
 {
-    int i;
-    const uint32_t tmp[] = {
-                             /* IDSEL 0x11 J17 Slot 1 */
-                             0x8800, 0x0, 0x0, 0x1, mpic, 0x2, 0x1,
-                             0x8800, 0x0, 0x0, 0x2, mpic, 0x3, 0x1,
-                             0x8800, 0x0, 0x0, 0x3, mpic, 0x4, 0x1,
-                             0x8800, 0x0, 0x0, 0x4, mpic, 0x1, 0x1,
-
-                             /* IDSEL 0x12 J16 Slot 2 */
-                             0x9000, 0x0, 0x0, 0x1, mpic, 0x3, 0x1,
-                             0x9000, 0x0, 0x0, 0x2, mpic, 0x4, 0x1,
-                             0x9000, 0x0, 0x0, 0x3, mpic, 0x2, 0x1,
-                             0x9000, 0x0, 0x0, 0x4, mpic, 0x1, 0x1,
-                           };
-    for (i = 0; i < (7 * 8); i++) {
-        pci_map[i] = cpu_to_be32(tmp[i]);
+    int i = 0;
+    int slot;
+    int pci_irq;
+    int host_irq;
+    int last_slot = first_slot + nr_slots;
+    uint32_t *pci_map;
+
+    *len = nr_slots * 4 * 7 * sizeof(uint32_t);
+    pci_map = g_malloc(*len);
+
+    for (slot = first_slot; slot < last_slot; slot++) {
+        for (pci_irq = 0; pci_irq < 4; pci_irq++) {
+            pci_map[i++] = cpu_to_be32(slot << 11);
+            pci_map[i++] = cpu_to_be32(0x0);
+            pci_map[i++] = cpu_to_be32(0x0);
+            pci_map[i++] = cpu_to_be32(pci_irq + 1);
+            pci_map[i++] = cpu_to_be32(mpic);
+            host_irq = ppce500_pci_map_irq_slot(slot, pci_irq);
+            pci_map[i++] = cpu_to_be32(host_irq + 1);
+            pci_map[i++] = cpu_to_be32(0x1);
+        }
     }
+
+    assert((i * sizeof(uint32_t)) == *len);
+
+    return pci_map;
 }
 
 static void dt_serial_create(void *fdt, unsigned long long offset,
@@ -124,9 +141,12 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     char soc[128];
     char mpic[128];
     uint32_t mpic_ph;
+    uint32_t msi_ph;
     char gutil[128];
     char pci[128];
-    uint32_t pci_map[7 * 8];
+    char msi[128];
+    uint32_t *pci_map = NULL;
+    int len;
     uint32_t pci_ranges[14] =
         {
             0x2000000, 0x0, 0xc0000000,
@@ -206,6 +226,10 @@ static int ppce500_load_device_tree(CPUPPCState *env,
         kvmppc_get_hypercall(env, hypercall, sizeof(hypercall));
         qemu_devtree_setprop(fdt, "/hypervisor", "hcall-instructions",
                              hypercall, sizeof(hypercall));
+        /* if KVM supports the idle hcall, set property indicating this */
+        if (kvmppc_get_hasidle(env)) {
+            qemu_devtree_setprop(fdt, "/hypervisor", "has-idle", NULL, 0);
+        }
     }
 
     /* Create CPU nodes */
@@ -216,25 +240,28 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     /* We need to generate the cpu nodes in reverse order, so Linux can pick
        the first node as boot node and be happy */
     for (i = smp_cpus - 1; i >= 0; i--) {
+        CPUState *cpu = NULL;
         char cpu_name[128];
         uint64_t cpu_release_addr = MPC8544_SPIN_BASE + (i * 0x20);
 
         for (env = first_cpu; env != NULL; env = env->next_cpu) {
-            if (env->cpu_index == i) {
+            cpu = ENV_GET_CPU(env);
+            if (cpu->cpu_index == i) {
                 break;
             }
         }
 
-        if (!env) {
+        if (cpu == NULL) {
             continue;
         }
 
-        snprintf(cpu_name, sizeof(cpu_name), "/cpus/PowerPC,8544@%x", env->cpu_index);
+        snprintf(cpu_name, sizeof(cpu_name), "/cpus/PowerPC,8544@%x",
+                 cpu->cpu_index);
         qemu_devtree_add_subnode(fdt, cpu_name);
         qemu_devtree_setprop_cell(fdt, cpu_name, "clock-frequency", clock_freq);
         qemu_devtree_setprop_cell(fdt, cpu_name, "timebase-frequency", tb_freq);
         qemu_devtree_setprop_string(fdt, cpu_name, "device_type", "cpu");
-        qemu_devtree_setprop_cell(fdt, cpu_name, "reg", env->cpu_index);
+        qemu_devtree_setprop_cell(fdt, cpu_name, "reg", cpu->cpu_index);
         qemu_devtree_setprop_cell(fdt, cpu_name, "d-cache-line-size",
                                   env->dcache_line_size);
         qemu_devtree_setprop_cell(fdt, cpu_name, "i-cache-line-size",
@@ -242,7 +269,7 @@ static int ppce500_load_device_tree(CPUPPCState *env,
         qemu_devtree_setprop_cell(fdt, cpu_name, "d-cache-size", 0x8000);
         qemu_devtree_setprop_cell(fdt, cpu_name, "i-cache-size", 0x8000);
         qemu_devtree_setprop_cell(fdt, cpu_name, "bus-frequency", 0);
-        if (env->cpu_index) {
+        if (cpu->cpu_index) {
             qemu_devtree_setprop_string(fdt, cpu_name, "status", "disabled");
             qemu_devtree_setprop_string(fdt, cpu_name, "enable-method", "spin-table");
             qemu_devtree_setprop_u64(fdt, cpu_name, "cpu-release-addr",
@@ -267,13 +294,12 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     /* XXX should contain a reasonable value */
     qemu_devtree_setprop_cell(fdt, soc, "bus-frequency", 0);
 
-    snprintf(mpic, sizeof(mpic), "%s/pic@%llx", soc,
-             MPC8544_MPIC_REGS_BASE - MPC8544_CCSRBAR_BASE);
+    snprintf(mpic, sizeof(mpic), "%s/pic@%llx", soc, MPC8544_MPIC_REGS_OFFSET);
     qemu_devtree_add_subnode(fdt, mpic);
     qemu_devtree_setprop_string(fdt, mpic, "device_type", "open-pic");
-    qemu_devtree_setprop_string(fdt, mpic, "compatible", "chrp,open-pic");
-    qemu_devtree_setprop_cells(fdt, mpic, "reg", MPC8544_MPIC_REGS_BASE -
-                               MPC8544_CCSRBAR_BASE, 0x40000);
+    qemu_devtree_setprop_string(fdt, mpic, "compatible", "fsl,mpic");
+    qemu_devtree_setprop_cells(fdt, mpic, "reg", MPC8544_MPIC_REGS_OFFSET,
+                               0x40000);
     qemu_devtree_setprop_cell(fdt, mpic, "#address-cells", 0);
     qemu_devtree_setprop_cell(fdt, mpic, "#interrupt-cells", 2);
     mpic_ph = qemu_devtree_alloc_phandle(fdt);
@@ -286,19 +312,37 @@ static int ppce500_load_device_tree(CPUPPCState *env,
      * device it finds in the dt as serial output device. And we generate
      * devices in reverse order to the dt.
      */
-    dt_serial_create(fdt, MPC8544_SERIAL1_REGS_BASE - MPC8544_CCSRBAR_BASE,
+    dt_serial_create(fdt, MPC8544_SERIAL1_REGS_OFFSET,
                      soc, mpic, "serial1", 1, false);
-    dt_serial_create(fdt, MPC8544_SERIAL0_REGS_BASE - MPC8544_CCSRBAR_BASE,
+    dt_serial_create(fdt, MPC8544_SERIAL0_REGS_OFFSET,
                      soc, mpic, "serial0", 0, true);
 
     snprintf(gutil, sizeof(gutil), "%s/global-utilities@%llx", soc,
-             MPC8544_UTIL_BASE - MPC8544_CCSRBAR_BASE);
+             MPC8544_UTIL_OFFSET);
     qemu_devtree_add_subnode(fdt, gutil);
     qemu_devtree_setprop_string(fdt, gutil, "compatible", "fsl,mpc8544-guts");
-    qemu_devtree_setprop_cells(fdt, gutil, "reg", MPC8544_UTIL_BASE -
-                               MPC8544_CCSRBAR_BASE, 0x1000);
+    qemu_devtree_setprop_cells(fdt, gutil, "reg", MPC8544_UTIL_OFFSET, 0x1000);
     qemu_devtree_setprop(fdt, gutil, "fsl,has-rstcr", NULL, 0);
 
+    snprintf(msi, sizeof(msi), "/%s/msi@%llx", soc, MPC8544_MSI_REGS_OFFSET);
+    qemu_devtree_add_subnode(fdt, msi);
+    qemu_devtree_setprop_string(fdt, msi, "compatible", "fsl,mpic-msi");
+    qemu_devtree_setprop_cells(fdt, msi, "reg", MPC8544_MSI_REGS_OFFSET, 0x200);
+    msi_ph = qemu_devtree_alloc_phandle(fdt);
+    qemu_devtree_setprop_cells(fdt, msi, "msi-available-ranges", 0x0, 0x100);
+    qemu_devtree_setprop_phandle(fdt, msi, "interrupt-parent", mpic);
+    qemu_devtree_setprop_cells(fdt, msi, "interrupts",
+        0xe0, 0x0,
+        0xe1, 0x0,
+        0xe2, 0x0,
+        0xe3, 0x0,
+        0xe4, 0x0,
+        0xe5, 0x0,
+        0xe6, 0x0,
+        0xe7, 0x0);
+    qemu_devtree_setprop_cell(fdt, msi, "phandle", msi_ph);
+    qemu_devtree_setprop_cell(fdt, msi, "linux,phandle", msi_ph);
+
     snprintf(pci, sizeof(pci), "/pci@%llx", MPC8544_PCI_REGS_BASE);
     qemu_devtree_add_subnode(fdt, pci);
     qemu_devtree_setprop_cell(fdt, pci, "cell-index", 0);
@@ -306,14 +350,17 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     qemu_devtree_setprop_string(fdt, pci, "device_type", "pci");
     qemu_devtree_setprop_cells(fdt, pci, "interrupt-map-mask", 0xf800, 0x0,
                                0x0, 0x7);
-    pci_map_create(fdt, pci_map, qemu_devtree_get_phandle(fdt, mpic));
-    qemu_devtree_setprop(fdt, pci, "interrupt-map", pci_map, sizeof(pci_map));
+    pci_map = pci_map_create(fdt, qemu_devtree_get_phandle(fdt, mpic),
+                             params->pci_first_slot, params->pci_nr_slots,
+                             &len);
+    qemu_devtree_setprop(fdt, pci, "interrupt-map", pci_map, len);
     qemu_devtree_setprop_phandle(fdt, pci, "interrupt-parent", mpic);
     qemu_devtree_setprop_cells(fdt, pci, "interrupts", 24, 2);
     qemu_devtree_setprop_cells(fdt, pci, "bus-range", 0, 255);
     for (i = 0; i < 14; i++) {
         pci_ranges[i] = cpu_to_be32(pci_ranges[i]);
     }
+    qemu_devtree_setprop_cell(fdt, pci, "fsl,msi", msi_ph);
     qemu_devtree_setprop(fdt, pci, "ranges", pci_ranges, sizeof(pci_ranges));
     qemu_devtree_setprop_cells(fdt, pci, "reg", MPC8544_PCI_REGS_BASE >> 32,
                                MPC8544_PCI_REGS_BASE, 0, 0x1000);
@@ -340,6 +387,7 @@ done:
     ret = fdt_size;
 
 out:
+    g_free(pci_map);
 
     return ret;
 }
@@ -416,12 +464,16 @@ void ppce500_init(PPCE500Params *params)
     target_long kernel_size=0;
     target_ulong dt_base = 0;
     target_ulong initrd_base = 0;
-    target_long initrd_size=0;
-    int i=0;
+    target_long initrd_size = 0;
+    target_ulong cur_base = 0;
+    int i = 0, j, k;
     unsigned int pci_irq_nrs[4] = {1, 2, 3, 4};
     qemu_irq **irqs, *mpic;
     DeviceState *dev;
     CPUPPCState *firstenv = NULL;
+    MemoryRegion *ccsr_addr_space;
+    SysBusDevice *s;
+    PPCE500CCSRState *ccsr;
 
     /* Setup CPUs */
     if (params->cpu_model == NULL) {
@@ -432,6 +484,7 @@ void ppce500_init(PPCE500Params *params)
     irqs[0] = g_malloc0(smp_cpus * sizeof(qemu_irq) * OPENPIC_OUTPUT_NB);
     for (i = 0; i < smp_cpus; i++) {
         PowerPCCPU *cpu;
+        CPUState *cs;
         qemu_irq *input;
 
         cpu = cpu_ppc_init(params->cpu_model);
@@ -440,6 +493,7 @@ void ppce500_init(PPCE500Params *params)
             exit(1);
         }
         env = &cpu->env;
+        cs = CPU(cpu);
 
         if (!firstenv) {
             firstenv = env;
@@ -449,10 +503,11 @@ void ppce500_init(PPCE500Params *params)
         input = (qemu_irq *)env->irq_inputs;
         irqs[i][OPENPIC_OUTPUT_INT] = input[PPCE500_INPUT_INT];
         irqs[i][OPENPIC_OUTPUT_CINT] = input[PPCE500_INPUT_CINT];
-        env->spr[SPR_BOOKE_PIR] = env->cpu_index = i;
-        env->mpic_cpu_base = MPC8544_MPIC_REGS_BASE + 0x20000;
+        env->spr[SPR_BOOKE_PIR] = cs->cpu_index = i;
+        env->mpic_iack = MPC8544_CCSRBAR_BASE +
+                         MPC8544_MPIC_REGS_OFFSET + 0xa0;
 
-        ppc_booke_timers_init(env, 400000000, PPC_TIMER_E500);
+        ppc_booke_timers_init(cpu, 400000000, PPC_TIMER_E500);
 
         /* Register reset handler */
         if (!i) {
@@ -477,40 +532,74 @@ void ppce500_init(PPCE500Params *params)
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
+    dev = qdev_create(NULL, "e500-ccsr");
+    object_property_add_child(qdev_get_machine(), "e500-ccsr",
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+    ccsr = CCSR(dev);
+    ccsr_addr_space = &ccsr->ccsr_space;
+    memory_region_add_subregion(address_space_mem, MPC8544_CCSRBAR_BASE,
+                                ccsr_addr_space);
+
     /* MPIC */
-    mpic = mpic_init(address_space_mem, MPC8544_MPIC_REGS_BASE,
-                     smp_cpus, irqs, NULL);
+    mpic = g_new(qemu_irq, 256);
+    dev = qdev_create(NULL, "openpic");
+    qdev_prop_set_uint32(dev, "nb_cpus", smp_cpus);
+    qdev_prop_set_uint32(dev, "model", params->mpic_version);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+
+    k = 0;
+    for (i = 0; i < smp_cpus; i++) {
+        for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
+            sysbus_connect_irq(s, k++, irqs[i][j]);
+        }
+    }
 
-    if (!mpic) {
-        cpu_abort(env, "MPIC failed to initialize\n");
+    for (i = 0; i < 256; i++) {
+        mpic[i] = qdev_get_gpio_in(dev, i);
     }
 
+    memory_region_add_subregion(ccsr_addr_space, MPC8544_MPIC_REGS_OFFSET,
+                                s->mmio[0].memory);
+
     /* Serial */
     if (serial_hds[0]) {
-        serial_mm_init(address_space_mem, MPC8544_SERIAL0_REGS_BASE,
-                       0, mpic[12+26], 399193,
+        serial_mm_init(ccsr_addr_space, MPC8544_SERIAL0_REGS_OFFSET,
+                       0, mpic[42], 399193,
                        serial_hds[0], DEVICE_BIG_ENDIAN);
     }
 
     if (serial_hds[1]) {
-        serial_mm_init(address_space_mem, MPC8544_SERIAL1_REGS_BASE,
-                       0, mpic[12+26], 399193,
+        serial_mm_init(ccsr_addr_space, MPC8544_SERIAL1_REGS_OFFSET,
+                       0, mpic[42], 399193,
                        serial_hds[1], DEVICE_BIG_ENDIAN);
     }
 
     /* General Utility device */
-    sysbus_create_simple("mpc8544-guts", MPC8544_UTIL_BASE, NULL);
+    dev = qdev_create(NULL, "mpc8544-guts");
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    memory_region_add_subregion(ccsr_addr_space, MPC8544_UTIL_OFFSET,
+                                sysbus_mmio_get_region(s, 0));
 
     /* PCI */
-    dev = sysbus_create_varargs("e500-pcihost", MPC8544_PCI_REGS_BASE,
-                                mpic[pci_irq_nrs[0]], mpic[pci_irq_nrs[1]],
-                                mpic[pci_irq_nrs[2]], mpic[pci_irq_nrs[3]],
-                                NULL);
+    dev = qdev_create(NULL, "e500-pcihost");
+    qdev_prop_set_uint32(dev, "first_slot", params->pci_first_slot);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    sysbus_connect_irq(s, 0, mpic[pci_irq_nrs[0]]);
+    sysbus_connect_irq(s, 1, mpic[pci_irq_nrs[1]]);
+    sysbus_connect_irq(s, 2, mpic[pci_irq_nrs[2]]);
+    sysbus_connect_irq(s, 3, mpic[pci_irq_nrs[3]]);
+    memory_region_add_subregion(ccsr_addr_space, MPC8544_PCI_REGS_OFFSET,
+                                sysbus_mmio_get_region(s, 0));
+
     pci_bus = (PCIBus *)qdev_get_child_bus(dev, "pci.0");
     if (!pci_bus)
         printf("couldn't create PCI controller!\n");
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 1, MPC8544_PCI_IO);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 1, MPC8544_PCI_IO);
 
     if (pci_bus) {
         /* Register network interfaces. */
@@ -539,12 +628,17 @@ void ppce500_init(PPCE500Params *params)
                     params->kernel_filename);
             exit(1);
         }
+
+        cur_base = loadaddr + kernel_size;
+
+        /* Reserve space for dtb */
+        dt_base = (cur_base + DTC_LOAD_PAD) & ~DTC_PAD_MASK;
+        cur_base += DTB_MAX_SIZE;
     }
 
     /* Load initrd. */
     if (params->initrd_filename) {
-        initrd_base = (loadaddr + kernel_size + INITRD_LOAD_PAD) &
-            ~INITRD_PAD_MASK;
+        initrd_base = (cur_base + INITRD_LOAD_PAD) & ~INITRD_PAD_MASK;
         initrd_size = load_image_targphys(params->initrd_filename, initrd_base,
                                           ram_size - initrd_base);
 
@@ -553,6 +647,8 @@ void ppce500_init(PPCE500Params *params)
                     params->initrd_filename);
             exit(1);
         }
+
+        cur_base = initrd_base + initrd_size;
     }
 
     /* If we're loading a kernel directly, we must load the device tree too. */
@@ -560,13 +656,13 @@ void ppce500_init(PPCE500Params *params)
         struct boot_info *boot_info;
         int dt_size;
 
-        dt_base = (loadaddr + kernel_size + DTC_LOAD_PAD) & ~DTC_PAD_MASK;
         dt_size = ppce500_load_device_tree(env, params, dt_base, initrd_base,
                                            initrd_size);
         if (dt_size < 0) {
             fprintf(stderr, "couldn't load device tree\n");
             exit(1);
         }
+        assert(dt_size < DTB_MAX_SIZE);
 
         boot_info = env->load_info;
         boot_info->entry = entry;
@@ -578,3 +674,33 @@ void ppce500_init(PPCE500Params *params)
         kvmppc_init();
     }
 }
+
+static int e500_ccsr_initfn(SysBusDevice *dev)
+{
+    PPCE500CCSRState *ccsr;
+
+    ccsr = CCSR(dev);
+    memory_region_init(&ccsr->ccsr_space, "e500-ccsr",
+                       MPC8544_CCSRBAR_SIZE);
+    return 0;
+}
+
+static void e500_ccsr_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    k->init = e500_ccsr_initfn;
+}
+
+static const TypeInfo e500_ccsr_info = {
+    .name          = TYPE_CCSR,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(PPCE500CCSRState),
+    .class_init    = e500_ccsr_class_init,
+};
+
+static void e500_register_types(void)
+{
+    type_register_static(&e500_ccsr_info);
+}
+
+type_init(e500_register_types)
diff --git a/hw/ppc/e500.h b/hw/ppc/e500.h
index 7ae87f4..226c93d 100644
--- a/hw/ppc/e500.h
+++ b/hw/ppc/e500.h
@@ -9,11 +9,15 @@ typedef struct PPCE500Params {
     const char *kernel_cmdline;
     const char *initrd_filename;
     const char *cpu_model;
+    int pci_first_slot;
+    int pci_nr_slots;
 
     /* e500-specific params */
 
     /* required -- must at least add toplevel board compatible */
     void (*fixup_devtree)(struct PPCE500Params *params, void *fdt);
+
+    int mpic_version;
 } PPCE500Params;
 
 void ppce500_init(PPCE500Params *params);
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index 4cfb940..25ac4b1 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -13,7 +13,9 @@
 #include "qemu-common.h"
 #include "e500.h"
 #include "../boards.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
+#include "hw/pci/pci.h"
+#include "hw/openpic.h"
 
 static void e500plat_fixup_devtree(PPCE500Params *params, void *fdt)
 {
@@ -40,7 +42,10 @@ static void e500plat_init(QEMUMachineInitArgs *args)
         .kernel_cmdline = kernel_cmdline,
         .initrd_filename = initrd_filename,
         .cpu_model = cpu_model,
+        .pci_first_slot = 0x1,
+        .pci_nr_slots = PCI_SLOT_MAX - 1,
         .fixup_devtree = e500plat_fixup_devtree,
+        .mpic_version = OPENPIC_MODEL_FSL_MPIC_42,
     };
 
     ppce500_init(&params);
@@ -50,7 +55,8 @@ static QEMUMachine e500plat_machine = {
     .name = "ppce500",
     .desc = "generic paravirt e500 platform",
     .init = e500plat_init,
-    .max_cpus = 15,
+    .max_cpus = 32,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void e500plat_machine_init(void)
diff --git a/hw/ppc/mac.h b/hw/ppc/mac.h
new file mode 100644
index 0000000..b17107b
--- /dev/null
+++ b/hw/ppc/mac.h
@@ -0,0 +1,181 @@
+/*
+ * QEMU PowerMac emulation shared definitions and prototypes
+ *
+ * Copyright (c) 2004-2007 Fabrice Bellard
+ * Copyright (c) 2007 Jocelyn Mayer
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#if !defined(__PPC_MAC_H__)
+#define __PPC_MAC_H__
+
+#include "exec/memory.h"
+#include "hw/sysbus.h"
+#include "hw/ide/internal.h"
+#include "hw/adb.h"
+
+/* SMP is not enabled, for now */
+#define MAX_CPUS 1
+
+#define BIOS_SIZE     (1024 * 1024)
+#define BIOS_FILENAME "ppc_rom.bin"
+#define NVRAM_SIZE        0x2000
+#define PROM_FILENAME    "openbios-ppc"
+#define PROM_ADDR         0xfff00000
+
+#define KERNEL_LOAD_ADDR 0x01000000
+#define KERNEL_GAP       0x00100000
+
+#define ESCC_CLOCK 3686400
+
+/* Cuda */
+#define TYPE_CUDA "cuda"
+#define CUDA(obj) OBJECT_CHECK(CUDAState, (obj), TYPE_CUDA)
+
+/**
+ * CUDATimer:
+ * @counter_value: counter value at load time
+ */
+typedef struct CUDATimer {
+    int index;
+    uint16_t latch;
+    uint16_t counter_value;
+    int64_t load_time;
+    int64_t next_irq_time;
+    QEMUTimer *timer;
+} CUDATimer;
+
+/**
+ * CUDAState:
+ * @b: B-side data
+ * @a: A-side data
+ * @dirb: B-side direction (1=output)
+ * @dira: A-side direction (1=output)
+ * @sr: Shift register
+ * @acr: Auxiliary control register
+ * @pcr: Peripheral control register
+ * @ifr: Interrupt flag register
+ * @ier: Interrupt enable register
+ * @anh: A-side data, no handshake
+ * @last_b: last value of B register
+ * @last_acr: last value of ACR register
+ */
+typedef struct CUDAState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion mem;
+    /* cuda registers */
+    uint8_t b;
+    uint8_t a;
+    uint8_t dirb;
+    uint8_t dira;
+    uint8_t sr;
+    uint8_t acr;
+    uint8_t pcr;
+    uint8_t ifr;
+    uint8_t ier;
+    uint8_t anh;
+
+    ADBBusState adb_bus;
+    CUDATimer timers[2];
+
+    uint32_t tick_offset;
+
+    uint8_t last_b;
+    uint8_t last_acr;
+
+    int data_in_size;
+    int data_in_index;
+    int data_out_index;
+
+    qemu_irq irq;
+    uint8_t autopoll;
+    uint8_t data_in[128];
+    uint8_t data_out[16];
+    QEMUTimer *adb_poll_timer;
+} CUDAState;
+
+/* MacIO */
+#define TYPE_OLDWORLD_MACIO "macio-oldworld"
+#define TYPE_NEWWORLD_MACIO "macio-newworld"
+
+#define TYPE_MACIO_IDE "macio-ide"
+#define MACIO_IDE(obj) OBJECT_CHECK(MACIOIDEState, (obj), TYPE_MACIO_IDE)
+
+typedef struct MACIOIDEState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    qemu_irq irq;
+    qemu_irq dma_irq;
+
+    MemoryRegion mem;
+    IDEBus bus;
+    BlockDriverAIOCB *aiocb;
+} MACIOIDEState;
+
+void macio_ide_init_drives(MACIOIDEState *ide, DriveInfo **hd_table);
+void macio_ide_register_dma(MACIOIDEState *ide, void *dbdma, int channel);
+
+void macio_init(PCIDevice *dev,
+                MemoryRegion *pic_mem,
+                MemoryRegion *escc_mem);
+
+/* Heathrow PIC */
+qemu_irq *heathrow_pic_init(MemoryRegion **pmem,
+                            int nb_cpus, qemu_irq **irqs);
+
+/* Grackle PCI */
+#define TYPE_GRACKLE_PCI_HOST_BRIDGE "grackle-pcihost"
+PCIBus *pci_grackle_init(uint32_t base, qemu_irq *pic,
+                         MemoryRegion *address_space_mem,
+                         MemoryRegion *address_space_io);
+
+/* UniNorth PCI */
+PCIBus *pci_pmac_init(qemu_irq *pic,
+                      MemoryRegion *address_space_mem,
+                      MemoryRegion *address_space_io);
+PCIBus *pci_pmac_u3_init(qemu_irq *pic,
+                         MemoryRegion *address_space_mem,
+                         MemoryRegion *address_space_io);
+
+/* Mac NVRAM */
+#define TYPE_MACIO_NVRAM "macio-nvram"
+#define MACIO_NVRAM(obj) \
+    OBJECT_CHECK(MacIONVRAMState, (obj), TYPE_MACIO_NVRAM)
+
+typedef struct MacIONVRAMState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    uint32_t size;
+    uint32_t it_shift;
+
+    MemoryRegion mem;
+    uint8_t *data;
+} MacIONVRAMState;
+
+void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
+uint8_t macio_nvram_read(MacIONVRAMState *s, uint32_t addr);
+void macio_nvram_write(MacIONVRAMState *s, uint32_t addr, uint8_t val);
+#endif /* !defined(__PPC_MAC_H__) */
diff --git a/hw/ppc_newworld.c b/hw/ppc/mac_newworld.c
index 664747e..065ea87 100644
--- a/hw/ppc_newworld.c
+++ b/hw/ppc/mac_newworld.c
@@ -46,27 +46,28 @@
  * 0001:05:0c.0 IDE interface [0101]: Broadcom K2 SATA [1166:0240]
  *
  */
-#include "hw.h"
-#include "ppc.h"
-#include "ppc_mac.h"
-#include "adb.h"
-#include "mac_dbdma.h"
-#include "nvram.h"
-#include "pci.h"
-#include "net.h"
-#include "sysemu.h"
-#include "boards.h"
-#include "fw_cfg.h"
-#include "escc.h"
-#include "openpic.h"
-#include "ide.h"
-#include "loader.h"
+#include "hw/hw.h"
+#include "hw/ppc.h"
+#include "hw/ppc/mac.h"
+#include "hw/adb.h"
+#include "hw/mac_dbdma.h"
+#include "hw/nvram.h"
+#include "hw/pci/pci.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "hw/boards.h"
+#include "hw/fw_cfg.h"
+#include "hw/escc.h"
+#include "hw/openpic.h"
+#include "hw/ide.h"
+#include "hw/loader.h"
 #include "elf.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "hw/usb.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
+#include "hw/sysbus.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -141,21 +142,24 @@ static void ppc_core99_init(QEMUMachineInitArgs *args)
     char *filename;
     qemu_irq *pic, **openpic_irqs;
     MemoryRegion *unin_memory = g_new(MemoryRegion, 1);
-    int linux_boot, i;
+    int linux_boot, i, j, k;
     MemoryRegion *ram = g_new(MemoryRegion, 1), *bios = g_new(MemoryRegion, 1);
     hwaddr kernel_base, initrd_base, cmdline_base = 0;
     long kernel_size, initrd_size;
     PCIBus *pci_bus;
+    PCIDevice *macio;
+    MACIOIDEState *macio_ide;
+    BusState *adb_bus;
     MacIONVRAMState *nvr;
     int bios_size;
-    MemoryRegion *pic_mem, *dbdma_mem, *cuda_mem, *escc_mem;
+    MemoryRegion *pic_mem, *escc_mem;
     MemoryRegion *escc_bar = g_new(MemoryRegion, 1);
-    MemoryRegion *ide_mem[3];
     int ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     void *fw_cfg;
-    void *dbdma;
     int machine_arch;
+    SysBusDevice *s;
+    DeviceState *dev;
 
     linux_boot = (kernel_filename != NULL);
 
@@ -320,7 +324,25 @@ static void ppc_core99_init(QEMUMachineInitArgs *args)
             exit(1);
         }
     }
-    pic = openpic_init(&pic_mem, smp_cpus, openpic_irqs, NULL);
+
+    pic = g_new(qemu_irq, 64);
+
+    dev = qdev_create(NULL, "openpic");
+    qdev_prop_set_uint32(dev, "model", OPENPIC_MODEL_RAVEN);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    pic_mem = s->mmio[0].memory;
+    k = 0;
+    for (i = 0; i < smp_cpus; i++) {
+        for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
+            sysbus_connect_irq(s, k++, openpic_irqs[i][j]);
+        }
+    }
+
+    for (i = 0; i < 64; i++) {
+        pic[i] = qdev_get_gpio_in(dev, i);
+    }
+
     if (PPC_INPUT(env) == PPC_FLAGS_INPUT_970) {
         /* 970 gets a U3 bus */
         pci_bus = pci_pmac_u3_init(pic, get_system_memory(), get_system_io());
@@ -341,20 +363,31 @@ static void ppc_core99_init(QEMUMachineInitArgs *args)
         pci_nic_init_nofail(&nd_table[i], "ne2k_pci", NULL);
 
     ide_drive_get(hd, MAX_IDE_BUS);
-    dbdma = DBDMA_init(&dbdma_mem);
 
-    /* We only emulate 2 out of 3 IDE controllers for now */
-    ide_mem[0] = NULL;
-    ide_mem[1] = pmac_ide_init(hd, pic[0x0d], dbdma, 0x16, pic[0x02]);
-    ide_mem[2] = pmac_ide_init(&hd[MAX_IDE_DEVS], pic[0x0e], dbdma, 0x1a, pic[0x02]);
+    macio = pci_create(pci_bus, -1, TYPE_NEWWORLD_MACIO);
+    dev = DEVICE(macio);
+    qdev_connect_gpio_out(dev, 0, pic[0x19]); /* CUDA */
+    qdev_connect_gpio_out(dev, 1, pic[0x0d]); /* IDE */
+    qdev_connect_gpio_out(dev, 2, pic[0x02]); /* IDE DMA */
+    qdev_connect_gpio_out(dev, 3, pic[0x0e]); /* IDE */
+    qdev_connect_gpio_out(dev, 4, pic[0x02]); /* IDE DMA */
+    macio_init(macio, pic_mem, escc_bar);
 
-    cuda_init(&cuda_mem, pic[0x19]);
+    /* We only emulate 2 out of 3 IDE controllers for now */
+    macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
+                                                        "ide[0]"));
+    macio_ide_init_drives(macio_ide, hd);
 
-    adb_kbd_init(&adb_bus);
-    adb_mouse_init(&adb_bus);
+    macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
+                                                        "ide[1]"));
+    macio_ide_init_drives(macio_ide, &hd[MAX_IDE_DEVS]);
 
-    macio_init(pci_bus, PCI_DEVICE_ID_APPLE_UNI_N_KEYL, 0, pic_mem,
-               dbdma_mem, cuda_mem, NULL, 3, ide_mem, escc_bar);
+    dev = DEVICE(object_resolve_path_component(OBJECT(macio), "cuda"));
+    adb_bus = qdev_get_child_bus(dev, "adb.0");
+    dev = qdev_create(adb_bus, TYPE_ADB_KEYBOARD);
+    qdev_init_nofail(dev);
+    dev = qdev_create(adb_bus, TYPE_ADB_MOUSE);
+    qdev_init_nofail(dev);
 
     if (usb_enabled(machine_arch == ARCH_MAC99_U3)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
@@ -370,12 +403,17 @@ static void ppc_core99_init(QEMUMachineInitArgs *args)
         graphic_depth = 15;
 
     /* The NewWorld NVRAM is not located in the MacIO device */
-    nvr = macio_nvram_init(0x2000, 1);
+    dev = qdev_create(NULL, TYPE_MACIO_NVRAM);
+    qdev_prop_set_uint32(dev, "size", 0x2000);
+    qdev_prop_set_uint32(dev, "it_shift", 1);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xFFF04000);
+    nvr = MACIO_NVRAM(dev);
     pmac_format_nvram_partition(nvr, 0x2000);
-    macio_nvram_setup_bar(nvr, get_system_memory(), 0xFFF04000);
     /* No PCI init: the BIOS will do it */
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, machine_arch);
@@ -421,6 +459,7 @@ static QEMUMachine core99_machine = {
 #ifdef TARGET_PPC64
     .is_default = 1,
 #endif
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void core99_machine_init(void)
diff --git a/hw/ppc_oldworld.c b/hw/ppc/mac_oldworld.c
index e8138c0..2778e45 100644
--- a/hw/ppc_oldworld.c
+++ b/hw/ppc/mac_oldworld.c
@@ -23,26 +23,25 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
-#include "ppc.h"
-#include "ppc_mac.h"
-#include "adb.h"
-#include "mac_dbdma.h"
-#include "nvram.h"
-#include "sysemu.h"
-#include "net.h"
-#include "isa.h"
-#include "pci.h"
-#include "boards.h"
-#include "fw_cfg.h"
-#include "escc.h"
-#include "ide.h"
-#include "loader.h"
+#include "hw/hw.h"
+#include "hw/ppc.h"
+#include "mac.h"
+#include "hw/adb.h"
+#include "hw/nvram.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
+#include "hw/isa.h"
+#include "hw/pci/pci.h"
+#include "hw/boards.h"
+#include "hw/fw_cfg.h"
+#include "hw/escc.h"
+#include "hw/ide.h"
+#include "hw/loader.h"
 #include "elf.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -90,14 +89,16 @@ static void ppc_heathrow_init(QEMUMachineInitArgs *args)
     uint32_t kernel_base, initrd_base, cmdline_base = 0;
     int32_t kernel_size, initrd_size;
     PCIBus *pci_bus;
-    MacIONVRAMState *nvr;
+    PCIDevice *macio;
+    MACIOIDEState *macio_ide;
+    DeviceState *dev;
+    BusState *adb_bus;
     int bios_size;
-    MemoryRegion *pic_mem, *dbdma_mem, *cuda_mem;
-    MemoryRegion *escc_mem, *escc_bar = g_new(MemoryRegion, 1), *ide_mem[2];
+    MemoryRegion *pic_mem;
+    MemoryRegion *escc_mem, *escc_bar = g_new(MemoryRegion, 1);
     uint16_t ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     void *fw_cfg;
-    void *dbdma;
 
     linux_boot = (kernel_filename != NULL);
 
@@ -263,10 +264,17 @@ static void ppc_heathrow_init(QEMUMachineInitArgs *args)
 
     ide_drive_get(hd, MAX_IDE_BUS);
 
+    macio = pci_create(pci_bus, -1, TYPE_OLDWORLD_MACIO);
+    dev = DEVICE(macio);
+    qdev_connect_gpio_out(dev, 0, pic[0x12]); /* CUDA */
+    qdev_connect_gpio_out(dev, 1, pic[0x0D]); /* IDE */
+    qdev_connect_gpio_out(dev, 2, pic[0x02]); /* IDE DMA */
+    macio_init(macio, pic_mem, escc_bar);
+
     /* First IDE channel is a MAC IDE on the MacIO bus */
-    dbdma = DBDMA_init(&dbdma_mem);
-    ide_mem[0] = NULL;
-    ide_mem[1] = pmac_ide_init(hd, pic[0x0D], dbdma, 0x16, pic[0x02]);
+    macio_ide = MACIO_IDE(object_resolve_path_component(OBJECT(macio),
+                                                        "ide"));
+    macio_ide_init_drives(macio_ide, hd);
 
     /* Second IDE channel is a CMD646 on the PCI bus */
     hd[0] = hd[MAX_IDE_DEVS];
@@ -274,17 +282,12 @@ static void ppc_heathrow_init(QEMUMachineInitArgs *args)
     hd[3] = hd[2] = NULL;
     pci_cmd646_ide_init(pci_bus, hd, 0);
 
-    /* cuda also initialize ADB */
-    cuda_init(&cuda_mem, pic[0x12]);
-
-    adb_kbd_init(&adb_bus);
-    adb_mouse_init(&adb_bus);
-
-    nvr = macio_nvram_init(0x2000, 4);
-    pmac_format_nvram_partition(nvr, 0x2000);
-
-    macio_init(pci_bus, PCI_DEVICE_ID_APPLE_343S1201, 1, pic_mem,
-               dbdma_mem, cuda_mem, nvr, 2, ide_mem, escc_bar);
+    dev = DEVICE(object_resolve_path_component(OBJECT(macio), "cuda"));
+    adb_bus = qdev_get_child_bus(dev, "adb.0");
+    dev = qdev_create(adb_bus, TYPE_ADB_KEYBOARD);
+    qdev_init_nofail(dev);
+    dev = qdev_create(adb_bus, TYPE_ADB_MOUSE);
+    qdev_init_nofail(dev);
 
     if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
@@ -296,6 +299,7 @@ static void ppc_heathrow_init(QEMUMachineInitArgs *args)
     /* No PCI init: the BIOS will do it */
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, ARCH_HEATHROW);
@@ -341,6 +345,7 @@ static QEMUMachine heathrow_machine = {
 #ifndef TARGET_PPC64
     .is_default = 1,
 #endif
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void heathrow_machine_init(void)
diff --git a/hw/ppc/mpc8544ds.c b/hw/ppc/mpc8544ds.c
index e651661..e25c70b 100644
--- a/hw/ppc/mpc8544ds.c
+++ b/hw/ppc/mpc8544ds.c
@@ -13,7 +13,8 @@
 #include "qemu-common.h"
 #include "e500.h"
 #include "../boards.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
+#include "hw/openpic.h"
 
 static void mpc8544ds_fixup_devtree(PPCE500Params *params, void *fdt)
 {
@@ -40,7 +41,10 @@ static void mpc8544ds_init(QEMUMachineInitArgs *args)
         .kernel_cmdline = kernel_cmdline,
         .initrd_filename = initrd_filename,
         .cpu_model = cpu_model,
+        .pci_first_slot = 0x11,
+        .pci_nr_slots = 2,
         .fixup_devtree = mpc8544ds_fixup_devtree,
+        .mpic_version = OPENPIC_MODEL_FSL_MPIC_20,
     };
 
     ppce500_init(&params);
@@ -52,6 +56,7 @@ static QEMUMachine ppce500_machine = {
     .desc = "mpc8544ds",
     .init = mpc8544ds_init,
     .max_cpus = 15,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void ppce500_machine_init(void)
diff --git a/hw/ppc_prep.c b/hw/ppc/prep.c
index bf15730..e06dded 100644
--- a/hw/ppc_prep.c
+++ b/hw/ppc/prep.c
@@ -21,25 +21,26 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
-#include "nvram.h"
-#include "pc.h"
-#include "serial.h"
-#include "fdc.h"
-#include "net.h"
-#include "sysemu.h"
-#include "isa.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "ppc.h"
-#include "boards.h"
-#include "qemu-log.h"
-#include "ide.h"
-#include "loader.h"
-#include "mc146818rtc.h"
-#include "blockdev.h"
-#include "arch_init.h"
-#include "exec-memory.h"
+#include "hw/hw.h"
+#include "hw/nvram.h"
+#include "hw/pc.h"
+#include "hw/serial.h"
+#include "hw/fdc.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "hw/isa.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_host.h"
+#include "hw/ppc.h"
+#include "hw/boards.h"
+#include "qemu/log.h"
+#include "hw/ide.h"
+#include "hw/loader.h"
+#include "hw/mc146818rtc.h"
+#include "hw/pc87312.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/arch_init.h"
+#include "exec/address-spaces.h"
 
 //#define HARD_DEBUG_PPC_IO
 //#define DEBUG_PPC_IO
@@ -181,7 +182,6 @@ typedef struct sysctrl_t {
     M48t59State *nvram;
     uint8_t state;
     uint8_t syscontrol;
-    uint8_t fake_io[2];
     int contiguous_map;
     int endian;
 } sysctrl_t;
@@ -192,24 +192,6 @@ enum {
 
 static sysctrl_t *sysctrl;
 
-static void PREP_io_write (void *opaque, uint32_t addr, uint32_t val)
-{
-    sysctrl_t *sysctrl = opaque;
-
-    PPC_IO_DPRINTF("0x%08" PRIx32 " => 0x%02" PRIx32 "\n", addr - PPC_IO_BASE,
-                   val);
-    sysctrl->fake_io[addr - 0x0398] = val;
-}
-
-static uint32_t PREP_io_read (void *opaque, uint32_t addr)
-{
-    sysctrl_t *sysctrl = opaque;
-
-    PPC_IO_DPRINTF("0x%08" PRIx32 " <= 0x%02" PRIx32 "\n", addr - PPC_IO_BASE,
-                   sysctrl->fake_io[addr - 0x0398]);
-    return sysctrl->fake_io[addr - 0x0398];
-}
-
 static void PREP_io_800_writeb (void *opaque, uint32_t addr, uint32_t val)
 {
     sysctrl_t *sysctrl = opaque;
@@ -476,10 +458,10 @@ static void ppc_prep_init(QEMUMachineInitArgs *args)
     PCIBus *pci_bus;
     PCIDevice *pci;
     ISABus *isa_bus;
+    ISADevice *isa;
     qemu_irq *cpu_exit_irq;
     int ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
-    DriveInfo *fd[MAX_FD];
 
     sysctrl = g_malloc0(sizeof(sysctrl_t));
 
@@ -606,6 +588,11 @@ static void ppc_prep_init(QEMUMachineInitArgs *args)
     sysbus_connect_irq(&pcihost->busdev, 3, qdev_get_gpio_in(&pci->qdev, 11));
     isa_bus = DO_UPCAST(ISABus, qbus, qdev_get_child_bus(&pci->qdev, "isa.0"));
 
+    /* Super I/O (parallel + serial ports) */
+    isa = isa_create(isa_bus, TYPE_PC87312);
+    qdev_prop_set_uint8(&isa->qdev, "config", 13); /* fdc, ser0, ser1, par0 */
+    qdev_init_nofail(&isa->qdev);
+
     /* Register 8 MB of ISA IO space (needed for non-contiguous map) */
     memory_region_init_io(PPC_io_memory, &PPC_prep_io_ops, sysctrl,
                           "ppc-io", 0x00800000);
@@ -614,8 +601,6 @@ static void ppc_prep_init(QEMUMachineInitArgs *args)
     /* init basic PC hardware */
     pci_vga_init(pci_bus);
 
-    if (serial_hds[0])
-        serial_isa_init(isa_bus, 0, serial_hds[0]);
     nb_nics1 = nb_nics;
     if (nb_nics1 > NE2000_NB_MAX)
         nb_nics1 = NE2000_NB_MAX;
@@ -639,17 +624,7 @@ static void ppc_prep_init(QEMUMachineInitArgs *args)
     }
     isa_create_simple(isa_bus, "i8042");
 
-    //    SB16_init();
-
-    for(i = 0; i < MAX_FD; i++) {
-        fd[i] = drive_get(IF_FLOPPY, 0, i);
-    }
-    fdctrl_init_isa(isa_bus, fd);
-
-    /* Register fake IO ports for PREP */
     sysctrl->reset_irq = first_cpu->irq_inputs[PPC6xx_INPUT_HRESET];
-    register_ioport_read(0x398, 2, 1, &PREP_io_read, sysctrl);
-    register_ioport_write(0x398, 2, 1, &PREP_io_write, sysctrl);
     /* System control ports */
     register_ioport_read(0x0092, 0x01, 1, &PREP_io_800_readb, sysctrl);
     register_ioport_write(0x0092, 0x01, 1, &PREP_io_800_writeb, sysctrl);
@@ -694,6 +669,7 @@ static QEMUMachine prep_machine = {
     .desc = "PowerPC PREP platform",
     .init = ppc_prep_init,
     .max_cpus = MAX_CPUS,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void prep_machine_init(void)
diff --git a/hw/ppc405_boards.c b/hw/ppc405_boards.c
index 8dc693f..cf371db 100644
--- a/hw/ppc405_boards.c
+++ b/hw/ppc405_boards.c
@@ -26,13 +26,13 @@
 #include "ppc405.h"
 #include "nvram.h"
 #include "flash.h"
-#include "sysemu.h"
-#include "block.h"
+#include "sysemu/sysemu.h"
+#include "block/block.h"
 #include "boards.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define BIOS_FILENAME "ppc405_rom.bin"
 #define BIOS_SIZE (2048 * 1024)
@@ -362,6 +362,7 @@ static QEMUMachine ref405ep_machine = {
     .name = "ref405ep",
     .desc = "ref405ep",
     .init = ref405ep_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 /*****************************************************************************/
@@ -649,6 +650,7 @@ static QEMUMachine taihu_machine = {
     .name = "taihu",
     .desc = "taihu",
     .init = taihu_405ep_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void ppc405_machine_init(void)
diff --git a/hw/ppc405_uc.c b/hw/ppc405_uc.c
index 0f458ef..c96d103 100644
--- a/hw/ppc405_uc.c
+++ b/hw/ppc405_uc.c
@@ -25,10 +25,10 @@
 #include "ppc.h"
 #include "ppc405.h"
 #include "serial.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "qemu-log.h"
-#include "exec-memory.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
 
 #define DEBUG_OPBA
 #define DEBUG_SDRAM
@@ -2111,12 +2111,14 @@ CPUPPCState *ppc405cr_init(MemoryRegion *address_space_mem,
 {
     clk_setup_t clk_setup[PPC405CR_CLK_NB];
     qemu_irq dma_irqs[4];
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     qemu_irq *pic, *irqs;
 
     memset(clk_setup, 0, sizeof(clk_setup));
-    env = ppc4xx_init("405cr", &clk_setup[PPC405CR_CPU_CLK],
+    cpu = ppc4xx_init("405cr", &clk_setup[PPC405CR_CPU_CLK],
                       &clk_setup[PPC405CR_TMR_CLK], sysclk);
+    env = &cpu->env;
     /* Memory mapped devices registers */
     /* PLB arbitrer */
     ppc4xx_plb_init(env);
@@ -2460,13 +2462,15 @@ CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
 {
     clk_setup_t clk_setup[PPC405EP_CLK_NB], tlb_clk_setup;
     qemu_irq dma_irqs[4], gpt_irqs[5], mal_irqs[4];
+    PowerPCCPU *cpu;
     CPUPPCState *env;
     qemu_irq *pic, *irqs;
 
     memset(clk_setup, 0, sizeof(clk_setup));
     /* init CPUs */
-    env = ppc4xx_init("405ep", &clk_setup[PPC405EP_CPU_CLK],
+    cpu = ppc4xx_init("405ep", &clk_setup[PPC405EP_CPU_CLK],
                       &tlb_clk_setup, sysclk);
+    env = &cpu->env;
     clk_setup[PPC405EP_CPU_CLK].cb = tlb_clk_setup.cb;
     clk_setup[PPC405EP_CPU_CLK].opaque = tlb_clk_setup.opaque;
     /* Internal devices init */
@@ -2478,7 +2482,7 @@ CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
     /* OBP arbitrer */
     ppc4xx_opba_init(0xef600600);
     /* Initialize timers */
-    ppc_booke_timers_init(env, sysclk, 0);
+    ppc_booke_timers_init(cpu, sysclk, 0);
     /* Universal interrupt controller */
     irqs = g_malloc0(sizeof(qemu_irq) * PPCUIC_OUTPUT_NB);
     irqs[PPCUIC_OUTPUT_INT] =
diff --git a/hw/ppc440_bamboo.c b/hw/ppc440_bamboo.c
index cc85607..73b5ac7 100644
--- a/hw/ppc440_bamboo.c
+++ b/hw/ppc440_bamboo.c
@@ -13,20 +13,20 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "net.h"
+#include "net/net.h"
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "boards.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "loader.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "serial.h"
 #include "ppc.h"
 #include "ppc405.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 
 #define BINARY_DEVICE_TREE_FILE "bamboo.dtb"
@@ -195,7 +195,7 @@ static void bamboo_init(QEMUMachineInitArgs *args)
     env = &cpu->env;
 
     qemu_register_reset(main_cpu_reset, cpu);
-    ppc_booke_timers_init(env, 400000000, 0);
+    ppc_booke_timers_init(cpu, 400000000, 0);
     ppc_dcr_init(env, NULL, NULL);
 
     /* interrupt controller */
@@ -295,6 +295,7 @@ static QEMUMachine bamboo_machine = {
     .name = "bamboo",
     .desc = "bamboo",
     .init = bamboo_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void bamboo_machine_init(void)
diff --git a/hw/ppc4xx.h b/hw/ppc4xx.h
index d795ced..59dba9e 100644
--- a/hw/ppc4xx.h
+++ b/hw/ppc4xx.h
@@ -25,12 +25,12 @@
 #if !defined(PPC_4XX_H)
 #define PPC_4XX_H
 
-#include "pci.h"
+#include "pci/pci.h"
 
 /* PowerPC 4xx core initialization */
-CPUPPCState *ppc4xx_init (const char *cpu_model,
-                       clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
-                       uint32_t sysclk);
+PowerPCCPU *ppc4xx_init(const char *cpu_model,
+                        clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
+                        uint32_t sysclk);
 
 /* PowerPC 4xx universal interrupt controller */
 enum {
diff --git a/hw/ppc4xx_devs.c b/hw/ppc4xx_devs.c
index bac8d87..5e491bc 100644
--- a/hw/ppc4xx_devs.c
+++ b/hw/ppc4xx_devs.c
@@ -24,8 +24,8 @@
 #include "hw.h"
 #include "ppc.h"
 #include "ppc4xx.h"
-#include "qemu-log.h"
-#include "exec-memory.h"
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_MMIO
 //#define DEBUG_UNASSIGNED
@@ -47,9 +47,9 @@ static void ppc4xx_reset(void *opaque)
 
 /*****************************************************************************/
 /* Generic PowerPC 4xx processor instantiation */
-CPUPPCState *ppc4xx_init (const char *cpu_model,
-                       clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
-                       uint32_t sysclk)
+PowerPCCPU *ppc4xx_init(const char *cpu_model,
+                        clk_setup_t *cpu_clk, clk_setup_t *tb_clk,
+                        uint32_t sysclk)
 {
     PowerPCCPU *cpu;
     CPUPPCState *env;
@@ -72,7 +72,7 @@ CPUPPCState *ppc4xx_init (const char *cpu_model,
     /* Register qemu callbacks */
     qemu_register_reset(ppc4xx_reset, cpu);
 
-    return env;
+    return cpu;
 }
 
 /*****************************************************************************/
diff --git a/hw/ppc4xx_pci.c b/hw/ppc4xx_pci.c
index d3ad6a0..ba2d669 100644
--- a/hw/ppc4xx_pci.c
+++ b/hw/ppc4xx_pci.c
@@ -22,9 +22,9 @@
 #include "hw.h"
 #include "ppc.h"
 #include "ppc4xx.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "exec-memory.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "exec/address-spaces.h"
 
 #undef DEBUG
 #ifdef DEBUG
diff --git a/hw/ppc_booke.c b/hw/ppc_booke.c
index d51e7fa..25a4e91 100644
--- a/hw/ppc_booke.c
+++ b/hw/ppc_booke.c
@@ -23,10 +23,10 @@
  */
 #include "hw.h"
 #include "ppc.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "nvram.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "loader.h"
 
 
@@ -71,17 +71,19 @@ struct booke_timer_t {
     uint32_t flags;
 };
 
-static void booke_update_irq(CPUPPCState *env)
+static void booke_update_irq(PowerPCCPU *cpu)
 {
-    ppc_set_irq(env, PPC_INTERRUPT_DECR,
+    CPUPPCState *env = &cpu->env;
+
+    ppc_set_irq(cpu, PPC_INTERRUPT_DECR,
                 (env->spr[SPR_BOOKE_TSR] & TSR_DIS
                  && env->spr[SPR_BOOKE_TCR] & TCR_DIE));
 
-    ppc_set_irq(env, PPC_INTERRUPT_WDT,
+    ppc_set_irq(cpu, PPC_INTERRUPT_WDT,
                 (env->spr[SPR_BOOKE_TSR] & TSR_WIS
                  && env->spr[SPR_BOOKE_TCR] & TCR_WIE));
 
-    ppc_set_irq(env, PPC_INTERRUPT_FIT,
+    ppc_set_irq(cpu, PPC_INTERRUPT_FIT,
                 (env->spr[SPR_BOOKE_TSR] & TSR_FIS
                  && env->spr[SPR_BOOKE_TCR] & TCR_FIE));
 }
@@ -153,10 +155,11 @@ static void booke_update_fixed_timer(CPUPPCState         *env,
 
 static void booke_decr_cb(void *opaque)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
 
     env->spr[SPR_BOOKE_TSR] |= TSR_DIS;
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     if (env->spr[SPR_BOOKE_TCR] & TCR_ARE) {
         /* Auto Reload */
@@ -166,16 +169,16 @@ static void booke_decr_cb(void *opaque)
 
 static void booke_fit_cb(void *opaque)
 {
-    CPUPPCState *env;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     ppc_tb_t *tb_env;
     booke_timer_t *booke_timer;
 
-    env = opaque;
     tb_env = env->tb_env;
     booke_timer = tb_env->opaque;
     env->spr[SPR_BOOKE_TSR] |= TSR_FIS;
 
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     booke_update_fixed_timer(env,
                              booke_get_fit_target(env, tb_env),
@@ -185,17 +188,17 @@ static void booke_fit_cb(void *opaque)
 
 static void booke_wdt_cb(void *opaque)
 {
-    CPUPPCState *env;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     ppc_tb_t *tb_env;
     booke_timer_t *booke_timer;
 
-    env = opaque;
     tb_env = env->tb_env;
     booke_timer = tb_env->opaque;
 
     /* TODO: There's lots of complicated stuff to do here */
 
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     booke_update_fixed_timer(env,
                              booke_get_wdt_target(env, tb_env),
@@ -205,19 +208,22 @@ static void booke_wdt_cb(void *opaque)
 
 void store_booke_tsr(CPUPPCState *env, target_ulong val)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     env->spr[SPR_BOOKE_TSR] &= ~val;
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 }
 
 void store_booke_tcr(CPUPPCState *env, target_ulong val)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_tb_t *tb_env = env->tb_env;
     booke_timer_t *booke_timer = tb_env->opaque;
 
     tb_env = env->tb_env;
     env->spr[SPR_BOOKE_TCR] = val;
 
-    booke_update_irq(env);
+    booke_update_irq(cpu);
 
     booke_update_fixed_timer(env,
                              booke_get_fit_target(env, tb_env),
@@ -231,7 +237,18 @@ void store_booke_tcr(CPUPPCState *env, target_ulong val)
 
 }
 
-void ppc_booke_timers_init(CPUPPCState *env, uint32_t freq, uint32_t flags)
+static void ppc_booke_timer_reset_handle(void *opaque)
+{
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
+
+    env->spr[SPR_BOOKE_TSR] = 0;
+    env->spr[SPR_BOOKE_TCR] = 0;
+
+    booke_update_irq(cpu);
+}
+
+void ppc_booke_timers_init(PowerPCCPU *cpu, uint32_t freq, uint32_t flags)
 {
     ppc_tb_t *tb_env;
     booke_timer_t *booke_timer;
@@ -239,16 +256,18 @@ void ppc_booke_timers_init(CPUPPCState *env, uint32_t freq, uint32_t flags)
     tb_env      = g_malloc0(sizeof(ppc_tb_t));
     booke_timer = g_malloc0(sizeof(booke_timer_t));
 
-    env->tb_env = tb_env;
+    cpu->env.tb_env = tb_env;
     tb_env->flags = flags | PPC_TIMER_BOOKE | PPC_DECR_ZERO_TRIGGERED;
 
     tb_env->tb_freq    = freq;
     tb_env->decr_freq  = freq;
     tb_env->opaque     = booke_timer;
-    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &booke_decr_cb, env);
+    tb_env->decr_timer = qemu_new_timer_ns(vm_clock, &booke_decr_cb, cpu);
 
     booke_timer->fit_timer =
-        qemu_new_timer_ns(vm_clock, &booke_fit_cb, env);
+        qemu_new_timer_ns(vm_clock, &booke_fit_cb, cpu);
     booke_timer->wdt_timer =
-        qemu_new_timer_ns(vm_clock, &booke_wdt_cb, env);
+        qemu_new_timer_ns(vm_clock, &booke_wdt_cb, cpu);
+
+    qemu_register_reset(ppc_booke_timer_reset_handle, cpu);
 }
diff --git a/hw/ppc_mac.h b/hw/ppc_mac.h
deleted file mode 100644
index 524b236..0000000
--- a/hw/ppc_mac.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * QEMU PowerMac emulation shared definitions and prototypes
- *
- * Copyright (c) 2004-2007 Fabrice Bellard
- * Copyright (c) 2007 Jocelyn Mayer
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#if !defined(__PPC_MAC_H__)
-#define __PPC_MAC_H__
-
-#include "memory.h"
-
-/* SMP is not enabled, for now */
-#define MAX_CPUS 1
-
-#define BIOS_SIZE     (1024 * 1024)
-#define BIOS_FILENAME "ppc_rom.bin"
-#define NVRAM_SIZE        0x2000
-#define PROM_FILENAME    "openbios-ppc"
-#define PROM_ADDR         0xfff00000
-
-#define KERNEL_LOAD_ADDR 0x01000000
-#define KERNEL_GAP       0x00100000
-
-#define ESCC_CLOCK 3686400
-
-/* Cuda */
-void cuda_init (MemoryRegion **cuda_mem, qemu_irq irq);
-
-/* MacIO */
-void macio_init (PCIBus *bus, int device_id, int is_oldworld,
-                 MemoryRegion *pic_mem, MemoryRegion *dbdma_mem,
-                 MemoryRegion *cuda_mem, void *nvram,
-                 int nb_ide, MemoryRegion **ide_mem, MemoryRegion *escc_mem);
-
-/* Heathrow PIC */
-qemu_irq *heathrow_pic_init(MemoryRegion **pmem,
-                            int nb_cpus, qemu_irq **irqs);
-
-/* Grackle PCI */
-#define TYPE_GRACKLE_PCI_HOST_BRIDGE "grackle-pcihost"
-PCIBus *pci_grackle_init(uint32_t base, qemu_irq *pic,
-                         MemoryRegion *address_space_mem,
-                         MemoryRegion *address_space_io);
-
-/* UniNorth PCI */
-PCIBus *pci_pmac_init(qemu_irq *pic,
-                      MemoryRegion *address_space_mem,
-                      MemoryRegion *address_space_io);
-PCIBus *pci_pmac_u3_init(qemu_irq *pic,
-                         MemoryRegion *address_space_mem,
-                         MemoryRegion *address_space_io);
-
-/* Mac NVRAM */
-typedef struct MacIONVRAMState MacIONVRAMState;
-
-MacIONVRAMState *macio_nvram_init (hwaddr size,
-                                   unsigned int it_shift);
-void macio_nvram_setup_bar(MacIONVRAMState *s, MemoryRegion *bar,
-                           hwaddr mem_base);
-void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
-uint32_t macio_nvram_read (void *opaque, uint32_t addr);
-void macio_nvram_write (void *opaque, uint32_t addr, uint32_t val);
-#endif /* !defined(__PPC_MAC_H__) */
diff --git a/hw/ppce500_pci.c b/hw/ppce500_pci.c
index 2ff7438..1e1ade3 100644
--- a/hw/ppce500_pci.c
+++ b/hw/ppce500_pci.c
@@ -15,9 +15,11 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "bswap.h"
+#include "hw/ppc/e500-ccsr.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "qemu/bswap.h"
+#include "ppce500_pci.h"
 
 #ifdef DEBUG_PCI
 #define pci_debug(fmt, ...) fprintf(stderr, fmt, ## __VA_ARGS__)
@@ -86,12 +88,26 @@ struct PPCE500PCIState {
     struct pci_inbound pib[PPCE500_PCI_NR_PIBS];
     uint32_t gasket_time;
     qemu_irq irq[4];
+    uint32_t first_slot;
     /* mmio maps */
     MemoryRegion container;
     MemoryRegion iomem;
     MemoryRegion pio;
 };
 
+#define TYPE_PPC_E500_PCI_BRIDGE "e500-host-bridge"
+#define PPC_E500_PCI_BRIDGE(obj) \
+    OBJECT_CHECK(PPCE500PCIBridgeState, (obj), TYPE_PPC_E500_PCI_BRIDGE)
+
+struct PPCE500PCIBridgeState {
+    /*< private >*/
+    PCIDevice parent;
+    /*< public >*/
+
+    MemoryRegion bar0;
+};
+
+typedef struct PPCE500PCIBridgeState PPCE500PCIBridgeState;
 typedef struct PPCE500PCIState PPCE500PCIState;
 
 static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
@@ -238,17 +254,10 @@ static const MemoryRegionOps e500_pci_reg_ops = {
 
 static int mpc85xx_pci_map_irq(PCIDevice *pci_dev, int irq_num)
 {
-    int devno = pci_dev->devfn >> 3, ret = 0;
+    int devno = pci_dev->devfn >> 3;
+    int ret;
 
-    switch (devno) {
-        /* Two PCI slot */
-        case 0x11:
-        case 0x12:
-            ret = (irq_num + devno - 0x10) % 4;
-            break;
-        default:
-            printf("Error:%s:unknown dev number\n", __func__);
-    }
+    ret = ppce500_pci_map_irq_slot(devno, irq_num);
 
     pci_debug("%s: devfn %x irq %d -> %d  devno:%x\n", __func__,
            pci_dev->devfn, irq_num, ret, devno);
@@ -308,7 +317,25 @@ static const VMStateDescription vmstate_ppce500_pci = {
     }
 };
 
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
+
+static int e500_pcihost_bridge_initfn(PCIDevice *d)
+{
+    PPCE500PCIBridgeState *b = PPC_E500_PCI_BRIDGE(d);
+    PPCE500CCSRState *ccsr = CCSR(container_get(qdev_get_machine(),
+                                  "/e500-ccsr"));
+
+    pci_config_set_class(d->config, PCI_CLASS_BRIDGE_PCI);
+    d->config[PCI_HEADER_TYPE] =
+        (d->config[PCI_HEADER_TYPE] & PCI_HEADER_TYPE_MULTI_FUNCTION) |
+        PCI_HEADER_TYPE_BRIDGE;
+
+    memory_region_init_alias(&b->bar0, "e500-pci-bar0", &ccsr->ccsr_space,
+                             0, int128_get64(ccsr->ccsr_space.size));
+    pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &b->bar0);
+
+    return 0;
+}
 
 static int e500_pcihost_initfn(SysBusDevice *dev)
 {
@@ -329,7 +356,7 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
 
     b = pci_register_bus(DEVICE(dev), NULL, mpc85xx_pci_set_irq,
                          mpc85xx_pci_map_irq, s->irq, address_space_mem,
-                         &s->pio, PCI_DEVFN(0x11, 0), 4);
+                         &s->pio, PCI_DEVFN(s->first_slot, 0), 4);
     h->bus = b;
 
     pci_create_simple(b, 0, "e500-host-bridge");
@@ -355,6 +382,7 @@ static void e500_host_bridge_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
+    k->init = e500_pcihost_bridge_initfn;
     k->vendor_id = PCI_VENDOR_ID_FREESCALE;
     k->device_id = PCI_DEVICE_ID_MPC8533E;
     k->class_id = PCI_CLASS_PROCESSOR_POWERPC;
@@ -364,16 +392,22 @@ static void e500_host_bridge_class_init(ObjectClass *klass, void *data)
 static const TypeInfo e500_host_bridge_info = {
     .name          = "e500-host-bridge",
     .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PCIDevice),
+    .instance_size = sizeof(PPCE500PCIBridgeState),
     .class_init    = e500_host_bridge_class_init,
 };
 
+static Property pcihost_properties[] = {
+    DEFINE_PROP_UINT32("first_slot", PPCE500PCIState, first_slot, 0x11),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void e500_pcihost_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
     k->init = e500_pcihost_initfn;
+    dc->props = pcihost_properties;
     dc->vmsd = &vmstate_ppce500_pci;
 }
 
diff --git a/hw/ppce500_pci.h b/hw/ppce500_pci.h
new file mode 100644
index 0000000..61f773e
--- /dev/null
+++ b/hw/ppce500_pci.h
@@ -0,0 +1,9 @@
+#ifndef PPCE500_PCI_H
+#define PPCE500_PCI_H
+
+static inline int ppce500_pci_map_irq_slot(int devno, int irq_num)
+{
+    return (devno + irq_num) % 4;
+}
+
+#endif
diff --git a/hw/ppce500_spin.c b/hw/ppce500_spin.c
index c1a155b..7e90fb9 100644
--- a/hw/ppce500_spin.c
+++ b/hw/ppce500_spin.c
@@ -28,9 +28,9 @@
  */
 
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #define MAX_CPUS 32
 
@@ -124,21 +124,23 @@ static void spin_write(void *opaque, hwaddr addr, uint64_t value,
     SpinState *s = opaque;
     int env_idx = addr / sizeof(SpinInfo);
     CPUPPCState *env;
+    CPUState *cpu = NULL;
     SpinInfo *curspin = &s->spin[env_idx];
     uint8_t *curspin_p = (uint8_t*)curspin;
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->cpu_index == env_idx) {
+        cpu = CPU(ppc_env_get_cpu(env));
+        if (cpu->cpu_index == env_idx) {
             break;
         }
     }
 
-    if (!env) {
+    if (cpu == NULL) {
         /* Unknown CPU */
         return;
     }
 
-    if (!env->cpu_index) {
+    if (cpu->cpu_index == 0) {
         /* primary CPU doesn't spin */
         return;
     }
@@ -194,7 +196,7 @@ static int ppce500_spin_initfn(SysBusDevice *dev)
 {
     SpinState *s;
 
-    s = FROM_SYSBUS(SpinState, sysbus_from_qdev(dev));
+    s = FROM_SYSBUS(SpinState, SYS_BUS_DEVICE(dev));
 
     memory_region_init_io(&s->iomem, &spin_rw_ops, s, "e500 spin pv device",
                           sizeof(SpinInfo) * MAX_CPUS);
@@ -212,7 +214,7 @@ static void ppce500_spin_class_init(ObjectClass *klass, void *data)
     k->init = ppce500_spin_initfn;
 }
 
-static TypeInfo ppce500_spin_info = {
+static const TypeInfo ppce500_spin_info = {
     .name          = "e500-spin",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SpinState),
diff --git a/hw/prep_pci.c b/hw/prep_pci.c
index 0bc479c..52ee5d9 100644
--- a/hw/prep_pci.c
+++ b/hw/prep_pci.c
@@ -2,6 +2,7 @@
  * QEMU PREP PCI host
  *
  * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2011-2013 Andreas Färber
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,13 +24,22 @@
  */
 
 #include "hw.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/pci_bus.h"
+#include "pci/pci_host.h"
 #include "pc.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
+#define TYPE_RAVEN_PCI_DEVICE "raven"
 #define TYPE_RAVEN_PCI_HOST_BRIDGE "raven-pcihost"
 
+#define RAVEN_PCI_DEVICE(obj) \
+    OBJECT_CHECK(RavenPCIState, (obj), TYPE_RAVEN_PCI_DEVICE)
+
+typedef struct RavenPCIState {
+    PCIDevice dev;
+} RavenPCIState;
+
 #define RAVEN_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(PREPPCIState, (obj), TYPE_RAVEN_PCI_HOST_BRIDGE)
 
@@ -38,12 +48,10 @@ typedef struct PRePPCIState {
 
     MemoryRegion intack;
     qemu_irq irq[4];
+    PCIBus pci_bus;
+    RavenPCIState pci_dev;
 } PREPPCIState;
 
-typedef struct RavenPCIState {
-    PCIDevice dev;
-} RavenPCIState;
-
 static inline uint32_t PPC_PCIIO_config(hwaddr addr)
 {
     int i;
@@ -103,23 +111,19 @@ static void prep_set_irq(void *opaque, int irq_num, int level)
     qemu_set_irq(pic[irq_num] , level);
 }
 
-static int raven_pcihost_init(SysBusDevice *dev)
+static void raven_pcihost_realizefn(DeviceState *d, Error **errp)
 {
+    SysBusDevice *dev = SYS_BUS_DEVICE(d);
     PCIHostState *h = PCI_HOST_BRIDGE(dev);
     PREPPCIState *s = RAVEN_PCI_HOST_BRIDGE(dev);
     MemoryRegion *address_space_mem = get_system_memory();
-    MemoryRegion *address_space_io = get_system_io();
-    PCIBus *bus;
     int i;
 
     for (i = 0; i < 4; i++) {
         sysbus_init_irq(dev, &s->irq[i]);
     }
 
-    bus = pci_register_bus(DEVICE(dev), NULL,
-                           prep_set_irq, prep_map_irq, s->irq,
-                           address_space_mem, address_space_io, 0, 4);
-    h->bus = bus;
+    pci_bus_irqs(&s->pci_bus, prep_set_irq, prep_map_irq, s->irq, 4);
 
     memory_region_init_io(&h->conf_mem, &pci_host_conf_be_ops, s,
                           "pci-conf-idx", 1);
@@ -136,9 +140,29 @@ static int raven_pcihost_init(SysBusDevice *dev)
 
     memory_region_init_io(&s->intack, &PPC_intack_ops, s, "pci-intack", 1);
     memory_region_add_subregion(address_space_mem, 0xbffffff0, &s->intack);
-    pci_create_simple(bus, 0, "raven");
 
-    return 0;
+    /* TODO Remove once realize propagates to child devices. */
+    object_property_set_bool(OBJECT(&s->pci_dev), true, "realized", errp);
+}
+
+static void raven_pcihost_initfn(Object *obj)
+{
+    PCIHostState *h = PCI_HOST_BRIDGE(obj);
+    PREPPCIState *s = RAVEN_PCI_HOST_BRIDGE(obj);
+    MemoryRegion *address_space_mem = get_system_memory();
+    MemoryRegion *address_space_io = get_system_io();
+    DeviceState *pci_dev;
+
+    pci_bus_new_inplace(&s->pci_bus, DEVICE(obj), NULL,
+                        address_space_mem, address_space_io, 0);
+    h->bus = &s->pci_bus;
+
+    object_initialize(&s->pci_dev, TYPE_RAVEN_PCI_DEVICE);
+    pci_dev = DEVICE(&s->pci_dev);
+    qdev_set_parent_bus(pci_dev, BUS(&s->pci_bus));
+    object_property_set_int(OBJECT(&s->pci_dev), PCI_DEVFN(0, 0), "addr",
+                            NULL);
+    qdev_prop_set_bit(pci_dev, "multifunction", false);
 }
 
 static int raven_init(PCIDevice *d)
@@ -176,7 +200,7 @@ static void raven_class_init(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo raven_info = {
-    .name = "raven",
+    .name = TYPE_RAVEN_PCI_DEVICE,
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(RavenPCIState),
     .class_init = raven_class_init,
@@ -184,10 +208,9 @@ static const TypeInfo raven_info = {
 
 static void raven_pcihost_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = raven_pcihost_init;
+    dc->realize = raven_pcihost_realizefn;
     dc->fw_name = "pci";
     dc->no_user = 1;
 }
@@ -196,6 +219,7 @@ static const TypeInfo raven_pcihost_info = {
     .name = TYPE_RAVEN_PCI_HOST_BRIDGE,
     .parent = TYPE_PCI_HOST_BRIDGE,
     .instance_size = sizeof(PREPPCIState),
+    .instance_init = raven_pcihost_initfn,
     .class_init = raven_pcihost_class_init,
 };
 
diff --git a/hw/ps2.c b/hw/ps2.c
index f93cd24..15cfd5b 100644
--- a/hw/ps2.c
+++ b/hw/ps2.c
@@ -23,8 +23,8 @@
  */
 #include "hw.h"
 #include "ps2.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 /* debug PC keyboard */
 //#define DEBUG_KBD
diff --git a/hw/ptimer.c b/hw/ptimer.c
index bc0b3f8..24af6a2 100644
--- a/hw/ptimer.c
+++ b/hw/ptimer.c
@@ -6,9 +6,9 @@
  * This code is licensed under the GNU LGPL.
  */
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 struct ptimer_state
 {
diff --git a/hw/ptimer.h b/hw/ptimer.h
index 6638f61..28fcaf1 100644
--- a/hw/ptimer.h
+++ b/hw/ptimer.h
@@ -9,8 +9,8 @@
 #define PTIMER_H
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "vmstate.h"
+#include "qemu/timer.h"
+#include "migration/vmstate.h"
 
 /* ptimer.c */
 typedef struct ptimer_state ptimer_state;
diff --git a/hw/puv3.c b/hw/puv3.c
index 764799c..c722510 100644
--- a/hw/puv3.c
+++ b/hw/puv3.c
@@ -8,9 +8,11 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
-#include "console.h"
+
+#include "qemu-common.h"
+#include "ui/console.h"
 #include "elf.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "sysbus.h"
 #include "boards.h"
 #include "loader.h"
@@ -122,7 +124,7 @@ static QEMUMachine puv3_machine = {
     .desc = "PKUnity Version-3 based on UniCore32",
     .init = puv3_init,
     .is_default = 1,
-    .use_scsi = 0,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void puv3_machine_init(void)
diff --git a/hw/pxa.h b/hw/pxa.h
index 49ac820..668232c 100644
--- a/hw/pxa.h
+++ b/hw/pxa.h
@@ -9,7 +9,7 @@
 #ifndef PXA_H
 # define PXA_H			"pxa.h"
 
-#include "memory.h"
+#include "exec/memory.h"
 
 /* Interrupt numbers */
 # define PXA2XX_PIC_SSP3	0
@@ -69,7 +69,7 @@ DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu);
 
 /* pxa2xx_gpio.c */
 DeviceState *pxa2xx_gpio_init(hwaddr base,
-                CPUARMState *env, DeviceState *pic, int lines);
+                              ARMCPU *cpu, DeviceState *pic, int lines);
 void pxa2xx_gpio_read_notifier(DeviceState *dev, qemu_irq handler);
 
 /* pxa2xx_dma.c */
diff --git a/hw/pxa2xx.c b/hw/pxa2xx.c
index e616979..d303320 100644
--- a/hw/pxa2xx.c
+++ b/hw/pxa2xx.c
@@ -9,12 +9,12 @@
 
 #include "sysbus.h"
 #include "pxa.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "serial.h"
 #include "i2c.h"
 #include "ssi.h"
-#include "qemu-char.h"
-#include "blockdev.h"
+#include "char/char.h"
+#include "sysemu/blockdev.h"
 
 static struct {
     hwaddr io_base;
@@ -343,23 +343,23 @@ static int pxa2xx_cpccnt_read(CPUARMState *env, const ARMCPRegInfo *ri,
 }
 
 static const ARMCPRegInfo pxa_cp_reginfo[] = {
-    /* cp14 crn==1: perf registers */
-    { .name = "CPPMNC", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
+    /* cp14 crm==1: perf registers */
+    { .name = "CPPMNC", .cp = 14, .crn = 0, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW,
       .readfn = pxa2xx_cppmnc_read, .writefn = pxa2xx_cppmnc_write },
     { .name = "CPCCNT", .cp = 14, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW,
       .readfn = pxa2xx_cpccnt_read, .writefn = arm_cp_write_ignore },
-    { .name = "CPINTEN", .cp = 14, .crn = 1, .crm = 4, .opc1 = 0, .opc2 = 0,
+    { .name = "CPINTEN", .cp = 14, .crn = 4, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
-    { .name = "CPFLAG", .cp = 14, .crn = 1, .crm = 5, .opc1 = 0, .opc2 = 0,
+    { .name = "CPFLAG", .cp = 14, .crn = 5, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
-    { .name = "CPEVTSEL", .cp = 14, .crn = 1, .crm = 8, .opc1 = 0, .opc2 = 0,
+    { .name = "CPEVTSEL", .cp = 14, .crn = 8, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
-    /* cp14 crn==2: performance count registers */
-    { .name = "CPPMN0", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
+    /* cp14 crm==2: performance count registers */
+    { .name = "CPPMN0", .cp = 14, .crn = 0, .crm = 2, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
-    { .name = "CPPMN1", .cp = 14, .crn = 2, .crm = 1, .opc1 = 0, .opc2 = 0,
+    { .name = "CPPMN1", .cp = 14, .crn = 1, .crm = 2, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "CPPMN2", .cp = 14, .crn = 2, .crm = 2, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
@@ -1194,7 +1194,7 @@ static void pxa2xx_rtc_sysbus_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pxa2xx_rtc_regs;
 }
 
-static TypeInfo pxa2xx_rtc_sysbus_info = {
+static const TypeInfo pxa2xx_rtc_sysbus_info = {
     .name          = "pxa2xx_rtc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxRTCState),
@@ -1442,7 +1442,7 @@ static void pxa2xx_i2c_slave_class_init(ObjectClass *klass, void *data)
     k->send = pxa2xx_i2c_tx;
 }
 
-static TypeInfo pxa2xx_i2c_slave_info = {
+static const TypeInfo pxa2xx_i2c_slave_info = {
     .name          = "pxa2xx-i2c-slave",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(PXA2xxI2CSlaveState),
@@ -1456,7 +1456,7 @@ PXA2xxI2CState *pxa2xx_i2c_init(hwaddr base,
     SysBusDevice *i2c_dev;
     PXA2xxI2CState *s;
 
-    i2c_dev = sysbus_from_qdev(qdev_create(NULL, "pxa2xx_i2c"));
+    i2c_dev = SYS_BUS_DEVICE(qdev_create(NULL, "pxa2xx_i2c"));
     qdev_prop_set_uint32(&i2c_dev->qdev, "size", region_size + 1);
     qdev_prop_set_uint32(&i2c_dev->qdev, "offset", base & region_size);
 
@@ -1468,7 +1468,7 @@ PXA2xxI2CState *pxa2xx_i2c_init(hwaddr base,
     s = FROM_SYSBUS(PXA2xxI2CState, i2c_dev);
     /* FIXME: Should the slave device really be on a separate bus?  */
     dev = i2c_create_slave(i2c_init_bus(NULL, "dummy"), "pxa2xx-i2c-slave", 0);
-    s->slave = FROM_I2C_SLAVE(PXA2xxI2CSlaveState, I2C_SLAVE_FROM_QDEV(dev));
+    s->slave = FROM_I2C_SLAVE(PXA2xxI2CSlaveState, I2C_SLAVE(dev));
     s->slave->host = s;
 
     return s;
@@ -1510,7 +1510,7 @@ static void pxa2xx_i2c_class_init(ObjectClass *klass, void *data)
     dc->props = pxa2xx_i2c_properties;
 }
 
-static TypeInfo pxa2xx_i2c_info = {
+static const TypeInfo pxa2xx_i2c_info = {
     .name          = "pxa2xx_i2c",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxI2CState),
@@ -2045,7 +2045,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
                     qdev_get_gpio_in(s->pic, PXA27X_PIC_OST_4_11),
                     NULL);
 
-    s->gpio = pxa2xx_gpio_init(0x40e00000, &s->cpu->env, s->pic, 121);
+    s->gpio = pxa2xx_gpio_init(0x40e00000, s->cpu, s->pic, 121);
 
     dinfo = drive_get(IF_SD, 0, 0);
     if (!dinfo) {
@@ -2176,7 +2176,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
                     qdev_get_gpio_in(s->pic, PXA2XX_PIC_OST_0 + 3),
                     NULL);
 
-    s->gpio = pxa2xx_gpio_init(0x40e00000, &s->cpu->env, s->pic, 85);
+    s->gpio = pxa2xx_gpio_init(0x40e00000, s->cpu, s->pic, 85);
 
     dinfo = drive_get(IF_SD, 0, 0);
     if (!dinfo) {
@@ -2273,7 +2273,7 @@ static void pxa2xx_ssp_class_init(ObjectClass *klass, void *data)
     sdc->init = pxa2xx_ssp_init;
 }
 
-static TypeInfo pxa2xx_ssp_info = {
+static const TypeInfo pxa2xx_ssp_info = {
     .name          = "pxa2xx-ssp",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxSSPState),
diff --git a/hw/pxa2xx_dma.c b/hw/pxa2xx_dma.c
index dbea1d2..c0dba45 100644
--- a/hw/pxa2xx_dma.c
+++ b/hw/pxa2xx_dma.c
@@ -481,8 +481,8 @@ DeviceState *pxa27x_dma_init(hwaddr base, qemu_irq irq)
     qdev_prop_set_int32(dev, "channels", PXA27X_DMA_NUM_CHANNELS);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
@@ -495,8 +495,8 @@ DeviceState *pxa255_dma_init(hwaddr base, qemu_irq irq)
     qdev_prop_set_int32(dev, "channels", PXA27X_DMA_NUM_CHANNELS);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     return dev;
 }
@@ -559,7 +559,7 @@ static void pxa2xx_dma_class_init(ObjectClass *klass, void *data)
     dc->props = pxa2xx_dma_properties;
 }
 
-static TypeInfo pxa2xx_dma_info = {
+static const TypeInfo pxa2xx_dma_info = {
     .name          = "pxa2xx-dma",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxDMAState),
diff --git a/hw/pxa2xx_gpio.c b/hw/pxa2xx_gpio.c
index 7aaf409..05d2ad2 100644
--- a/hw/pxa2xx_gpio.c
+++ b/hw/pxa2xx_gpio.c
@@ -250,21 +250,22 @@ static const MemoryRegionOps pxa_gpio_ops = {
 };
 
 DeviceState *pxa2xx_gpio_init(hwaddr base,
-                CPUARMState *env, DeviceState *pic, int lines)
+                              ARMCPU *cpu, DeviceState *pic, int lines)
 {
+    CPUState *cs = CPU(cpu);
     DeviceState *dev;
 
     dev = qdev_create(NULL, "pxa2xx-gpio");
     qdev_prop_set_int32(dev, "lines", lines);
-    qdev_prop_set_int32(dev, "ncpu", env->cpu_index);
+    qdev_prop_set_int32(dev, "ncpu", cs->cpu_index);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0,
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0,
                     qdev_get_gpio_in(pic, PXA2XX_PIC_GPIO_0));
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1,
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1,
                     qdev_get_gpio_in(pic, PXA2XX_PIC_GPIO_1));
-    sysbus_connect_irq(sysbus_from_qdev(dev), 2,
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 2,
                     qdev_get_gpio_in(pic, PXA2XX_PIC_GPIO_X));
 
     return dev;
@@ -276,7 +277,7 @@ static int pxa2xx_gpio_initfn(SysBusDevice *dev)
 
     s = FROM_SYSBUS(PXA2xxGPIOInfo, dev);
 
-    s->cpu = arm_env_get_cpu(qemu_get_cpu(s->ncpu));
+    s->cpu = ARM_CPU(qemu_get_cpu(s->ncpu));
 
     qdev_init_gpio_in(&dev->qdev, pxa2xx_gpio_set, s->lines);
     qdev_init_gpio_out(&dev->qdev, s->handler, s->lines);
@@ -296,7 +297,7 @@ static int pxa2xx_gpio_initfn(SysBusDevice *dev)
  */
 void pxa2xx_gpio_read_notifier(DeviceState *dev, qemu_irq handler)
 {
-    PXA2xxGPIOInfo *s = FROM_SYSBUS(PXA2xxGPIOInfo, sysbus_from_qdev(dev));
+    PXA2xxGPIOInfo *s = FROM_SYSBUS(PXA2xxGPIOInfo, SYS_BUS_DEVICE(dev));
     s->read_notify = handler;
 }
 
@@ -334,7 +335,7 @@ static void pxa2xx_gpio_class_init(ObjectClass *klass, void *data)
     dc->props = pxa2xx_gpio_properties;
 }
 
-static TypeInfo pxa2xx_gpio_info = {
+static const TypeInfo pxa2xx_gpio_info = {
     .name          = "pxa2xx-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxGPIOInfo),
diff --git a/hw/pxa2xx_keypad.c b/hw/pxa2xx_keypad.c
index 257984c..4ff04ad 100644
--- a/hw/pxa2xx_keypad.c
+++ b/hw/pxa2xx_keypad.c
@@ -13,7 +13,7 @@
 
 #include "hw.h"
 #include "pxa.h"
-#include "console.h"
+#include "ui/console.h"
 
 /*
  * Keypad
diff --git a/hw/pxa2xx_lcd.c b/hw/pxa2xx_lcd.c
index b53dfaf..512a27e 100644
--- a/hw/pxa2xx_lcd.c
+++ b/hw/pxa2xx_lcd.c
@@ -11,11 +11,11 @@
  */
 
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pxa.h"
-#include "pixel_ops.h"
+#include "ui/pixel_ops.h"
 /* FIXME: For graphic_rotate. Should probably be done in common code.  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "framebuffer.h"
 
 struct DMAChannel {
diff --git a/hw/pxa2xx_pic.c b/hw/pxa2xx_pic.c
index 70b2b79..90b8fef 100644
--- a/hw/pxa2xx_pic.c
+++ b/hw/pxa2xx_pic.c
@@ -261,7 +261,7 @@ DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu)
 {
     CPUARMState *env = &cpu->env;
     DeviceState *dev = qdev_create(NULL, "pxa2xx_pic");
-    PXA2xxPICState *s = FROM_SYSBUS(PXA2xxPICState, sysbus_from_qdev(dev));
+    PXA2xxPICState *s = FROM_SYSBUS(PXA2xxPICState, SYS_BUS_DEVICE(dev));
 
     s->cpu = cpu;
 
@@ -279,8 +279,8 @@ DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu)
     /* Enable IC memory-mapped registers access.  */
     memory_region_init_io(&s->iomem, &pxa2xx_pic_ops, s,
                           "pxa2xx-pic", 0x00100000);
-    sysbus_init_mmio(sysbus_from_qdev(dev), &s->iomem);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
     /* Enable IC coprocessor access.  */
     define_arm_cp_regs_with_opaque(arm_env_get_cpu(env), pxa_pic_cp_reginfo, s);
@@ -319,7 +319,7 @@ static void pxa2xx_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_pxa2xx_pic_regs;
 }
 
-static TypeInfo pxa2xx_pic_info = {
+static const TypeInfo pxa2xx_pic_info = {
     .name          = "pxa2xx_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxPICState),
diff --git a/hw/pxa2xx_timer.c b/hw/pxa2xx_timer.c
index 8242d26..5c9d2e8 100644
--- a/hw/pxa2xx_timer.c
+++ b/hw/pxa2xx_timer.c
@@ -8,8 +8,8 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "pxa.h"
 #include "sysbus.h"
 
@@ -157,17 +157,27 @@ static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
 
     switch (offset) {
     case OSMR3:  tm ++;
+        /* fall through */
     case OSMR2:  tm ++;
+        /* fall through */
     case OSMR1:  tm ++;
+        /* fall through */
     case OSMR0:
         return s->timer[tm].value;
     case OSMR11: tm ++;
+        /* fall through */
     case OSMR10: tm ++;
+        /* fall through */
     case OSMR9:  tm ++;
+        /* fall through */
     case OSMR8:  tm ++;
+        /* fall through */
     case OSMR7:  tm ++;
+        /* fall through */
     case OSMR6:  tm ++;
+        /* fall through */
     case OSMR5:  tm ++;
+        /* fall through */
     case OSMR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -176,12 +186,19 @@ static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
         return s->clock + muldiv64(qemu_get_clock_ns(vm_clock) -
                         s->lastload, s->freq, get_ticks_per_sec());
     case OSCR11: tm ++;
+        /* fall through */
     case OSCR10: tm ++;
+        /* fall through */
     case OSCR9:  tm ++;
+        /* fall through */
     case OSCR8:  tm ++;
+        /* fall through */
     case OSCR7:  tm ++;
+        /* fall through */
     case OSCR6:  tm ++;
+        /* fall through */
     case OSCR5:  tm ++;
+        /* fall through */
     case OSCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -207,12 +224,19 @@ static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
     case OWER:
         return s->reset3;
     case OMCR11: tm ++;
+        /* fall through */
     case OMCR10: tm ++;
+        /* fall through */
     case OMCR9:  tm ++;
+        /* fall through */
     case OMCR8:  tm ++;
+        /* fall through */
     case OMCR7:  tm ++;
+        /* fall through */
     case OMCR6:  tm ++;
+        /* fall through */
     case OMCR5:  tm ++;
+        /* fall through */
     case OMCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -235,19 +259,29 @@ static void pxa2xx_timer_write(void *opaque, hwaddr offset,
 
     switch (offset) {
     case OSMR3:  tm ++;
+        /* fall through */
     case OSMR2:  tm ++;
+        /* fall through */
     case OSMR1:  tm ++;
+        /* fall through */
     case OSMR0:
         s->timer[tm].value = value;
         pxa2xx_timer_update(s, qemu_get_clock_ns(vm_clock));
         break;
     case OSMR11: tm ++;
+        /* fall through */
     case OSMR10: tm ++;
+        /* fall through */
     case OSMR9:  tm ++;
+        /* fall through */
     case OSMR8:  tm ++;
+        /* fall through */
     case OSMR7:  tm ++;
+        /* fall through */
     case OSMR6:  tm ++;
+        /* fall through */
     case OSMR5:  tm ++;
+        /* fall through */
     case OSMR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -261,12 +295,19 @@ static void pxa2xx_timer_write(void *opaque, hwaddr offset,
         pxa2xx_timer_update(s, s->lastload);
         break;
     case OSCR11: tm ++;
+        /* fall through */
     case OSCR10: tm ++;
+        /* fall through */
     case OSCR9:  tm ++;
+        /* fall through */
     case OSCR8:  tm ++;
+        /* fall through */
     case OSCR7:  tm ++;
+        /* fall through */
     case OSCR6:  tm ++;
+        /* fall through */
     case OSCR5:  tm ++;
+        /* fall through */
     case OSCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -291,8 +332,11 @@ static void pxa2xx_timer_write(void *opaque, hwaddr offset,
         s->reset3 = value;
         break;
     case OMCR7:  tm ++;
+        /* fall through */
     case OMCR6:  tm ++;
+        /* fall through */
     case OMCR5:  tm ++;
+        /* fall through */
     case OMCR4:
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -306,8 +350,11 @@ static void pxa2xx_timer_write(void *opaque, hwaddr offset,
         }
         break;
     case OMCR11: tm ++;
+        /* fall through */
     case OMCR10: tm ++;
+        /* fall through */
     case OMCR9:  tm ++;
+        /* fall through */
     case OMCR8:  tm += 4;
         if (!pxa2xx_timer_has_tm4(s))
             goto badreg;
@@ -495,7 +542,7 @@ static void pxa25x_timer_dev_class_init(ObjectClass *klass, void *data)
     dc->props = pxa25x_timer_dev_properties;
 }
 
-static TypeInfo pxa25x_timer_dev_info = {
+static const TypeInfo pxa25x_timer_dev_info = {
     .name          = "pxa25x-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxTimerInfo),
@@ -520,7 +567,7 @@ static void pxa27x_timer_dev_class_init(ObjectClass *klass, void *data)
     dc->props = pxa27x_timer_dev_properties;
 }
 
-static TypeInfo pxa27x_timer_dev_info = {
+static const TypeInfo pxa27x_timer_dev_info = {
     .name          = "pxa27x-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxTimerInfo),
diff --git a/hw/q35.h b/hw/q35.h
index e34f7c1..246c12c 100644
--- a/hw/q35.h
+++ b/hw/q35.h
@@ -23,14 +23,14 @@
 #define HW_Q35_H
 
 #include "hw.h"
-#include "range.h"
+#include "qemu/range.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "pc.h"
 #include "apm.h"
 #include "apic.h"
-#include "pci.h"
-#include "pcie_host.h"
+#include "pci/pci.h"
+#include "pci/pcie_host.h"
 #include "acpi.h"
 #include "acpi_ich9.h"
 #include "pam.h"
diff --git a/hw/qdev-addr.c b/hw/qdev-addr.c
index ea32c31..b4388f6 100644
--- a/hw/qdev-addr.c
+++ b/hw/qdev-addr.c
@@ -1,7 +1,7 @@
 #include "qdev.h"
 #include "qdev-addr.h"
-#include "hwaddr.h"
-#include "qapi/qapi-visit-core.h"
+#include "exec/hwaddr.h"
+#include "qapi/visitor.h"
 
 /* --- target physical address --- */
 
@@ -40,7 +40,7 @@ static void set_taddr(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     int64_t value;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
diff --git a/hw/qdev-addr.h b/hw/qdev-addr.h
index ea5ecb4..79708e6 100644
--- a/hw/qdev-addr.h
+++ b/hw/qdev-addr.h
@@ -1,5 +1,10 @@
+#ifndef HW_QDEV_ADDR_H
+#define HW_QDEV_ADDR_H 1
+
 #define DEFINE_PROP_TADDR(_n, _s, _f, _d)                               \
     DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_taddr, hwaddr)
 
 extern PropertyInfo qdev_prop_taddr;
 void qdev_prop_set_taddr(DeviceState *dev, const char *name, hwaddr value);
+
+#endif
diff --git a/hw/qdev-core.h b/hw/qdev-core.h
index fff7f0f..2486f36 100644
--- a/hw/qdev-core.h
+++ b/hw/qdev-core.h
@@ -1,26 +1,12 @@
 #ifndef QDEV_CORE_H
 #define QDEV_CORE_H
 
-#include "qemu-queue.h"
-#include "qemu-option.h"
-#include "qemu/object.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/typedefs.h"
+#include "qom/object.h"
 #include "hw/irq.h"
-#include "error.h"
-
-typedef struct Property Property;
-
-typedef struct PropertyInfo PropertyInfo;
-
-typedef struct CompatProperty CompatProperty;
-
-typedef struct BusState BusState;
-
-typedef struct BusClass BusClass;
-
-enum DevState {
-    DEV_STATE_CREATED = 1,
-    DEV_STATE_INITIALIZED,
-};
+#include "qapi/error.h"
 
 enum {
     DEV_NVECTORS_UNSPECIFIED = -1,
@@ -34,11 +20,65 @@ enum {
 typedef int (*qdev_initfn)(DeviceState *dev);
 typedef int (*qdev_event)(DeviceState *dev);
 typedef void (*qdev_resetfn)(DeviceState *dev);
+typedef void (*DeviceRealize)(DeviceState *dev, Error **errp);
+typedef void (*DeviceUnrealize)(DeviceState *dev, Error **errp);
 
 struct VMStateDescription;
 
+/**
+ * DeviceClass:
+ * @props: Properties accessing state fields.
+ * @realize: Callback function invoked when the #DeviceState:realized
+ * property is changed to %true. The default invokes @init if not %NULL.
+ * @unrealize: Callback function invoked when the #DeviceState:realized
+ * property is changed to %false.
+ * @init: Callback function invoked when the #DeviceState::realized property
+ * is changed to %true. Deprecated, new types inheriting directly from
+ * TYPE_DEVICE should use @realize instead, new leaf types should consult
+ * their respective parent type.
+ *
+ * # Realization #
+ * Devices are constructed in two stages,
+ * 1) object instantiation via object_initialize() and
+ * 2) device realization via #DeviceState:realized property.
+ * The former may not fail (it might assert or exit), the latter may return
+ * error information to the caller and must be re-entrant.
+ * Trivial field initializations should go into #TypeInfo.instance_init.
+ * Operations depending on @props static properties should go into @realize.
+ * After successful realization, setting static properties will fail.
+ *
+ * As an interim step, the #DeviceState:realized property is set by deprecated
+ * functions qdev_init() and qdev_init_nofail().
+ * In the future, devices will propagate this state change to their children
+ * and along busses they expose.
+ * The point in time will be deferred to machine creation, so that values
+ * set in @realize will not be introspectable beforehand. Therefore devices
+ * must not create children during @realize; they should initialize them via
+ * object_initialize() in their own #TypeInfo.instance_init and forward the
+ * realization events appropriately.
+ *
+ * The @init callback is considered private to a particular bus implementation
+ * (immediate abstract child types of TYPE_DEVICE). Derived leaf types set an
+ * "init" callback on their parent class instead.
+ *
+ * Any type may override the @realize and/or @unrealize callbacks but needs
+ * to call the parent type's implementation if keeping their functionality
+ * is desired. Refer to QOM documentation for further discussion and examples.
+ *
+ * <note>
+ *   <para>
+ * If a type derived directly from TYPE_DEVICE implements @realize, it does
+ * not need to implement @init and therefore does not need to store and call
+ * #DeviceClass' default @realize callback.
+ * For other types consult the documentation and implementation of the
+ * respective parent types.
+ *   </para>
+ * </note>
+ */
 typedef struct DeviceClass {
+    /*< private >*/
     ObjectClass parent_class;
+    /*< public >*/
 
     const char *fw_name;
     const char *desc;
@@ -47,24 +87,33 @@ typedef struct DeviceClass {
 
     /* callbacks */
     void (*reset)(DeviceState *dev);
+    DeviceRealize realize;
+    DeviceUnrealize unrealize;
 
     /* device state */
     const struct VMStateDescription *vmsd;
 
     /* Private to qdev / bus.  */
-    qdev_initfn init;
+    qdev_initfn init; /* TODO remove, once users are converted to realize */
     qdev_event unplug;
     qdev_event exit;
     const char *bus_type;
 } DeviceClass;
 
-/* This structure should not be accessed directly.  We declare it here
-   so that it can be embedded in individual device state structures.  */
+/**
+ * DeviceState:
+ * @realized: Indicates whether the device has been fully constructed.
+ *
+ * This structure should not be accessed directly.  We declare it here
+ * so that it can be embedded in individual device state structures.
+ */
 struct DeviceState {
+    /*< private >*/
     Object parent_obj;
+    /*< public >*/
 
     const char *id;
-    enum DevState state;
+    bool realized;
     QemuOpts *opts;
     int hotplugged;
     BusState *parent_bus;
@@ -96,6 +145,8 @@ struct BusClass {
      */
     char *(*get_fw_dev_path)(DeviceState *dev);
     int (*reset)(BusState *bus);
+    /* maximum devices allowed on the bus, 0: no limit. */
+    int max_dev;
 };
 
 typedef struct BusChild {
@@ -180,7 +231,7 @@ DeviceState *qdev_find_recursive(BusState *bus, const char *id);
 typedef int (qbus_walkerfn)(BusState *bus, void *opaque);
 typedef int (qdev_walkerfn)(DeviceState *dev, void *opaque);
 
-void qbus_create_inplace(BusState *bus, const char *typename,
+void qbus_create_inplace(void *bus, const char *typename,
                          DeviceState *parent, const char *name);
 BusState *qbus_create(const char *typename, DeviceState *parent, const char *name);
 /* Returns > 0 if either devfn or busfn skip walk somewhere in cursion,
@@ -191,6 +242,18 @@ int qbus_walk_children(BusState *bus, qdev_walkerfn *devfn,
 int qdev_walk_children(DeviceState *dev, qdev_walkerfn *devfn,
                        qbus_walkerfn *busfn, void *opaque);
 void qdev_reset_all(DeviceState *dev);
+
+/**
+ * @qbus_reset_all:
+ * @bus: Bus to be reset.
+ *
+ * Reset @bus and perform a bus-level ("hard") reset of all devices connected
+ * to it, including recursive processing of all buses below @bus itself.  A
+ * hard reset means that qbus_reset_all will reset all state of the device.
+ * For PCI devices, for example, this will include the base address registers
+ * or configuration space.
+ */
+void qbus_reset_all(BusState *bus);
 void qbus_reset_all_fn(void *opaque);
 
 void qbus_free(BusState *bus);
diff --git a/hw/qdev-monitor.c b/hw/qdev-monitor.c
index a1b4d6a..4f9a6eb 100644
--- a/hw/qdev-monitor.c
+++ b/hw/qdev-monitor.c
@@ -18,9 +18,10 @@
  */
 
 #include "qdev.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "qmp-commands.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
+#include "qemu/config-file.h"
 
 /*
  * Aliases were a bad idea from the start.  Let's keep them
@@ -282,6 +283,7 @@ static DeviceState *qbus_find_dev(BusState *bus, char *elem)
 static BusState *qbus_find_recursive(BusState *bus, const char *name,
                                      const char *bus_typename)
 {
+    BusClass *bus_class = BUS_GET_CLASS(bus);
     BusChild *kid;
     BusState *child, *ret;
     int match = 1;
@@ -292,6 +294,17 @@ static BusState *qbus_find_recursive(BusState *bus, const char *name,
     if (bus_typename && !object_dynamic_cast(OBJECT(bus), bus_typename)) {
         match = 0;
     }
+    if ((bus_class->max_dev != 0) && (bus_class->max_dev <= bus->max_index)) {
+        if (name != NULL) {
+            /* bus was explicitly specified: return an error. */
+            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Bus '%s' is full",
+                          bus->name);
+            return NULL;
+        } else {
+            /* bus was not specified: try to find another one. */
+            match = 0;
+        }
+    }
     if (match) {
         return bus;
     }
@@ -563,13 +576,13 @@ static void qbus_print(Monitor *mon, BusState *bus, int indent)
 }
 #undef qdev_printf
 
-void do_info_qtree(Monitor *mon)
+void do_info_qtree(Monitor *mon, const QDict *qdict)
 {
     if (sysbus_get_default())
         qbus_print(mon, sysbus_get_default(), 0);
 }
 
-void do_info_qdm(Monitor *mon)
+void do_info_qdm(Monitor *mon, const QDict *qdict)
 {
     object_class_foreach(qdev_print_devinfo, TYPE_DEVICE, false, NULL);
 }
@@ -578,6 +591,7 @@ int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data)
 {
     Error *local_err = NULL;
     QemuOpts *opts;
+    DeviceState *dev;
 
     opts = qemu_opts_from_qdict(qemu_find_opts("device"), qdict, &local_err);
     if (error_is_set(&local_err)) {
@@ -589,10 +603,12 @@ int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data)
         qemu_opts_del(opts);
         return 0;
     }
-    if (!qdev_device_add(opts)) {
+    dev = qdev_device_add(opts);
+    if (!dev) {
         qemu_opts_del(opts);
         return -1;
     }
+    object_unref(OBJECT(dev));
     return 0;
 }
 
@@ -614,3 +630,54 @@ void qdev_machine_init(void)
     qdev_get_peripheral_anon();
     qdev_get_peripheral();
 }
+
+QemuOptsList qemu_device_opts = {
+    .name = "device",
+    .implied_opt_name = "driver",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_device_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any
+         * sanity checking will happen later
+         * when setting device properties
+         */
+        { /* end of list */ }
+    },
+};
+
+QemuOptsList qemu_global_opts = {
+    .name = "global",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_global_opts.head),
+    .desc = {
+        {
+            .name = "driver",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "property",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "value",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+int qemu_global_option(const char *str)
+{
+    char driver[64], property[64];
+    QemuOpts *opts;
+    int rc, offset;
+
+    rc = sscanf(str, "%63[^.].%63[^=]%n", driver, property, &offset);
+    if (rc < 2 || str[offset] != '=') {
+        error_report("can't parse: \"%s\"", str);
+        return -1;
+    }
+
+    opts = qemu_opts_create_nofail(&qemu_global_opts);
+    qemu_opt_set(opts, "driver", driver);
+    qemu_opt_set(opts, "property", property);
+    qemu_opt_set(opts, "value", str+offset+1);
+    return 0;
+}
diff --git a/hw/qdev-monitor.h b/hw/qdev-monitor.h
index 220ceba..9ec4850 100644
--- a/hw/qdev-monitor.h
+++ b/hw/qdev-monitor.h
@@ -2,12 +2,12 @@
 #define QEMU_QDEV_MONITOR_H
 
 #include "qdev-core.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 /*** monitor commands ***/
 
-void do_info_qtree(Monitor *mon);
-void do_info_qdm(Monitor *mon);
+void do_info_qtree(Monitor *mon, const QDict *qdict);
+void do_info_qdm(Monitor *mon, const QDict *qdict);
 int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data);
 int do_device_del(Monitor *mon, const QDict *qdict, QObject **ret_data);
 int qdev_device_help(QemuOpts *opts);
diff --git a/hw/qdev-properties-system.c b/hw/qdev-properties-system.c
new file mode 100644
index 0000000..ce3af22
--- /dev/null
+++ b/hw/qdev-properties-system.c
@@ -0,0 +1,390 @@
+/*
+ * qdev property parsing and global properties
+ * (parts specific for qemu-system-*)
+ *
+ * This file is based on code from hw/qdev-properties.c from
+ * commit 074a86fccd185616469dfcdc0e157f438aebba18,
+ * Copyright (c) Gerd Hoffmann <kraxel@redhat.com> and other contributors.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "net/net.h"
+#include "qdev.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/blockdev.h"
+#include "hw/block-common.h"
+#include "net/hub.h"
+#include "qapi/visitor.h"
+#include "char/char.h"
+
+static void get_pointer(Object *obj, Visitor *v, Property *prop,
+                        const char *(*print)(void *ptr),
+                        const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    void **ptr = qdev_get_prop_ptr(dev, prop);
+    char *p;
+
+    p = (char *) (*ptr ? print(*ptr) : "");
+    visit_type_str(v, &p, name, errp);
+}
+
+static void set_pointer(Object *obj, Visitor *v, Property *prop,
+                        int (*parse)(DeviceState *dev, const char *str,
+                                     void **ptr),
+                        const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Error *local_err = NULL;
+    void **ptr = qdev_get_prop_ptr(dev, prop);
+    char *str;
+    int ret;
+
+    if (dev->realized) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    visit_type_str(v, &str, name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    if (!*str) {
+        g_free(str);
+        *ptr = NULL;
+        return;
+    }
+    ret = parse(dev, str, ptr);
+    error_set_from_qdev_prop_error(errp, ret, dev, prop, str);
+    g_free(str);
+}
+
+/* --- drive --- */
+
+static int parse_drive(DeviceState *dev, const char *str, void **ptr)
+{
+    BlockDriverState *bs;
+
+    bs = bdrv_find(str);
+    if (bs == NULL) {
+        return -ENOENT;
+    }
+    if (bdrv_attach_dev(bs, dev) < 0) {
+        return -EEXIST;
+    }
+    *ptr = bs;
+    return 0;
+}
+
+static void release_drive(Object *obj, const char *name, void *opaque)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    BlockDriverState **ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (*ptr) {
+        bdrv_detach_dev(*ptr, dev);
+        blockdev_auto_del(*ptr);
+    }
+}
+
+static const char *print_drive(void *ptr)
+{
+    return bdrv_get_device_name(ptr);
+}
+
+static void get_drive(Object *obj, Visitor *v, void *opaque,
+                      const char *name, Error **errp)
+{
+    get_pointer(obj, v, opaque, print_drive, name, errp);
+}
+
+static void set_drive(Object *obj, Visitor *v, void *opaque,
+                      const char *name, Error **errp)
+{
+    set_pointer(obj, v, opaque, parse_drive, name, errp);
+}
+
+PropertyInfo qdev_prop_drive = {
+    .name  = "drive",
+    .get   = get_drive,
+    .set   = set_drive,
+    .release = release_drive,
+};
+
+/* --- character device --- */
+
+static int parse_chr(DeviceState *dev, const char *str, void **ptr)
+{
+    CharDriverState *chr = qemu_chr_find(str);
+    if (chr == NULL) {
+        return -ENOENT;
+    }
+    if (chr->avail_connections < 1) {
+        return -EEXIST;
+    }
+    *ptr = chr;
+    --chr->avail_connections;
+    return 0;
+}
+
+static void release_chr(Object *obj, const char *name, void *opaque)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    CharDriverState **ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (*ptr) {
+        qemu_chr_add_handlers(*ptr, NULL, NULL, NULL, NULL);
+    }
+}
+
+
+static const char *print_chr(void *ptr)
+{
+    CharDriverState *chr = ptr;
+
+    return chr->label ? chr->label : "";
+}
+
+static void get_chr(Object *obj, Visitor *v, void *opaque,
+                    const char *name, Error **errp)
+{
+    get_pointer(obj, v, opaque, print_chr, name, errp);
+}
+
+static void set_chr(Object *obj, Visitor *v, void *opaque,
+                    const char *name, Error **errp)
+{
+    set_pointer(obj, v, opaque, parse_chr, name, errp);
+}
+
+PropertyInfo qdev_prop_chr = {
+    .name  = "chr",
+    .get   = get_chr,
+    .set   = set_chr,
+    .release = release_chr,
+};
+
+/* --- netdev device --- */
+
+static int parse_netdev(DeviceState *dev, const char *str, void **ptr)
+{
+    NICPeers *peers_ptr = (NICPeers *)ptr;
+    NICConf *conf = container_of(peers_ptr, NICConf, peers);
+    NetClientState **ncs = peers_ptr->ncs;
+    NetClientState *peers[MAX_QUEUE_NUM];
+    int queues, i = 0;
+    int ret;
+
+    queues = qemu_find_net_clients_except(str, peers,
+                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          MAX_QUEUE_NUM);
+    if (queues == 0) {
+        ret = -ENOENT;
+        goto err;
+    }
+
+    if (queues > MAX_QUEUE_NUM) {
+        ret = -E2BIG;
+        goto err;
+    }
+
+    for (i = 0; i < queues; i++) {
+        if (peers[i] == NULL) {
+            ret = -ENOENT;
+            goto err;
+        }
+
+        if (peers[i]->peer) {
+            ret = -EEXIST;
+            goto err;
+        }
+
+        ncs[i] = peers[i];
+        ncs[i]->queue_index = i;
+    }
+
+    conf->queues = queues;
+
+    return 0;
+
+err:
+    return ret;
+}
+
+static const char *print_netdev(void *ptr)
+{
+    NetClientState *netdev = ptr;
+
+    return netdev->name ? netdev->name : "";
+}
+
+static void get_netdev(Object *obj, Visitor *v, void *opaque,
+                       const char *name, Error **errp)
+{
+    get_pointer(obj, v, opaque, print_netdev, name, errp);
+}
+
+static void set_netdev(Object *obj, Visitor *v, void *opaque,
+                       const char *name, Error **errp)
+{
+    set_pointer(obj, v, opaque, parse_netdev, name, errp);
+}
+
+PropertyInfo qdev_prop_netdev = {
+    .name  = "netdev",
+    .get   = get_netdev,
+    .set   = set_netdev,
+};
+
+/* --- vlan --- */
+
+static int print_vlan(DeviceState *dev, Property *prop, char *dest, size_t len)
+{
+    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (*ptr) {
+        int id;
+        if (!net_hub_id_for_client(*ptr, &id)) {
+            return snprintf(dest, len, "%d", id);
+        }
+    }
+
+    return snprintf(dest, len, "<null>");
+}
+
+static void get_vlan(Object *obj, Visitor *v, void *opaque,
+                     const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
+    int32_t id = -1;
+
+    if (*ptr) {
+        int hub_id;
+        if (!net_hub_id_for_client(*ptr, &hub_id)) {
+            id = hub_id;
+        }
+    }
+
+    visit_type_int32(v, &id, name, errp);
+}
+
+static void set_vlan(Object *obj, Visitor *v, void *opaque,
+                     const char *name, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    NICPeers *peers_ptr = qdev_get_prop_ptr(dev, prop);
+    NetClientState **ptr = &peers_ptr->ncs[0];
+    Error *local_err = NULL;
+    int32_t id;
+    NetClientState *hubport;
+
+    if (dev->realized) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    visit_type_int32(v, &id, name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    if (id == -1) {
+        *ptr = NULL;
+        return;
+    }
+
+    hubport = net_hub_port_find(id);
+    if (!hubport) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
+                  name, prop->info->name);
+        return;
+    }
+    *ptr = hubport;
+}
+
+PropertyInfo qdev_prop_vlan = {
+    .name  = "vlan",
+    .print = print_vlan,
+    .get   = get_vlan,
+    .set   = set_vlan,
+};
+
+int qdev_prop_set_drive(DeviceState *dev, const char *name,
+                        BlockDriverState *value)
+{
+    Error *errp = NULL;
+    const char *bdrv_name = value ? bdrv_get_device_name(value) : "";
+    object_property_set_str(OBJECT(dev), bdrv_name,
+                            name, &errp);
+    if (errp) {
+        qerror_report_err(errp);
+        error_free(errp);
+        return -1;
+    }
+    return 0;
+}
+
+void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
+                                BlockDriverState *value)
+{
+    if (qdev_prop_set_drive(dev, name, value) < 0) {
+        exit(1);
+    }
+}
+void qdev_prop_set_chr(DeviceState *dev, const char *name,
+                       CharDriverState *value)
+{
+    Error *errp = NULL;
+    assert(!value || value->label);
+    object_property_set_str(OBJECT(dev),
+                            value ? value->label : "", name, &errp);
+    assert_no_error(errp);
+}
+
+void qdev_prop_set_netdev(DeviceState *dev, const char *name,
+                          NetClientState *value)
+{
+    Error *errp = NULL;
+    assert(!value || value->name);
+    object_property_set_str(OBJECT(dev),
+                            value ? value->name : "", name, &errp);
+    assert_no_error(errp);
+}
+
+void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd)
+{
+    qdev_prop_set_macaddr(dev, "mac", nd->macaddr.a);
+    if (nd->netdev) {
+        qdev_prop_set_netdev(dev, "netdev", nd->netdev);
+    }
+    if (nd->nvectors != DEV_NVECTORS_UNSPECIFIED &&
+        object_property_find(OBJECT(dev), "vectors", NULL)) {
+        qdev_prop_set_uint32(dev, "vectors", nd->nvectors);
+    }
+    nd->instantiated = 1;
+}
+
+static int qdev_add_one_global(QemuOpts *opts, void *opaque)
+{
+    GlobalProperty *g;
+
+    g = g_malloc0(sizeof(*g));
+    g->driver   = qemu_opt_get(opts, "driver");
+    g->property = qemu_opt_get(opts, "property");
+    g->value    = qemu_opt_get(opts, "value");
+    qdev_prop_register_global(g);
+    return 0;
+}
+
+void qemu_add_globals(void)
+{
+    qemu_opts_foreach(qemu_find_opts("global"), qdev_add_one_global, NULL, 0);
+}
diff --git a/hw/qdev-properties.c b/hw/qdev-properties.c
index 81d901c..a8a31f5 100644
--- a/hw/qdev-properties.c
+++ b/hw/qdev-properties.c
@@ -1,10 +1,11 @@
-#include "net.h"
+#include "net/net.h"
 #include "qdev.h"
-#include "qerror.h"
-#include "blockdev.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
 #include "net/hub.h"
-#include "qapi/qapi-visit-core.h"
+#include "qapi/visitor.h"
+#include "char/char.h"
 
 void *qdev_get_prop_ptr(DeviceState *dev, Property *prop)
 {
@@ -13,49 +14,6 @@ void *qdev_get_prop_ptr(DeviceState *dev, Property *prop)
     return ptr;
 }
 
-static void get_pointer(Object *obj, Visitor *v, Property *prop,
-                        const char *(*print)(void *ptr),
-                        const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    void **ptr = qdev_get_prop_ptr(dev, prop);
-    char *p;
-
-    p = (char *) (*ptr ? print(*ptr) : "");
-    visit_type_str(v, &p, name, errp);
-}
-
-static void set_pointer(Object *obj, Visitor *v, Property *prop,
-                        int (*parse)(DeviceState *dev, const char *str,
-                                     void **ptr),
-                        const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    Error *local_err = NULL;
-    void **ptr = qdev_get_prop_ptr(dev, prop);
-    char *str;
-    int ret;
-
-    if (dev->state != DEV_STATE_CREATED) {
-        error_set(errp, QERR_PERMISSION_DENIED);
-        return;
-    }
-
-    visit_type_str(v, &str, name, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-    if (!*str) {
-        g_free(str);
-        *ptr = NULL;
-        return;
-    }
-    ret = parse(dev, str, ptr);
-    error_set_from_qdev_prop_error(errp, ret, dev, prop, str);
-    g_free(str);
-}
-
 static void get_enum(Object *obj, Visitor *v, void *opaque,
                      const char *name, Error **errp)
 {
@@ -74,7 +32,7 @@ static void set_enum(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     int *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -95,10 +53,11 @@ static void bit_prop_set(DeviceState *dev, Property *props, bool val)
 {
     uint32_t *p = qdev_get_prop_ptr(dev, props);
     uint32_t mask = qdev_get_prop_mask(props);
-    if (val)
+    if (val) {
         *p |= mask;
-    else
+    } else {
         *p &= ~mask;
+    }
 }
 
 static int print_bit(DeviceState *dev, Property *prop, char *dest, size_t len)
@@ -126,7 +85,7 @@ static void set_bit(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     bool value;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -166,7 +125,7 @@ static void set_uint8(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint8_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -233,7 +192,7 @@ static void set_uint16(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint16_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -266,7 +225,7 @@ static void set_uint32(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -291,7 +250,7 @@ static void set_int32(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     int32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -364,7 +323,7 @@ static void set_uint64(Object *obj, Visitor *v, void *opaque,
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -420,11 +379,13 @@ static void release_string(Object *obj, const char *name, void *opaque)
     g_free(*(char **)qdev_get_prop_ptr(DEVICE(obj), prop));
 }
 
-static int print_string(DeviceState *dev, Property *prop, char *dest, size_t len)
+static int print_string(DeviceState *dev, Property *prop, char *dest,
+                        size_t len)
 {
     char **ptr = qdev_get_prop_ptr(dev, prop);
-    if (!*ptr)
+    if (!*ptr) {
         return snprintf(dest, len, "<null>");
+    }
     return snprintf(dest, len, "\"%s\"", *ptr);
 }
 
@@ -452,7 +413,7 @@ static void set_string(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     char *str;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -476,227 +437,6 @@ PropertyInfo qdev_prop_string = {
     .set   = set_string,
 };
 
-/* --- drive --- */
-
-static int parse_drive(DeviceState *dev, const char *str, void **ptr)
-{
-    BlockDriverState *bs;
-
-    bs = bdrv_find(str);
-    if (bs == NULL)
-        return -ENOENT;
-    if (bdrv_attach_dev(bs, dev) < 0)
-        return -EEXIST;
-    *ptr = bs;
-    return 0;
-}
-
-static void release_drive(Object *obj, const char *name, void *opaque)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    BlockDriverState **ptr = qdev_get_prop_ptr(dev, prop);
-
-    if (*ptr) {
-        bdrv_detach_dev(*ptr, dev);
-        blockdev_auto_del(*ptr);
-    }
-}
-
-static const char *print_drive(void *ptr)
-{
-    return bdrv_get_device_name(ptr);
-}
-
-static void get_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_drive, name, errp);
-}
-
-static void set_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_drive, name, errp);
-}
-
-PropertyInfo qdev_prop_drive = {
-    .name  = "drive",
-    .get   = get_drive,
-    .set   = set_drive,
-    .release = release_drive,
-};
-
-/* --- character device --- */
-
-static int parse_chr(DeviceState *dev, const char *str, void **ptr)
-{
-    CharDriverState *chr = qemu_chr_find(str);
-    if (chr == NULL) {
-        return -ENOENT;
-    }
-    if (chr->avail_connections < 1) {
-        return -EEXIST;
-    }
-    *ptr = chr;
-    --chr->avail_connections;
-    return 0;
-}
-
-static void release_chr(Object *obj, const char *name, void *opaque)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    CharDriverState **ptr = qdev_get_prop_ptr(dev, prop);
-
-    if (*ptr) {
-        qemu_chr_add_handlers(*ptr, NULL, NULL, NULL, NULL);
-    }
-}
-
-
-static const char *print_chr(void *ptr)
-{
-    CharDriverState *chr = ptr;
-
-    return chr->label ? chr->label : "";
-}
-
-static void get_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_chr, name, errp);
-}
-
-static void set_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_chr, name, errp);
-}
-
-PropertyInfo qdev_prop_chr = {
-    .name  = "chr",
-    .get   = get_chr,
-    .set   = set_chr,
-    .release = release_chr,
-};
-
-/* --- netdev device --- */
-
-static int parse_netdev(DeviceState *dev, const char *str, void **ptr)
-{
-    NetClientState *netdev = qemu_find_netdev(str);
-
-    if (netdev == NULL) {
-        return -ENOENT;
-    }
-    if (netdev->peer) {
-        return -EEXIST;
-    }
-    *ptr = netdev;
-    return 0;
-}
-
-static const char *print_netdev(void *ptr)
-{
-    NetClientState *netdev = ptr;
-
-    return netdev->name ? netdev->name : "";
-}
-
-static void get_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_netdev, name, errp);
-}
-
-static void set_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_netdev, name, errp);
-}
-
-PropertyInfo qdev_prop_netdev = {
-    .name  = "netdev",
-    .get   = get_netdev,
-    .set   = set_netdev,
-};
-
-/* --- vlan --- */
-
-static int print_vlan(DeviceState *dev, Property *prop, char *dest, size_t len)
-{
-    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
-
-    if (*ptr) {
-        int id;
-        if (!net_hub_id_for_client(*ptr, &id)) {
-            return snprintf(dest, len, "%d", id);
-        }
-    }
-
-    return snprintf(dest, len, "<null>");
-}
-
-static void get_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
-    int32_t id = -1;
-
-    if (*ptr) {
-        int hub_id;
-        if (!net_hub_id_for_client(*ptr, &hub_id)) {
-            id = hub_id;
-        }
-    }
-
-    visit_type_int32(v, &id, name, errp);
-}
-
-static void set_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
-{
-    DeviceState *dev = DEVICE(obj);
-    Property *prop = opaque;
-    NetClientState **ptr = qdev_get_prop_ptr(dev, prop);
-    Error *local_err = NULL;
-    int32_t id;
-    NetClientState *hubport;
-
-    if (dev->state != DEV_STATE_CREATED) {
-        error_set(errp, QERR_PERMISSION_DENIED);
-        return;
-    }
-
-    visit_type_int32(v, &id, name, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-    if (id == -1) {
-        *ptr = NULL;
-        return;
-    }
-
-    hubport = net_hub_port_find(id);
-    if (!hubport) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  name, prop->info->name);
-        return;
-    }
-    *ptr = hubport;
-}
-
-PropertyInfo qdev_prop_vlan = {
-    .name  = "vlan",
-    .print = print_vlan,
-    .get   = get_vlan,
-    .set   = set_vlan,
-};
-
 /* --- pointer --- */
 
 /* Not a proper property, just for dirty hacks.  TODO Remove it!  */
@@ -737,7 +477,7 @@ static void set_mac(Object *obj, Visitor *v, void *opaque,
     int i, pos;
     char *str, *p;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -749,16 +489,20 @@ static void set_mac(Object *obj, Visitor *v, void *opaque,
     }
 
     for (i = 0, pos = 0; i < 6; i++, pos += 3) {
-        if (!qemu_isxdigit(str[pos]))
+        if (!qemu_isxdigit(str[pos])) {
             goto inval;
-        if (!qemu_isxdigit(str[pos+1]))
+        }
+        if (!qemu_isxdigit(str[pos+1])) {
             goto inval;
+        }
         if (i == 5) {
-            if (str[pos+2] != '\0')
+            if (str[pos+2] != '\0') {
                 goto inval;
+            }
         } else {
-            if (str[pos+2] != ':' && str[pos+2] != '-')
+            if (str[pos+2] != ':' && str[pos+2] != '-') {
                 goto inval;
+            }
         }
         mac->a[i] = strtol(str+pos, &p, 16);
     }
@@ -825,7 +569,7 @@ static void set_pci_devfn(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     char *str;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -864,7 +608,8 @@ invalid:
     g_free(str);
 }
 
-static int print_pci_devfn(DeviceState *dev, Property *prop, char *dest, size_t len)
+static int print_pci_devfn(DeviceState *dev, Property *prop, char *dest,
+                           size_t len)
 {
     int32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
@@ -895,7 +640,7 @@ static void set_blocksize(Object *obj, Visitor *v, void *opaque,
     const int64_t min = 512;
     const int64_t max = 32768;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -963,7 +708,7 @@ static void set_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
     unsigned long dom = 0, bus = 0;
     unsigned int slot = 0, func = 0;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -1038,11 +783,13 @@ PropertyInfo qdev_prop_pci_host_devaddr = {
 
 static Property *qdev_prop_walk(Property *props, const char *name)
 {
-    if (!props)
+    if (!props) {
         return NULL;
+    }
     while (props->name) {
-        if (strcmp(props->name, name) == 0)
+        if (strcmp(props->name, name) == 0) {
             return props;
+        }
         props++;
     }
     return NULL;
@@ -1158,44 +905,6 @@ void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value)
     assert_no_error(errp);
 }
 
-int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value)
-{
-    Error *errp = NULL;
-    const char *bdrv_name = value ? bdrv_get_device_name(value) : "";
-    object_property_set_str(OBJECT(dev), bdrv_name,
-                            name, &errp);
-    if (errp) {
-        qerror_report_err(errp);
-        error_free(errp);
-        return -1;
-    }
-    return 0;
-}
-
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value)
-{
-    if (qdev_prop_set_drive(dev, name, value) < 0) {
-        exit(1);
-    }
-}
-void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value)
-{
-    Error *errp = NULL;
-    assert(!value || value->label);
-    object_property_set_str(OBJECT(dev),
-                            value ? value->label : "", name, &errp);
-    assert_no_error(errp);
-}
-
-void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value)
-{
-    Error *errp = NULL;
-    assert(!value || value->name);
-    object_property_set_str(OBJECT(dev),
-                            value ? value->name : "", name, &errp);
-    assert_no_error(errp);
-}
-
 void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value)
 {
     Error *errp = NULL;
@@ -1229,9 +938,10 @@ void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value)
     *ptr = value;
 }
 
-static QTAILQ_HEAD(, GlobalProperty) global_props = QTAILQ_HEAD_INITIALIZER(global_props);
+static QTAILQ_HEAD(, GlobalProperty) global_props =
+        QTAILQ_HEAD_INITIALIZER(global_props);
 
-static void qdev_prop_register_global(GlobalProperty *prop)
+void qdev_prop_register_global(GlobalProperty *prop)
 {
     QTAILQ_INSERT_TAIL(&global_props, prop, next);
 }
@@ -1262,20 +972,3 @@ void qdev_prop_set_globals(DeviceState *dev)
         class = object_class_get_parent(class);
     } while (class);
 }
-
-static int qdev_add_one_global(QemuOpts *opts, void *opaque)
-{
-    GlobalProperty *g;
-
-    g = g_malloc0(sizeof(*g));
-    g->driver   = qemu_opt_get(opts, "driver");
-    g->property = qemu_opt_get(opts, "property");
-    g->value    = qemu_opt_get(opts, "value");
-    qdev_prop_register_global(g);
-    return 0;
-}
-
-void qemu_add_globals(void)
-{
-    qemu_opts_foreach(qemu_find_opts("global"), qdev_add_one_global, NULL, 0);
-}
diff --git a/hw/qdev-properties.h b/hw/qdev-properties.h
index 5b046ab..20c67f3 100644
--- a/hw/qdev-properties.h
+++ b/hw/qdev-properties.h
@@ -31,7 +31,7 @@ extern PropertyInfo qdev_prop_pci_host_devaddr;
         .name      = (_name),                                    \
         .info      = &(_prop),                                   \
         .offset    = offsetof(_state, _field)                    \
-            + type_check(_type,typeof_field(_state, _field)),    \
+            + type_check(_type, typeof_field(_state, _field)),   \
         }
 #define DEFINE_PROP_DEFAULT(_name, _state, _field, _defval, _prop, _type) { \
         .name      = (_name),                                           \
@@ -77,9 +77,9 @@ extern PropertyInfo qdev_prop_pci_host_devaddr;
 #define DEFINE_PROP_STRING(_n, _s, _f)             \
     DEFINE_PROP(_n, _s, _f, qdev_prop_string, char*)
 #define DEFINE_PROP_NETDEV(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_netdev, NetClientState*)
+    DEFINE_PROP(_n, _s, _f, qdev_prop_netdev, NICPeers)
 #define DEFINE_PROP_VLAN(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NetClientState*)
+    DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NICPeers)
 #define DEFINE_PROP_DRIVE(_n, _s, _f) \
     DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockDriverState *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
@@ -116,6 +116,7 @@ void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
 /* FIXME: Remove opaque pointer properties.  */
 void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value);
 
+void qdev_prop_register_global(GlobalProperty *prop);
 void qdev_prop_register_global_list(GlobalProperty *props);
 void qdev_prop_set_globals(DeviceState *dev);
 void error_set_from_qdev_prop_error(Error **errp, int ret, DeviceState *dev,
diff --git a/hw/qdev.c b/hw/qdev.c
index 788b4da..689cd54 100644
--- a/hw/qdev.c
+++ b/hw/qdev.c
@@ -25,11 +25,10 @@
    inherit from a particular bus (e.g. PCI or I2C) rather than
    this API directly.  */
 
-#include "net.h"
 #include "qdev.h"
-#include "sysemu.h"
-#include "error.h"
-#include "qapi/qapi-visit-core.h"
+#include "sysemu/sysemu.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
 
 int qdev_hotplug = 0;
 static bool qdev_hot_added = false;
@@ -65,7 +64,10 @@ static void bus_remove_child(BusState *bus, DeviceState *child)
 
             snprintf(name, sizeof(name), "child[%d]", kid->index);
             QTAILQ_REMOVE(&bus->children, kid, sibling);
+
+            /* This gives back ownership of kid->child back to us.  */
             object_property_del(OBJECT(bus), name, NULL);
+            object_unref(OBJECT(kid->child));
             g_free(kid);
             return;
         }
@@ -83,9 +85,11 @@ static void bus_add_child(BusState *bus, DeviceState *child)
 
     kid->index = bus->max_index++;
     kid->child = child;
+    object_ref(OBJECT(kid->child));
 
     QTAILQ_INSERT_HEAD(&bus->children, kid, sibling);
 
+    /* This transfers ownership of kid->child to the property.  */
     snprintf(name, sizeof(name), "child[%d]", kid->index);
     object_property_add_link(OBJECT(bus), name,
                              object_get_typename(OBJECT(child)),
@@ -96,6 +100,7 @@ static void bus_add_child(BusState *bus, DeviceState *child)
 void qdev_set_parent_bus(DeviceState *dev, BusState *bus)
 {
     dev->parent_bus = bus;
+    object_ref(OBJECT(bus));
     bus_add_child(bus, dev);
 }
 
@@ -109,10 +114,12 @@ DeviceState *qdev_create(BusState *bus, const char *name)
     dev = qdev_try_create(bus, name);
     if (!dev) {
         if (bus) {
-            hw_error("Unknown device '%s' for bus '%s'\n", name,
-                     object_get_typename(OBJECT(bus)));
+            error_report("Unknown device '%s' for bus '%s'", name,
+                         object_get_typename(OBJECT(bus)));
+            abort();
         } else {
-            hw_error("Unknown device '%s' for default sysbus\n", name);
+            error_report("Unknown device '%s' for default sysbus", name);
+            abort();
         }
     }
 
@@ -136,7 +143,7 @@ DeviceState *qdev_try_create(BusState *bus, const char *type)
     }
 
     qdev_set_parent_bus(dev, bus);
-
+    object_unref(OBJECT(dev));
     return dev;
 }
 
@@ -147,43 +154,36 @@ DeviceState *qdev_try_create(BusState *bus, const char *type)
    Return 0 on success.  */
 int qdev_init(DeviceState *dev)
 {
-    DeviceClass *dc = DEVICE_GET_CLASS(dev);
-    int rc;
+    Error *local_err = NULL;
 
-    assert(dev->state == DEV_STATE_CREATED);
+    assert(!dev->realized);
 
-    rc = dc->init(dev);
-    if (rc < 0) {
+    object_property_set_bool(OBJECT(dev), true, "realized", &local_err);
+    if (local_err != NULL) {
+        error_free(local_err);
         qdev_free(dev);
-        return rc;
+        return -1;
     }
+    return 0;
+}
 
-    if (!OBJECT(dev)->parent) {
-        static int unattached_count = 0;
-        gchar *name = g_strdup_printf("device[%d]", unattached_count++);
-
-        object_property_add_child(container_get(qdev_get_machine(),
-                                                "/unattached"),
-                                  name, OBJECT(dev), NULL);
-        g_free(name);
-    }
+static void device_realize(DeviceState *dev, Error **err)
+{
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
 
-    if (qdev_get_vmsd(dev)) {
-        vmstate_register_with_alias_id(dev, -1, qdev_get_vmsd(dev), dev,
-                                       dev->instance_id_alias,
-                                       dev->alias_required_for_version);
-    }
-    dev->state = DEV_STATE_INITIALIZED;
-    if (dev->hotplugged) {
-        device_reset(dev);
+    if (dc->init) {
+        int rc = dc->init(dev);
+        if (rc < 0) {
+            error_setg(err, "Device initialization failed.");
+            return;
+        }
     }
-    return 0;
 }
 
 void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
                                  int required_for_version)
 {
-    assert(dev->state == DEV_STATE_CREATED);
+    assert(!dev->realized);
     dev->instance_id_alias = alias_id;
     dev->alias_required_for_version = required_for_version;
 }
@@ -227,10 +227,15 @@ void qdev_reset_all(DeviceState *dev)
     qdev_walk_children(dev, qdev_reset_one, qbus_reset_one, NULL);
 }
 
+void qbus_reset_all(BusState *bus)
+{
+    qbus_walk_children(bus, qdev_reset_one, qbus_reset_one, NULL);
+}
+
 void qbus_reset_all_fn(void *opaque)
 {
     BusState *bus = opaque;
-    qbus_walk_children(bus, qdev_reset_one, qbus_reset_one, NULL);
+    qbus_reset_all(bus);
 }
 
 /* can be used as ->unplug() callback for the simple cases */
@@ -262,7 +267,7 @@ void qdev_init_nofail(DeviceState *dev)
 /* Unlink device from bus and free the structure.  */
 void qdev_free(DeviceState *dev)
 {
-    object_delete(OBJECT(dev));
+    object_unparent(OBJECT(dev));
 }
 
 void qdev_machine_creation_done(void)
@@ -310,18 +315,6 @@ void qdev_connect_gpio_out(DeviceState * dev, int n, qemu_irq pin)
     dev->gpio_out[n] = pin;
 }
 
-void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd)
-{
-    qdev_prop_set_macaddr(dev, "mac", nd->macaddr.a);
-    if (nd->netdev)
-        qdev_prop_set_netdev(dev, "netdev", nd->netdev);
-    if (nd->nvectors != DEV_NVECTORS_UNSPECIFIED &&
-        object_property_find(OBJECT(dev), "vectors", NULL)) {
-        qdev_prop_set_uint32(dev, "vectors", nd->nvectors);
-    }
-    nd->instantiated = 1;
-}
-
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name)
 {
     BusState *bus;
@@ -403,14 +396,16 @@ DeviceState *qdev_find_recursive(BusState *bus, const char *id)
     return NULL;
 }
 
-static void qbus_realize(BusState *bus)
+static void qbus_realize(BusState *bus, DeviceState *parent, const char *name)
 {
     const char *typename = object_get_typename(OBJECT(bus));
     char *buf;
     int i,len;
 
-    if (bus->name) {
-        /* use supplied name */
+    bus->parent = parent;
+
+    if (name) {
+        bus->name = g_strdup(name);
     } else if (bus->parent && bus->parent->id) {
         /* parent device has id -> use it for bus name */
         len = strlen(bus->parent->id) + 16;
@@ -432,6 +427,7 @@ static void qbus_realize(BusState *bus)
         QLIST_INSERT_HEAD(&bus->parent->child_bus, bus, sibling);
         bus->parent->num_child_bus++;
         object_property_add_child(OBJECT(bus->parent), bus->name, OBJECT(bus), NULL);
+        object_unref(OBJECT(bus));
     } else if (bus != sysbus_get_default()) {
         /* TODO: once all bus devices are qdevified,
            only reset handler for main_system_bus should be registered here. */
@@ -439,14 +435,30 @@ static void qbus_realize(BusState *bus)
     }
 }
 
-void qbus_create_inplace(BusState *bus, const char *typename,
+static void bus_unparent(Object *obj)
+{
+    BusState *bus = BUS(obj);
+    BusChild *kid;
+
+    while ((kid = QTAILQ_FIRST(&bus->children)) != NULL) {
+        DeviceState *dev = kid->child;
+        qdev_free(dev);
+    }
+    if (bus->parent) {
+        QLIST_REMOVE(bus, sibling);
+        bus->parent->num_child_bus--;
+        bus->parent = NULL;
+    } else {
+        assert(bus != sysbus_get_default()); /* main_system_bus is never freed */
+        qemu_unregister_reset(qbus_reset_all_fn, bus);
+    }
+}
+
+void qbus_create_inplace(void *bus, const char *typename,
                          DeviceState *parent, const char *name)
 {
     object_initialize(bus, typename);
-
-    bus->parent = parent;
-    bus->name = name ? g_strdup(name) : NULL;
-    qbus_realize(bus);
+    qbus_realize(bus, parent, name);
 }
 
 BusState *qbus_create(const char *typename, DeviceState *parent, const char *name)
@@ -454,17 +466,14 @@ BusState *qbus_create(const char *typename, DeviceState *parent, const char *nam
     BusState *bus;
 
     bus = BUS(object_new(typename));
-
-    bus->parent = parent;
-    bus->name = name ? g_strdup(name) : NULL;
-    qbus_realize(bus);
+    qbus_realize(bus, parent, name);
 
     return bus;
 }
 
 void qbus_free(BusState *bus)
 {
-    object_delete(OBJECT(bus));
+    object_unparent(OBJECT(bus));
 }
 
 static char *bus_get_fw_dev_path(BusState *bus, DeviceState *dev)
@@ -552,7 +561,7 @@ static void qdev_set_legacy_property(Object *obj, Visitor *v, void *opaque,
     char *ptr = NULL;
     int ret;
 
-    if (dev->state != DEV_STATE_CREATED) {
+    if (dev->realized) {
         error_set(errp, QERR_PERMISSION_DENIED);
         return;
     }
@@ -647,6 +656,55 @@ void qdev_property_add_static(DeviceState *dev, Property *prop,
     assert_no_error(local_err);
 }
 
+static bool device_get_realized(Object *obj, Error **err)
+{
+    DeviceState *dev = DEVICE(obj);
+    return dev->realized;
+}
+
+static void device_set_realized(Object *obj, bool value, Error **err)
+{
+    DeviceState *dev = DEVICE(obj);
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
+    Error *local_err = NULL;
+
+    if (value && !dev->realized) {
+        if (dc->realize) {
+            dc->realize(dev, &local_err);
+        }
+
+        if (!obj->parent && local_err == NULL) {
+            static int unattached_count;
+            gchar *name = g_strdup_printf("device[%d]", unattached_count++);
+
+            object_property_add_child(container_get(qdev_get_machine(),
+                                                    "/unattached"),
+                                      name, obj, &local_err);
+            g_free(name);
+        }
+
+        if (qdev_get_vmsd(dev) && local_err == NULL) {
+            vmstate_register_with_alias_id(dev, -1, qdev_get_vmsd(dev), dev,
+                                           dev->instance_id_alias,
+                                           dev->alias_required_for_version);
+        }
+        if (dev->hotplugged && local_err == NULL) {
+            device_reset(dev);
+        }
+    } else if (!value && dev->realized) {
+        if (dc->unrealize) {
+            dc->unrealize(dev, &local_err);
+        }
+    }
+
+    if (local_err != NULL) {
+        error_propagate(err, local_err);
+        return;
+    }
+
+    dev->realized = value;
+}
+
 static void device_initfn(Object *obj)
 {
     DeviceState *dev = DEVICE(obj);
@@ -659,7 +717,10 @@ static void device_initfn(Object *obj)
     }
 
     dev->instance_id_alias = -1;
-    dev->state = DEV_STATE_CREATED;
+    dev->realized = false;
+
+    object_property_add_bool(obj, "realized",
+                             device_get_realized, device_set_realized, NULL);
 
     class = object_get_class(OBJECT(dev));
     do {
@@ -679,23 +740,8 @@ static void device_initfn(Object *obj)
 static void device_finalize(Object *obj)
 {
     DeviceState *dev = DEVICE(obj);
-    BusState *bus;
-    DeviceClass *dc = DEVICE_GET_CLASS(dev);
-
-    if (dev->state == DEV_STATE_INITIALIZED) {
-        while (dev->num_child_bus) {
-            bus = QLIST_FIRST(&dev->child_bus);
-            qbus_free(bus);
-        }
-        if (qdev_get_vmsd(dev)) {
-            vmstate_unregister(dev, qdev_get_vmsd(dev), dev);
-        }
-        if (dc->exit) {
-            dc->exit(dev);
-        }
-        if (dev->opts) {
-            qemu_opts_del(dev->opts);
-        }
+    if (dev->opts) {
+        qemu_opts_del(dev->opts);
     }
 }
 
@@ -709,16 +755,37 @@ static void device_class_base_init(ObjectClass *class, void *data)
     klass->props = NULL;
 }
 
-static void qdev_remove_from_bus(Object *obj)
+static void device_unparent(Object *obj)
 {
     DeviceState *dev = DEVICE(obj);
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
+    BusState *bus;
 
-    bus_remove_child(dev->parent_bus, dev);
+    while (dev->num_child_bus) {
+        bus = QLIST_FIRST(&dev->child_bus);
+        qbus_free(bus);
+    }
+    if (dev->realized) {
+        if (qdev_get_vmsd(dev)) {
+            vmstate_unregister(dev, qdev_get_vmsd(dev), dev);
+        }
+        if (dc->exit) {
+            dc->exit(dev);
+        }
+    }
+    if (dev->parent_bus) {
+        bus_remove_child(dev->parent_bus, dev);
+        object_unref(OBJECT(dev->parent_bus));
+        dev->parent_bus = NULL;
+    }
 }
 
 static void device_class_init(ObjectClass *class, void *data)
 {
-    class->unparent = qdev_remove_from_bus;
+    DeviceClass *dc = DEVICE_CLASS(class);
+
+    class->unparent = device_unparent;
+    dc->realize = device_realize;
 }
 
 void device_reset(DeviceState *dev)
@@ -741,7 +808,7 @@ Object *qdev_get_machine(void)
     return dev;
 }
 
-static TypeInfo device_type_info = {
+static const TypeInfo device_type_info = {
     .name = TYPE_DEVICE,
     .parent = TYPE_OBJECT,
     .instance_size = sizeof(DeviceState),
@@ -760,22 +827,15 @@ static void qbus_initfn(Object *obj)
     QTAILQ_INIT(&bus->children);
 }
 
+static void bus_class_init(ObjectClass *class, void *data)
+{
+    class->unparent = bus_unparent;
+}
+
 static void qbus_finalize(Object *obj)
 {
     BusState *bus = BUS(obj);
-    BusChild *kid;
 
-    while ((kid = QTAILQ_FIRST(&bus->children)) != NULL) {
-        DeviceState *dev = kid->child;
-        qdev_free(dev);
-    }
-    if (bus->parent) {
-        QLIST_REMOVE(bus, sibling);
-        bus->parent->num_child_bus--;
-    } else {
-        assert(bus != sysbus_get_default()); /* main_system_bus is never freed */
-        qemu_unregister_reset(qbus_reset_all_fn, bus);
-    }
     g_free((char *)bus->name);
 }
 
@@ -787,6 +847,7 @@ static const TypeInfo bus_info = {
     .class_size = sizeof(BusClass),
     .instance_init = qbus_initfn,
     .instance_finalize = qbus_finalize,
+    .class_init = bus_class_init,
 };
 
 static void qdev_register_types(void)
diff --git a/hw/qxl-logger.c b/hw/qxl-logger.c
index fe2878c..3cd85d9 100644
--- a/hw/qxl-logger.c
+++ b/hw/qxl-logger.c
@@ -19,7 +19,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "qxl.h"
 
 static const char *qxl_type[] = {
diff --git a/hw/qxl-render.c b/hw/qxl-render.c
index 98ecb21..88e63f8 100644
--- a/hw/qxl-render.c
+++ b/hw/qxl-render.c
@@ -113,11 +113,12 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
                qxl->guest_primary.bits_pp);
         if (qxl->guest_primary.qxl_stride > 0) {
             qemu_free_displaysurface(vga->ds);
-            qemu_create_displaysurface_from(qxl->guest_primary.surface.width,
-                                            qxl->guest_primary.surface.height,
-                                            qxl->guest_primary.bits_pp,
-                                            qxl->guest_primary.abs_stride,
-                                            qxl->guest_primary.data);
+            vga->ds->surface = qemu_create_displaysurface_from
+                (qxl->guest_primary.surface.width,
+                 qxl->guest_primary.surface.height,
+                 qxl->guest_primary.bits_pp,
+                 qxl->guest_primary.abs_stride,
+                 qxl->guest_primary.data);
         } else {
             qemu_resize_displaysurface(vga->ds,
                     qxl->guest_primary.surface.width,
diff --git a/hw/qxl.c b/hw/qxl.c
index 96887c4..2e1c5e2 100644
--- a/hw/qxl.c
+++ b/hw/qxl.c
@@ -21,10 +21,10 @@
 #include <zlib.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "qemu-queue.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "qemu/queue.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
 #include "qxl.h"
@@ -37,33 +37,25 @@
  */
 #undef SPICE_RING_PROD_ITEM
 #define SPICE_RING_PROD_ITEM(qxl, r, ret) {                             \
-        typeof(r) start = r;                                            \
-        typeof(r) end = r + 1;                                          \
         uint32_t prod = (r)->prod & SPICE_RING_INDEX_MASK(r);           \
-        typeof(&(r)->items[prod]) m_item = &(r)->items[prod];           \
-        if (!((uint8_t*)m_item >= (uint8_t*)(start) && (uint8_t*)(m_item + 1) <= (uint8_t*)(end))) { \
+        if (prod >= ARRAY_SIZE((r)->items)) {                           \
             qxl_set_guest_bug(qxl, "SPICE_RING_PROD_ITEM indices mismatch " \
-                          "! %p <= %p < %p", (uint8_t *)start,          \
-                          (uint8_t *)m_item, (uint8_t *)end);           \
+                          "%u >= %zu", prod, ARRAY_SIZE((r)->items));   \
             ret = NULL;                                                 \
         } else {                                                        \
-            ret = &m_item->el;                                          \
+            ret = &(r)->items[prod].el;                                 \
         }                                                               \
     }
 
 #undef SPICE_RING_CONS_ITEM
 #define SPICE_RING_CONS_ITEM(qxl, r, ret) {                             \
-        typeof(r) start = r;                                            \
-        typeof(r) end = r + 1;                                          \
         uint32_t cons = (r)->cons & SPICE_RING_INDEX_MASK(r);           \
-        typeof(&(r)->items[cons]) m_item = &(r)->items[cons];           \
-        if (!((uint8_t*)m_item >= (uint8_t*)(start) && (uint8_t*)(m_item + 1) <= (uint8_t*)(end))) { \
+        if (cons >= ARRAY_SIZE((r)->items)) {                           \
             qxl_set_guest_bug(qxl, "SPICE_RING_CONS_ITEM indices mismatch " \
-                          "! %p <= %p < %p", (uint8_t *)start,          \
-                          (uint8_t *)m_item, (uint8_t *)end);           \
+                          "%u >= %zu", cons, ARRAY_SIZE((r)->items));   \
             ret = NULL;                                                 \
         } else {                                                        \
-            ret = &m_item->el;                                          \
+            ret = &(r)->items[cons].el;                                 \
         }                                                               \
     }
 
@@ -88,9 +80,7 @@
 
 #define QXL_MODE_EX(x_res, y_res)                 \
     QXL_MODE_16_32(x_res, y_res, 0),              \
-    QXL_MODE_16_32(y_res, x_res, 1),              \
-    QXL_MODE_16_32(x_res, y_res, 2),              \
-    QXL_MODE_16_32(y_res, x_res, 3)
+    QXL_MODE_16_32(x_res, y_res, 1)
 
 static QXLMode qxl_modes[] = {
     QXL_MODE_EX(640, 480),
@@ -314,10 +304,13 @@ static inline uint32_t msb_mask(uint32_t val)
 
 static ram_addr_t qxl_rom_size(void)
 {
-    uint32_t rom_size = sizeof(QXLRom) + sizeof(QXLModes) + sizeof(qxl_modes);
+    uint32_t required_rom_size = sizeof(QXLRom) + sizeof(QXLModes) +
+                                 sizeof(qxl_modes);
+    uint32_t rom_size = 8192; /* two pages */
 
-    rom_size = MAX(rom_size, TARGET_PAGE_SIZE);
-    rom_size = msb_mask(rom_size * 2 - 1);
+    required_rom_size = MAX(required_rom_size, TARGET_PAGE_SIZE);
+    required_rom_size = msb_mask(required_rom_size * 2 - 1);
+    assert(required_rom_size <= rom_size);
     return rom_size;
 }
 
@@ -953,15 +946,23 @@ static void interface_set_client_capabilities(QXLInstance *sin,
 {
     PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
 
+    if (qxl->revision < 4) {
+        trace_qxl_set_client_capabilities_unsupported_by_revision(qxl->id,
+                                                              qxl->revision);
+        return;
+    }
+
     if (runstate_check(RUN_STATE_INMIGRATE) ||
         runstate_check(RUN_STATE_POSTMIGRATE)) {
         return;
     }
 
     qxl->shadow_rom.client_present = client_present;
-    memcpy(qxl->shadow_rom.client_capabilities, caps, sizeof(caps));
+    memcpy(qxl->shadow_rom.client_capabilities, caps,
+           sizeof(qxl->shadow_rom.client_capabilities));
     qxl->rom->client_present = client_present;
-    memcpy(qxl->rom->client_capabilities, caps, sizeof(caps));
+    memcpy(qxl->rom->client_capabilities, caps,
+           sizeof(qxl->rom->client_capabilities));
     qxl_rom_set_dirty(qxl);
 
     qxl_send_events(qxl, QXL_INTERRUPT_CLIENT);
@@ -985,6 +986,11 @@ static int interface_client_monitors_config(QXLInstance *sin,
     QXLRom *rom = memory_region_get_ram_ptr(&qxl->rom_bar);
     int i;
 
+    if (qxl->revision < 4) {
+        trace_qxl_client_monitors_config_unsupported_by_device(qxl->id,
+                                                               qxl->revision);
+        return 0;
+    }
     /*
      * Older windows drivers set int_mask to 0 when their ISR is called,
      * then later set it to ~0. So it doesn't relate to the actual interrupts
@@ -2030,7 +2036,7 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     qxl->ssd.qxl.base.sif = &qxl_interface.base;
     qxl->ssd.qxl.id = qxl->id;
     if (qemu_spice_add_interface(&qxl->ssd.qxl.base) != 0) {
-        error_report("qxl interface %d.%d not supported by spice-server\n",
+        error_report("qxl interface %d.%d not supported by spice-server",
                      SPICE_INTERFACE_QXL_MAJOR, SPICE_INTERFACE_QXL_MINOR);
         return -1;
     }
@@ -2310,7 +2316,7 @@ static void qxl_primary_class_init(ObjectClass *klass, void *data)
     dc->props = qxl_properties;
 }
 
-static TypeInfo qxl_primary_info = {
+static const TypeInfo qxl_primary_info = {
     .name          = "qxl-vga",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIQXLDevice),
@@ -2332,7 +2338,7 @@ static void qxl_secondary_class_init(ObjectClass *klass, void *data)
     dc->props = qxl_properties;
 }
 
-static TypeInfo qxl_secondary_info = {
+static const TypeInfo qxl_secondary_info = {
     .name          = "qxl",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIQXLDevice),
diff --git a/hw/qxl.h b/hw/qxl.h
index e583cfb..f867a1d 100644
--- a/hw/qxl.h
+++ b/hw/qxl.h
@@ -1,10 +1,13 @@
+#ifndef HW_QXL_H
+#define HW_QXL_H 1
+
 #include "qemu-common.h"
 
-#include "console.h"
+#include "ui/console.h"
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "vga_int.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "ui/qemu-spice.h"
 #include "ui/spice-display.h"
@@ -158,3 +161,5 @@ void qxl_render_update(PCIQXLDevice *qxl);
 int qxl_render_cursor(PCIQXLDevice *qxl, QXLCommandExt *ext);
 void qxl_render_update_area_done(PCIQXLDevice *qxl, QXLCookie *cookie);
 void qxl_render_update_area_bh(void *opaque);
+
+#endif
diff --git a/hw/r2d.c b/hw/r2d.c
index 66212e9..2d0dd1f 100644
--- a/hw/r2d.c
+++ b/hw/r2d.c
@@ -27,17 +27,17 @@
 #include "hw.h"
 #include "sh.h"
 #include "devices.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "pci.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "net/net.h"
 #include "sh7750_regs.h"
 #include "ide.h"
 #include "loader.h"
 #include "usb.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define FLASH_BASE 0x00000000
 #define FLASH_SIZE 0x02000000
@@ -262,7 +262,7 @@ static void r2d_init(QEMUMachineInitArgs *args)
     irq = r2d_fpga_init(address_space_mem, 0x04000000, sh7750_irl(s));
 
     dev = qdev_create(NULL, "sh_pci");
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     sysbus_mmio_map(busdev, 0, P4ADDR(0x1e200000));
     sysbus_mmio_map(busdev, 1, A7ADDR(0x1e200000));
@@ -276,8 +276,14 @@ static void r2d_init(QEMUMachineInitArgs *args)
 
     /* onboard CF (True IDE mode, Master only). */
     dinfo = drive_get(IF_IDE, 0, 0);
-    mmio_ide_init(0x14001000, 0x1400080c, address_space_mem, irq[CF_IDE], 1,
-                  dinfo, NULL);
+    dev = qdev_create(NULL, "mmio-ide");
+    busdev = SYS_BUS_DEVICE(dev);
+    sysbus_connect_irq(busdev, 0, irq[CF_IDE]);
+    qdev_prop_set_uint32(dev, "shift", 1);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(busdev, 0, 0x14001000);
+    sysbus_mmio_map(busdev, 1, 0x1400080c);
+    mmio_ide_init_drives(dev, dinfo, NULL);
 
     /* onboard flash memory */
     dinfo = drive_get(IF_PFLASH, 0, 0);
@@ -347,6 +353,7 @@ static QEMUMachine r2d_machine = {
     .name = "r2d",
     .desc = "r2d-plus board",
     .init = r2d_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void r2d_machine_init(void)
diff --git a/hw/rc4030.c b/hw/rc4030.c
index e0024c8..a0358a3 100644
--- a/hw/rc4030.c
+++ b/hw/rc4030.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "mips.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /********************************************************/
 /* debug rc4030 */
diff --git a/hw/realview.c b/hw/realview.c
index e789c15..78da767 100644
--- a/hw/realview.c
+++ b/hw/realview.c
@@ -11,13 +11,13 @@
 #include "arm-misc.h"
 #include "primecell.h"
 #include "devices.h"
-#include "pci.h"
-#include "net.h"
-#include "sysemu.h"
+#include "pci/pci.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "i2c.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 #define SMP_BOOT_ADDR 0xe0000000
 #define SMP_BOOTREG_ADDR 0x10000030
@@ -140,14 +140,14 @@ static void realview_init(QEMUMachineInitArgs *args,
     qdev_prop_set_uint32(sysctl, "sys_id", sys_id);
     qdev_prop_set_uint32(sysctl, "proc_id", proc_id);
     qdev_init_nofail(sysctl);
-    sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, 0x10000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(sysctl), 0, 0x10000000);
 
     if (is_mpcore) {
         hwaddr periphbase;
         dev = qdev_create(NULL, is_pb ? "a9mpcore_priv": "realview_mpcore");
         qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
         qdev_init_nofail(dev);
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         if (is_pb) {
             periphbase = 0x1f000000;
         } else {
@@ -172,8 +172,8 @@ static void realview_init(QEMUMachineInitArgs *args,
     pl041 = qdev_create(NULL, "pl041");
     qdev_prop_set_uint32(pl041, "nc_fifo_depth", 512);
     qdev_init_nofail(pl041);
-    sysbus_mmio_map(sysbus_from_qdev(pl041), 0, 0x10004000);
-    sysbus_connect_irq(sysbus_from_qdev(pl041), 0, pic[19]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(pl041), 0, 0x10004000);
+    sysbus_connect_irq(SYS_BUS_DEVICE(pl041), 0, pic[19]);
 
     sysbus_create_simple("pl050_keyboard", 0x10006000, pic[20]);
     sysbus_create_simple("pl050_mouse", 0x10007000, pic[21]);
@@ -215,7 +215,7 @@ static void realview_init(QEMUMachineInitArgs *args,
 
     if (!is_pb) {
         dev = qdev_create(NULL, "realview_pci");
-        busdev = sysbus_from_qdev(dev);
+        busdev = SYS_BUS_DEVICE(dev);
         qdev_init_nofail(dev);
         sysbus_mmio_map(busdev, 0, 0x61000000); /* PCI self-config */
         sysbus_mmio_map(busdev, 1, 0x62000000); /* PCI config */
@@ -364,29 +364,33 @@ static QEMUMachine realview_eb_machine = {
     .name = "realview-eb",
     .desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)",
     .init = realview_eb_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine realview_eb_mpcore_machine = {
     .name = "realview-eb-mpcore",
     .desc = "ARM RealView Emulation Baseboard (ARM11MPCore)",
     .init = realview_eb_mpcore_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine realview_pb_a8_machine = {
     .name = "realview-pb-a8",
     .desc = "ARM RealView Platform Baseboard for Cortex-A8",
     .init = realview_pb_a8_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine realview_pbx_a9_machine = {
     .name = "realview-pbx-a9",
     .desc = "ARM RealView Platform Baseboard Explore for Cortex-A9",
     .init = realview_pbx_a9_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void realview_machine_init(void)
diff --git a/hw/realview_gic.c b/hw/realview_gic.c
index 5bc37a7..8f2a7e2 100644
--- a/hw/realview_gic.c
+++ b/hw/realview_gic.c
@@ -35,7 +35,7 @@ static int realview_gic_init(SysBusDevice *dev)
     qdev_prop_set_uint32(s->gic, "num-cpu", 1);
     qdev_prop_set_uint32(s->gic, "num-irq", numirq);
     qdev_init_nofail(s->gic);
-    busdev = sysbus_from_qdev(s->gic);
+    busdev = SYS_BUS_DEVICE(s->gic);
 
     /* Pass through outbound IRQ lines from the GIC */
     sysbus_pass_irq(dev, busdev);
@@ -59,7 +59,7 @@ static void realview_gic_class_init(ObjectClass *klass, void *data)
     sdc->init = realview_gic_init;
 }
 
-static TypeInfo realview_gic_info = {
+static const TypeInfo realview_gic_info = {
     .name          = "realview_gic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RealViewGICState),
diff --git a/hw/rtl8139.c b/hw/rtl8139.c
index e3aa8bf..d7716be 100644
--- a/hw/rtl8139.c
+++ b/hw/rtl8139.c
@@ -52,13 +52,13 @@
 #include <zlib.h>
 
 #include "hw.h"
-#include "pci.h"
-#include "dma.h"
-#include "qemu-timer.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "sysemu/dma.h"
+#include "qemu/timer.h"
+#include "net/net.h"
 #include "loader.h"
-#include "sysemu.h"
-#include "iov.h"
+#include "sysemu/sysemu.h"
+#include "qemu/iov.h"
 
 /* debug RTL8139 card */
 //#define DEBUG_RTL8139 1
@@ -786,7 +786,7 @@ static bool rtl8139_cp_rx_valid(RTL8139State *s)
 
 static int rtl8139_can_receive(NetClientState *nc)
 {
-    RTL8139State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    RTL8139State *s = qemu_get_nic_opaque(nc);
     int avail;
 
     /* Receive (drop) packets if card is disabled.  */
@@ -808,7 +808,7 @@ static int rtl8139_can_receive(NetClientState *nc)
 
 static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t size_, int do_interrupt)
 {
-    RTL8139State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    RTL8139State *s = qemu_get_nic_opaque(nc);
     /* size is the length of the buffer passed to the driver */
     int size = size_;
     const uint8_t *dot1q_buf = NULL;
@@ -1258,7 +1258,8 @@ static void rtl8139_reset(DeviceState *d)
     s->BasicModeStatus  = 0x7809;
     //s->BasicModeStatus |= 0x0040; /* UTP medium */
     s->BasicModeStatus |= 0x0020; /* autonegotiation completed */
-    s->BasicModeStatus |= 0x0004; /* link is up */
+    /* preserve link state */
+    s->BasicModeStatus |= qemu_get_queue(s->nic)->link_down ? 0 : 0x04;
 
     s->NWayAdvert    = 0x05e1; /* all modes, full duplex */
     s->NWayLPAR      = 0x05e1; /* all modes, full duplex */
@@ -1786,7 +1787,7 @@ static void rtl8139_transfer_frame(RTL8139State *s, uint8_t *buf, int size,
         }
 
         DPRINTF("+++ transmit loopback mode\n");
-        rtl8139_do_receive(&s->nic->nc, buf, size, do_interrupt);
+        rtl8139_do_receive(qemu_get_queue(s->nic), buf, size, do_interrupt);
 
         if (iov) {
             g_free(buf2);
@@ -1795,9 +1796,9 @@ static void rtl8139_transfer_frame(RTL8139State *s, uint8_t *buf, int size,
     else
     {
         if (iov) {
-            qemu_sendv_packet(&s->nic->nc, iov, 3);
+            qemu_sendv_packet(qemu_get_queue(s->nic), iov, 3);
         } else {
-            qemu_send_packet(&s->nic->nc, buf, size);
+            qemu_send_packet(qemu_get_queue(s->nic), buf, size);
         }
     }
 }
@@ -3229,7 +3230,7 @@ static int rtl8139_post_load(void *opaque, int version_id)
 
     /* nc.link_down can't be migrated, so infer link_down according
      * to link status bit in BasicModeStatus */
-    s->nic->nc.link_down = (s->BasicModeStatus & 0x04) == 0;
+    qemu_get_queue(s->nic)->link_down = (s->BasicModeStatus & 0x04) == 0;
 
     return 0;
 }
@@ -3428,7 +3429,7 @@ static void rtl8139_timer(void *opaque)
 
 static void rtl8139_cleanup(NetClientState *nc)
 {
-    RTL8139State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    RTL8139State *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -3445,12 +3446,12 @@ static void pci_rtl8139_uninit(PCIDevice *dev)
     }
     qemu_del_timer(s->timer);
     qemu_free_timer(s->timer);
-    qemu_del_net_client(&s->nic->nc);
+    qemu_del_nic(s->nic);
 }
 
 static void rtl8139_set_link_status(NetClientState *nc)
 {
-    RTL8139State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    RTL8139State *s = qemu_get_nic_opaque(nc);
 
     if (nc->link_down) {
         s->BasicModeStatus &= ~0x04;
@@ -3502,7 +3503,7 @@ static int pci_rtl8139_init(PCIDevice *dev)
 
     s->nic = qemu_new_nic(&net_rtl8139_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     s->cplus_txbuffer = NULL;
     s->cplus_txbuffer_len = 0;
@@ -3539,7 +3540,7 @@ static void rtl8139_class_init(ObjectClass *klass, void *data)
     dc->props = rtl8139_properties;
 }
 
-static TypeInfo rtl8139_info = {
+static const TypeInfo rtl8139_info = {
     .name          = "rtl8139",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(RTL8139State),
diff --git a/hw/s390-virtio.c b/hw/s390-virtio.c
deleted file mode 100644
index ca1bb09..0000000
--- a/hw/s390-virtio.c
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * QEMU S390 virtio target
- *
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "hw.h"
-#include "block.h"
-#include "blockdev.h"
-#include "sysemu.h"
-#include "net.h"
-#include "boards.h"
-#include "monitor.h"
-#include "loader.h"
-#include "elf.h"
-#include "hw/virtio.h"
-#include "hw/sysbus.h"
-#include "kvm.h"
-#include "exec-memory.h"
-
-#include "hw/s390-virtio-bus.h"
-#include "hw/s390x/sclp.h"
-
-//#define DEBUG_S390
-
-#ifdef DEBUG_S390
-#define dprintf(fmt, ...) \
-    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
-#else
-#define dprintf(fmt, ...) \
-    do { } while (0)
-#endif
-
-#define KVM_S390_VIRTIO_NOTIFY          0
-#define KVM_S390_VIRTIO_RESET           1
-#define KVM_S390_VIRTIO_SET_STATUS      2
-
-#define KERN_IMAGE_START                0x010000UL
-#define KERN_PARM_AREA                  0x010480UL
-#define INITRD_START                    0x800000UL
-#define INITRD_PARM_START               0x010408UL
-#define INITRD_PARM_SIZE                0x010410UL
-#define PARMFILE_START                  0x001000UL
-
-#define ZIPL_START			0x009000UL
-#define ZIPL_LOAD_ADDR			0x009000UL
-#define ZIPL_FILENAME			"s390-zipl.rom"
-
-#define MAX_BLK_DEVS                    10
-
-static VirtIOS390Bus *s390_bus;
-static S390CPU **ipi_states;
-
-S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
-{
-    if (cpu_addr >= smp_cpus) {
-        return NULL;
-    }
-
-    return ipi_states[cpu_addr];
-}
-
-int s390_virtio_hypercall(CPUS390XState *env, uint64_t mem, uint64_t hypercall)
-{
-    int r = 0, i;
-
-    dprintf("KVM hypercall: %ld\n", hypercall);
-    switch (hypercall) {
-    case KVM_S390_VIRTIO_NOTIFY:
-        if (mem > ram_size) {
-            VirtIOS390Device *dev = s390_virtio_bus_find_vring(s390_bus,
-                                                               mem, &i);
-            if (dev) {
-                virtio_queue_notify(dev->vdev, i);
-            } else {
-                r = -EINVAL;
-            }
-        } else {
-            /* Early printk */
-        }
-        break;
-    case KVM_S390_VIRTIO_RESET:
-    {
-        VirtIOS390Device *dev;
-
-        dev = s390_virtio_bus_find_mem(s390_bus, mem);
-        virtio_reset(dev->vdev);
-        stb_phys(dev->dev_offs + VIRTIO_DEV_OFFS_STATUS, 0);
-        s390_virtio_device_sync(dev);
-        s390_virtio_reset_idx(dev);
-        break;
-    }
-    case KVM_S390_VIRTIO_SET_STATUS:
-    {
-        VirtIOS390Device *dev;
-
-        dev = s390_virtio_bus_find_mem(s390_bus, mem);
-        if (dev) {
-            s390_virtio_device_update_status(dev);
-        } else {
-            r = -EINVAL;
-        }
-        break;
-    }
-    default:
-        r = -EINVAL;
-        break;
-    }
-
-    return r;
-}
-
-/*
- * The number of running CPUs. On s390 a shutdown is the state of all CPUs
- * being either stopped or disabled (for interrupts) waiting. We have to
- * track this number to call the shutdown sequence accordingly. This
- * number is modified either on startup or while holding the big qemu lock.
- */
-static unsigned s390_running_cpus;
-
-void s390_add_running_cpu(CPUS390XState *env)
-{
-    if (env->halted) {
-        s390_running_cpus++;
-        env->halted = 0;
-        env->exception_index = -1;
-    }
-}
-
-unsigned s390_del_running_cpu(CPUS390XState *env)
-{
-    if (env->halted == 0) {
-        assert(s390_running_cpus >= 1);
-        s390_running_cpus--;
-        env->halted = 1;
-        env->exception_index = EXCP_HLT;
-    }
-    return s390_running_cpus;
-}
-
-/* PC hardware initialisation */
-static void s390_init(QEMUMachineInitArgs *args)
-{
-    ram_addr_t my_ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
-    CPUS390XState *env = NULL;
-    MemoryRegion *sysmem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    ram_addr_t kernel_size = 0;
-    ram_addr_t initrd_offset;
-    ram_addr_t initrd_size = 0;
-    int shift = 0;
-    uint8_t *storage_keys;
-    void *virtio_region;
-    hwaddr virtio_region_len;
-    hwaddr virtio_region_start;
-    int i;
-
-    /* s390x ram size detection needs a 16bit multiplier + an increment. So
-       guests > 64GB can be specified in 2MB steps etc. */
-    while ((my_ram_size >> (20 + shift)) > 65535) {
-        shift++;
-    }
-    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
-
-    /* lets propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
-
-    /* get a BUS */
-    s390_bus = s390_virtio_bus_init(&my_ram_size);
-    s390_sclp_init();
-
-    /* allocate RAM */
-    memory_region_init_ram(ram, "s390.ram", my_ram_size);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
-
-    /* clear virtio region */
-    virtio_region_len = my_ram_size - ram_size;
-    virtio_region_start = ram_size;
-    virtio_region = cpu_physical_memory_map(virtio_region_start,
-                                            &virtio_region_len, true);
-    memset(virtio_region, 0, virtio_region_len);
-    cpu_physical_memory_unmap(virtio_region, virtio_region_len, 1,
-                              virtio_region_len);
-
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
-    /* init CPUs */
-    if (cpu_model == NULL) {
-        cpu_model = "host";
-    }
-
-    ipi_states = g_malloc(sizeof(S390CPU *) * smp_cpus);
-
-    for (i = 0; i < smp_cpus; i++) {
-        S390CPU *cpu;
-        CPUS390XState *tmp_env;
-
-        cpu = cpu_s390x_init(cpu_model);
-        tmp_env = &cpu->env;
-        if (!env) {
-            env = tmp_env;
-        }
-        ipi_states[i] = cpu;
-        tmp_env->halted = 1;
-        tmp_env->exception_index = EXCP_HLT;
-        tmp_env->storage_keys = storage_keys;
-    }
-
-    /* One CPU has to run */
-    s390_add_running_cpu(env);
-
-    if (kernel_filename) {
-
-        kernel_size = load_elf(kernel_filename, NULL, NULL, NULL, NULL,
-                               NULL, 1, ELF_MACHINE, 0);
-        if (kernel_size == -1UL) {
-            kernel_size = load_image_targphys(kernel_filename, 0, ram_size);
-        }
-        if (kernel_size == -1UL) {
-            fprintf(stderr, "qemu: could not load kernel '%s'\n",
-                    kernel_filename);
-            exit(1);
-        }
-        /*
-         * we can not rely on the ELF entry point, since up to 3.2 this
-         * value was 0x800 (the SALIPL loader) and it wont work. For
-         * all (Linux) cases 0x10000 (KERN_IMAGE_START) should be fine.
-         */
-        env->psw.addr = KERN_IMAGE_START;
-        env->psw.mask = 0x0000000180000000ULL;
-    } else {
-        ram_addr_t bios_size = 0;
-        char *bios_filename;
-
-        /* Load zipl bootloader */
-        if (bios_name == NULL) {
-            bios_name = ZIPL_FILENAME;
-        }
-
-        bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
-        bios_size = load_image_targphys(bios_filename, ZIPL_LOAD_ADDR, 4096);
-        g_free(bios_filename);
-
-        if ((long)bios_size < 0) {
-            hw_error("could not load bootloader '%s'\n", bios_name);
-        }
-
-        if (bios_size > 4096) {
-            hw_error("stage1 bootloader is > 4k\n");
-        }
-
-        env->psw.addr = ZIPL_START;
-        env->psw.mask = 0x0000000180000000ULL;
-    }
-
-    if (initrd_filename) {
-        initrd_offset = INITRD_START;
-        while (kernel_size + 0x100000 > initrd_offset) {
-            initrd_offset += 0x100000;
-        }
-        initrd_size = load_image_targphys(initrd_filename, initrd_offset,
-                                          ram_size - initrd_offset);
-        if (initrd_size == -1UL) {
-            fprintf(stderr, "qemu: could not load initrd '%s'\n",
-                    initrd_filename);
-            exit(1);
-        }
-
-        /* we have to overwrite values in the kernel image, which are "rom" */
-        stq_p(rom_ptr(INITRD_PARM_START), initrd_offset);
-        stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
-    }
-
-    if (rom_ptr(KERN_PARM_AREA)) {
-        /* we have to overwrite values in the kernel image, which are "rom" */
-        memcpy(rom_ptr(KERN_PARM_AREA), kernel_cmdline,
-               strlen(kernel_cmdline) + 1);
-    }
-
-    /* Create VirtIO network adapters */
-    for(i = 0; i < nb_nics; i++) {
-        NICInfo *nd = &nd_table[i];
-        DeviceState *dev;
-
-        if (!nd->model) {
-            nd->model = g_strdup("virtio");
-        }
-
-        if (strcmp(nd->model, "virtio")) {
-            fprintf(stderr, "S390 only supports VirtIO nics\n");
-            exit(1);
-        }
-
-        dev = qdev_create((BusState *)s390_bus, "virtio-net-s390");
-        qdev_set_nic_properties(dev, nd);
-        qdev_init_nofail(dev);
-    }
-
-    /* Create VirtIO disk drives */
-    for(i = 0; i < MAX_BLK_DEVS; i++) {
-        DriveInfo *dinfo;
-        DeviceState *dev;
-
-        dinfo = drive_get(IF_IDE, 0, i);
-        if (!dinfo) {
-            continue;
-        }
-
-        dev = qdev_create((BusState *)s390_bus, "virtio-blk-s390");
-        qdev_prop_set_drive_nofail(dev, "drive", dinfo->bdrv);
-        qdev_init_nofail(dev);
-    }
-}
-
-static QEMUMachine s390_machine = {
-    .name = "s390-virtio",
-    .alias = "s390",
-    .desc = "VirtIO based S390 machine",
-    .init = s390_init,
-    .no_cdrom = 1,
-    .no_floppy = 1,
-    .no_serial = 1,
-    .no_parallel = 1,
-    .no_sdcard = 1,
-    .use_virtcon = 1,
-    .max_cpus = 255,
-    .is_default = 1,
-};
-
-static void s390_machine_init(void)
-{
-    qemu_register_machine(&s390_machine);
-}
-
-machine_init(s390_machine_init);
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 096dfcd..9f2f419 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -1,6 +1,9 @@
 obj-y = s390-virtio-bus.o s390-virtio.o
-
-obj-y := $(addprefix ../,$(obj-y))
+obj-y += s390-virtio-hcall.o
 obj-y += sclp.o
 obj-y += event-facility.o
 obj-y += sclpquiesce.o sclpconsole.o
+obj-y += ipl.o
+obj-y += css.o
+obj-y += s390-virtio-ccw.o
+obj-y += virtio-ccw.o
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
new file mode 100644
index 0000000..85f6f22
--- /dev/null
+++ b/hw/s390x/css.c
@@ -0,0 +1,1277 @@
+/*
+ * Channel subsystem base support.
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include <hw/qdev.h>
+#include "qemu/bitops.h"
+#include "cpu.h"
+#include "ioinst.h"
+#include "css.h"
+#include "trace.h"
+
+typedef struct CrwContainer {
+    CRW crw;
+    QTAILQ_ENTRY(CrwContainer) sibling;
+} CrwContainer;
+
+typedef struct ChpInfo {
+    uint8_t in_use;
+    uint8_t type;
+    uint8_t is_virtual;
+} ChpInfo;
+
+typedef struct SubchSet {
+    SubchDev *sch[MAX_SCHID + 1];
+    unsigned long schids_used[BITS_TO_LONGS(MAX_SCHID + 1)];
+    unsigned long devnos_used[BITS_TO_LONGS(MAX_SCHID + 1)];
+} SubchSet;
+
+typedef struct CssImage {
+    SubchSet *sch_set[MAX_SSID + 1];
+    ChpInfo chpids[MAX_CHPID + 1];
+} CssImage;
+
+typedef struct ChannelSubSys {
+    QTAILQ_HEAD(, CrwContainer) pending_crws;
+    bool do_crw_mchk;
+    bool crws_lost;
+    uint8_t max_cssid;
+    uint8_t max_ssid;
+    bool chnmon_active;
+    uint64_t chnmon_area;
+    CssImage *css[MAX_CSSID + 1];
+    uint8_t default_cssid;
+} ChannelSubSys;
+
+static ChannelSubSys *channel_subsys;
+
+int css_create_css_image(uint8_t cssid, bool default_image)
+{
+    trace_css_new_image(cssid, default_image ? "(default)" : "");
+    if (cssid > MAX_CSSID) {
+        return -EINVAL;
+    }
+    if (channel_subsys->css[cssid]) {
+        return -EBUSY;
+    }
+    channel_subsys->css[cssid] = g_malloc0(sizeof(CssImage));
+    if (default_image) {
+        channel_subsys->default_cssid = cssid;
+    }
+    return 0;
+}
+
+static uint16_t css_build_subchannel_id(SubchDev *sch)
+{
+    if (channel_subsys->max_cssid > 0) {
+        return (sch->cssid << 8) | (1 << 3) | (sch->ssid << 1) | 1;
+    }
+    return (sch->ssid << 1) | 1;
+}
+
+static void css_inject_io_interrupt(SubchDev *sch)
+{
+    S390CPU *cpu = s390_cpu_addr2state(0);
+    uint8_t isc = (sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ISC) >> 11;
+
+    trace_css_io_interrupt(sch->cssid, sch->ssid, sch->schid,
+                           sch->curr_status.pmcw.intparm, isc, "");
+    s390_io_interrupt(cpu,
+                      css_build_subchannel_id(sch),
+                      sch->schid,
+                      sch->curr_status.pmcw.intparm,
+                      isc << 27);
+}
+
+void css_conditional_io_interrupt(SubchDev *sch)
+{
+    /*
+     * If the subchannel is not currently status pending, make it pending
+     * with alert status.
+     */
+    if (!(sch->curr_status.scsw.ctrl & SCSW_STCTL_STATUS_PEND)) {
+        S390CPU *cpu = s390_cpu_addr2state(0);
+        uint8_t isc = (sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ISC) >> 11;
+
+        trace_css_io_interrupt(sch->cssid, sch->ssid, sch->schid,
+                               sch->curr_status.pmcw.intparm, isc,
+                               "(unsolicited)");
+        sch->curr_status.scsw.ctrl &= ~SCSW_CTRL_MASK_STCTL;
+        sch->curr_status.scsw.ctrl |=
+            SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND;
+        /* Inject an I/O interrupt. */
+        s390_io_interrupt(cpu,
+                          css_build_subchannel_id(sch),
+                          sch->schid,
+                          sch->curr_status.pmcw.intparm,
+                          isc << 27);
+    }
+}
+
+static void sch_handle_clear_func(SubchDev *sch)
+{
+    PMCW *p = &sch->curr_status.pmcw;
+    SCSW *s = &sch->curr_status.scsw;
+    int path;
+
+    /* Path management: In our simple css, we always choose the only path. */
+    path = 0x80;
+
+    /* Reset values prior to 'issueing the clear signal'. */
+    p->lpum = 0;
+    p->pom = 0xff;
+    s->flags &= ~SCSW_FLAGS_MASK_PNO;
+
+    /* We always 'attempt to issue the clear signal', and we always succeed. */
+    sch->orb = NULL;
+    sch->channel_prog = 0x0;
+    sch->last_cmd_valid = false;
+    s->ctrl &= ~SCSW_ACTL_CLEAR_PEND;
+    s->ctrl |= SCSW_STCTL_STATUS_PEND;
+
+    s->dstat = 0;
+    s->cstat = 0;
+    p->lpum = path;
+
+}
+
+static void sch_handle_halt_func(SubchDev *sch)
+{
+
+    PMCW *p = &sch->curr_status.pmcw;
+    SCSW *s = &sch->curr_status.scsw;
+    int path;
+
+    /* Path management: In our simple css, we always choose the only path. */
+    path = 0x80;
+
+    /* We always 'attempt to issue the halt signal', and we always succeed. */
+    sch->orb = NULL;
+    sch->channel_prog = 0x0;
+    sch->last_cmd_valid = false;
+    s->ctrl &= ~SCSW_ACTL_HALT_PEND;
+    s->ctrl |= SCSW_STCTL_STATUS_PEND;
+
+    if ((s->ctrl & (SCSW_ACTL_SUBCH_ACTIVE | SCSW_ACTL_DEVICE_ACTIVE)) ||
+        !((s->ctrl & SCSW_ACTL_START_PEND) ||
+          (s->ctrl & SCSW_ACTL_SUSP))) {
+        s->dstat = SCSW_DSTAT_DEVICE_END;
+    }
+    s->cstat = 0;
+    p->lpum = path;
+
+}
+
+static void copy_sense_id_to_guest(SenseId *dest, SenseId *src)
+{
+    int i;
+
+    dest->reserved = src->reserved;
+    dest->cu_type = cpu_to_be16(src->cu_type);
+    dest->cu_model = src->cu_model;
+    dest->dev_type = cpu_to_be16(src->dev_type);
+    dest->dev_model = src->dev_model;
+    dest->unused = src->unused;
+    for (i = 0; i < ARRAY_SIZE(dest->ciw); i++) {
+        dest->ciw[i].type = src->ciw[i].type;
+        dest->ciw[i].command = src->ciw[i].command;
+        dest->ciw[i].count = cpu_to_be16(src->ciw[i].count);
+    }
+}
+
+static CCW1 copy_ccw_from_guest(hwaddr addr)
+{
+    CCW1 tmp;
+    CCW1 ret;
+
+    cpu_physical_memory_read(addr, &tmp, sizeof(tmp));
+    ret.cmd_code = tmp.cmd_code;
+    ret.flags = tmp.flags;
+    ret.count = be16_to_cpu(tmp.count);
+    ret.cda = be32_to_cpu(tmp.cda);
+
+    return ret;
+}
+
+static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
+{
+    int ret;
+    bool check_len;
+    int len;
+    CCW1 ccw;
+
+    if (!ccw_addr) {
+        return -EIO;
+    }
+
+    ccw = copy_ccw_from_guest(ccw_addr);
+
+    /* Check for invalid command codes. */
+    if ((ccw.cmd_code & 0x0f) == 0) {
+        return -EINVAL;
+    }
+    if (((ccw.cmd_code & 0x0f) == CCW_CMD_TIC) &&
+        ((ccw.cmd_code & 0xf0) != 0)) {
+        return -EINVAL;
+    }
+
+    if (ccw.flags & CCW_FLAG_SUSPEND) {
+        return -EINPROGRESS;
+    }
+
+    check_len = !((ccw.flags & CCW_FLAG_SLI) && !(ccw.flags & CCW_FLAG_DC));
+
+    /* Look at the command. */
+    switch (ccw.cmd_code) {
+    case CCW_CMD_NOOP:
+        /* Nothing to do. */
+        ret = 0;
+        break;
+    case CCW_CMD_BASIC_SENSE:
+        if (check_len) {
+            if (ccw.count != sizeof(sch->sense_data)) {
+                ret = -EINVAL;
+                break;
+            }
+        }
+        len = MIN(ccw.count, sizeof(sch->sense_data));
+        cpu_physical_memory_write(ccw.cda, sch->sense_data, len);
+        sch->curr_status.scsw.count = ccw.count - len;
+        memset(sch->sense_data, 0, sizeof(sch->sense_data));
+        ret = 0;
+        break;
+    case CCW_CMD_SENSE_ID:
+    {
+        SenseId sense_id;
+
+        copy_sense_id_to_guest(&sense_id, &sch->id);
+        /* Sense ID information is device specific. */
+        if (check_len) {
+            if (ccw.count != sizeof(sense_id)) {
+                ret = -EINVAL;
+                break;
+            }
+        }
+        len = MIN(ccw.count, sizeof(sense_id));
+        /*
+         * Only indicate 0xff in the first sense byte if we actually
+         * have enough place to store at least bytes 0-3.
+         */
+        if (len >= 4) {
+            sense_id.reserved = 0xff;
+        } else {
+            sense_id.reserved = 0;
+        }
+        cpu_physical_memory_write(ccw.cda, &sense_id, len);
+        sch->curr_status.scsw.count = ccw.count - len;
+        ret = 0;
+        break;
+    }
+    case CCW_CMD_TIC:
+        if (sch->last_cmd_valid && (sch->last_cmd.cmd_code == CCW_CMD_TIC)) {
+            ret = -EINVAL;
+            break;
+        }
+        if (ccw.flags & (CCW_FLAG_CC | CCW_FLAG_DC)) {
+            ret = -EINVAL;
+            break;
+        }
+        sch->channel_prog = ccw.cda;
+        ret = -EAGAIN;
+        break;
+    default:
+        if (sch->ccw_cb) {
+            /* Handle device specific commands. */
+            ret = sch->ccw_cb(sch, ccw);
+        } else {
+            ret = -ENOSYS;
+        }
+        break;
+    }
+    sch->last_cmd = ccw;
+    sch->last_cmd_valid = true;
+    if (ret == 0) {
+        if (ccw.flags & CCW_FLAG_CC) {
+            sch->channel_prog += 8;
+            ret = -EAGAIN;
+        }
+    }
+
+    return ret;
+}
+
+static void sch_handle_start_func(SubchDev *sch)
+{
+
+    PMCW *p = &sch->curr_status.pmcw;
+    SCSW *s = &sch->curr_status.scsw;
+    ORB *orb = sch->orb;
+    int path;
+    int ret;
+
+    /* Path management: In our simple css, we always choose the only path. */
+    path = 0x80;
+
+    if (!(s->ctrl & SCSW_ACTL_SUSP)) {
+        /* Look at the orb and try to execute the channel program. */
+        p->intparm = orb->intparm;
+        if (!(orb->lpm & path)) {
+            /* Generate a deferred cc 3 condition. */
+            s->flags |= SCSW_FLAGS_MASK_CC;
+            s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+            s->ctrl |= (SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND);
+            return;
+        }
+    } else {
+        s->ctrl &= ~(SCSW_ACTL_SUSP | SCSW_ACTL_RESUME_PEND);
+    }
+    sch->last_cmd_valid = false;
+    do {
+        ret = css_interpret_ccw(sch, sch->channel_prog);
+        switch (ret) {
+        case -EAGAIN:
+            /* ccw chain, continue processing */
+            break;
+        case 0:
+            /* success */
+            s->ctrl &= ~SCSW_ACTL_START_PEND;
+            s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+            s->ctrl |= SCSW_STCTL_PRIMARY | SCSW_STCTL_SECONDARY |
+                    SCSW_STCTL_STATUS_PEND;
+            s->dstat = SCSW_DSTAT_CHANNEL_END | SCSW_DSTAT_DEVICE_END;
+            break;
+        case -ENOSYS:
+            /* unsupported command, generate unit check (command reject) */
+            s->ctrl &= ~SCSW_ACTL_START_PEND;
+            s->dstat = SCSW_DSTAT_UNIT_CHECK;
+            /* Set sense bit 0 in ecw0. */
+            sch->sense_data[0] = 0x80;
+            s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+            s->ctrl |= SCSW_STCTL_PRIMARY | SCSW_STCTL_SECONDARY |
+                    SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND;
+            break;
+        case -EFAULT:
+            /* memory problem, generate channel data check */
+            s->ctrl &= ~SCSW_ACTL_START_PEND;
+            s->cstat = SCSW_CSTAT_DATA_CHECK;
+            s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+            s->ctrl |= SCSW_STCTL_PRIMARY | SCSW_STCTL_SECONDARY |
+                    SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND;
+            break;
+        case -EBUSY:
+            /* subchannel busy, generate deferred cc 1 */
+            s->flags &= ~SCSW_FLAGS_MASK_CC;
+            s->flags |= (1 << 8);
+            s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+            s->ctrl |= SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND;
+            break;
+        case -EINPROGRESS:
+            /* channel program has been suspended */
+            s->ctrl &= ~SCSW_ACTL_START_PEND;
+            s->ctrl |= SCSW_ACTL_SUSP;
+            break;
+        default:
+            /* error, generate channel program check */
+            s->ctrl &= ~SCSW_ACTL_START_PEND;
+            s->cstat = SCSW_CSTAT_PROG_CHECK;
+            s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+            s->ctrl |= SCSW_STCTL_PRIMARY | SCSW_STCTL_SECONDARY |
+                    SCSW_STCTL_ALERT | SCSW_STCTL_STATUS_PEND;
+            break;
+        }
+    } while (ret == -EAGAIN);
+
+}
+
+/*
+ * On real machines, this would run asynchronously to the main vcpus.
+ * We might want to make some parts of the ssch handling (interpreting
+ * read/writes) asynchronous later on if we start supporting more than
+ * our current very simple devices.
+ */
+static void do_subchannel_work(SubchDev *sch)
+{
+
+    SCSW *s = &sch->curr_status.scsw;
+
+    if (s->ctrl & SCSW_FCTL_CLEAR_FUNC) {
+        sch_handle_clear_func(sch);
+    } else if (s->ctrl & SCSW_FCTL_HALT_FUNC) {
+        sch_handle_halt_func(sch);
+    } else if (s->ctrl & SCSW_FCTL_START_FUNC) {
+        sch_handle_start_func(sch);
+    } else {
+        /* Cannot happen. */
+        return;
+    }
+    css_inject_io_interrupt(sch);
+}
+
+static void copy_pmcw_to_guest(PMCW *dest, const PMCW *src)
+{
+    int i;
+
+    dest->intparm = cpu_to_be32(src->intparm);
+    dest->flags = cpu_to_be16(src->flags);
+    dest->devno = cpu_to_be16(src->devno);
+    dest->lpm = src->lpm;
+    dest->pnom = src->pnom;
+    dest->lpum = src->lpum;
+    dest->pim = src->pim;
+    dest->mbi = cpu_to_be16(src->mbi);
+    dest->pom = src->pom;
+    dest->pam = src->pam;
+    for (i = 0; i < ARRAY_SIZE(dest->chpid); i++) {
+        dest->chpid[i] = src->chpid[i];
+    }
+    dest->chars = cpu_to_be32(src->chars);
+}
+
+static void copy_scsw_to_guest(SCSW *dest, const SCSW *src)
+{
+    dest->flags = cpu_to_be16(src->flags);
+    dest->ctrl = cpu_to_be16(src->ctrl);
+    dest->cpa = cpu_to_be32(src->cpa);
+    dest->dstat = src->dstat;
+    dest->cstat = src->cstat;
+    dest->count = cpu_to_be16(src->count);
+}
+
+static void copy_schib_to_guest(SCHIB *dest, const SCHIB *src)
+{
+    int i;
+
+    copy_pmcw_to_guest(&dest->pmcw, &src->pmcw);
+    copy_scsw_to_guest(&dest->scsw, &src->scsw);
+    dest->mba = cpu_to_be64(src->mba);
+    for (i = 0; i < ARRAY_SIZE(dest->mda); i++) {
+        dest->mda[i] = src->mda[i];
+    }
+}
+
+int css_do_stsch(SubchDev *sch, SCHIB *schib)
+{
+    /* Use current status. */
+    copy_schib_to_guest(schib, &sch->curr_status);
+    return 0;
+}
+
+static void copy_pmcw_from_guest(PMCW *dest, const PMCW *src)
+{
+    int i;
+
+    dest->intparm = be32_to_cpu(src->intparm);
+    dest->flags = be16_to_cpu(src->flags);
+    dest->devno = be16_to_cpu(src->devno);
+    dest->lpm = src->lpm;
+    dest->pnom = src->pnom;
+    dest->lpum = src->lpum;
+    dest->pim = src->pim;
+    dest->mbi = be16_to_cpu(src->mbi);
+    dest->pom = src->pom;
+    dest->pam = src->pam;
+    for (i = 0; i < ARRAY_SIZE(dest->chpid); i++) {
+        dest->chpid[i] = src->chpid[i];
+    }
+    dest->chars = be32_to_cpu(src->chars);
+}
+
+static void copy_scsw_from_guest(SCSW *dest, const SCSW *src)
+{
+    dest->flags = be16_to_cpu(src->flags);
+    dest->ctrl = be16_to_cpu(src->ctrl);
+    dest->cpa = be32_to_cpu(src->cpa);
+    dest->dstat = src->dstat;
+    dest->cstat = src->cstat;
+    dest->count = be16_to_cpu(src->count);
+}
+
+static void copy_schib_from_guest(SCHIB *dest, const SCHIB *src)
+{
+    int i;
+
+    copy_pmcw_from_guest(&dest->pmcw, &src->pmcw);
+    copy_scsw_from_guest(&dest->scsw, &src->scsw);
+    dest->mba = be64_to_cpu(src->mba);
+    for (i = 0; i < ARRAY_SIZE(dest->mda); i++) {
+        dest->mda[i] = src->mda[i];
+    }
+}
+
+int css_do_msch(SubchDev *sch, SCHIB *orig_schib)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    int ret;
+    SCHIB schib;
+
+    if (!(sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_DNV)) {
+        ret = 0;
+        goto out;
+    }
+
+    if (s->ctrl & SCSW_STCTL_STATUS_PEND) {
+        ret = -EINPROGRESS;
+        goto out;
+    }
+
+    if (s->ctrl &
+        (SCSW_FCTL_START_FUNC|SCSW_FCTL_HALT_FUNC|SCSW_FCTL_CLEAR_FUNC)) {
+        ret = -EBUSY;
+        goto out;
+    }
+
+    copy_schib_from_guest(&schib, orig_schib);
+    /* Only update the program-modifiable fields. */
+    p->intparm = schib.pmcw.intparm;
+    p->flags &= ~(PMCW_FLAGS_MASK_ISC | PMCW_FLAGS_MASK_ENA |
+                  PMCW_FLAGS_MASK_LM | PMCW_FLAGS_MASK_MME |
+                  PMCW_FLAGS_MASK_MP);
+    p->flags |= schib.pmcw.flags &
+            (PMCW_FLAGS_MASK_ISC | PMCW_FLAGS_MASK_ENA |
+             PMCW_FLAGS_MASK_LM | PMCW_FLAGS_MASK_MME |
+             PMCW_FLAGS_MASK_MP);
+    p->lpm = schib.pmcw.lpm;
+    p->mbi = schib.pmcw.mbi;
+    p->pom = schib.pmcw.pom;
+    p->chars &= ~(PMCW_CHARS_MASK_MBFC | PMCW_CHARS_MASK_CSENSE);
+    p->chars |= schib.pmcw.chars &
+            (PMCW_CHARS_MASK_MBFC | PMCW_CHARS_MASK_CSENSE);
+    sch->curr_status.mba = schib.mba;
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int css_do_xsch(SubchDev *sch)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    int ret;
+
+    if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
+        ret = -ENODEV;
+        goto out;
+    }
+
+    if (!(s->ctrl & SCSW_CTRL_MASK_FCTL) ||
+        ((s->ctrl & SCSW_CTRL_MASK_FCTL) != SCSW_FCTL_START_FUNC) ||
+        (!(s->ctrl &
+           (SCSW_ACTL_RESUME_PEND | SCSW_ACTL_START_PEND | SCSW_ACTL_SUSP))) ||
+        (s->ctrl & SCSW_ACTL_SUBCH_ACTIVE)) {
+        ret = -EINPROGRESS;
+        goto out;
+    }
+
+    if (s->ctrl & SCSW_CTRL_MASK_STCTL) {
+        ret = -EBUSY;
+        goto out;
+    }
+
+    /* Cancel the current operation. */
+    s->ctrl &= ~(SCSW_FCTL_START_FUNC |
+                 SCSW_ACTL_RESUME_PEND |
+                 SCSW_ACTL_START_PEND |
+                 SCSW_ACTL_SUSP);
+    sch->channel_prog = 0x0;
+    sch->last_cmd_valid = false;
+    sch->orb = NULL;
+    s->dstat = 0;
+    s->cstat = 0;
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int css_do_csch(SubchDev *sch)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    int ret;
+
+    if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
+        ret = -ENODEV;
+        goto out;
+    }
+
+    /* Trigger the clear function. */
+    s->ctrl &= ~(SCSW_CTRL_MASK_FCTL | SCSW_CTRL_MASK_ACTL);
+    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_FCTL_CLEAR_FUNC;
+
+    do_subchannel_work(sch);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int css_do_hsch(SubchDev *sch)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    int ret;
+
+    if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
+        ret = -ENODEV;
+        goto out;
+    }
+
+    if (((s->ctrl & SCSW_CTRL_MASK_STCTL) == SCSW_STCTL_STATUS_PEND) ||
+        (s->ctrl & (SCSW_STCTL_PRIMARY |
+                    SCSW_STCTL_SECONDARY |
+                    SCSW_STCTL_ALERT))) {
+        ret = -EINPROGRESS;
+        goto out;
+    }
+
+    if (s->ctrl & (SCSW_FCTL_HALT_FUNC | SCSW_FCTL_CLEAR_FUNC)) {
+        ret = -EBUSY;
+        goto out;
+    }
+
+    /* Trigger the halt function. */
+    s->ctrl |= SCSW_FCTL_HALT_FUNC;
+    s->ctrl &= ~SCSW_FCTL_START_FUNC;
+    if (((s->ctrl & SCSW_CTRL_MASK_ACTL) ==
+         (SCSW_ACTL_SUBCH_ACTIVE | SCSW_ACTL_DEVICE_ACTIVE)) &&
+        ((s->ctrl & SCSW_CTRL_MASK_STCTL) == SCSW_STCTL_INTERMEDIATE)) {
+        s->ctrl &= ~SCSW_STCTL_STATUS_PEND;
+    }
+    s->ctrl |= SCSW_ACTL_HALT_PEND;
+
+    do_subchannel_work(sch);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+static void css_update_chnmon(SubchDev *sch)
+{
+    if (!(sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_MME)) {
+        /* Not active. */
+        return;
+    }
+    /* The counter is conveniently located at the beginning of the struct. */
+    if (sch->curr_status.pmcw.chars & PMCW_CHARS_MASK_MBFC) {
+        /* Format 1, per-subchannel area. */
+        uint32_t count;
+
+        count = ldl_phys(sch->curr_status.mba);
+        count++;
+        stl_phys(sch->curr_status.mba, count);
+    } else {
+        /* Format 0, global area. */
+        uint32_t offset;
+        uint16_t count;
+
+        offset = sch->curr_status.pmcw.mbi << 5;
+        count = lduw_phys(channel_subsys->chnmon_area + offset);
+        count++;
+        stw_phys(channel_subsys->chnmon_area + offset, count);
+    }
+}
+
+int css_do_ssch(SubchDev *sch, ORB *orb)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    int ret;
+
+    if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
+        ret = -ENODEV;
+        goto out;
+    }
+
+    if (s->ctrl & SCSW_STCTL_STATUS_PEND) {
+        ret = -EINPROGRESS;
+        goto out;
+    }
+
+    if (s->ctrl & (SCSW_FCTL_START_FUNC |
+                   SCSW_FCTL_HALT_FUNC |
+                   SCSW_FCTL_CLEAR_FUNC)) {
+        ret = -EBUSY;
+        goto out;
+    }
+
+    /* If monitoring is active, update counter. */
+    if (channel_subsys->chnmon_active) {
+        css_update_chnmon(sch);
+    }
+    sch->orb = orb;
+    sch->channel_prog = orb->cpa;
+    /* Trigger the start function. */
+    s->ctrl |= (SCSW_FCTL_START_FUNC | SCSW_ACTL_START_PEND);
+    s->flags &= ~SCSW_FLAGS_MASK_PNO;
+
+    do_subchannel_work(sch);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+static void copy_irb_to_guest(IRB *dest, const IRB *src)
+{
+    int i;
+
+    copy_scsw_to_guest(&dest->scsw, &src->scsw);
+
+    for (i = 0; i < ARRAY_SIZE(dest->esw); i++) {
+        dest->esw[i] = cpu_to_be32(src->esw[i]);
+    }
+    for (i = 0; i < ARRAY_SIZE(dest->ecw); i++) {
+        dest->ecw[i] = cpu_to_be32(src->ecw[i]);
+    }
+    for (i = 0; i < ARRAY_SIZE(dest->emw); i++) {
+        dest->emw[i] = cpu_to_be32(src->emw[i]);
+    }
+}
+
+int css_do_tsch(SubchDev *sch, IRB *target_irb)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    uint16_t stctl;
+    uint16_t fctl;
+    uint16_t actl;
+    IRB irb;
+    int ret;
+
+    if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
+        ret = 3;
+        goto out;
+    }
+
+    stctl = s->ctrl & SCSW_CTRL_MASK_STCTL;
+    fctl = s->ctrl & SCSW_CTRL_MASK_FCTL;
+    actl = s->ctrl & SCSW_CTRL_MASK_ACTL;
+
+    /* Prepare the irb for the guest. */
+    memset(&irb, 0, sizeof(IRB));
+
+    /* Copy scsw from current status. */
+    memcpy(&irb.scsw, s, sizeof(SCSW));
+    if (stctl & SCSW_STCTL_STATUS_PEND) {
+        if (s->cstat & (SCSW_CSTAT_DATA_CHECK |
+                        SCSW_CSTAT_CHN_CTRL_CHK |
+                        SCSW_CSTAT_INTF_CTRL_CHK)) {
+            irb.scsw.flags |= SCSW_FLAGS_MASK_ESWF;
+            irb.esw[0] = 0x04804000;
+        } else {
+            irb.esw[0] = 0x00800000;
+        }
+        /* If a unit check is pending, copy sense data. */
+        if ((s->dstat & SCSW_DSTAT_UNIT_CHECK) &&
+            (p->chars & PMCW_CHARS_MASK_CSENSE)) {
+            irb.scsw.flags |= SCSW_FLAGS_MASK_ESWF | SCSW_FLAGS_MASK_ECTL;
+            memcpy(irb.ecw, sch->sense_data, sizeof(sch->sense_data));
+            irb.esw[1] = 0x02000000 | (sizeof(sch->sense_data) << 8);
+        }
+    }
+    /* Store the irb to the guest. */
+    copy_irb_to_guest(target_irb, &irb);
+
+    /* Clear conditions on subchannel, if applicable. */
+    if (stctl & SCSW_STCTL_STATUS_PEND) {
+        s->ctrl &= ~SCSW_CTRL_MASK_STCTL;
+        if ((stctl != (SCSW_STCTL_INTERMEDIATE | SCSW_STCTL_STATUS_PEND)) ||
+            ((fctl & SCSW_FCTL_HALT_FUNC) &&
+             (actl & SCSW_ACTL_SUSP))) {
+            s->ctrl &= ~SCSW_CTRL_MASK_FCTL;
+        }
+        if (stctl != (SCSW_STCTL_INTERMEDIATE | SCSW_STCTL_STATUS_PEND)) {
+            s->flags &= ~SCSW_FLAGS_MASK_PNO;
+            s->ctrl &= ~(SCSW_ACTL_RESUME_PEND |
+                         SCSW_ACTL_START_PEND |
+                         SCSW_ACTL_HALT_PEND |
+                         SCSW_ACTL_CLEAR_PEND |
+                         SCSW_ACTL_SUSP);
+        } else {
+            if ((actl & SCSW_ACTL_SUSP) &&
+                (fctl & SCSW_FCTL_START_FUNC)) {
+                s->flags &= ~SCSW_FLAGS_MASK_PNO;
+                if (fctl & SCSW_FCTL_HALT_FUNC) {
+                    s->ctrl &= ~(SCSW_ACTL_RESUME_PEND |
+                                 SCSW_ACTL_START_PEND |
+                                 SCSW_ACTL_HALT_PEND |
+                                 SCSW_ACTL_CLEAR_PEND |
+                                 SCSW_ACTL_SUSP);
+                } else {
+                    s->ctrl &= ~SCSW_ACTL_RESUME_PEND;
+                }
+            }
+        }
+        /* Clear pending sense data. */
+        if (p->chars & PMCW_CHARS_MASK_CSENSE) {
+            memset(sch->sense_data, 0 , sizeof(sch->sense_data));
+        }
+    }
+
+    ret = ((stctl & SCSW_STCTL_STATUS_PEND) == 0);
+
+out:
+    return ret;
+}
+
+static void copy_crw_to_guest(CRW *dest, const CRW *src)
+{
+    dest->flags = cpu_to_be16(src->flags);
+    dest->rsid = cpu_to_be16(src->rsid);
+}
+
+int css_do_stcrw(CRW *crw)
+{
+    CrwContainer *crw_cont;
+    int ret;
+
+    crw_cont = QTAILQ_FIRST(&channel_subsys->pending_crws);
+    if (crw_cont) {
+        QTAILQ_REMOVE(&channel_subsys->pending_crws, crw_cont, sibling);
+        copy_crw_to_guest(crw, &crw_cont->crw);
+        g_free(crw_cont);
+        ret = 0;
+    } else {
+        /* List was empty, turn crw machine checks on again. */
+        memset(crw, 0, sizeof(*crw));
+        channel_subsys->do_crw_mchk = true;
+        ret = 1;
+    }
+
+    return ret;
+}
+
+int css_do_tpi(IOIntCode *int_code, int lowcore)
+{
+    /* No pending interrupts for !KVM. */
+    return 0;
+ }
+
+int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid, uint8_t l_chpid,
+                         int rfmt, void *buf)
+{
+    int i, desc_size;
+    uint32_t words[8];
+    uint32_t chpid_type_word;
+    CssImage *css;
+
+    if (!m && !cssid) {
+        css = channel_subsys->css[channel_subsys->default_cssid];
+    } else {
+        css = channel_subsys->css[cssid];
+    }
+    if (!css) {
+        return 0;
+    }
+    desc_size = 0;
+    for (i = f_chpid; i <= l_chpid; i++) {
+        if (css->chpids[i].in_use) {
+            chpid_type_word = 0x80000000 | (css->chpids[i].type << 8) | i;
+            if (rfmt == 0) {
+                words[0] = cpu_to_be32(chpid_type_word);
+                words[1] = 0;
+                memcpy(buf + desc_size, words, 8);
+                desc_size += 8;
+            } else if (rfmt == 1) {
+                words[0] = cpu_to_be32(chpid_type_word);
+                words[1] = 0;
+                words[2] = 0;
+                words[3] = 0;
+                words[4] = 0;
+                words[5] = 0;
+                words[6] = 0;
+                words[7] = 0;
+                memcpy(buf + desc_size, words, 32);
+                desc_size += 32;
+            }
+        }
+    }
+    return desc_size;
+}
+
+void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo)
+{
+    /* dct is currently ignored (not really meaningful for our devices) */
+    /* TODO: Don't ignore mbk. */
+    if (update && !channel_subsys->chnmon_active) {
+        /* Enable measuring. */
+        channel_subsys->chnmon_area = mbo;
+        channel_subsys->chnmon_active = true;
+    }
+    if (!update && channel_subsys->chnmon_active) {
+        /* Disable measuring. */
+        channel_subsys->chnmon_area = 0;
+        channel_subsys->chnmon_active = false;
+    }
+}
+
+int css_do_rsch(SubchDev *sch)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    int ret;
+
+    if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
+        ret = -ENODEV;
+        goto out;
+    }
+
+    if (s->ctrl & SCSW_STCTL_STATUS_PEND) {
+        ret = -EINPROGRESS;
+        goto out;
+    }
+
+    if (((s->ctrl & SCSW_CTRL_MASK_FCTL) != SCSW_FCTL_START_FUNC) ||
+        (s->ctrl & SCSW_ACTL_RESUME_PEND) ||
+        (!(s->ctrl & SCSW_ACTL_SUSP))) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    /* If monitoring is active, update counter. */
+    if (channel_subsys->chnmon_active) {
+        css_update_chnmon(sch);
+    }
+
+    s->ctrl |= SCSW_ACTL_RESUME_PEND;
+    do_subchannel_work(sch);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int css_do_rchp(uint8_t cssid, uint8_t chpid)
+{
+    uint8_t real_cssid;
+
+    if (cssid > channel_subsys->max_cssid) {
+        return -EINVAL;
+    }
+    if (channel_subsys->max_cssid == 0) {
+        real_cssid = channel_subsys->default_cssid;
+    } else {
+        real_cssid = cssid;
+    }
+    if (!channel_subsys->css[real_cssid]) {
+        return -EINVAL;
+    }
+
+    if (!channel_subsys->css[real_cssid]->chpids[chpid].in_use) {
+        return -ENODEV;
+    }
+
+    if (!channel_subsys->css[real_cssid]->chpids[chpid].is_virtual) {
+        fprintf(stderr,
+                "rchp unsupported for non-virtual chpid %x.%02x!\n",
+                real_cssid, chpid);
+        return -ENODEV;
+    }
+
+    /* We don't really use a channel path, so we're done here. */
+    css_queue_crw(CRW_RSC_CHP, CRW_ERC_INIT,
+                  channel_subsys->max_cssid > 0 ? 1 : 0, chpid);
+    if (channel_subsys->max_cssid > 0) {
+        css_queue_crw(CRW_RSC_CHP, CRW_ERC_INIT, 0, real_cssid << 8);
+    }
+    return 0;
+}
+
+bool css_schid_final(uint8_t cssid, uint8_t ssid, uint16_t schid)
+{
+    SubchSet *set;
+
+    if (cssid > MAX_CSSID || ssid > MAX_SSID || !channel_subsys->css[cssid] ||
+        !channel_subsys->css[cssid]->sch_set[ssid]) {
+        return true;
+    }
+    set = channel_subsys->css[cssid]->sch_set[ssid];
+    return schid > find_last_bit(set->schids_used,
+                                 (MAX_SCHID + 1) / sizeof(unsigned long));
+}
+
+static int css_add_virtual_chpid(uint8_t cssid, uint8_t chpid, uint8_t type)
+{
+    CssImage *css;
+
+    trace_css_chpid_add(cssid, chpid, type);
+    if (cssid > MAX_CSSID) {
+        return -EINVAL;
+    }
+    css = channel_subsys->css[cssid];
+    if (!css) {
+        return -EINVAL;
+    }
+    if (css->chpids[chpid].in_use) {
+        return -EEXIST;
+    }
+    css->chpids[chpid].in_use = 1;
+    css->chpids[chpid].type = type;
+    css->chpids[chpid].is_virtual = 1;
+
+    css_generate_chp_crws(cssid, chpid);
+
+    return 0;
+}
+
+void css_sch_build_virtual_schib(SubchDev *sch, uint8_t chpid, uint8_t type)
+{
+    PMCW *p = &sch->curr_status.pmcw;
+    SCSW *s = &sch->curr_status.scsw;
+    int i;
+    CssImage *css = channel_subsys->css[sch->cssid];
+
+    assert(css != NULL);
+    memset(p, 0, sizeof(PMCW));
+    p->flags |= PMCW_FLAGS_MASK_DNV;
+    p->devno = sch->devno;
+    /* single path */
+    p->pim = 0x80;
+    p->pom = 0xff;
+    p->pam = 0x80;
+    p->chpid[0] = chpid;
+    if (!css->chpids[chpid].in_use) {
+        css_add_virtual_chpid(sch->cssid, chpid, type);
+    }
+
+    memset(s, 0, sizeof(SCSW));
+    sch->curr_status.mba = 0;
+    for (i = 0; i < ARRAY_SIZE(sch->curr_status.mda); i++) {
+        sch->curr_status.mda[i] = 0;
+    }
+}
+
+SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid, uint16_t schid)
+{
+    uint8_t real_cssid;
+
+    real_cssid = (!m && (cssid == 0)) ? channel_subsys->default_cssid : cssid;
+
+    if (!channel_subsys->css[real_cssid]) {
+        return NULL;
+    }
+
+    if (!channel_subsys->css[real_cssid]->sch_set[ssid]) {
+        return NULL;
+    }
+
+    return channel_subsys->css[real_cssid]->sch_set[ssid]->sch[schid];
+}
+
+bool css_subch_visible(SubchDev *sch)
+{
+    if (sch->ssid > channel_subsys->max_ssid) {
+        return false;
+    }
+
+    if (sch->cssid != channel_subsys->default_cssid) {
+        return (channel_subsys->max_cssid > 0);
+    }
+
+    return true;
+}
+
+bool css_present(uint8_t cssid)
+{
+    return (channel_subsys->css[cssid] != NULL);
+}
+
+bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno)
+{
+    if (!channel_subsys->css[cssid]) {
+        return false;
+    }
+    if (!channel_subsys->css[cssid]->sch_set[ssid]) {
+        return false;
+    }
+
+    return !!test_bit(devno,
+                      channel_subsys->css[cssid]->sch_set[ssid]->devnos_used);
+}
+
+void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
+                      uint16_t devno, SubchDev *sch)
+{
+    CssImage *css;
+    SubchSet *s_set;
+
+    trace_css_assign_subch(sch ? "assign" : "deassign", cssid, ssid, schid,
+                           devno);
+    if (!channel_subsys->css[cssid]) {
+        fprintf(stderr,
+                "Suspicious call to %s (%x.%x.%04x) for non-existing css!\n",
+                __func__, cssid, ssid, schid);
+        return;
+    }
+    css = channel_subsys->css[cssid];
+
+    if (!css->sch_set[ssid]) {
+        css->sch_set[ssid] = g_malloc0(sizeof(SubchSet));
+    }
+    s_set = css->sch_set[ssid];
+
+    s_set->sch[schid] = sch;
+    if (sch) {
+        set_bit(schid, s_set->schids_used);
+        set_bit(devno, s_set->devnos_used);
+    } else {
+        clear_bit(schid, s_set->schids_used);
+        clear_bit(devno, s_set->devnos_used);
+    }
+}
+
+void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid)
+{
+    CrwContainer *crw_cont;
+
+    trace_css_crw(rsc, erc, rsid, chain ? "(chained)" : "");
+    /* TODO: Maybe use a static crw pool? */
+    crw_cont = g_try_malloc0(sizeof(CrwContainer));
+    if (!crw_cont) {
+        channel_subsys->crws_lost = true;
+        return;
+    }
+    crw_cont->crw.flags = (rsc << 8) | erc;
+    if (chain) {
+        crw_cont->crw.flags |= CRW_FLAGS_MASK_C;
+    }
+    crw_cont->crw.rsid = rsid;
+    if (channel_subsys->crws_lost) {
+        crw_cont->crw.flags |= CRW_FLAGS_MASK_R;
+        channel_subsys->crws_lost = false;
+    }
+
+    QTAILQ_INSERT_TAIL(&channel_subsys->pending_crws, crw_cont, sibling);
+
+    if (channel_subsys->do_crw_mchk) {
+        S390CPU *cpu = s390_cpu_addr2state(0);
+
+        channel_subsys->do_crw_mchk = false;
+        /* Inject crw pending machine check. */
+        s390_crw_mchk(cpu);
+    }
+}
+
+void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
+                           int hotplugged, int add)
+{
+    uint8_t guest_cssid;
+    bool chain_crw;
+
+    if (add && !hotplugged) {
+        return;
+    }
+    if (channel_subsys->max_cssid == 0) {
+        /* Default cssid shows up as 0. */
+        guest_cssid = (cssid == channel_subsys->default_cssid) ? 0 : cssid;
+    } else {
+        /* Show real cssid to the guest. */
+        guest_cssid = cssid;
+    }
+    /*
+     * Only notify for higher subchannel sets/channel subsystems if the
+     * guest has enabled it.
+     */
+    if ((ssid > channel_subsys->max_ssid) ||
+        (guest_cssid > channel_subsys->max_cssid) ||
+        ((channel_subsys->max_cssid == 0) &&
+         (cssid != channel_subsys->default_cssid))) {
+        return;
+    }
+    chain_crw = (channel_subsys->max_ssid > 0) ||
+            (channel_subsys->max_cssid > 0);
+    css_queue_crw(CRW_RSC_SUBCH, CRW_ERC_IPI, chain_crw ? 1 : 0, schid);
+    if (chain_crw) {
+        css_queue_crw(CRW_RSC_SUBCH, CRW_ERC_IPI, 0,
+                      (guest_cssid << 8) | (ssid << 4));
+    }
+}
+
+void css_generate_chp_crws(uint8_t cssid, uint8_t chpid)
+{
+    /* TODO */
+}
+
+int css_enable_mcsse(void)
+{
+    trace_css_enable_facility("mcsse");
+    channel_subsys->max_cssid = MAX_CSSID;
+    return 0;
+}
+
+int css_enable_mss(void)
+{
+    trace_css_enable_facility("mss");
+    channel_subsys->max_ssid = MAX_SSID;
+    return 0;
+}
+
+static void css_init(void)
+{
+    channel_subsys = g_malloc0(sizeof(*channel_subsys));
+    QTAILQ_INIT(&channel_subsys->pending_crws);
+    channel_subsys->do_crw_mchk = true;
+    channel_subsys->crws_lost = false;
+    channel_subsys->chnmon_active = false;
+}
+machine_init(css_init);
+
+void css_reset_sch(SubchDev *sch)
+{
+    PMCW *p = &sch->curr_status.pmcw;
+
+    p->intparm = 0;
+    p->flags &= ~(PMCW_FLAGS_MASK_ISC | PMCW_FLAGS_MASK_ENA |
+                  PMCW_FLAGS_MASK_LM | PMCW_FLAGS_MASK_MME |
+                  PMCW_FLAGS_MASK_MP | PMCW_FLAGS_MASK_TF);
+    p->flags |= PMCW_FLAGS_MASK_DNV;
+    p->devno = sch->devno;
+    p->pim = 0x80;
+    p->lpm = p->pim;
+    p->pnom = 0;
+    p->lpum = 0;
+    p->mbi = 0;
+    p->pom = 0xff;
+    p->pam = 0x80;
+    p->chars &= ~(PMCW_CHARS_MASK_MBFC | PMCW_CHARS_MASK_XMWME |
+                  PMCW_CHARS_MASK_CSENSE);
+
+    memset(&sch->curr_status.scsw, 0, sizeof(sch->curr_status.scsw));
+    sch->curr_status.mba = 0;
+
+    sch->channel_prog = 0x0;
+    sch->last_cmd_valid = false;
+    sch->orb = NULL;
+}
+
+void css_reset(void)
+{
+    CrwContainer *crw_cont;
+
+    /* Clean up monitoring. */
+    channel_subsys->chnmon_active = false;
+    channel_subsys->chnmon_area = 0;
+
+    /* Clear pending CRWs. */
+    while ((crw_cont = QTAILQ_FIRST(&channel_subsys->pending_crws))) {
+        QTAILQ_REMOVE(&channel_subsys->pending_crws, crw_cont, sibling);
+        g_free(crw_cont);
+    }
+    channel_subsys->do_crw_mchk = true;
+    channel_subsys->crws_lost = false;
+
+    /* Reset maximum ids. */
+    channel_subsys->max_cssid = 0;
+    channel_subsys->max_ssid = 0;
+}
diff --git a/hw/s390x/css.h b/hw/s390x/css.h
new file mode 100644
index 0000000..85ed05d
--- /dev/null
+++ b/hw/s390x/css.h
@@ -0,0 +1,99 @@
+/*
+ * Channel subsystem structures and definitions.
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef CSS_H
+#define CSS_H
+
+#include "ioinst.h"
+
+/* Channel subsystem constants. */
+#define MAX_SCHID 65535
+#define MAX_SSID 3
+#define MAX_CSSID 254 /* 255 is reserved */
+#define MAX_CHPID 255
+
+#define MAX_CIWS 62
+
+typedef struct CIW {
+    uint8_t type;
+    uint8_t command;
+    uint16_t count;
+} QEMU_PACKED CIW;
+
+typedef struct SenseId {
+    /* common part */
+    uint8_t reserved;        /* always 0x'FF' */
+    uint16_t cu_type;        /* control unit type */
+    uint8_t cu_model;        /* control unit model */
+    uint16_t dev_type;       /* device type */
+    uint8_t dev_model;       /* device model */
+    uint8_t unused;          /* padding byte */
+    /* extended part */
+    CIW ciw[MAX_CIWS];       /* variable # of CIWs */
+} QEMU_PACKED SenseId;
+
+/* Channel measurements, from linux/drivers/s390/cio/cmf.c. */
+typedef struct CMB {
+    uint16_t ssch_rsch_count;
+    uint16_t sample_count;
+    uint32_t device_connect_time;
+    uint32_t function_pending_time;
+    uint32_t device_disconnect_time;
+    uint32_t control_unit_queuing_time;
+    uint32_t device_active_only_time;
+    uint32_t reserved[2];
+} QEMU_PACKED CMB;
+
+typedef struct CMBE {
+    uint32_t ssch_rsch_count;
+    uint32_t sample_count;
+    uint32_t device_connect_time;
+    uint32_t function_pending_time;
+    uint32_t device_disconnect_time;
+    uint32_t control_unit_queuing_time;
+    uint32_t device_active_only_time;
+    uint32_t device_busy_time;
+    uint32_t initial_command_response_time;
+    uint32_t reserved[7];
+} QEMU_PACKED CMBE;
+
+struct SubchDev {
+    /* channel-subsystem related things: */
+    uint8_t cssid;
+    uint8_t ssid;
+    uint16_t schid;
+    uint16_t devno;
+    SCHIB curr_status;
+    uint8_t sense_data[32];
+    hwaddr channel_prog;
+    CCW1 last_cmd;
+    bool last_cmd_valid;
+    ORB *orb;
+    /* transport-provided data: */
+    int (*ccw_cb) (SubchDev *, CCW1);
+    SenseId id;
+    void *driver_data;
+};
+
+typedef SubchDev *(*css_subch_cb_func)(uint8_t m, uint8_t cssid, uint8_t ssid,
+                                       uint16_t schid);
+int css_create_css_image(uint8_t cssid, bool default_image);
+bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno);
+void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
+                      uint16_t devno, SubchDev *sch);
+void css_sch_build_virtual_schib(SubchDev *sch, uint8_t chpid, uint8_t type);
+void css_reset(void);
+void css_reset_sch(SubchDev *sch);
+void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid);
+void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
+                           int hotplugged, int add);
+void css_generate_chp_crws(uint8_t cssid, uint8_t chpid);
+#endif
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
index bc9cea9..6b56995 100644
--- a/hw/s390x/event-facility.c
+++ b/hw/s390x/event-facility.c
@@ -15,8 +15,8 @@
  *
  */
 
-#include "monitor.h"
-#include "sysemu.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 
 #include "sclp.h"
 #include "event-facility.h"
@@ -345,7 +345,7 @@ static void init_event_facility_class(ObjectClass *klass, void *data)
     k->init = init_event_facility;
 }
 
-static TypeInfo s390_sclp_event_facility_info = {
+static const TypeInfo s390_sclp_event_facility_info = {
     .name          = "s390-sclp-event-facility",
     .parent        = TYPE_DEVICE_S390_SCLP,
     .instance_size = sizeof(S390SCLPDevice),
@@ -380,7 +380,7 @@ static void event_class_init(ObjectClass *klass, void *data)
     dc->exit = event_qdev_exit;
 }
 
-static TypeInfo s390_sclp_event_type_info = {
+static const TypeInfo s390_sclp_event_type_info = {
     .name = TYPE_SCLP_EVENT,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(SCLPEvent),
diff --git a/hw/s390x/event-facility.h b/hw/s390x/event-facility.h
index 30af0a7..791ab2a 100644
--- a/hw/s390x/event-facility.h
+++ b/hw/s390x/event-facility.h
@@ -16,7 +16,7 @@
 #define HW_S390_SCLP_EVENT_FACILITY_H
 
 #include <hw/qdev.h>
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 /* SCLP event types */
 #define SCLP_EVENT_ASCII_CONSOLE_DATA           0x1a
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
new file mode 100644
index 0000000..206d552
--- /dev/null
+++ b/hw/s390x/ipl.c
@@ -0,0 +1,176 @@
+/*
+ * bootloader support
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Christian Borntraeger <borntraeger@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "sysemu/sysemu.h"
+#include "cpu.h"
+#include "elf.h"
+#include "hw/loader.h"
+#include "hw/sysbus.h"
+
+#define KERN_IMAGE_START                0x010000UL
+#define KERN_PARM_AREA                  0x010480UL
+#define INITRD_START                    0x800000UL
+#define INITRD_PARM_START               0x010408UL
+#define INITRD_PARM_SIZE                0x010410UL
+#define PARMFILE_START                  0x001000UL
+#define ZIPL_FILENAME                   "s390-zipl.rom"
+#define ZIPL_IMAGE_START                0x009000UL
+#define IPL_PSW_MASK                    (PSW_MASK_32 | PSW_MASK_64)
+
+#define TYPE_S390_IPL "s390-ipl"
+#define S390_IPL(obj) \
+    OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
+#if 0
+#define S390_IPL_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390IPLState, (klass), TYPE_S390_IPL)
+#define S390_IPL_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390IPLState, (obj), TYPE_S390_IPL)
+#endif
+
+typedef struct S390IPLClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    void (*parent_reset) (SysBusDevice *dev);
+} S390IPLClass;
+
+typedef struct S390IPLState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    char *kernel;
+    char *initrd;
+    char *cmdline;
+} S390IPLState;
+
+
+static void s390_ipl_cpu(uint64_t pswaddr)
+{
+    S390CPU *cpu = S390_CPU(qemu_get_cpu(0));
+    CPUS390XState *env = &cpu->env;
+
+    env->psw.addr = pswaddr;
+    env->psw.mask = IPL_PSW_MASK;
+    s390_add_running_cpu(cpu);
+}
+
+static int s390_ipl_init(SysBusDevice *dev)
+{
+    S390IPLState *ipl = S390_IPL(dev);
+    ram_addr_t kernel_size = 0;
+
+    if (!ipl->kernel) {
+        ram_addr_t bios_size = 0;
+        char *bios_filename;
+
+        /* Load zipl bootloader */
+        if (bios_name == NULL) {
+            bios_name = ZIPL_FILENAME;
+        }
+
+        bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+        bios_size = load_image_targphys(bios_filename, ZIPL_IMAGE_START, 4096);
+        g_free(bios_filename);
+
+        if ((long)bios_size < 0) {
+            hw_error("could not load bootloader '%s'\n", bios_name);
+        }
+
+        if (bios_size > 4096) {
+            hw_error("stage1 bootloader is > 4k\n");
+        }
+        return 0;
+    } else {
+        kernel_size = load_elf(ipl->kernel, NULL, NULL, NULL, NULL,
+                               NULL, 1, ELF_MACHINE, 0);
+        if (kernel_size == -1UL) {
+            kernel_size = load_image_targphys(ipl->kernel, 0, ram_size);
+        }
+        if (kernel_size == -1UL) {
+            fprintf(stderr, "could not load kernel '%s'\n", ipl->kernel);
+            return -1;
+        }
+        /* we have to overwrite values in the kernel image, which are "rom" */
+        strcpy(rom_ptr(KERN_PARM_AREA), ipl->cmdline);
+    }
+    if (ipl->initrd) {
+        ram_addr_t initrd_offset, initrd_size;
+
+        initrd_offset = INITRD_START;
+        while (kernel_size + 0x100000 > initrd_offset) {
+            initrd_offset += 0x100000;
+        }
+        initrd_size = load_image_targphys(ipl->initrd, initrd_offset,
+                                          ram_size - initrd_offset);
+        if (initrd_size == -1UL) {
+            fprintf(stderr, "qemu: could not load initrd '%s'\n", ipl->initrd);
+            exit(1);
+        }
+
+        /* we have to overwrite values in the kernel image, which are "rom" */
+        stq_p(rom_ptr(INITRD_PARM_START), initrd_offset);
+        stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
+    }
+
+    return 0;
+}
+
+static Property s390_ipl_properties[] = {
+    DEFINE_PROP_STRING("kernel", S390IPLState, kernel),
+    DEFINE_PROP_STRING("initrd", S390IPLState, initrd),
+    DEFINE_PROP_STRING("cmdline", S390IPLState, cmdline),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_ipl_reset(DeviceState *dev)
+{
+    S390IPLState *ipl = S390_IPL(dev);
+
+    if (ipl->kernel) {
+        /*
+         * we can not rely on the ELF entry point, since up to 3.2 this
+         * value was 0x800 (the SALIPL loader) and it wont work. For
+         * all (Linux) cases 0x10000 (KERN_IMAGE_START) should be fine.
+         */
+        return s390_ipl_cpu(KERN_IMAGE_START);
+    } else {
+        return s390_ipl_cpu(ZIPL_IMAGE_START);
+    }
+}
+
+static void s390_ipl_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = s390_ipl_init;
+    dc->props = s390_ipl_properties;
+    dc->reset = s390_ipl_reset;
+    dc->no_user = 1;
+}
+
+static const TypeInfo s390_ipl_info = {
+    .class_init = s390_ipl_class_init,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .name  = "s390-ipl",
+    .instance_size  = sizeof(S390IPLState),
+};
+
+static void s390_ipl_register_types(void)
+{
+    type_register_static(&s390_ipl_info);
+}
+
+type_init(s390_ipl_register_types)
diff --git a/hw/s390-virtio-bus.c b/hw/s390x/s390-virtio-bus.c
index e0ac2d1..089ed92 100644
--- a/hw/s390-virtio-bus.c
+++ b/hw/s390x/s390-virtio-bus.c
@@ -17,22 +17,22 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "hw.h"
-#include "block.h"
-#include "sysemu.h"
-#include "net.h"
-#include "boards.h"
-#include "monitor.h"
-#include "loader.h"
+#include "hw/hw.h"
+#include "block/block.h"
+#include "sysemu/sysemu.h"
+#include "hw/boards.h"
+#include "monitor/monitor.h"
+#include "hw/loader.h"
 #include "elf.h"
 #include "hw/virtio.h"
 #include "hw/virtio-rng.h"
 #include "hw/virtio-serial.h"
 #include "hw/virtio-net.h"
 #include "hw/sysbus.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
-#include "hw/s390-virtio-bus.h"
+#include "hw/s390x/s390-virtio-bus.h"
+#include "hw/virtio-bus.h"
 
 /* #define DEBUG_S390 */
 
@@ -111,12 +111,12 @@ VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size)
     return bus;
 }
 
-static void s390_virtio_irq(CPUS390XState *env, int config_change, uint64_t token)
+static void s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token)
 {
     if (kvm_enabled()) {
-        kvm_s390_virtio_irq(env, config_change, token);
+        kvm_s390_virtio_irq(cpu, config_change, token);
     } else {
-        cpu_inject_ext(env, VIRTIO_EXT_CODE, config_change, token);
+        cpu_inject_ext(cpu, VIRTIO_EXT_CODE, config_change, token);
     }
 }
 
@@ -137,14 +137,13 @@ static int s390_virtio_device_init(VirtIOS390Device *dev, VirtIODevice *vdev)
 
     bus->dev_offs += dev_len;
 
-    virtio_bind_device(vdev, &virtio_s390_bindings, dev);
+    virtio_bind_device(vdev, &virtio_s390_bindings, DEVICE(dev));
     dev->host_features = vdev->get_features(vdev, dev->host_features);
     s390_virtio_device_sync(dev);
     s390_virtio_reset_idx(dev);
     if (dev->qdev.hotplugged) {
         S390CPU *cpu = s390_cpu_addr2state(0);
-        CPUS390XState *env = &cpu->env;
-        s390_virtio_irq(env, VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
+        s390_virtio_irq(cpu, VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
     }
 
     return 0;
@@ -154,7 +153,8 @@ static int s390_virtio_net_init(VirtIOS390Device *dev)
 {
     VirtIODevice *vdev;
 
-    vdev = virtio_net_init((DeviceState *)dev, &dev->nic, &dev->net);
+    vdev = virtio_net_init((DeviceState *)dev, &dev->nic, &dev->net,
+                           dev->host_features);
     if (!vdev) {
         return -1;
     }
@@ -364,19 +364,32 @@ VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem)
     return NULL;
 }
 
-static void virtio_s390_notify(void *opaque, uint16_t vector)
+/* DeviceState to VirtIOS390Device. Note: used on datapath,
+ * be careful and test performance if you change this.
+ */
+static inline VirtIOS390Device *to_virtio_s390_device_fast(DeviceState *d)
+{
+    return container_of(d, VirtIOS390Device, qdev);
+}
+
+/* DeviceState to VirtIOS390Device. TODO: use QOM. */
+static inline VirtIOS390Device *to_virtio_s390_device(DeviceState *d)
 {
-    VirtIOS390Device *dev = (VirtIOS390Device*)opaque;
+    return container_of(d, VirtIOS390Device, qdev);
+}
+
+static void virtio_s390_notify(DeviceState *d, uint16_t vector)
+{
+    VirtIOS390Device *dev = to_virtio_s390_device_fast(d);
     uint64_t token = s390_virtio_device_vq_token(dev, vector);
     S390CPU *cpu = s390_cpu_addr2state(0);
-    CPUS390XState *env = &cpu->env;
 
-    s390_virtio_irq(env, 0, token);
+    s390_virtio_irq(cpu, 0, token);
 }
 
-static unsigned virtio_s390_get_features(void *opaque)
+static unsigned virtio_s390_get_features(DeviceState *d)
 {
-    VirtIOS390Device *dev = (VirtIOS390Device*)opaque;
+    VirtIOS390Device *dev = to_virtio_s390_device(d);
     return dev->host_features;
 }
 
@@ -406,7 +419,7 @@ static void s390_virtio_net_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_net_properties;
 }
 
-static TypeInfo s390_virtio_net = {
+static const TypeInfo s390_virtio_net = {
     .name          = "virtio-net-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -432,7 +445,7 @@ static void s390_virtio_blk_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_blk_properties;
 }
 
-static TypeInfo s390_virtio_blk = {
+static const TypeInfo s390_virtio_blk = {
     .name          = "virtio-blk-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -454,7 +467,7 @@ static void s390_virtio_serial_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_serial_properties;
 }
 
-static TypeInfo s390_virtio_serial = {
+static const TypeInfo s390_virtio_serial = {
     .name          = "virtio-serial-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -476,7 +489,7 @@ static void s390_virtio_rng_class_init(ObjectClass *klass, void *data)
     k->init = s390_virtio_rng_init;
 }
 
-static TypeInfo s390_virtio_rng = {
+static const TypeInfo s390_virtio_rng = {
     .name          = "virtio-rng-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -489,9 +502,18 @@ static int s390_virtio_busdev_init(DeviceState *dev)
     VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
     VirtIOS390DeviceClass *_info = VIRTIO_S390_DEVICE_GET_CLASS(dev);
 
+    virtio_s390_bus_new(&_dev->bus, _dev);
+
     return _info->init(_dev);
 }
 
+static void s390_virtio_busdev_reset(DeviceState *dev)
+{
+    VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
+
+    virtio_reset(_dev->vdev);
+}
+
 static void virtio_s390_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -499,9 +521,10 @@ static void virtio_s390_device_class_init(ObjectClass *klass, void *data)
     dc->init = s390_virtio_busdev_init;
     dc->bus_type = TYPE_S390_VIRTIO_BUS;
     dc->unplug = qdev_simple_unplug_cb;
+    dc->reset = s390_virtio_busdev_reset;
 }
 
-static TypeInfo virtio_s390_device_info = {
+static const TypeInfo virtio_s390_device_info = {
     .name = TYPE_VIRTIO_S390_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -524,7 +547,7 @@ static void s390_virtio_scsi_class_init(ObjectClass *klass, void *data)
     dc->props = s390_virtio_scsi_properties;
 }
 
-static TypeInfo s390_virtio_scsi = {
+static const TypeInfo s390_virtio_scsi = {
     .name          = "virtio-scsi-s390",
     .parent        = TYPE_VIRTIO_S390_DEVICE,
     .instance_size = sizeof(VirtIOS390Device),
@@ -549,15 +572,43 @@ static void s390_virtio_bridge_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo s390_virtio_bridge_info = {
+static const TypeInfo s390_virtio_bridge_info = {
     .name          = "s390-virtio-bridge",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
     .class_init    = s390_virtio_bridge_class_init,
 };
 
+/* virtio-s390-bus */
+
+void virtio_s390_bus_new(VirtioBusState *bus, VirtIOS390Device *dev)
+{
+    DeviceState *qdev = DEVICE(dev);
+    BusState *qbus;
+    qbus_create_inplace((BusState *)bus, TYPE_VIRTIO_S390_BUS, qdev, NULL);
+    qbus = BUS(bus);
+    qbus->allow_hotplug = 0;
+}
+
+static void virtio_s390_bus_class_init(ObjectClass *klass, void *data)
+{
+    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
+    BusClass *bus_class = BUS_CLASS(klass);
+    bus_class->max_dev = 1;
+    k->notify = virtio_s390_notify;
+    k->get_features = virtio_s390_get_features;
+}
+
+static const TypeInfo virtio_s390_bus_info = {
+    .name          = TYPE_VIRTIO_S390_BUS,
+    .parent        = TYPE_VIRTIO_BUS,
+    .instance_size = sizeof(VirtioS390BusState),
+    .class_init    = virtio_s390_bus_class_init,
+};
+
 static void s390_virtio_register_types(void)
 {
+    type_register_static(&virtio_s390_bus_info);
     type_register_static(&s390_virtio_bus_info);
     type_register_static(&virtio_s390_device_info);
     type_register_static(&s390_virtio_serial);
diff --git a/hw/s390-virtio-bus.h b/hw/s390x/s390-virtio-bus.h
index a83afe7..4aacf83 100644
--- a/hw/s390-virtio-bus.h
+++ b/hw/s390x/s390-virtio-bus.h
@@ -16,12 +16,15 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef HW_S390_VIRTIO_BUS_H
+#define HW_S390_VIRTIO_BUS_H 1
 
-#include "virtio-blk.h"
-#include "virtio-net.h"
-#include "virtio-rng.h"
-#include "virtio-serial.h"
-#include "virtio-scsi.h"
+#include "hw/virtio-blk.h"
+#include "hw/virtio-net.h"
+#include "hw/virtio-rng.h"
+#include "hw/virtio-serial.h"
+#include "hw/virtio-scsi.h"
+#include "hw/virtio-bus.h"
 
 #define VIRTIO_DEV_OFFS_TYPE		0	/* 8 bits */
 #define VIRTIO_DEV_OFFS_NUM_VQ		1	/* 8 bits */
@@ -57,8 +60,24 @@
 #define S390_VIRTIO_BUS(obj) \
      OBJECT_CHECK(VirtIOS390Bus, (obj), TYPE_S390_VIRTIO_BUS)
 
+/* virtio-s390-bus */
+
+typedef struct VirtioBusState VirtioS390BusState;
+typedef struct VirtioBusClass VirtioS390BusClass;
+
+#define TYPE_VIRTIO_S390_BUS "virtio-s390-bus"
+#define VIRTIO_S390_BUS(obj) \
+        OBJECT_CHECK(VirtioS390BusState, (obj), TYPE_VIRTIO_S390_BUS)
+#define VIRTIO_S390_BUS_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioS390BusClass, obj, TYPE_VIRTIO_S390_BUS)
+#define VIRTIO_S390_BUS_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioS390BusClass, klass, TYPE_VIRTIO_S390_BUS)
+
+
 typedef struct VirtIOS390Device VirtIOS390Device;
 
+void virtio_s390_bus_new(VirtioBusState *bus, VirtIOS390Device *dev);
+
 typedef struct VirtIOS390DeviceClass {
     DeviceClass qdev;
     int (*init)(VirtIOS390Device *dev);
@@ -77,6 +96,7 @@ struct VirtIOS390Device {
     virtio_net_conf net;
     VirtIOSCSIConf scsi;
     VirtIORNGConf rng;
+    VirtioBusState bus;
 };
 
 typedef struct VirtIOS390Bus {
@@ -100,3 +120,5 @@ VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem);
 void s390_virtio_device_sync(VirtIOS390Device *dev);
 void s390_virtio_reset_idx(VirtIOS390Device *dev);
 
+
+#endif
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
new file mode 100644
index 0000000..6549211
--- /dev/null
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -0,0 +1,134 @@
+/*
+ * virtio ccw machine
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "hw/boards.h"
+#include "exec/address-spaces.h"
+#include "s390-virtio.h"
+#include "sclp.h"
+#include "ioinst.h"
+#include "css.h"
+#include "virtio-ccw.h"
+
+static int virtio_ccw_hcall_notify(const uint64_t *args)
+{
+    uint64_t subch_id = args[0];
+    uint64_t queue = args[1];
+    SubchDev *sch;
+    int cssid, ssid, schid, m;
+
+    if (ioinst_disassemble_sch_ident(subch_id, &m, &cssid, &ssid, &schid)) {
+        return -EINVAL;
+    }
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (!sch || !css_subch_visible(sch)) {
+        return -EINVAL;
+    }
+    virtio_queue_notify(virtio_ccw_get_vdev(sch), queue);
+    return 0;
+
+}
+
+static int virtio_ccw_hcall_early_printk(const uint64_t *args)
+{
+    uint64_t mem = args[0];
+
+    if (mem < ram_size) {
+        /* Early printk */
+        return 0;
+    }
+    return -EINVAL;
+}
+
+static void virtio_ccw_register_hcalls(void)
+{
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_CCW_NOTIFY,
+                                   virtio_ccw_hcall_notify);
+    /* Tolerate early printk. */
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_NOTIFY,
+                                   virtio_ccw_hcall_early_printk);
+}
+
+static void ccw_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t my_ram_size = args->ram_size;
+    MemoryRegion *sysmem = get_system_memory();
+    MemoryRegion *ram = g_new(MemoryRegion, 1);
+    int shift = 0;
+    uint8_t *storage_keys;
+    int ret;
+    VirtualCssBus *css_bus;
+
+    /* s390x ram size detection needs a 16bit multiplier + an increment. So
+       guests > 64GB can be specified in 2MB steps etc. */
+    while ((my_ram_size >> (20 + shift)) > 65535) {
+        shift++;
+    }
+    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
+
+    /* lets propagate the changed ram size into the global variable. */
+    ram_size = my_ram_size;
+
+    /* get a BUS */
+    css_bus = virtual_css_bus_init();
+    s390_sclp_init();
+    s390_init_ipl_dev(args->kernel_filename, args->kernel_cmdline,
+                      args->initrd_filename);
+
+    /* register hypercalls */
+    virtio_ccw_register_hcalls();
+
+    /* allocate RAM */
+    memory_region_init_ram(ram, "s390.ram", my_ram_size);
+    vmstate_register_ram_global(ram);
+    memory_region_add_subregion(sysmem, 0, ram);
+
+    /* allocate storage keys */
+    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
+
+    /* init CPUs */
+    s390_init_cpus(args->cpu_model, storage_keys);
+
+    if (kvm_enabled()) {
+        kvm_s390_enable_css_support(s390_cpu_addr2state(0));
+    }
+    /*
+     * Create virtual css and set it as default so that non mcss-e
+     * enabled guests only see virtio devices.
+     */
+    ret = css_create_css_image(VIRTUAL_CSSID, true);
+    assert(ret == 0);
+
+    /* Create VirtIO network adapters */
+    s390_create_virtio_net(BUS(css_bus), "virtio-net-ccw");
+}
+
+static QEMUMachine ccw_machine = {
+    .name = "s390-ccw-virtio",
+    .alias = "s390-ccw",
+    .desc = "VirtIO-ccw based S390 machine",
+    .init = ccw_init,
+    .block_default_type = IF_VIRTIO,
+    .no_cdrom = 1,
+    .no_floppy = 1,
+    .no_serial = 1,
+    .no_parallel = 1,
+    .no_sdcard = 1,
+    .use_sclp = 1,
+    .max_cpus = 255,
+    DEFAULT_MACHINE_OPTIONS,
+};
+
+static void ccw_machine_init(void)
+{
+    qemu_register_machine(&ccw_machine);
+}
+
+machine_init(ccw_machine_init)
diff --git a/hw/s390x/s390-virtio-hcall.c b/hw/s390x/s390-virtio-hcall.c
new file mode 100644
index 0000000..ee62649
--- /dev/null
+++ b/hw/s390x/s390-virtio-hcall.c
@@ -0,0 +1,36 @@
+/*
+ * Support for virtio hypercalls on s390
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "cpu.h"
+#include "hw/s390x/s390-virtio.h"
+
+#define MAX_DIAG_SUBCODES 255
+
+static s390_virtio_fn s390_diag500_table[MAX_DIAG_SUBCODES];
+
+void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn)
+{
+    assert(code < MAX_DIAG_SUBCODES);
+    assert(!s390_diag500_table[code]);
+
+    s390_diag500_table[code] = fn;
+}
+
+int s390_virtio_hypercall(CPUS390XState *env)
+{
+    s390_virtio_fn fn = s390_diag500_table[env->regs[1]];
+
+    if (!fn) {
+        return -EINVAL;
+    }
+
+    return fn(&env->regs[2]);
+}
diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
new file mode 100644
index 0000000..e25c330
--- /dev/null
+++ b/hw/s390x/s390-virtio.c
@@ -0,0 +1,297 @@
+/*
+ * QEMU S390 virtio target
+ *
+ * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ * Copyright IBM Corp 2012
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * Contributions after 2012-10-29 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU (Lesser) General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/hw.h"
+#include "block/block.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
+#include "hw/boards.h"
+#include "monitor/monitor.h"
+#include "hw/loader.h"
+#include "hw/virtio.h"
+#include "hw/sysbus.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
+
+#include "hw/s390x/s390-virtio-bus.h"
+#include "hw/s390x/sclp.h"
+#include "hw/s390x/s390-virtio.h"
+
+//#define DEBUG_S390
+
+#ifdef DEBUG_S390
+#define dprintf(fmt, ...) \
+    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
+#else
+#define dprintf(fmt, ...) \
+    do { } while (0)
+#endif
+
+#define MAX_BLK_DEVS                    10
+
+static VirtIOS390Bus *s390_bus;
+static S390CPU **ipi_states;
+
+S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
+{
+    if (cpu_addr >= smp_cpus) {
+        return NULL;
+    }
+
+    return ipi_states[cpu_addr];
+}
+
+static int s390_virtio_hcall_notify(const uint64_t *args)
+{
+    uint64_t mem = args[0];
+    int r = 0, i;
+
+    if (mem > ram_size) {
+        VirtIOS390Device *dev = s390_virtio_bus_find_vring(s390_bus, mem, &i);
+        if (dev) {
+            virtio_queue_notify(dev->vdev, i);
+        } else {
+            r = -EINVAL;
+        }
+    } else {
+        /* Early printk */
+    }
+    return r;
+}
+
+static int s390_virtio_hcall_reset(const uint64_t *args)
+{
+    uint64_t mem = args[0];
+    VirtIOS390Device *dev;
+
+    dev = s390_virtio_bus_find_mem(s390_bus, mem);
+    if (dev == NULL) {
+        return -EINVAL;
+    }
+    virtio_reset(dev->vdev);
+    stb_phys(dev->dev_offs + VIRTIO_DEV_OFFS_STATUS, 0);
+    s390_virtio_device_sync(dev);
+    s390_virtio_reset_idx(dev);
+
+    return 0;
+}
+
+static int s390_virtio_hcall_set_status(const uint64_t *args)
+{
+    uint64_t mem = args[0];
+    int r = 0;
+    VirtIOS390Device *dev;
+
+    dev = s390_virtio_bus_find_mem(s390_bus, mem);
+    if (dev) {
+        s390_virtio_device_update_status(dev);
+    } else {
+        r = -EINVAL;
+    }
+    return r;
+}
+
+static void s390_virtio_register_hcalls(void)
+{
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_NOTIFY,
+                                   s390_virtio_hcall_notify);
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_RESET,
+                                   s390_virtio_hcall_reset);
+    s390_register_virtio_hypercall(KVM_S390_VIRTIO_SET_STATUS,
+                                   s390_virtio_hcall_set_status);
+}
+
+/*
+ * The number of running CPUs. On s390 a shutdown is the state of all CPUs
+ * being either stopped or disabled (for interrupts) waiting. We have to
+ * track this number to call the shutdown sequence accordingly. This
+ * number is modified either on startup or while holding the big qemu lock.
+ */
+static unsigned s390_running_cpus;
+
+void s390_add_running_cpu(S390CPU *cpu)
+{
+    CPUS390XState *env = &cpu->env;
+
+    if (env->halted) {
+        s390_running_cpus++;
+        env->halted = 0;
+        env->exception_index = -1;
+    }
+}
+
+unsigned s390_del_running_cpu(S390CPU *cpu)
+{
+    CPUS390XState *env = &cpu->env;
+
+    if (env->halted == 0) {
+        assert(s390_running_cpus >= 1);
+        s390_running_cpus--;
+        env->halted = 1;
+        env->exception_index = EXCP_HLT;
+    }
+    return s390_running_cpus;
+}
+
+void s390_init_ipl_dev(const char *kernel_filename,
+                       const char *kernel_cmdline,
+                       const char *initrd_filename)
+{
+    DeviceState *dev;
+
+    dev  = qdev_create(NULL, "s390-ipl");
+    if (kernel_filename) {
+        qdev_prop_set_string(dev, "kernel", kernel_filename);
+    }
+    if (initrd_filename) {
+        qdev_prop_set_string(dev, "initrd", initrd_filename);
+    }
+    qdev_prop_set_string(dev, "cmdline", kernel_cmdline);
+    qdev_init_nofail(dev);
+}
+
+void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys)
+{
+    int i;
+
+    if (cpu_model == NULL) {
+        cpu_model = "host";
+    }
+
+    ipi_states = g_malloc(sizeof(S390CPU *) * smp_cpus);
+
+    for (i = 0; i < smp_cpus; i++) {
+        S390CPU *cpu;
+
+        cpu = cpu_s390x_init(cpu_model);
+
+        ipi_states[i] = cpu;
+        cpu->env.halted = 1;
+        cpu->env.exception_index = EXCP_HLT;
+        cpu->env.storage_keys = storage_keys;
+    }
+}
+
+
+void s390_create_virtio_net(BusState *bus, const char *name)
+{
+    int i;
+
+    for (i = 0; i < nb_nics; i++) {
+        NICInfo *nd = &nd_table[i];
+        DeviceState *dev;
+
+        if (!nd->model) {
+            nd->model = g_strdup("virtio");
+        }
+
+        if (strcmp(nd->model, "virtio")) {
+            fprintf(stderr, "S390 only supports VirtIO nics\n");
+            exit(1);
+        }
+
+        dev = qdev_create(bus, name);
+        qdev_set_nic_properties(dev, nd);
+        qdev_init_nofail(dev);
+    }
+}
+
+/* PC hardware initialisation */
+static void s390_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t my_ram_size = args->ram_size;
+    MemoryRegion *sysmem = get_system_memory();
+    MemoryRegion *ram = g_new(MemoryRegion, 1);
+    int shift = 0;
+    uint8_t *storage_keys;
+    void *virtio_region;
+    hwaddr virtio_region_len;
+    hwaddr virtio_region_start;
+
+    /* s390x ram size detection needs a 16bit multiplier + an increment. So
+       guests > 64GB can be specified in 2MB steps etc. */
+    while ((my_ram_size >> (20 + shift)) > 65535) {
+        shift++;
+    }
+    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
+
+    /* lets propagate the changed ram size into the global variable. */
+    ram_size = my_ram_size;
+
+    /* get a BUS */
+    s390_bus = s390_virtio_bus_init(&my_ram_size);
+    s390_sclp_init();
+    s390_init_ipl_dev(args->kernel_filename, args->kernel_cmdline,
+                      args->initrd_filename);
+
+    /* register hypercalls */
+    s390_virtio_register_hcalls();
+
+    /* allocate RAM */
+    memory_region_init_ram(ram, "s390.ram", my_ram_size);
+    vmstate_register_ram_global(ram);
+    memory_region_add_subregion(sysmem, 0, ram);
+
+    /* clear virtio region */
+    virtio_region_len = my_ram_size - ram_size;
+    virtio_region_start = ram_size;
+    virtio_region = cpu_physical_memory_map(virtio_region_start,
+                                            &virtio_region_len, true);
+    memset(virtio_region, 0, virtio_region_len);
+    cpu_physical_memory_unmap(virtio_region, virtio_region_len, 1,
+                              virtio_region_len);
+
+    /* allocate storage keys */
+    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
+
+    /* init CPUs */
+    s390_init_cpus(args->cpu_model, storage_keys);
+
+    /* Create VirtIO network adapters */
+    s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
+}
+
+static QEMUMachine s390_machine = {
+    .name = "s390-virtio",
+    .alias = "s390",
+    .desc = "VirtIO based S390 machine",
+    .init = s390_init,
+    .block_default_type = IF_VIRTIO,
+    .no_cdrom = 1,
+    .no_floppy = 1,
+    .no_serial = 1,
+    .no_parallel = 1,
+    .no_sdcard = 1,
+    .use_virtcon = 1,
+    .max_cpus = 255,
+    .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
+};
+
+static void s390_machine_init(void)
+{
+    qemu_register_machine(&s390_machine);
+}
+
+machine_init(s390_machine_init);
diff --git a/hw/s390x/s390-virtio.h b/hw/s390x/s390-virtio.h
new file mode 100644
index 0000000..a6c4c19
--- /dev/null
+++ b/hw/s390x/s390-virtio.h
@@ -0,0 +1,28 @@
+/*
+ * Virtio interfaces for s390
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_VIRTIO_H
+#define HW_S390_VIRTIO_H 1
+
+#define KVM_S390_VIRTIO_NOTIFY          0
+#define KVM_S390_VIRTIO_RESET           1
+#define KVM_S390_VIRTIO_SET_STATUS      2
+#define KVM_S390_VIRTIO_CCW_NOTIFY      3
+
+typedef int (*s390_virtio_fn)(const uint64_t *args);
+void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn);
+
+void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys);
+void s390_init_ipl_dev(const char *kernel_filename,
+                       const char *kernel_cmdline,
+                       const char *initrd_filename);
+void s390_create_virtio_net(BusState *bus, const char *name);
+#endif
diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index 5c274fa..a9d3a6a 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -13,8 +13,8 @@
  */
 
 #include "cpu.h"
-#include "kvm.h"
-#include "memory.h"
+#include "sysemu/kvm.h"
+#include "exec/memory.h"
 
 #include "sclp.h"
 
@@ -146,7 +146,7 @@ static void s390_sclp_device_class_init(ObjectClass *klass, void *data)
     dc->init = s390_sclp_dev_init;
 }
 
-static TypeInfo s390_sclp_device_info = {
+static const TypeInfo s390_sclp_device_info = {
     .name = TYPE_DEVICE_S390_SCLP,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(S390SCLPDevice),
diff --git a/hw/s390x/sclp.h b/hw/s390x/sclp.h
index fe89dad..231a38a 100644
--- a/hw/s390x/sclp.h
+++ b/hw/s390x/sclp.h
@@ -51,7 +51,7 @@
 
 /*
  * Normally packed structures are not the right thing to do, since all code
- * must take care of endianess. We cant use ldl_phys and friends for two
+ * must take care of endianness. We cannot use ldl_phys and friends for two
  * reasons, though:
  * - some of the embedded structures below the SCCB can appear multiple times
  *   at different locations, so there is no fixed offset
@@ -60,7 +60,7 @@
  *   alter the structure while we parse it. We cannot use ldl_p and friends
  *   either without doing pointer arithmetics
  * So we have to double check that all users of sclp data structures use the
- * right endianess wrappers.
+ * right endianness wrappers.
  */
 typedef struct SCCBHeader {
     uint16_t length;
diff --git a/hw/s390x/sclpconsole.c b/hw/s390x/sclpconsole.c
index 0ec5623..effe511 100644
--- a/hw/s390x/sclpconsole.c
+++ b/hw/s390x/sclpconsole.c
@@ -13,10 +13,11 @@
  */
 
 #include <hw/qdev.h>
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "sclp.h"
 #include "event-facility.h"
+#include "char/char.h"
 
 typedef struct ASCIIConsoleData {
     EventBufferHeader ebh;
@@ -43,12 +44,9 @@ typedef struct SCLPConsole {
 /* Return number of bytes that fit into iov buffer */
 static int chr_can_read(void *opaque)
 {
-    int can_read;
     SCLPConsole *scon = opaque;
 
-    can_read = SIZE_BUFFER_VT220 - scon->iov_data_len;
-
-    return can_read;
+    return scon->iov ? SIZE_BUFFER_VT220 - scon->iov_data_len : 0;
 }
 
 /* Receive n bytes from character layer, save in iov buffer,
@@ -179,8 +177,8 @@ static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
 }
 
 /* triggered by SCLP's write_event_data
- *  - write console data into character layer
- *  returns < 0 if an error occured
+ *  - write console data to character layer
+ *  returns < 0 if an error occurred
  */
 static ssize_t write_console_data(SCLPEvent *event, const uint8_t *buf,
                                   size_t len)
@@ -290,7 +288,7 @@ static void console_class_init(ObjectClass *klass, void *data)
     ec->write_event_data = write_event_data;
 }
 
-static TypeInfo sclp_console_info = {
+static const TypeInfo sclp_console_info = {
     .name          = "sclpconsole",
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPConsole),
diff --git a/hw/s390x/sclpquiesce.c b/hw/s390x/sclpquiesce.c
index 9a773b8..2538498 100644
--- a/hw/s390x/sclpquiesce.c
+++ b/hw/s390x/sclpquiesce.c
@@ -12,7 +12,7 @@
  *
  */
 #include <hw/qdev.h>
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sclp.h"
 #include "event-facility.h"
 
@@ -107,7 +107,7 @@ static void quiesce_class_init(ObjectClass *klass, void *data)
     k->write_event_data = NULL;
 }
 
-static TypeInfo sclp_quiesce_info = {
+static const TypeInfo sclp_quiesce_info = {
     .name          = "sclpquiesce",
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
new file mode 100644
index 0000000..d92e427
--- /dev/null
+++ b/hw/s390x/virtio-ccw.c
@@ -0,0 +1,961 @@
+/*
+ * virtio ccw target implementation
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "hw/hw.h"
+#include "block/block.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
+#include "monitor/monitor.h"
+#include "hw/virtio.h"
+#include "hw/virtio-serial.h"
+#include "hw/virtio-net.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/virtio-bus.h"
+
+#include "ioinst.h"
+#include "css.h"
+#include "virtio-ccw.h"
+#include "trace.h"
+
+static int virtual_css_bus_reset(BusState *qbus)
+{
+    /* This should actually be modelled via the generic css */
+    css_reset();
+
+    /* we dont traverse ourself, return 0 */
+    return 0;
+}
+
+
+static void virtual_css_bus_class_init(ObjectClass *klass, void *data)
+{
+    BusClass *k = BUS_CLASS(klass);
+
+    k->reset = virtual_css_bus_reset;
+}
+
+static const TypeInfo virtual_css_bus_info = {
+    .name = TYPE_VIRTUAL_CSS_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(VirtualCssBus),
+    .class_init = virtual_css_bus_class_init,
+};
+
+static const VirtIOBindings virtio_ccw_bindings;
+
+VirtIODevice *virtio_ccw_get_vdev(SubchDev *sch)
+{
+    VirtIODevice *vdev = NULL;
+
+    if (sch->driver_data) {
+        vdev = ((VirtioCcwDevice *)sch->driver_data)->vdev;
+    }
+    return vdev;
+}
+
+VirtualCssBus *virtual_css_bus_init(void)
+{
+    VirtualCssBus *cbus;
+    BusState *bus;
+    DeviceState *dev;
+
+    /* Create bridge device */
+    dev = qdev_create(NULL, "virtual-css-bridge");
+    qdev_init_nofail(dev);
+
+    /* Create bus on bridge device */
+    bus = qbus_create(TYPE_VIRTUAL_CSS_BUS, dev, "virtual-css");
+    cbus = VIRTUAL_CSS_BUS(bus);
+
+    /* Enable hotplugging */
+    bus->allow_hotplug = 1;
+
+    return cbus;
+}
+
+/* Communication blocks used by several channel commands. */
+typedef struct VqInfoBlock {
+    uint64_t queue;
+    uint32_t align;
+    uint16_t index;
+    uint16_t num;
+} QEMU_PACKED VqInfoBlock;
+
+typedef struct VqConfigBlock {
+    uint16_t index;
+    uint16_t num_max;
+} QEMU_PACKED VqConfigBlock;
+
+typedef struct VirtioFeatDesc {
+    uint32_t features;
+    uint8_t index;
+} QEMU_PACKED VirtioFeatDesc;
+
+/* Specify where the virtqueues for the subchannel are in guest memory. */
+static int virtio_ccw_set_vqs(SubchDev *sch, uint64_t addr, uint32_t align,
+                              uint16_t index, uint16_t num)
+{
+    VirtioCcwDevice *dev = sch->driver_data;
+
+    if (index > VIRTIO_PCI_QUEUE_MAX) {
+        return -EINVAL;
+    }
+
+    /* Current code in virtio.c relies on 4K alignment. */
+    if (addr && (align != 4096)) {
+        return -EINVAL;
+    }
+
+    if (!dev) {
+        return -EINVAL;
+    }
+
+    virtio_queue_set_addr(dev->vdev, index, addr);
+    if (!addr) {
+        virtio_queue_set_vector(dev->vdev, index, 0);
+    } else {
+        /* Fail if we don't have a big enough queue. */
+        /* TODO: Add interface to handle vring.num changing */
+        if (virtio_queue_get_num(dev->vdev, index) > num) {
+            return -EINVAL;
+        }
+        virtio_queue_set_vector(dev->vdev, index, index);
+    }
+    /* tell notify handler in case of config change */
+    dev->vdev->config_vector = VIRTIO_PCI_QUEUE_MAX;
+    return 0;
+}
+
+static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
+{
+    int ret;
+    VqInfoBlock info;
+    uint8_t status;
+    VirtioFeatDesc features;
+    void *config;
+    hwaddr indicators;
+    VqConfigBlock vq_config;
+    VirtioCcwDevice *dev = sch->driver_data;
+    bool check_len;
+    int len;
+    hwaddr hw_len;
+
+    if (!dev) {
+        return -EINVAL;
+    }
+
+    trace_virtio_ccw_interpret_ccw(sch->cssid, sch->ssid, sch->schid,
+                                   ccw.cmd_code);
+    check_len = !((ccw.flags & CCW_FLAG_SLI) && !(ccw.flags & CCW_FLAG_DC));
+
+    /* Look at the command. */
+    switch (ccw.cmd_code) {
+    case CCW_CMD_SET_VQ:
+        if (check_len) {
+            if (ccw.count != sizeof(info)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(info)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            info.queue = ldq_phys(ccw.cda);
+            info.align = ldl_phys(ccw.cda + sizeof(info.queue));
+            info.index = lduw_phys(ccw.cda + sizeof(info.queue)
+                                   + sizeof(info.align));
+            info.num = lduw_phys(ccw.cda + sizeof(info.queue)
+                                 + sizeof(info.align)
+                                 + sizeof(info.index));
+            ret = virtio_ccw_set_vqs(sch, info.queue, info.align, info.index,
+                                     info.num);
+            sch->curr_status.scsw.count = 0;
+        }
+        break;
+    case CCW_CMD_VDEV_RESET:
+        virtio_reset(dev->vdev);
+        ret = 0;
+        break;
+    case CCW_CMD_READ_FEAT:
+        if (check_len) {
+            if (ccw.count != sizeof(features)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(features)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            features.index = ldub_phys(ccw.cda + sizeof(features.features));
+            if (features.index < ARRAY_SIZE(dev->host_features)) {
+                features.features = dev->host_features[features.index];
+            } else {
+                /* Return zeroes if the guest supports more feature bits. */
+                features.features = 0;
+            }
+            stl_le_phys(ccw.cda, features.features);
+            sch->curr_status.scsw.count = ccw.count - sizeof(features);
+            ret = 0;
+        }
+        break;
+    case CCW_CMD_WRITE_FEAT:
+        if (check_len) {
+            if (ccw.count != sizeof(features)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(features)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            features.index = ldub_phys(ccw.cda + sizeof(features.features));
+            features.features = ldl_le_phys(ccw.cda);
+            if (features.index < ARRAY_SIZE(dev->host_features)) {
+                if (dev->vdev->set_features) {
+                    dev->vdev->set_features(dev->vdev, features.features);
+                }
+                dev->vdev->guest_features = features.features;
+            } else {
+                /*
+                 * If the guest supports more feature bits, assert that it
+                 * passes us zeroes for those we don't support.
+                 */
+                if (features.features) {
+                    fprintf(stderr, "Guest bug: features[%i]=%x (expected 0)\n",
+                            features.index, features.features);
+                    /* XXX: do a unit check here? */
+                }
+            }
+            sch->curr_status.scsw.count = ccw.count - sizeof(features);
+            ret = 0;
+        }
+        break;
+    case CCW_CMD_READ_CONF:
+        if (check_len) {
+            if (ccw.count > dev->vdev->config_len) {
+                ret = -EINVAL;
+                break;
+            }
+        }
+        len = MIN(ccw.count, dev->vdev->config_len);
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            dev->vdev->get_config(dev->vdev, dev->vdev->config);
+            /* XXX config space endianness */
+            cpu_physical_memory_write(ccw.cda, dev->vdev->config, len);
+            sch->curr_status.scsw.count = ccw.count - len;
+            ret = 0;
+        }
+        break;
+    case CCW_CMD_WRITE_CONF:
+        if (check_len) {
+            if (ccw.count > dev->vdev->config_len) {
+                ret = -EINVAL;
+                break;
+            }
+        }
+        len = MIN(ccw.count, dev->vdev->config_len);
+        hw_len = len;
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            config = cpu_physical_memory_map(ccw.cda, &hw_len, 0);
+            if (!config) {
+                ret = -EFAULT;
+            } else {
+                len = hw_len;
+                /* XXX config space endianness */
+                memcpy(dev->vdev->config, config, len);
+                cpu_physical_memory_unmap(config, hw_len, 0, hw_len);
+                if (dev->vdev->set_config) {
+                    dev->vdev->set_config(dev->vdev, dev->vdev->config);
+                }
+                sch->curr_status.scsw.count = ccw.count - len;
+                ret = 0;
+            }
+        }
+        break;
+    case CCW_CMD_WRITE_STATUS:
+        if (check_len) {
+            if (ccw.count != sizeof(status)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(status)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            status = ldub_phys(ccw.cda);
+            virtio_set_status(dev->vdev, status);
+            if (dev->vdev->status == 0) {
+                virtio_reset(dev->vdev);
+            }
+            sch->curr_status.scsw.count = ccw.count - sizeof(status);
+            ret = 0;
+        }
+        break;
+    case CCW_CMD_SET_IND:
+        if (check_len) {
+            if (ccw.count != sizeof(indicators)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(indicators)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        indicators = ldq_phys(ccw.cda);
+        if (!indicators) {
+            ret = -EFAULT;
+        } else {
+            dev->indicators = indicators;
+            sch->curr_status.scsw.count = ccw.count - sizeof(indicators);
+            ret = 0;
+        }
+        break;
+    case CCW_CMD_SET_CONF_IND:
+        if (check_len) {
+            if (ccw.count != sizeof(indicators)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(indicators)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        indicators = ldq_phys(ccw.cda);
+        if (!indicators) {
+            ret = -EFAULT;
+        } else {
+            dev->indicators2 = indicators;
+            sch->curr_status.scsw.count = ccw.count - sizeof(indicators);
+            ret = 0;
+        }
+        break;
+    case CCW_CMD_READ_VQ_CONF:
+        if (check_len) {
+            if (ccw.count != sizeof(vq_config)) {
+                ret = -EINVAL;
+                break;
+            }
+        } else if (ccw.count < sizeof(vq_config)) {
+            /* Can't execute command. */
+            ret = -EINVAL;
+            break;
+        }
+        if (!ccw.cda) {
+            ret = -EFAULT;
+        } else {
+            vq_config.index = lduw_phys(ccw.cda);
+            vq_config.num_max = virtio_queue_get_num(dev->vdev,
+                                                     vq_config.index);
+            stw_phys(ccw.cda + sizeof(vq_config.index), vq_config.num_max);
+            sch->curr_status.scsw.count = ccw.count - sizeof(vq_config);
+            ret = 0;
+        }
+        break;
+    default:
+        ret = -ENOSYS;
+        break;
+    }
+    return ret;
+}
+
+static int virtio_ccw_device_init(VirtioCcwDevice *dev, VirtIODevice *vdev)
+{
+    unsigned int cssid = 0;
+    unsigned int ssid = 0;
+    unsigned int schid;
+    unsigned int devno;
+    bool have_devno = false;
+    bool found = false;
+    SubchDev *sch;
+    int ret;
+    int num;
+    DeviceState *parent = DEVICE(dev);
+
+    sch = g_malloc0(sizeof(SubchDev));
+
+    sch->driver_data = dev;
+    dev->sch = sch;
+
+    dev->vdev = vdev;
+    dev->indicators = 0;
+
+    /* Initialize subchannel structure. */
+    sch->channel_prog = 0x0;
+    sch->last_cmd_valid = false;
+    sch->orb = NULL;
+    /*
+     * Use a device number if provided. Otherwise, fall back to subchannel
+     * number.
+     */
+    if (dev->bus_id) {
+        num = sscanf(dev->bus_id, "%x.%x.%04x", &cssid, &ssid, &devno);
+        if (num == 3) {
+            if ((cssid > MAX_CSSID) || (ssid > MAX_SSID)) {
+                ret = -EINVAL;
+                error_report("Invalid cssid or ssid: cssid %x, ssid %x",
+                             cssid, ssid);
+                goto out_err;
+            }
+            /* Enforce use of virtual cssid. */
+            if (cssid != VIRTUAL_CSSID) {
+                ret = -EINVAL;
+                error_report("cssid %x not valid for virtio devices", cssid);
+                goto out_err;
+            }
+            if (css_devno_used(cssid, ssid, devno)) {
+                ret = -EEXIST;
+                error_report("Device %x.%x.%04x already exists", cssid, ssid,
+                             devno);
+                goto out_err;
+            }
+            sch->cssid = cssid;
+            sch->ssid = ssid;
+            sch->devno = devno;
+            have_devno = true;
+        } else {
+            ret = -EINVAL;
+            error_report("Malformed devno parameter '%s'", dev->bus_id);
+            goto out_err;
+        }
+    }
+
+    /* Find the next free id. */
+    if (have_devno) {
+        for (schid = 0; schid <= MAX_SCHID; schid++) {
+            if (!css_find_subch(1, cssid, ssid, schid)) {
+                sch->schid = schid;
+                css_subch_assign(cssid, ssid, schid, devno, sch);
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            ret = -ENODEV;
+            error_report("No free subchannel found for %x.%x.%04x", cssid, ssid,
+                         devno);
+            goto out_err;
+        }
+        trace_virtio_ccw_new_device(cssid, ssid, schid, devno,
+                                    "user-configured");
+    } else {
+        cssid = VIRTUAL_CSSID;
+        for (ssid = 0; ssid <= MAX_SSID; ssid++) {
+            for (schid = 0; schid <= MAX_SCHID; schid++) {
+                if (!css_find_subch(1, cssid, ssid, schid)) {
+                    sch->cssid = cssid;
+                    sch->ssid = ssid;
+                    sch->schid = schid;
+                    devno = schid;
+                    /*
+                     * If the devno is already taken, look further in this
+                     * subchannel set.
+                     */
+                    while (css_devno_used(cssid, ssid, devno)) {
+                        if (devno == MAX_SCHID) {
+                            devno = 0;
+                        } else if (devno == schid - 1) {
+                            ret = -ENODEV;
+                            error_report("No free devno found");
+                            goto out_err;
+                        } else {
+                            devno++;
+                        }
+                    }
+                    sch->devno = devno;
+                    css_subch_assign(cssid, ssid, schid, devno, sch);
+                    found = true;
+                    break;
+                }
+            }
+            if (found) {
+                break;
+            }
+        }
+        if (!found) {
+            ret = -ENODEV;
+            error_report("Virtual channel subsystem is full!");
+            goto out_err;
+        }
+        trace_virtio_ccw_new_device(cssid, ssid, schid, devno,
+                                    "auto-configured");
+    }
+
+    /* Build initial schib. */
+    css_sch_build_virtual_schib(sch, 0, VIRTIO_CCW_CHPID_TYPE);
+
+    sch->ccw_cb = virtio_ccw_cb;
+
+    /* Build senseid data. */
+    memset(&sch->id, 0, sizeof(SenseId));
+    sch->id.reserved = 0xff;
+    sch->id.cu_type = VIRTIO_CCW_CU_TYPE;
+    sch->id.cu_model = dev->vdev->device_id;
+
+    virtio_bind_device(vdev, &virtio_ccw_bindings, DEVICE(dev));
+    /* Only the first 32 feature bits are used. */
+    dev->host_features[0] = vdev->get_features(vdev, dev->host_features[0]);
+    dev->host_features[0] |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
+    dev->host_features[0] |= 0x1 << VIRTIO_F_BAD_FEATURE;
+
+    css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid,
+                          parent->hotplugged, 1);
+    return 0;
+
+out_err:
+    dev->sch = NULL;
+    g_free(sch);
+    return ret;
+}
+
+static int virtio_ccw_exit(VirtioCcwDevice *dev)
+{
+    SubchDev *sch = dev->sch;
+
+    if (sch) {
+        css_subch_assign(sch->cssid, sch->ssid, sch->schid, sch->devno, NULL);
+        g_free(sch);
+    }
+    dev->indicators = 0;
+    return 0;
+}
+
+static int virtio_ccw_net_init(VirtioCcwDevice *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_net_init((DeviceState *)dev, &dev->nic, &dev->net,
+                           dev->host_features[0]);
+    if (!vdev) {
+        return -1;
+    }
+
+    return virtio_ccw_device_init(dev, vdev);
+}
+
+static int virtio_ccw_net_exit(VirtioCcwDevice *dev)
+{
+    virtio_net_exit(dev->vdev);
+    return virtio_ccw_exit(dev);
+}
+
+static int virtio_ccw_blk_init(VirtioCcwDevice *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_blk_init((DeviceState *)dev, &dev->blk);
+    if (!vdev) {
+        return -1;
+    }
+
+    return virtio_ccw_device_init(dev, vdev);
+}
+
+static int virtio_ccw_blk_exit(VirtioCcwDevice *dev)
+{
+    virtio_blk_exit(dev->vdev);
+    blockdev_mark_auto_del(dev->blk.conf.bs);
+    return virtio_ccw_exit(dev);
+}
+
+static int virtio_ccw_serial_init(VirtioCcwDevice *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_serial_init((DeviceState *)dev, &dev->serial);
+    if (!vdev) {
+        return -1;
+    }
+
+    return virtio_ccw_device_init(dev, vdev);
+}
+
+static int virtio_ccw_serial_exit(VirtioCcwDevice *dev)
+{
+    virtio_serial_exit(dev->vdev);
+    return virtio_ccw_exit(dev);
+}
+
+static int virtio_ccw_balloon_init(VirtioCcwDevice *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_balloon_init((DeviceState *)dev);
+    if (!vdev) {
+        return -1;
+    }
+
+    return virtio_ccw_device_init(dev, vdev);
+}
+
+static int virtio_ccw_balloon_exit(VirtioCcwDevice *dev)
+{
+    virtio_balloon_exit(dev->vdev);
+    return virtio_ccw_exit(dev);
+}
+
+static int virtio_ccw_scsi_init(VirtioCcwDevice *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_scsi_init((DeviceState *)dev, &dev->scsi);
+    if (!vdev) {
+        return -1;
+    }
+
+    return virtio_ccw_device_init(dev, vdev);
+}
+
+static int virtio_ccw_scsi_exit(VirtioCcwDevice *dev)
+{
+    virtio_scsi_exit(dev->vdev);
+    return virtio_ccw_exit(dev);
+}
+
+/* DeviceState to VirtioCcwDevice. Note: used on datapath,
+ * be careful and test performance if you change this.
+ */
+static inline VirtioCcwDevice *to_virtio_ccw_dev_fast(DeviceState *d)
+{
+    return container_of(d, VirtioCcwDevice, parent_obj);
+}
+
+static void virtio_ccw_notify(DeviceState *d, uint16_t vector)
+{
+    VirtioCcwDevice *dev = to_virtio_ccw_dev_fast(d);
+    SubchDev *sch = dev->sch;
+    uint64_t indicators;
+
+    if (vector >= 128) {
+        return;
+    }
+
+    if (vector < VIRTIO_PCI_QUEUE_MAX) {
+        indicators = ldq_phys(dev->indicators);
+        indicators |= 1ULL << vector;
+        stq_phys(dev->indicators, indicators);
+    } else {
+        vector = 0;
+        indicators = ldq_phys(dev->indicators2);
+        indicators |= 1ULL << vector;
+        stq_phys(dev->indicators2, indicators);
+    }
+
+    css_conditional_io_interrupt(sch);
+
+}
+
+static unsigned virtio_ccw_get_features(DeviceState *d)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+    /* Only the first 32 feature bits are used. */
+    return dev->host_features[0];
+}
+
+static void virtio_ccw_reset(DeviceState *d)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+    virtio_reset(dev->vdev);
+    css_reset_sch(dev->sch);
+}
+
+/**************** Virtio-ccw Bus Device Descriptions *******************/
+
+static const VirtIOBindings virtio_ccw_bindings = {
+    .notify = virtio_ccw_notify,
+    .get_features = virtio_ccw_get_features,
+};
+
+static Property virtio_ccw_net_properties[] = {
+    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_VIRTIO_NET_FEATURES(VirtioCcwDevice, host_features[0]),
+    DEFINE_NIC_PROPERTIES(VirtioCcwDevice, nic),
+    DEFINE_PROP_UINT32("x-txtimer", VirtioCcwDevice,
+                       net.txtimer, TX_TIMER_INTERVAL),
+    DEFINE_PROP_INT32("x-txburst", VirtioCcwDevice,
+                      net.txburst, TX_BURST),
+    DEFINE_PROP_STRING("tx", VirtioCcwDevice, net.tx),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_net_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+
+    k->init = virtio_ccw_net_init;
+    k->exit = virtio_ccw_net_exit;
+    dc->reset = virtio_ccw_reset;
+    dc->props = virtio_ccw_net_properties;
+}
+
+static const TypeInfo virtio_ccw_net = {
+    .name          = "virtio-net-ccw",
+    .parent        = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(VirtioCcwDevice),
+    .class_init    = virtio_ccw_net_class_init,
+};
+
+static Property virtio_ccw_blk_properties[] = {
+    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_BLOCK_PROPERTIES(VirtioCcwDevice, blk.conf),
+    DEFINE_PROP_STRING("serial", VirtioCcwDevice, blk.serial),
+#ifdef __linux__
+    DEFINE_PROP_BIT("scsi", VirtioCcwDevice, blk.scsi, 0, true),
+#endif
+    DEFINE_VIRTIO_BLK_FEATURES(VirtioCcwDevice, host_features[0]),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_blk_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+
+    k->init = virtio_ccw_blk_init;
+    k->exit = virtio_ccw_blk_exit;
+    dc->reset = virtio_ccw_reset;
+    dc->props = virtio_ccw_blk_properties;
+}
+
+static const TypeInfo virtio_ccw_blk = {
+    .name          = "virtio-blk-ccw",
+    .parent        = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(VirtioCcwDevice),
+    .class_init    = virtio_ccw_blk_class_init,
+};
+
+static Property virtio_ccw_serial_properties[] = {
+    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_UINT32("max_ports", VirtioCcwDevice,
+                       serial.max_virtserial_ports, 31),
+    DEFINE_VIRTIO_COMMON_FEATURES(VirtioCcwDevice, host_features[0]),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_serial_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+
+    k->init = virtio_ccw_serial_init;
+    k->exit = virtio_ccw_serial_exit;
+    dc->reset = virtio_ccw_reset;
+    dc->props = virtio_ccw_serial_properties;
+}
+
+static const TypeInfo virtio_ccw_serial = {
+    .name          = "virtio-serial-ccw",
+    .parent        = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(VirtioCcwDevice),
+    .class_init    = virtio_ccw_serial_class_init,
+};
+
+static Property virtio_ccw_balloon_properties[] = {
+    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_VIRTIO_COMMON_FEATURES(VirtioCcwDevice, host_features[0]),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_balloon_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+
+    k->init = virtio_ccw_balloon_init;
+    k->exit = virtio_ccw_balloon_exit;
+    dc->reset = virtio_ccw_reset;
+    dc->props = virtio_ccw_balloon_properties;
+}
+
+static const TypeInfo virtio_ccw_balloon = {
+    .name          = "virtio-balloon-ccw",
+    .parent        = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(VirtioCcwDevice),
+    .class_init    = virtio_ccw_balloon_class_init,
+};
+
+static Property virtio_ccw_scsi_properties[] = {
+    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtioCcwDevice, host_features[0], scsi),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_scsi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+
+    k->init = virtio_ccw_scsi_init;
+    k->exit = virtio_ccw_scsi_exit;
+    dc->reset = virtio_ccw_reset;
+    dc->props = virtio_ccw_scsi_properties;
+}
+
+static const TypeInfo virtio_ccw_scsi = {
+    .name          = "virtio-scsi-ccw",
+    .parent        = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(VirtioCcwDevice),
+    .class_init    = virtio_ccw_scsi_class_init,
+};
+
+static int virtio_ccw_busdev_init(DeviceState *dev)
+{
+    VirtioCcwDevice *_dev = (VirtioCcwDevice *)dev;
+    VirtIOCCWDeviceClass *_info = VIRTIO_CCW_DEVICE_GET_CLASS(dev);
+
+    virtio_ccw_bus_new(&_dev->bus, _dev);
+
+    return _info->init(_dev);
+}
+
+static int virtio_ccw_busdev_exit(DeviceState *dev)
+{
+    VirtioCcwDevice *_dev = (VirtioCcwDevice *)dev;
+    VirtIOCCWDeviceClass *_info = VIRTIO_CCW_DEVICE_GET_CLASS(dev);
+
+    return _info->exit(_dev);
+}
+
+static int virtio_ccw_busdev_unplug(DeviceState *dev)
+{
+    VirtioCcwDevice *_dev = (VirtioCcwDevice *)dev;
+    SubchDev *sch = _dev->sch;
+
+    /*
+     * We should arrive here only for device_del, since we don't support
+     * direct hot(un)plug of channels, but only through virtio.
+     */
+    assert(sch != NULL);
+    /* Subchannel is now disabled and no longer valid. */
+    sch->curr_status.pmcw.flags &= ~(PMCW_FLAGS_MASK_ENA |
+                                     PMCW_FLAGS_MASK_DNV);
+
+    css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid, 1, 0);
+
+    object_unparent(OBJECT(dev));
+    qdev_free(dev);
+    return 0;
+}
+
+static void virtio_ccw_device_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->init = virtio_ccw_busdev_init;
+    dc->exit = virtio_ccw_busdev_exit;
+    dc->unplug = virtio_ccw_busdev_unplug;
+    dc->bus_type = TYPE_VIRTUAL_CSS_BUS;
+
+}
+
+static const TypeInfo virtio_ccw_device_info = {
+    .name = TYPE_VIRTIO_CCW_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(VirtioCcwDevice),
+    .class_init = virtio_ccw_device_class_init,
+    .class_size = sizeof(VirtIOCCWDeviceClass),
+    .abstract = true,
+};
+
+/***************** Virtual-css Bus Bridge Device ********************/
+/* Only required to have the virtio bus as child in the system bus */
+
+static int virtual_css_bridge_init(SysBusDevice *dev)
+{
+    /* nothing */
+    return 0;
+}
+
+static void virtual_css_bridge_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = virtual_css_bridge_init;
+    dc->no_user = 1;
+}
+
+static const TypeInfo virtual_css_bridge_info = {
+    .name          = "virtual-css-bridge",
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(SysBusDevice),
+    .class_init    = virtual_css_bridge_class_init,
+};
+
+/* virtio-ccw-bus */
+
+void virtio_ccw_bus_new(VirtioBusState *bus, VirtioCcwDevice *dev)
+{
+    DeviceState *qdev = DEVICE(dev);
+    BusState *qbus;
+
+    qbus_create_inplace((BusState *)bus, TYPE_VIRTIO_CCW_BUS, qdev, NULL);
+    qbus = BUS(bus);
+    qbus->allow_hotplug = 0;
+}
+
+static void virtio_ccw_bus_class_init(ObjectClass *klass, void *data)
+{
+    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
+    BusClass *bus_class = BUS_CLASS(klass);
+
+    bus_class->max_dev = 1;
+    k->notify = virtio_ccw_notify;
+    k->get_features = virtio_ccw_get_features;
+}
+
+static const TypeInfo virtio_ccw_bus_info = {
+    .name = TYPE_VIRTIO_CCW_BUS,
+    .parent = TYPE_VIRTIO_BUS,
+    .instance_size = sizeof(VirtioCcwBusState),
+    .class_init = virtio_ccw_bus_class_init,
+};
+
+static void virtio_ccw_register(void)
+{
+    type_register_static(&virtio_ccw_bus_info);
+    type_register_static(&virtual_css_bus_info);
+    type_register_static(&virtio_ccw_device_info);
+    type_register_static(&virtio_ccw_serial);
+    type_register_static(&virtio_ccw_blk);
+    type_register_static(&virtio_ccw_net);
+    type_register_static(&virtio_ccw_balloon);
+    type_register_static(&virtio_ccw_scsi);
+    type_register_static(&virtual_css_bridge_info);
+}
+
+type_init(virtio_ccw_register)
diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
new file mode 100644
index 0000000..48474b3
--- /dev/null
+++ b/hw/s390x/virtio-ccw.h
@@ -0,0 +1,98 @@
+/*
+ * virtio ccw target definitions
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390X_VIRTIO_CCW_H
+#define HW_S390X_VIRTIO_CCW_H
+
+#include <hw/virtio-blk.h>
+#include <hw/virtio-net.h>
+#include <hw/virtio-serial.h>
+#include <hw/virtio-scsi.h>
+#include <hw/virtio-bus.h>
+
+#define VIRTUAL_CSSID 0xfe
+
+#define VIRTIO_CCW_CU_TYPE 0x3832
+#define VIRTIO_CCW_CHPID_TYPE 0x32
+
+#define CCW_CMD_SET_VQ       0x13
+#define CCW_CMD_VDEV_RESET   0x33
+#define CCW_CMD_READ_FEAT    0x12
+#define CCW_CMD_WRITE_FEAT   0x11
+#define CCW_CMD_READ_CONF    0x22
+#define CCW_CMD_WRITE_CONF   0x21
+#define CCW_CMD_WRITE_STATUS 0x31
+#define CCW_CMD_SET_IND      0x43
+#define CCW_CMD_SET_CONF_IND 0x53
+#define CCW_CMD_READ_VQ_CONF 0x32
+
+#define TYPE_VIRTIO_CCW_DEVICE "virtio-ccw-device"
+#define VIRTIO_CCW_DEVICE(obj) \
+     OBJECT_CHECK(VirtioCcwDevice, (obj), TYPE_VIRTIO_CCW_DEVICE)
+#define VIRTIO_CCW_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(VirtIOCCWDeviceClass, (klass), TYPE_VIRTIO_CCW_DEVICE)
+#define VIRTIO_CCW_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(VirtIOCCWDeviceClass, (obj), TYPE_VIRTIO_CCW_DEVICE)
+
+typedef struct VirtioBusState VirtioCcwBusState;
+typedef struct VirtioBusClass VirtioCcwBusClass;
+
+#define TYPE_VIRTIO_CCW_BUS "virtio-ccw-bus"
+#define VIRTIO_CCW_BUS(obj) \
+     OBJECT_CHECK(VirtioCcwBus, (obj), TYPE_VIRTIO_CCW_BUS)
+#define VIRTIO_CCW_BUS_GET_CLASS(obj) \
+    OBJECT_CHECK(VirtioCcwBusState, (obj), TYPE_VIRTIO_CCW_BUS)
+#define VIRTIO_CCW_BUS_CLASS(klass) \
+    OBJECT_CLASS_CHECK(VirtioCcwBusClass, klass, TYPE_VIRTIO_CCW_BUS)
+
+typedef struct VirtioCcwDevice VirtioCcwDevice;
+
+void virtio_ccw_bus_new(VirtioBusState *bus, VirtioCcwDevice *dev);
+
+typedef struct VirtIOCCWDeviceClass {
+    DeviceClass parent_class;
+    int (*init)(VirtioCcwDevice *dev);
+    int (*exit)(VirtioCcwDevice *dev);
+} VirtIOCCWDeviceClass;
+
+/* Change here if we want to support more feature bits. */
+#define VIRTIO_CCW_FEATURE_SIZE 1
+
+struct VirtioCcwDevice {
+    DeviceState parent_obj;
+    SubchDev *sch;
+    VirtIODevice *vdev;
+    char *bus_id;
+    VirtIOBlkConf blk;
+    NICConf nic;
+    uint32_t host_features[VIRTIO_CCW_FEATURE_SIZE];
+    virtio_serial_conf serial;
+    virtio_net_conf net;
+    VirtIOSCSIConf scsi;
+    VirtioBusState bus;
+    /* Guest provided values: */
+    hwaddr indicators;
+    hwaddr indicators2;
+};
+
+/* virtual css bus type */
+typedef struct VirtualCssBus {
+    BusState parent_obj;
+} VirtualCssBus;
+
+#define TYPE_VIRTUAL_CSS_BUS "virtual-css-bus"
+#define VIRTUAL_CSS_BUS(obj) \
+     OBJECT_CHECK(VirtualCssBus, (obj), TYPE_VIRTUAL_CSS_BUS)
+
+VirtualCssBus *virtual_css_bus_init(void);
+void virtio_ccw_device_update_status(SubchDev *sch);
+VirtIODevice *virtio_ccw_get_vdev(SubchDev *sch);
+#endif
diff --git a/hw/sb16.c b/hw/sb16.c
index 523ab0d..52dfedf 100644
--- a/hw/sb16.c
+++ b/hw/sb16.c
@@ -26,8 +26,8 @@
 #include "audio/audio.h"
 #include "isa.h"
 #include "qdev.h"
-#include "qemu-timer.h"
-#include "host-utils.h"
+#include "qemu/timer.h"
+#include "qemu/host-utils.h"
 
 #define dolog(...) AUD_log ("sb16", __VA_ARGS__)
 
@@ -1409,7 +1409,7 @@ static void sb16_class_initfn (ObjectClass *klass, void *data)
     dc->props = sb16_properties;
 }
 
-static TypeInfo sb16_info = {
+static const TypeInfo sb16_info = {
     .name          = "sb16",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof (SB16State),
diff --git a/hw/sbi.c b/hw/sbi.c
index ca78a38..d58184a 100644
--- a/hw/sbi.c
+++ b/hw/sbi.c
@@ -141,7 +141,7 @@ static void sbi_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_sbi;
 }
 
-static TypeInfo sbi_info = {
+static const TypeInfo sbi_info = {
     .name          = "sbi",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SBIState),
diff --git a/hw/scsi-bus.c b/hw/scsi-bus.c
index dfb2631..a97f1cd 100644
--- a/hw/scsi-bus.c
+++ b/hw/scsi-bus.c
@@ -1,11 +1,11 @@
 #include "hw.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "scsi.h"
 #include "scsi-defs.h"
 #include "qdev.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "trace.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 static char *scsibus_get_dev_path(DeviceState *dev);
 static char *scsibus_get_fw_dev_path(DeviceState *dev);
@@ -282,7 +282,7 @@ static const struct SCSIReqOps reqops_invalid_opcode = {
 
 static int32_t scsi_unit_attention(SCSIRequest *req, uint8_t *buf)
 {
-    if (req->dev && req->dev->unit_attention.key == UNIT_ATTENTION) {
+    if (req->dev->unit_attention.key == UNIT_ATTENTION) {
         scsi_req_build_sense(req, req->dev->unit_attention);
     } else if (req->bus->unit_attention.key == UNIT_ATTENTION) {
         scsi_req_build_sense(req, req->bus->unit_attention);
@@ -1863,7 +1863,7 @@ static void scsi_device_class_init(ObjectClass *klass, void *data)
     k->props    = scsi_props;
 }
 
-static TypeInfo scsi_device_type_info = {
+static const TypeInfo scsi_device_type_info = {
     .name = TYPE_SCSI_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(SCSIDevice),
diff --git a/hw/scsi-defs.h b/hw/scsi-defs.h
index d7a4019..9ab045b 100644
--- a/hw/scsi-defs.h
+++ b/hw/scsi-defs.h
@@ -19,6 +19,8 @@
  * This header file contains public constants and structures used by
  * the scsi code for linux.
  */
+#ifndef HW_SCSI_DEFS_H
+#define HW_SCSI_DEFS_H 1
 
 /*
  *      SCSI opcodes
@@ -301,3 +303,5 @@
 #define MMC_PROFILE_HDDVD_R_DL          0x0058
 #define MMC_PROFILE_HDDVD_RW_DL         0x005A
 #define MMC_PROFILE_INVALID             0xFFFF
+
+#endif
diff --git a/hw/scsi-disk.c b/hw/scsi-disk.c
index 49b5686..28e75bb 100644
--- a/hw/scsi-disk.c
+++ b/hw/scsi-disk.c
@@ -29,13 +29,13 @@ do { printf("scsi-disk: " fmt , ## __VA_ARGS__); } while (0)
 #endif
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "scsi.h"
 #include "scsi-defs.h"
-#include "sysemu.h"
-#include "blockdev.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 #ifdef __linux
 #include <scsi/sg.h>
@@ -85,9 +85,7 @@ static void scsi_free_request(SCSIRequest *req)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
 
-    if (r->iov.iov_base) {
-        qemu_vfree(r->iov.iov_base);
-    }
+    qemu_vfree(r->iov.iov_base);
 }
 
 /* Helper function for command completion with sense.  */
@@ -1682,7 +1680,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
         if (!nb_sectors) {
             scsi_check_condition(r, SENSE_CODE(LUN_NOT_READY));
-            return -1;
+            return 0;
         }
         if ((req->cmd.buf[8] & 1) == 0 && req->cmd.lba) {
             goto illegal_request;
@@ -1751,7 +1749,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
             bdrv_get_geometry(s->qdev.conf.bs, &nb_sectors);
             if (!nb_sectors) {
                 scsi_check_condition(r, SENSE_CODE(LUN_NOT_READY));
-                return -1;
+                return 0;
             }
             if ((req->cmd.buf[14] & 1) == 0 && req->cmd.lba) {
                 goto illegal_request;
@@ -2389,7 +2387,7 @@ static void scsi_hd_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_hd_info = {
+static const TypeInfo scsi_hd_info = {
     .name          = "scsi-hd",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
@@ -2418,7 +2416,7 @@ static void scsi_cd_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_cd_info = {
+static const TypeInfo scsi_cd_info = {
     .name          = "scsi-cd",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
@@ -2447,7 +2445,7 @@ static void scsi_block_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_block_info = {
+static const TypeInfo scsi_block_info = {
     .name          = "scsi-block",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
@@ -2481,7 +2479,7 @@ static void scsi_disk_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
-static TypeInfo scsi_disk_info = {
+static const TypeInfo scsi_disk_info = {
     .name          = "scsi-disk",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDiskState),
diff --git a/hw/scsi-generic.c b/hw/scsi-generic.c
index d904534..8175474 100644
--- a/hw/scsi-generic.c
+++ b/hw/scsi-generic.c
@@ -12,9 +12,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "scsi.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 #ifdef __linux__
 
@@ -499,7 +499,7 @@ static void scsi_generic_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd  = &vmstate_scsi_device;
 }
 
-static TypeInfo scsi_generic_info = {
+static const TypeInfo scsi_generic_info = {
     .name          = "scsi-generic",
     .parent        = TYPE_SCSI_DEVICE,
     .instance_size = sizeof(SCSIDevice),
diff --git a/hw/scsi.h b/hw/scsi.h
index b8f7357..a5b5b2e 100644
--- a/hw/scsi.h
+++ b/hw/scsi.h
@@ -2,9 +2,9 @@
 #define QEMU_HW_SCSI_H
 
 #include "qdev.h"
-#include "block.h"
+#include "block/block.h"
 #include "hw/block-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define MAX_SCSI_DEVS	255
 
diff --git a/hw/sd.c b/hw/sd.c
index 5928966..235963b 100644
--- a/hw/sd.c
+++ b/hw/sd.c
@@ -31,9 +31,9 @@
  */
 
 #include "hw.h"
-#include "block.h"
+#include "block/block.h"
 #include "sd.h"
-#include "bitmap.h"
+#include "qemu/bitmap.h"
 
 //#define DEBUG_SD 1
 
diff --git a/hw/serial-isa.c b/hw/serial-isa.c
index 96c78f7..5a6f51f 100644
--- a/hw/serial-isa.c
+++ b/hw/serial-isa.c
@@ -99,7 +99,7 @@ static void serial_isa_class_initfn(ObjectClass *klass, void *data)
     dc->props = serial_isa_properties;
 }
 
-static TypeInfo serial_isa_info = {
+static const TypeInfo serial_isa_info = {
     .name          = "isa-serial",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISASerialState),
diff --git a/hw/serial-pci.c b/hw/serial-pci.c
index 95dc5c8..1c31353 100644
--- a/hw/serial-pci.c
+++ b/hw/serial-pci.c
@@ -26,7 +26,7 @@
 /* see docs/specs/pci-serial.txt */
 
 #include "serial.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 #define PCI_SERIAL_MAX_PORTS 4
 
@@ -185,8 +185,8 @@ static void serial_pci_class_initfn(ObjectClass *klass, void *data)
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
     pc->init = serial_pci_init;
     pc->exit = serial_pci_exit;
-    pc->vendor_id = 0x1b36; /* Red Hat */
-    pc->device_id = 0x0002;
+    pc->vendor_id = PCI_VENDOR_ID_REDHAT;
+    pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL;
     pc->revision = 1;
     pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
     dc->vmsd = &vmstate_pci_serial;
@@ -199,8 +199,8 @@ static void multi_2x_serial_pci_class_initfn(ObjectClass *klass, void *data)
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
     pc->init = multi_serial_pci_init;
     pc->exit = multi_serial_pci_exit;
-    pc->vendor_id = 0x1b36; /* Red Hat */
-    pc->device_id = 0x0003;
+    pc->vendor_id = PCI_VENDOR_ID_REDHAT;
+    pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL2;
     pc->revision = 1;
     pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
     dc->vmsd = &vmstate_pci_multi_serial;
@@ -213,29 +213,29 @@ static void multi_4x_serial_pci_class_initfn(ObjectClass *klass, void *data)
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
     pc->init = multi_serial_pci_init;
     pc->exit = multi_serial_pci_exit;
-    pc->vendor_id = 0x1b36; /* Red Hat */
-    pc->device_id = 0x0004;
+    pc->vendor_id = PCI_VENDOR_ID_REDHAT;
+    pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL4;
     pc->revision = 1;
     pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
     dc->vmsd = &vmstate_pci_multi_serial;
     dc->props = multi_4x_serial_pci_properties;
 }
 
-static TypeInfo serial_pci_info = {
+static const TypeInfo serial_pci_info = {
     .name          = "pci-serial",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCISerialState),
     .class_init    = serial_pci_class_initfn,
 };
 
-static TypeInfo multi_2x_serial_pci_info = {
+static const TypeInfo multi_2x_serial_pci_info = {
     .name          = "pci-serial-2x",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIMultiSerialState),
     .class_init    = multi_2x_serial_pci_class_initfn,
 };
 
-static TypeInfo multi_4x_serial_pci_info = {
+static const TypeInfo multi_4x_serial_pci_info = {
     .name          = "pci-serial-4x",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIMultiSerialState),
diff --git a/hw/serial.c b/hw/serial.c
index 8f95d08..d2c38b2 100644
--- a/hw/serial.c
+++ b/hw/serial.c
@@ -24,9 +24,9 @@
  */
 
 #include "serial.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
-#include "exec-memory.h"
+#include "char/char.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_SERIAL
 
@@ -266,8 +266,6 @@ static void serial_xmit(void *opaque)
             s->tsr = fifo_get(s,XMIT_FIFO);
             if (!s->xmit_fifo.count)
                 s->lsr |= UART_LSR_THRE;
-        } else if ((s->lsr & UART_LSR_THRE)) {
-            return;
         } else {
             s->tsr = s->thr;
             s->lsr |= UART_LSR_THRE;
@@ -279,7 +277,7 @@ static void serial_xmit(void *opaque)
         /* in loopback mode, say that we just received a char */
         serial_receive1(s, &s->tsr, 1);
     } else if (qemu_chr_fe_write(s->chr, &s->tsr, 1) != 1) {
-        if ((s->tsr_retry >= 0) && (s->tsr_retry <= MAX_XMIT_RETRY)) {
+        if ((s->tsr_retry > 0) && (s->tsr_retry <= MAX_XMIT_RETRY)) {
             s->tsr_retry++;
             qemu_mod_timer(s->transmit_timer,  new_xmit_ts + s->char_transmit_time);
             return;
@@ -737,7 +735,7 @@ const MemoryRegionOps serial_io_ops = {
 };
 
 SerialState *serial_init(int base, qemu_irq irq, int baudbase,
-                         CharDriverState *chr)
+                         CharDriverState *chr, MemoryRegion *system_io)
 {
     SerialState *s;
 
@@ -751,7 +749,7 @@ SerialState *serial_init(int base, qemu_irq irq, int baudbase,
     vmstate_register(NULL, base, &vmstate_serial, s);
 
     memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
-    memory_region_add_subregion(get_system_io(), base, &s->io);
+    memory_region_add_subregion(system_io, base, &s->io);
 
     return s;
 }
diff --git a/hw/serial.h b/hw/serial.h
index 5261ec5..5a7776a 100644
--- a/hw/serial.h
+++ b/hw/serial.h
@@ -22,10 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_SERIAL_H
+#define HW_SERIAL_H 1
 
 #include "hw.h"
-#include "sysemu.h"
-#include "memory.h"
+#include "sysemu/sysemu.h"
+#include "exec/memory.h"
 
 #define UART_FIFO_LENGTH    16      /* 16550A Fifo Length */
 
@@ -95,7 +97,7 @@ unsigned serial_tx_fifo_count(SerialState *s);
 
 /* legacy pre qom */
 SerialState *serial_init(int base, qemu_irq irq, int baudbase,
-                         CharDriverState *chr);
+                         CharDriverState *chr, MemoryRegion *system_io);
 SerialState *serial_mm_init(MemoryRegion *address_space,
                             hwaddr base, int it_shift,
                             qemu_irq irq, int baudbase,
@@ -103,3 +105,5 @@ SerialState *serial_mm_init(MemoryRegion *address_space,
 
 /* serial-isa.c */
 bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr);
+
+#endif
diff --git a/hw/sga.c b/hw/sga.c
index a666349..29bc3e0 100644
--- a/hw/sga.c
+++ b/hw/sga.c
@@ -24,10 +24,10 @@
  * sgabios code originally available at code.google.com/p/sgabios
  *
  */
-#include "pci.h"
+#include "pci/pci.h"
 #include "pc.h"
 #include "loader.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define SGABIOS_FILENAME "sgabios.bin"
 
@@ -48,7 +48,7 @@ static void sga_class_initfn(ObjectClass *klass, void *data)
     dc->desc = "Serial Graphics Adapter";
 }
 
-static TypeInfo sga_info = {
+static const TypeInfo sga_info = {
     .name          = "sga",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISASGAState),
diff --git a/hw/sh7750.c b/hw/sh7750.c
index 8bcf0df..666f865 100644
--- a/hw/sh7750.c
+++ b/hw/sh7750.c
@@ -25,12 +25,12 @@
 #include <stdio.h>
 #include "hw.h"
 #include "sh.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sh7750_regs.h"
 #include "sh7750_regnames.h"
 #include "sh_intc.h"
 #include "cpu.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define NB_DEVICES 4
 
diff --git a/hw/sh_intc.h b/hw/sh_intc.h
index 80c9430..6f11bee 100644
--- a/hw/sh_intc.h
+++ b/hw/sh_intc.h
@@ -3,7 +3,7 @@
 
 #include "qemu-common.h"
 #include "irq.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 typedef unsigned char intc_enum;
 
diff --git a/hw/sh_pci.c b/hw/sh_pci.c
index fdec71b..077d957 100644
--- a/hw/sh_pci.c
+++ b/hw/sh_pci.c
@@ -23,10 +23,10 @@
  */
 #include "sysbus.h"
 #include "sh.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "bswap.h"
-#include "exec-memory.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "qemu/bswap.h"
+#include "exec/address-spaces.h"
 
 typedef struct SHPCIState {
     SysBusDevice busdev;
@@ -156,7 +156,7 @@ static void sh_pci_host_class_init(ObjectClass *klass, void *data)
     k->device_id = PCI_DEVICE_ID_HITACHI_SH7751R;
 }
 
-static TypeInfo sh_pci_host_info = {
+static const TypeInfo sh_pci_host_info = {
     .name          = "sh_pci_host",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIDevice),
@@ -170,7 +170,7 @@ static void sh_pci_device_class_init(ObjectClass *klass, void *data)
     sdc->init = sh_pci_device_init;
 }
 
-static TypeInfo sh_pci_device_info = {
+static const TypeInfo sh_pci_device_info = {
     .name          = "sh_pci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SHPCIState),
diff --git a/hw/sh_serial.c b/hw/sh_serial.c
index 9da5d08..21c5b13 100644
--- a/hw/sh_serial.c
+++ b/hw/sh_serial.c
@@ -26,8 +26,8 @@
  */
 #include "hw.h"
 #include "sh.h"
-#include "qemu-char.h"
-#include "exec-memory.h"
+#include "char/char.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_SERIAL
 
diff --git a/hw/sh_timer.c b/hw/sh_timer.c
index c0365b1..64ea23f 100644
--- a/hw/sh_timer.c
+++ b/hw/sh_timer.c
@@ -10,8 +10,8 @@
 
 #include "hw.h"
 #include "sh.h"
-#include "qemu-timer.h"
-#include "exec-memory.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
 #include "ptimer.h"
 
 //#define DEBUG_TIMER
diff --git a/hw/shix.c b/hw/shix.c
index b56dd54..6f2d55a 100644
--- a/hw/shix.c
+++ b/hw/shix.c
@@ -29,10 +29,10 @@
 */
 #include "hw.h"
 #include "sh.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define BIOS_FILENAME "shix_bios.bin"
 #define BIOS_ADDRESS 0xA0000000
@@ -92,6 +92,7 @@ static QEMUMachine shix_machine = {
     .desc = "shix card",
     .init = shix_init,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void shix_machine_init(void)
diff --git a/hw/slavio_intctl.c b/hw/slavio_intctl.c
index 6aafa8b..136ceeb 100644
--- a/hw/slavio_intctl.c
+++ b/hw/slavio_intctl.c
@@ -23,7 +23,7 @@
  */
 
 #include "sun4m.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
 #include "trace.h"
 
@@ -210,7 +210,7 @@ void slavio_pic_info(Monitor *mon, DeviceState *dev)
     SLAVIO_INTCTLState *s;
     int i;
 
-    sd = sysbus_from_qdev(dev);
+    sd = SYS_BUS_DEVICE(dev);
     s = FROM_SYSBUS(SLAVIO_INTCTLState, sd);
     for (i = 0; i < MAX_CPUS; i++) {
         monitor_printf(mon, "per-cpu %d: pending 0x%08x\n", i,
@@ -230,7 +230,7 @@ void slavio_irq_info(Monitor *mon, DeviceState *dev)
     int i;
     int64_t count;
 
-    sd = sysbus_from_qdev(dev);
+    sd = SYS_BUS_DEVICE(dev);
     s = FROM_SYSBUS(SLAVIO_INTCTLState, sd);
     monitor_printf(mon, "IRQ statistics:\n");
     for (i = 0; i < 32; i++) {
@@ -456,7 +456,7 @@ static void slavio_intctl_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_intctl;
 }
 
-static TypeInfo slavio_intctl_info = {
+static const TypeInfo slavio_intctl_info = {
     .name          = "slavio_intctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLAVIO_INTCTLState),
diff --git a/hw/slavio_misc.c b/hw/slavio_misc.c
index 682fb45..af24cc1 100644
--- a/hw/slavio_misc.c
+++ b/hw/slavio_misc.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "sysbus.h"
 #include "trace.h"
 
@@ -478,7 +478,7 @@ static void slavio_misc_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_misc;
 }
 
-static TypeInfo slavio_misc_info = {
+static const TypeInfo slavio_misc_info = {
     .name          = "slavio_misc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MiscState),
@@ -492,7 +492,7 @@ static void apc_class_init(ObjectClass *klass, void *data)
     k->init = apc_init1;
 }
 
-static TypeInfo apc_info = {
+static const TypeInfo apc_info = {
     .name          = "apc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MiscState),
diff --git a/hw/slavio_timer.c b/hw/slavio_timer.c
index c07ceb1..68a4c0c 100644
--- a/hw/slavio_timer.c
+++ b/hw/slavio_timer.c
@@ -23,7 +23,7 @@
  */
 
 #include "sun4m.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
 #include "sysbus.h"
 #include "trace.h"
@@ -420,7 +420,7 @@ static void slavio_timer_class_init(ObjectClass *klass, void *data)
     dc->props = slavio_timer_properties;
 }
 
-static TypeInfo slavio_timer_info = {
+static const TypeInfo slavio_timer_info = {
     .name          = "slavio_timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLAVIO_TIMERState),
diff --git a/hw/sm501.c b/hw/sm501.c
index 50324cd..b7ac7f9 100644
--- a/hw/sm501.c
+++ b/hw/sm501.c
@@ -25,11 +25,12 @@
 #include <stdio.h>
 #include "hw.h"
 #include "serial.h"
-#include "console.h"
+#include "ui/console.h"
 #include "devices.h"
 #include "sysbus.h"
 #include "qdev-addr.h"
-#include "range.h"
+#include "qemu/range.h"
+#include "ui/pixel_ops.h"
 
 /*
  * Status: 2010/05/07
@@ -1163,8 +1164,6 @@ static const MemoryRegionOps sm501_2d_engine_ops = {
 
 /* draw line functions for all console modes */
 
-#include "pixel_ops.h"
-
 typedef void draw_line_func(uint8_t *d, const uint8_t *s,
 			    int width, const uint32_t *pal);
 
@@ -1429,9 +1428,9 @@ void sm501_init(MemoryRegion *address_space_mem, uint32_t base,
     qdev_prop_set_uint32(dev, "num-ports", 2);
     qdev_prop_set_taddr(dev, "dma-offset", base);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0,
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0,
                     base + MMIO_BASE_OFFSET + SM501_USB_HOST);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 
     /* bridge to serial emulation module */
     if (chr) {
diff --git a/hw/smbios.c b/hw/smbios.c
index c57237d..a7b8bfc 100644
--- a/hw/smbios.c
+++ b/hw/smbios.c
@@ -13,7 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "smbios.h"
 #include "loader.h"
 
diff --git a/hw/smbus.c b/hw/smbus.c
index e3cf6a2..a908591 100644
--- a/hw/smbus.c
+++ b/hw/smbus.c
@@ -318,7 +318,7 @@ static void smbus_device_class_init(ObjectClass *klass, void *data)
     sc->send = smbus_i2c_send;
 }
 
-static TypeInfo smbus_device_type_info = {
+static const TypeInfo smbus_device_type_info = {
     .name = TYPE_SMBUS_DEVICE,
     .parent = TYPE_I2C_SLAVE,
     .instance_size = sizeof(SMBusDevice),
diff --git a/hw/smbus_eeprom.c b/hw/smbus_eeprom.c
index 11adab0..d36dc7b 100644
--- a/hw/smbus_eeprom.c
+++ b/hw/smbus_eeprom.c
@@ -123,7 +123,7 @@ static void smbus_eeprom_class_initfn(ObjectClass *klass, void *data)
     dc->props = smbus_eeprom_properties;
 }
 
-static TypeInfo smbus_eeprom_info = {
+static const TypeInfo smbus_eeprom_info = {
     .name          = "smbus-eeprom",
     .parent        = TYPE_SMBUS_DEVICE,
     .instance_size = sizeof(SMBusEEPROMDevice),
diff --git a/hw/smbus_ich9.c b/hw/smbus_ich9.c
index 6940583..16db3a7 100644
--- a/hw/smbus_ich9.c
+++ b/hw/smbus_ich9.c
@@ -2,6 +2,11 @@
  * ACPI implementation
  *
  * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on acpi.c, but heavily rewritten.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -14,19 +19,16 @@
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-/*
- *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
- *                     VA Linux Systems Japan K.K.
- *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
  *
- *  This is based on acpi.c, but heavily rewritten.
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
  */
 #include "hw.h"
 #include "pc.h"
 #include "pm_smbus.h"
-#include "pci.h"
-#include "sysemu.h"
+#include "pci/pci.h"
+#include "sysemu/sysemu.h"
 #include "i2c.h"
 #include "smbus.h"
 
@@ -40,7 +42,6 @@ typedef struct ICH9SMBState {
     PCIDevice dev;
 
     PMSMBus smb;
-    MemoryRegion mem_bar;
 } ICH9SMBState;
 
 static const VMStateDescription vmstate_ich9_smbus = {
@@ -54,42 +55,23 @@ static const VMStateDescription vmstate_ich9_smbus = {
     }
 };
 
-static void ich9_smb_ioport_writeb(void *opaque, hwaddr addr,
-                                   uint64_t val, unsigned size)
-{
-    ICH9SMBState *s = opaque;
-    uint8_t hostc = s->dev.config[ICH9_SMB_HOSTC];
-
-    if ((hostc & ICH9_SMB_HOSTC_HST_EN) && !(hostc & ICH9_SMB_HOSTC_I2C_EN)) {
-        uint64_t offset = addr - s->dev.io_regions[ICH9_SMB_SMB_BASE_BAR].addr;
-        smb_ioport_writeb(&s->smb, offset, val);
-    }
-}
-
-static uint64_t ich9_smb_ioport_readb(void *opaque, hwaddr addr,
-                                      unsigned size)
+static void ich9_smbus_write_config(PCIDevice *d, uint32_t address,
+                                    uint32_t val, int len)
 {
-    ICH9SMBState *s = opaque;
-    uint8_t hostc = s->dev.config[ICH9_SMB_HOSTC];
+    ICH9SMBState *s = ICH9_SMB_DEVICE(d);
 
-    if ((hostc & ICH9_SMB_HOSTC_HST_EN) && !(hostc & ICH9_SMB_HOSTC_I2C_EN)) {
-        uint64_t offset = addr - s->dev.io_regions[ICH9_SMB_SMB_BASE_BAR].addr;
-        return smb_ioport_readb(&s->smb, offset);
+    pci_default_write_config(d, address, val, len);
+    if (range_covers_byte(address, len, ICH9_SMB_HOSTC)) {
+        uint8_t hostc = s->dev.config[ICH9_SMB_HOSTC];
+        if ((hostc & ICH9_SMB_HOSTC_HST_EN) &&
+            !(hostc & ICH9_SMB_HOSTC_I2C_EN)) {
+            memory_region_set_enabled(&s->smb.io, true);
+        } else {
+            memory_region_set_enabled(&s->smb.io, false);
+        }
     }
-
-    return 0xff;
 }
 
-static const MemoryRegionOps lpc_smb_mmio_ops = {
-    .read = ich9_smb_ioport_readb,
-    .write = ich9_smb_ioport_writeb,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-    .impl = {
-        .min_access_size = 1,
-        .max_access_size = 1,
-    },
-};
-
 static int ich9_smbus_initfn(PCIDevice *d)
 {
     ICH9SMBState *s = ICH9_SMB_DEVICE(d);
@@ -98,26 +80,11 @@ static int ich9_smbus_initfn(PCIDevice *d)
     pci_config_set_interrupt_pin(d->config, 0x01); /* interrupt pin 1 */
 
     pci_set_byte(d->config + ICH9_SMB_HOSTC, 0);
-
-    /*
-     * update parameters based on
-     * paralell_hds[0]
-     * serial_hds[0]
-     * serial_hds[0]
-     * fdc
-     *
-     * Is there any OS that depends on them?
-     */
-
-    /* TODO smb_io_base */
-    pci_set_byte(d->config + ICH9_SMB_HOSTC, 0);
     /* TODO bar0, bar1: 64bit BAR support*/
 
-    memory_region_init_io(&s->mem_bar, &lpc_smb_mmio_ops, s, "ich9-smbus-bar",
-                            ICH9_SMB_SMB_BASE_SIZE);
-    pci_register_bar(d, ICH9_SMB_SMB_BASE_BAR, PCI_BASE_ADDRESS_SPACE_IO,
-                        &s->mem_bar);
     pm_smbus_init(&d->qdev, &s->smb);
+    pci_register_bar(d, ICH9_SMB_SMB_BASE_BAR, PCI_BASE_ADDRESS_SPACE_IO,
+                     &s->smb.io);
     return 0;
 }
 
@@ -134,6 +101,7 @@ static void ich9_smb_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_ich9_smbus;
     dc->desc = "ICH9 SMBUS Bridge";
     k->init = ich9_smbus_initfn;
+    k->config_write = ich9_smbus_write_config;
 }
 
 i2c_bus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base)
diff --git a/hw/smc91c111.c b/hw/smc91c111.c
index 4ceed01..67fd074 100644
--- a/hw/smc91c111.c
+++ b/hw/smc91c111.c
@@ -8,7 +8,7 @@
  */
 
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include "devices.h"
 /* For crc32 */
 #include <zlib.h>
@@ -237,7 +237,7 @@ static void smc91c111_do_tx(smc91c111_state *s)
             smc91c111_release_packet(s, packetnum);
         else if (s->tx_fifo_done_len < NUM_PACKETS)
             s->tx_fifo_done[s->tx_fifo_done_len++] = packetnum;
-        qemu_send_packet(&s->nic->nc, p, len);
+        qemu_send_packet(qemu_get_queue(s->nic), p, len);
     }
     s->tx_fifo_len = 0;
     smc91c111_update(s);
@@ -254,7 +254,7 @@ static void smc91c111_queue_tx(smc91c111_state *s, int packet)
 
 static void smc91c111_reset(DeviceState *dev)
 {
-    smc91c111_state *s = FROM_SYSBUS(smc91c111_state, sysbus_from_qdev(dev));
+    smc91c111_state *s = FROM_SYSBUS(smc91c111_state, SYS_BUS_DEVICE(dev));
     s->bank = 0;
     s->tx_fifo_len = 0;
     s->tx_fifo_done_len = 0;
@@ -442,6 +442,7 @@ static void smc91c111_writeb(void *opaque, hwaddr offset,
             return;
         case 12: /* Early receive.  */
             s->ercv = value & 0x1f;
+            return;
         case 13:
             /* Ignore.  */
             return;
@@ -630,7 +631,7 @@ static uint32_t smc91c111_readl(void *opaque, hwaddr offset)
 
 static int smc91c111_can_receive(NetClientState *nc)
 {
-    smc91c111_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    smc91c111_state *s = qemu_get_nic_opaque(nc);
 
     if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST))
         return 1;
@@ -641,7 +642,7 @@ static int smc91c111_can_receive(NetClientState *nc)
 
 static ssize_t smc91c111_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    smc91c111_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    smc91c111_state *s = qemu_get_nic_opaque(nc);
     int status;
     int packetsize;
     uint32_t crc;
@@ -730,7 +731,7 @@ static const MemoryRegionOps smc91c111_mem_ops = {
 
 static void smc91c111_cleanup(NetClientState *nc)
 {
-    smc91c111_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    smc91c111_state *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -753,7 +754,7 @@ static int smc91c111_init1(SysBusDevice *dev)
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_smc91c111_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
     /* ??? Save/restore.  */
     return 0;
 }
@@ -774,7 +775,7 @@ static void smc91c111_class_init(ObjectClass *klass, void *data)
     dc->props = smc91c111_properties;
 }
 
-static TypeInfo smc91c111_info = {
+static const TypeInfo smc91c111_info = {
     .name          = "smc91c111",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(smc91c111_state),
@@ -797,7 +798,7 @@ void smc91c111_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     dev = qdev_create(NULL, "smc91c111");
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, base);
     sysbus_connect_irq(s, 0, irq);
 }
diff --git a/hw/soc_dma.c b/hw/soc_dma.c
index 100e6ac..b5fc26c 100644
--- a/hw/soc_dma.c
+++ b/hw/soc_dma.c
@@ -18,7 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "soc_dma.h"
 
 static void transfer_mem2mem(struct soc_dma_ch_s *ch)
diff --git a/hw/soc_dma.h b/hw/soc_dma.h
index b84441b..94f1055 100644
--- a/hw/soc_dma.h
+++ b/hw/soc_dma.h
@@ -18,7 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "memory.h"
+#ifndef HW_SOC_DMA_H
+#define HW_SOC_DMA_H 1
+
+
+#include "exec/memory.h"
 #include "hw/irq.h"
 
 #define DMA_MAX_DRQ 96
@@ -110,3 +114,5 @@ static inline void soc_dma_port_add_fifo_out(struct soc_dma_s *dma,
 {
     return soc_dma_port_add_fifo(dma, virt_base, fn, opaque, 1);
 }
+
+#endif
diff --git a/hw/spapr.c b/hw/spapr.c
index ad3f0ea..e88a27a 100644
--- a/hw/spapr.c
+++ b/hw/spapr.c
@@ -24,13 +24,13 @@
  * THE SOFTWARE.
  *
  */
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "hw.h"
 #include "elf.h"
-#include "net.h"
-#include "blockdev.h"
-#include "cpus.h"
-#include "kvm.h"
+#include "net/net.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/cpus.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 #include "hw/boards.h"
@@ -41,14 +41,15 @@
 #include "hw/spapr_vio.h"
 #include "hw/spapr_pci.h"
 #include "hw/xics.h"
-#include "hw/msi.h"
+#include "hw/pci/msi.h"
 
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
-#include "pci.h"
+#include "pci/pci.h"
 
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include "hw/usb.h"
+#include "qemu/config-file.h"
 
 #include <libfdt.h>
 
@@ -76,12 +77,6 @@
 #define MAX_CPUS                256
 #define XICS_IRQS               1024
 
-#define SPAPR_PCI_BUID          0x800000020000001ULL
-#define SPAPR_PCI_MEM_WIN_ADDR  (0x10000000000ULL + 0xA0000000)
-#define SPAPR_PCI_MEM_WIN_SIZE  0x20000000
-#define SPAPR_PCI_IO_WIN_ADDR   (0x10000000000ULL + 0x80000000)
-#define SPAPR_PCI_MSI_WIN_ADDR  (0x10000000000ULL + 0x90000000)
-
 #define PHANDLE_XICP            0x00001111
 
 #define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
@@ -139,6 +134,7 @@ static int spapr_fixup_cpu_dt(void *fdt, sPAPREnvironment *spapr)
 {
     int ret = 0, offset;
     CPUPPCState *env;
+    CPUState *cpu;
     char cpu_model[32];
     int smt = kvmppc_smt_threads();
     uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
@@ -146,19 +142,20 @@ static int spapr_fixup_cpu_dt(void *fdt, sPAPREnvironment *spapr)
     assert(spapr->cpu_model);
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = CPU(ppc_env_get_cpu(env));
         uint32_t associativity[] = {cpu_to_be32(0x5),
                                     cpu_to_be32(0x0),
                                     cpu_to_be32(0x0),
                                     cpu_to_be32(0x0),
-                                    cpu_to_be32(env->numa_node),
-                                    cpu_to_be32(env->cpu_index)};
+                                    cpu_to_be32(cpu->numa_node),
+                                    cpu_to_be32(cpu->cpu_index)};
 
-        if ((env->cpu_index % smt) != 0) {
+        if ((cpu->cpu_index % smt) != 0) {
             continue;
         }
 
         snprintf(cpu_model, 32, "/cpus/%s@%x", spapr->cpu_model,
-                 env->cpu_index);
+                 cpu->cpu_index);
 
         offset = fdt_path_offset(fdt, cpu_model);
         if (offset < 0) {
@@ -284,7 +281,9 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
 
         _FDT((fdt_property(fdt, "qemu,boot-kernel", &kprop, sizeof(kprop))));
     }
-    _FDT((fdt_property_string(fdt, "qemu,boot-device", boot_device)));
+    if (boot_device) {
+        _FDT((fdt_property_string(fdt, "qemu,boot-device", boot_device)));
+    }
     _FDT((fdt_property_cell(fdt, "qemu,graphic-width", graphic_width)));
     _FDT((fdt_property_cell(fdt, "qemu,graphic-height", graphic_height)));
     _FDT((fdt_property_cell(fdt, "qemu,graphic-depth", graphic_depth)));
@@ -307,7 +306,8 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
     spapr->cpu_model = g_strdup(modelname);
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        int index = env->cpu_index;
+        CPUState *cpu = CPU(ppc_env_get_cpu(env));
+        int index = cpu->cpu_index;
         uint32_t servers_prop[smp_threads];
         uint32_t gservers_prop[smp_threads * 2];
         char *nodename;
@@ -322,14 +322,11 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
             continue;
         }
 
-        if (asprintf(&nodename, "%s@%x", modelname, index) < 0) {
-            fprintf(stderr, "Allocation failure\n");
-            exit(1);
-        }
+        nodename = g_strdup_printf("%s@%x", modelname, index);
 
         _FDT((fdt_begin_node(fdt, nodename)));
 
-        free(nodename);
+        g_free(nodename);
 
         _FDT((fdt_property_cell(fdt, "reg", index)));
         _FDT((fdt_property_string(fdt, "device_type", "cpu")));
@@ -657,6 +654,36 @@ static void spapr_cpu_reset(void *opaque)
         (spapr->htab_shift - 18);
 }
 
+static void spapr_create_nvram(sPAPREnvironment *spapr)
+{
+    QemuOpts *machine_opts;
+    DeviceState *dev;
+
+    dev = qdev_create(&spapr->vio_bus->bus, "spapr-nvram");
+
+    machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (machine_opts) {
+        const char *drivename;
+
+        drivename = qemu_opt_get(machine_opts, "nvram");
+        if (drivename) {
+            BlockDriverState *bs;
+
+            bs = bdrv_find(drivename);
+            if (!bs) {
+                fprintf(stderr, "No such block device \"%s\" for nvram\n",
+                        drivename);
+                exit(1);
+            }
+            qdev_prop_set_drive_nofail(dev, "drive", bs);
+        }
+    }
+
+    qdev_init_nofail(dev);
+
+    spapr->nvram = (struct sPAPRNVRAM *)dev;
+}
+
 /* Returns whether we want to use VGA or not */
 static int spapr_vga_init(PCIBus *pci_bus)
 {
@@ -767,7 +794,7 @@ static void ppc_spapr_init(QEMUMachineInitArgs *args)
 
         /* Tell KVM that we're in PAPR mode */
         if (kvm_enabled()) {
-            kvmppc_set_papr(env);
+            kvmppc_set_papr(cpu);
         }
 
         qemu_register_reset(spapr_cpu_reset, cpu);
@@ -801,7 +828,7 @@ static void ppc_spapr_init(QEMUMachineInitArgs *args)
 
     /* Set up Interrupt Controller */
     spapr->icp = xics_system_init(XICS_IRQS);
-    spapr->next_irq = 16;
+    spapr->next_irq = XICS_IRQ_BASE;
 
     /* Set up EPOW events infrastructure */
     spapr_events_init(spapr);
@@ -818,15 +845,13 @@ static void ppc_spapr_init(QEMUMachineInitArgs *args)
         }
     }
 
+    /* We always have at least the nvram device on VIO */
+    spapr_create_nvram(spapr);
+
     /* Set up PCI */
     spapr_pci_rtas_init();
 
-    spapr_create_phb(spapr, "pci", SPAPR_PCI_BUID,
-                     SPAPR_PCI_MEM_WIN_ADDR,
-                     SPAPR_PCI_MEM_WIN_SIZE,
-                     SPAPR_PCI_IO_WIN_ADDR,
-                     SPAPR_PCI_MSI_WIN_ADDR);
-    phb = PCI_HOST_BRIDGE(QLIST_FIRST(&spapr->phbs));
+    phb = spapr_create_phb(spapr, 0, "pci");
 
     for (i = 0; i < nb_nics; i++) {
         NICInfo *nd = &nd_table[i];
@@ -924,9 +949,10 @@ static QEMUMachine spapr_machine = {
     .desc = "pSeries Logical Partition (PAPR compliant)",
     .init = ppc_spapr_init,
     .reset = ppc_spapr_reset,
+    .block_default_type = IF_SCSI,
     .max_cpus = MAX_CPUS,
     .no_parallel = 1,
-    .use_scsi = 1,
+    .boot_order = NULL,
 };
 
 static void spapr_machine_init(void)
diff --git a/hw/spapr.h b/hw/spapr.h
index efe7f57..3a1f69f 100644
--- a/hw/spapr.h
+++ b/hw/spapr.h
@@ -1,16 +1,18 @@
 #if !defined(__HW_SPAPR_H__)
 #define __HW_SPAPR_H__
 
-#include "dma.h"
+#include "sysemu/dma.h"
 #include "hw/xics.h"
 
 struct VIOsPAPRBus;
 struct sPAPRPHBState;
+struct sPAPRNVRAM;
 struct icp_state;
 
 typedef struct sPAPREnvironment {
     struct VIOsPAPRBus *vio_bus;
     QLIST_HEAD(, sPAPRPHBState) phbs;
+    struct sPAPRNVRAM *nvram;
     struct icp_state *icp;
 
     hwaddr ram_limit;
@@ -320,7 +322,7 @@ static inline void rtas_st(target_ulong phys, int n, uint32_t val)
 typedef void (*spapr_rtas_fn)(sPAPREnvironment *spapr, uint32_t token,
                               uint32_t nargs, target_ulong args,
                               uint32_t nret, target_ulong rets);
-void spapr_rtas_register(const char *name, spapr_rtas_fn fn);
+int spapr_rtas_register(const char *name, spapr_rtas_fn fn);
 target_ulong spapr_rtas_call(sPAPREnvironment *spapr,
                              uint32_t token, uint32_t nargs, target_ulong args,
                              uint32_t nret, target_ulong rets);
diff --git a/hw/spapr_events.c b/hw/spapr_events.c
index 18ccd4a..ce78f09 100644
--- a/hw/spapr_events.c
+++ b/hw/spapr_events.c
@@ -25,10 +25,10 @@
  *
  */
 #include "cpu.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 #include "hw/qdev.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
diff --git a/hw/spapr_hcall.c b/hw/spapr_hcall.c
index 63cadb8..2889742 100644
--- a/hw/spapr_hcall.c
+++ b/hw/spapr_hcall.c
@@ -1,8 +1,6 @@
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "cpu.h"
-#include "qemu-char.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
 #include "helper_regs.h"
 #include "hw/spapr.h"
 
@@ -469,9 +467,11 @@ static target_ulong h_register_vpa(PowerPCCPU *cpu, sPAPREnvironment *spapr,
     target_ulong vpa = args[2];
     target_ulong ret = H_PARAMETER;
     CPUPPCState *tenv;
+    CPUState *tcpu;
 
     for (tenv = first_cpu; tenv; tenv = tenv->next_cpu) {
-        if (tenv->cpu_index == procno) {
+        tcpu = CPU(ppc_env_get_cpu(tenv));
+        if (tcpu->cpu_index == procno) {
             break;
         }
     }
diff --git a/hw/spapr_iommu.c b/hw/spapr_iommu.c
index 02d78cc..d8a098c 100644
--- a/hw/spapr_iommu.c
+++ b/hw/spapr_iommu.c
@@ -17,11 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "qdev.h"
 #include "kvm_ppc.h"
-#include "dma.h"
-#include "exec-memory.h"
+#include "sysemu/dma.h"
+#include "exec/address-spaces.h"
 
 #include "hw/spapr.h"
 
@@ -120,6 +120,12 @@ DMAContext *spapr_tce_new_dma_context(uint32_t liobn, size_t window_size)
 {
     sPAPRTCETable *tcet;
 
+    if (spapr_tce_find_by_liobn(liobn)) {
+        fprintf(stderr, "Attempted to create TCE table with duplicate"
+                " LIOBN 0x%x\n", liobn);
+        return NULL;
+    }
+
     if (!window_size) {
         return NULL;
     }
diff --git a/hw/spapr_llan.c b/hw/spapr_llan.c
index 09ad69f..6ef2936 100644
--- a/hw/spapr_llan.c
+++ b/hw/spapr_llan.c
@@ -25,7 +25,7 @@
  *
  */
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "hw/qdev.h"
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
@@ -85,7 +85,7 @@ typedef struct VIOsPAPRVLANDevice {
 
 static int spapr_vlan_can_receive(NetClientState *nc)
 {
-    VIOsPAPRVLANDevice *dev = DO_UPCAST(NICState, nc, nc)->opaque;
+    VIOsPAPRVLANDevice *dev = qemu_get_nic_opaque(nc);
 
     return (dev->isopen && dev->rx_bufs > 0);
 }
@@ -93,7 +93,7 @@ static int spapr_vlan_can_receive(NetClientState *nc)
 static ssize_t spapr_vlan_receive(NetClientState *nc, const uint8_t *buf,
                                   size_t size)
 {
-    VIOsPAPRDevice *sdev = DO_UPCAST(NICState, nc, nc)->opaque;
+    VIOsPAPRDevice *sdev = qemu_get_nic_opaque(nc);
     VIOsPAPRVLANDevice *dev = (VIOsPAPRVLANDevice *)sdev;
     vlan_bd_t rxq_bd = vio_ldq(sdev, dev->buf_list + VLAN_RXQ_BD_OFF);
     vlan_bd_t bd;
@@ -199,7 +199,7 @@ static int spapr_vlan_init(VIOsPAPRDevice *sdev)
 
     dev->nic = qemu_new_nic(&net_spapr_vlan_info, &dev->nicconf,
                             object_get_typename(OBJECT(sdev)), sdev->qdev.id, dev);
-    qemu_format_nic_info_str(&dev->nic->nc, dev->nicconf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(dev->nic), dev->nicconf.macaddr.a);
 
     return 0;
 }
@@ -462,7 +462,7 @@ static target_ulong h_send_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
         p += VLAN_BD_LEN(bufs[i]);
     }
 
-    qemu_send_packet(&dev->nic->nc, lbuf, total_len);
+    qemu_send_packet(qemu_get_queue(dev->nic), lbuf, total_len);
 
     return H_SUCCESS;
 }
@@ -502,7 +502,7 @@ static void spapr_vlan_class_init(ObjectClass *klass, void *data)
     k->rtce_window_size = 0x10000000;
 }
 
-static TypeInfo spapr_vlan_info = {
+static const TypeInfo spapr_vlan_info = {
     .name          = "spapr-vlan",
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VIOsPAPRVLANDevice),
diff --git a/hw/spapr_nvram.c b/hw/spapr_nvram.c
new file mode 100644
index 0000000..680cdba
--- /dev/null
+++ b/hw/spapr_nvram.c
@@ -0,0 +1,196 @@
+/*
+ * QEMU sPAPR NVRAM emulation
+ *
+ * Copyright (C) 2012 David Gibson, IBM Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <libfdt.h>
+
+#include "sysemu/device_tree.h"
+#include "hw/sysbus.h"
+#include "hw/spapr.h"
+#include "hw/spapr_vio.h"
+
+typedef struct sPAPRNVRAM {
+    VIOsPAPRDevice sdev;
+    uint32_t size;
+    uint8_t *buf;
+    BlockDriverState *drive;
+} sPAPRNVRAM;
+
+#define MIN_NVRAM_SIZE 8192
+#define DEFAULT_NVRAM_SIZE 65536
+#define MAX_NVRAM_SIZE (UINT16_MAX * 16)
+
+static void rtas_nvram_fetch(sPAPREnvironment *spapr,
+                             uint32_t token, uint32_t nargs,
+                             target_ulong args,
+                             uint32_t nret, target_ulong rets)
+{
+    sPAPRNVRAM *nvram = spapr->nvram;
+    hwaddr offset, buffer, len;
+    int alen;
+    void *membuf;
+
+    if ((nargs != 3) || (nret != 2)) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    if (!nvram) {
+        rtas_st(rets, 0, -1);
+        rtas_st(rets, 1, 0);
+        return;
+    }
+
+    offset = rtas_ld(args, 0);
+    buffer = rtas_ld(args, 1);
+    len = rtas_ld(args, 2);
+
+    if (((offset + len) < offset)
+        || ((offset + len) > nvram->size)) {
+        rtas_st(rets, 0, -3);
+        rtas_st(rets, 1, 0);
+        return;
+    }
+
+    membuf = cpu_physical_memory_map(buffer, &len, 1);
+    if (nvram->drive) {
+        alen = bdrv_pread(nvram->drive, offset, membuf, len);
+    } else {
+        assert(nvram->buf);
+
+        memcpy(membuf, nvram->buf + offset, len);
+        alen = len;
+    }
+    cpu_physical_memory_unmap(membuf, len, 1, len);
+
+    rtas_st(rets, 0, (alen < len) ? -1 : 0);
+    rtas_st(rets, 1, (alen < 0) ? 0 : alen);
+}
+
+static void rtas_nvram_store(sPAPREnvironment *spapr,
+                             uint32_t token, uint32_t nargs,
+                             target_ulong args,
+                             uint32_t nret, target_ulong rets)
+{
+    sPAPRNVRAM *nvram = spapr->nvram;
+    hwaddr offset, buffer, len;
+    int alen;
+    void *membuf;
+
+    if ((nargs != 3) || (nret != 2)) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    if (!nvram) {
+        rtas_st(rets, 0, -1);
+        return;
+    }
+
+    offset = rtas_ld(args, 0);
+    buffer = rtas_ld(args, 1);
+    len = rtas_ld(args, 2);
+
+    if (((offset + len) < offset)
+        || ((offset + len) > nvram->size)) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    membuf = cpu_physical_memory_map(buffer, &len, 0);
+    if (nvram->drive) {
+        alen = bdrv_pwrite(nvram->drive, offset, membuf, len);
+    } else {
+        assert(nvram->buf);
+
+        memcpy(nvram->buf + offset, membuf, len);
+        alen = len;
+    }
+    cpu_physical_memory_unmap(membuf, len, 0, len);
+
+    rtas_st(rets, 0, (alen < len) ? -1 : 0);
+    rtas_st(rets, 1, (alen < 0) ? 0 : alen);
+}
+
+static int spapr_nvram_init(VIOsPAPRDevice *dev)
+{
+    sPAPRNVRAM *nvram = (sPAPRNVRAM *)dev;
+
+    if (nvram->drive) {
+        nvram->size = bdrv_getlength(nvram->drive);
+    } else {
+        nvram->size = DEFAULT_NVRAM_SIZE;
+        nvram->buf = g_malloc0(nvram->size);
+    }
+
+    if ((nvram->size < MIN_NVRAM_SIZE) || (nvram->size > MAX_NVRAM_SIZE)) {
+        fprintf(stderr, "spapr-nvram must be between %d and %d bytes in size\n",
+                MIN_NVRAM_SIZE, MAX_NVRAM_SIZE);
+        return -1;
+    }
+
+    spapr_rtas_register("nvram-fetch", rtas_nvram_fetch);
+    spapr_rtas_register("nvram-store", rtas_nvram_store);
+
+    return 0;
+}
+
+static int spapr_nvram_devnode(VIOsPAPRDevice *dev, void *fdt, int node_off)
+{
+    sPAPRNVRAM *nvram = (sPAPRNVRAM *)dev;
+
+    return fdt_setprop_cell(fdt, node_off, "#bytes", nvram->size);
+}
+
+static Property spapr_nvram_properties[] = {
+    DEFINE_SPAPR_PROPERTIES(sPAPRNVRAM, sdev),
+    DEFINE_PROP_DRIVE("drive", sPAPRNVRAM, drive),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void spapr_nvram_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VIOsPAPRDeviceClass *k = VIO_SPAPR_DEVICE_CLASS(klass);
+
+    k->init = spapr_nvram_init;
+    k->devnode = spapr_nvram_devnode;
+    k->dt_name = "nvram";
+    k->dt_type = "nvram";
+    k->dt_compatible = "qemu,spapr-nvram";
+    dc->props = spapr_nvram_properties;
+}
+
+static const TypeInfo spapr_nvram_type_info = {
+    .name          = "spapr-nvram",
+    .parent        = TYPE_VIO_SPAPR_DEVICE,
+    .instance_size = sizeof(sPAPRNVRAM),
+    .class_init    = spapr_nvram_class_init,
+};
+
+static void spapr_nvram_register_types(void)
+{
+    type_register_static(&spapr_nvram_type_info);
+}
+
+type_init(spapr_nvram_register_types)
diff --git a/hw/spapr_pci.c b/hw/spapr_pci.c
index 3c5b855..4eacbcf 100644
--- a/hw/spapr_pci.c
+++ b/hw/spapr_pci.c
@@ -23,17 +23,17 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "pci.h"
-#include "msi.h"
-#include "msix.h"
-#include "pci_host.h"
+#include "pci/pci.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
+#include "pci/pci_host.h"
 #include "hw/spapr.h"
 #include "hw/spapr_pci.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 #include <libfdt.h>
 #include "trace.h"
 
-#include "hw/pci_internals.h"
+#include "hw/pci/pci_bus.h"
 
 /* Copied from the kernel arch/powerpc/platforms/pseries/msi.c */
 #define RTAS_QUERY_FN           0
@@ -435,7 +435,7 @@ static void pci_spapr_set_irq(void *opaque, int irq_num, int level)
      */
     sPAPRPHBState *phb = opaque;
 
-    trace_spapr_pci_lsi_set(phb->busname, irq_num, phb->lsi_table[irq_num].irq);
+    trace_spapr_pci_lsi_set(phb->dtbusname, irq_num, phb->lsi_table[irq_num].irq);
     qemu_set_irq(spapr_phb_lsi_qirq(phb, irq_num), level);
 }
 
@@ -522,7 +522,63 @@ static int spapr_phb_init(SysBusDevice *s)
     int i;
     PCIBus *bus;
 
+    if (sphb->index != -1) {
+        hwaddr windows_base;
+
+        if ((sphb->buid != -1) || (sphb->dma_liobn != -1)
+            || (sphb->mem_win_addr != -1)
+            || (sphb->io_win_addr != -1)
+            || (sphb->msi_win_addr != -1)) {
+            fprintf(stderr, "Either \"index\" or other parameters must"
+                    " be specified for PAPR PHB, not both\n");
+            return -1;
+        }
+
+        sphb->buid = SPAPR_PCI_BASE_BUID + sphb->index;
+        sphb->dma_liobn = SPAPR_PCI_BASE_LIOBN + sphb->index;
+
+        windows_base = SPAPR_PCI_WINDOW_BASE
+            + sphb->index * SPAPR_PCI_WINDOW_SPACING;
+        sphb->mem_win_addr = windows_base + SPAPR_PCI_MMIO_WIN_OFF;
+        sphb->io_win_addr = windows_base + SPAPR_PCI_IO_WIN_OFF;
+        sphb->msi_win_addr = windows_base + SPAPR_PCI_MSI_WIN_OFF;
+    }
+
+    if (sphb->buid == -1) {
+        fprintf(stderr, "BUID not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->dma_liobn == -1) {
+        fprintf(stderr, "LIOBN not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->mem_win_addr == -1) {
+        fprintf(stderr, "Memory window address not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->io_win_addr == -1) {
+        fprintf(stderr, "IO window address not specified for PHB\n");
+        return -1;
+    }
+
+    if (sphb->msi_win_addr == -1) {
+        fprintf(stderr, "MSI window address not specified for PHB\n");
+        return -1;
+    }
+
+    if (find_phb(spapr, sphb->buid)) {
+        fprintf(stderr, "PCI host bridges must have unique BUIDs\n");
+        return -1;
+    }
+
     sphb->dtbusname = g_strdup_printf("pci@%" PRIx64, sphb->buid);
+    if (!sphb->busname) {
+        sphb->busname = sphb->dtbusname;
+    }
+
     namebuf = alloca(strlen(sphb->dtbusname) + 32);
 
     /* Initialize memory regions */
@@ -565,17 +621,19 @@ static int spapr_phb_init(SysBusDevice *s)
                                     &sphb->msiwindow);
     }
 
-    bus = pci_register_bus(DEVICE(s),
-                           sphb->busname ? sphb->busname : sphb->dtbusname,
+    bus = pci_register_bus(DEVICE(s), sphb->busname,
                            pci_spapr_set_irq, pci_spapr_map_irq, sphb,
                            &sphb->memspace, &sphb->iospace,
                            PCI_DEVFN(0, 0), PCI_NUM_PINS);
     phb->bus = bus;
 
-    sphb->dma_liobn = SPAPR_PCI_BASE_LIOBN | (pci_find_domain(bus) << 16);
     sphb->dma_window_start = 0;
     sphb->dma_window_size = 0x40000000;
     sphb->dma = spapr_tce_new_dma_context(sphb->dma_liobn, sphb->dma_window_size);
+    if (!sphb->dma) {
+        fprintf(stderr, "Unable to create TCE table for %s\n", sphb->dtbusname);
+        return -1;
+    }
     pci_setup_iommu(bus, spapr_pci_dma_context_fn, sphb);
 
     QLIST_INSERT_HEAD(&spapr->phbs, sphb, list);
@@ -597,7 +655,7 @@ static int spapr_phb_init(SysBusDevice *s)
 
 static void spapr_phb_reset(DeviceState *qdev)
 {
-    SysBusDevice *s = sysbus_from_qdev(qdev);
+    SysBusDevice *s = SYS_BUS_DEVICE(qdev);
     sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(s);
 
     /* Reset the IOMMU state */
@@ -605,13 +663,17 @@ static void spapr_phb_reset(DeviceState *qdev)
 }
 
 static Property spapr_phb_properties[] = {
-    DEFINE_PROP_HEX64("buid", sPAPRPHBState, buid, 0),
     DEFINE_PROP_STRING("busname", sPAPRPHBState, busname),
-    DEFINE_PROP_HEX64("mem_win_addr", sPAPRPHBState, mem_win_addr, 0),
-    DEFINE_PROP_HEX64("mem_win_size", sPAPRPHBState, mem_win_size, 0x20000000),
-    DEFINE_PROP_HEX64("io_win_addr", sPAPRPHBState, io_win_addr, 0),
-    DEFINE_PROP_HEX64("io_win_size", sPAPRPHBState, io_win_size, 0x10000),
-    DEFINE_PROP_HEX64("msi_win_addr", sPAPRPHBState, msi_win_addr, 0),
+    DEFINE_PROP_INT32("index", sPAPRPHBState, index, -1),
+    DEFINE_PROP_HEX64("buid", sPAPRPHBState, buid, -1),
+    DEFINE_PROP_HEX32("liobn", sPAPRPHBState, dma_liobn, -1),
+    DEFINE_PROP_HEX64("mem_win_addr", sPAPRPHBState, mem_win_addr, -1),
+    DEFINE_PROP_HEX64("mem_win_size", sPAPRPHBState, mem_win_size,
+                      SPAPR_PCI_MMIO_WIN_SIZE),
+    DEFINE_PROP_HEX64("io_win_addr", sPAPRPHBState, io_win_addr, -1),
+    DEFINE_PROP_HEX64("io_win_size", sPAPRPHBState, io_win_size,
+                      SPAPR_PCI_IO_WIN_SIZE),
+    DEFINE_PROP_HEX64("msi_win_addr", sPAPRPHBState, msi_win_addr, -1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -632,25 +694,17 @@ static const TypeInfo spapr_phb_info = {
     .class_init    = spapr_phb_class_init,
 };
 
-void spapr_create_phb(sPAPREnvironment *spapr,
-                      const char *busname, uint64_t buid,
-                      uint64_t mem_win_addr, uint64_t mem_win_size,
-                      uint64_t io_win_addr, uint64_t msi_win_addr)
+PCIHostState *spapr_create_phb(sPAPREnvironment *spapr, int index,
+                               const char *busname)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, TYPE_SPAPR_PCI_HOST_BRIDGE);
-
-    if (busname) {
-        qdev_prop_set_string(dev, "busname", g_strdup(busname));
-    }
-    qdev_prop_set_uint64(dev, "buid", buid);
-    qdev_prop_set_uint64(dev, "mem_win_addr", mem_win_addr);
-    qdev_prop_set_uint64(dev, "mem_win_size", mem_win_size);
-    qdev_prop_set_uint64(dev, "io_win_addr", io_win_addr);
-    qdev_prop_set_uint64(dev, "msi_win_addr", msi_win_addr);
-
+    qdev_prop_set_uint32(dev, "index", index);
+    qdev_prop_set_string(dev, "busname", busname);
     qdev_init_nofail(dev);
+
+    return PCI_HOST_BRIDGE(dev);
 }
 
 /* Macros to operate with address in OF binding to PCI */
diff --git a/hw/spapr_pci.h b/hw/spapr_pci.h
index e307ac8..8bb3c62 100644
--- a/hw/spapr_pci.h
+++ b/hw/spapr_pci.h
@@ -23,8 +23,8 @@
 #if !defined(__HW_SPAPR_PCI_H__)
 #define __HW_SPAPR_PCI_H__
 
-#include "hw/pci.h"
-#include "hw/pci_host.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_host.h"
 #include "hw/xics.h"
 
 #define SPAPR_MSIX_MAX_DEVS 32
@@ -37,6 +37,7 @@
 typedef struct sPAPRPHBState {
     PCIHostState parent_obj;
 
+    int32_t index;
     uint64_t buid;
     char *busname;
     char *dtbusname;
@@ -64,18 +65,25 @@ typedef struct sPAPRPHBState {
     QLIST_ENTRY(sPAPRPHBState) list;
 } sPAPRPHBState;
 
+#define SPAPR_PCI_BASE_BUID          0x800000020000000ULL
+
+#define SPAPR_PCI_WINDOW_BASE        0x10000000000ULL
+#define SPAPR_PCI_WINDOW_SPACING     0x1000000000ULL
+#define SPAPR_PCI_MMIO_WIN_OFF       0xA0000000
+#define SPAPR_PCI_MMIO_WIN_SIZE      0x20000000
+#define SPAPR_PCI_IO_WIN_OFF         0x80000000
+#define SPAPR_PCI_IO_WIN_SIZE        0x10000
+#define SPAPR_PCI_MSI_WIN_OFF        0x90000000
+
+#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
+
 static inline qemu_irq spapr_phb_lsi_qirq(struct sPAPRPHBState *phb, int pin)
 {
     return xics_get_qirq(spapr->icp, phb->lsi_table[pin].irq);
 }
 
-#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
-#define SPAPR_PCI_IO_WIN_SIZE        0x10000
-
-void spapr_create_phb(sPAPREnvironment *spapr,
-                      const char *busname, uint64_t buid,
-                      uint64_t mem_win_addr, uint64_t mem_win_size,
-                      uint64_t io_win_addr, uint64_t msi_win_addr);
+PCIHostState *spapr_create_phb(sPAPREnvironment *spapr, int index,
+                               const char *busname);
 
 int spapr_populate_pci_dt(sPAPRPHBState *phb,
                           uint32_t xics_phandle,
diff --git a/hw/spapr_rtas.c b/hw/spapr_rtas.c
index 6d5c48a..5ec787f 100644
--- a/hw/spapr_rtas.c
+++ b/hw/spapr_rtas.c
@@ -25,10 +25,10 @@
  *
  */
 #include "cpu.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "sysemu/sysemu.h"
+#include "char/char.h"
 #include "hw/qdev.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
@@ -131,6 +131,7 @@ static void rtas_query_cpu_stopped_state(sPAPREnvironment *spapr,
 {
     target_ulong id;
     CPUPPCState *env;
+    CPUState *cpu;
 
     if (nargs != 1 || nret != 2) {
         rtas_st(rets, 0, -3);
@@ -139,7 +140,8 @@ static void rtas_query_cpu_stopped_state(sPAPREnvironment *spapr,
 
     id = rtas_ld(args, 0);
     for (env = first_cpu; env; env = env->next_cpu) {
-        if (env->cpu_index != id) {
+        cpu = CPU(ppc_env_get_cpu(env));
+        if (cpu->cpu_index != id) {
             continue;
         }
 
@@ -176,9 +178,9 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
     r3 = rtas_ld(args, 2);
 
     for (env = first_cpu; env; env = env->next_cpu) {
-        cpu = ENV_GET_CPU(env);
+        cpu = CPU(ppc_env_get_cpu(env));
 
-        if (env->cpu_index != id) {
+        if (cpu->cpu_index != id) {
             continue;
         }
 
@@ -242,7 +244,7 @@ target_ulong spapr_rtas_call(sPAPREnvironment *spapr,
     return H_PARAMETER;
 }
 
-void spapr_rtas_register(const char *name, spapr_rtas_fn fn)
+int spapr_rtas_register(const char *name, spapr_rtas_fn fn)
 {
     int i;
 
@@ -258,7 +260,7 @@ void spapr_rtas_register(const char *name, spapr_rtas_fn fn)
     rtas_next->name = name;
     rtas_next->fn = fn;
 
-    rtas_next++;
+    return (rtas_next++ - rtas_table) + TOKEN_BASE;
 }
 
 int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
@@ -301,7 +303,7 @@ int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
     for (i = 0; i < TOKEN_MAX; i++) {
         struct rtas_call *call = &rtas_table[i];
 
-        if (!call->fn) {
+        if (!call->name) {
             continue;
         }
 
diff --git a/hw/spapr_vio.c b/hw/spapr_vio.c
index 1f19fed..34c9ca6 100644
--- a/hw/spapr_vio.c
+++ b/hw/spapr_vio.c
@@ -20,14 +20,14 @@
  */
 
 #include "hw.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "loader.h"
 #include "elf.h"
 #include "hw/sysbus.h"
-#include "kvm.h"
-#include "device_tree.h"
+#include "sysemu/kvm.h"
+#include "sysemu/device_tree.h"
 #include "kvm_ppc.h"
 
 #include "hw/spapr.h"
@@ -80,9 +80,7 @@ static char *vio_format_dev_name(VIOsPAPRDevice *dev)
     char *name;
 
     /* Device tree style name device@reg */
-    if (asprintf(&name, "%s@%x", pc->dt_name, dev->reg) < 0) {
-        return NULL;
-    }
+    name = g_strdup_printf("%s@%x", pc->dt_name, dev->reg);
 
     return name;
 }
@@ -101,12 +99,8 @@ static int vio_make_devnode(VIOsPAPRDevice *dev,
     }
 
     dt_name = vio_format_dev_name(dev);
-    if (!dt_name) {
-        return -ENOMEM;
-    }
-
     node_off = fdt_add_subnode(fdt, vdevice_off, dt_name);
-    free(dt_name);
+    g_free(dt_name);
     if (node_off < 0) {
         return node_off;
     }
@@ -444,9 +438,6 @@ static int spapr_vio_busdev_init(DeviceState *qdev)
     /* Don't overwrite ids assigned on the command line */
     if (!dev->qdev.id) {
         id = vio_format_dev_name(dev);
-        if (!id) {
-            return -1;
-        }
         dev->qdev.id = id;
     }
 
@@ -501,7 +492,7 @@ VIOsPAPRBus *spapr_vio_bus_init(void)
 
     qbus = qbus_create(TYPE_SPAPR_VIO_BUS, dev, "spapr-vio");
     bus = DO_UPCAST(VIOsPAPRBus, bus, qbus);
-    bus->next_reg = 0x1000;
+    bus->next_reg = 0x71000000;
 
     /* hcall-vio */
     spapr_register_hypercall(H_VIO_SIGNAL, h_vio_signal);
@@ -536,7 +527,7 @@ static void spapr_vio_bridge_class_init(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo spapr_vio_bridge_info = {
+static const TypeInfo spapr_vio_bridge_info = {
     .name          = "spapr-vio-bridge",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysBusDevice),
@@ -552,7 +543,7 @@ static void vio_spapr_device_class_init(ObjectClass *klass, void *data)
     k->props = spapr_vio_props;
 }
 
-static TypeInfo spapr_vio_type_info = {
+static const TypeInfo spapr_vio_type_info = {
     .name = TYPE_VIO_SPAPR_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(VIOsPAPRDevice),
@@ -646,20 +637,12 @@ int spapr_populate_chosen_stdout(void *fdt, VIOsPAPRBus *bus)
     }
 
     name = vio_format_dev_name(dev);
-    if (!name) {
-        return -ENOMEM;
-    }
-
-    if (asprintf(&path, "/vdevice/%s", name) < 0) {
-        path = NULL;
-        ret = -ENOMEM;
-        goto out;
-    }
+    path = g_strdup_printf("/vdevice/%s", name);
 
     ret = fdt_setprop_string(fdt, offset, "linux,stdout-path", path);
-out:
-    free(name);
-    free(path);
+
+    g_free(name);
+    g_free(path);
 
     return ret;
 }
diff --git a/hw/spapr_vio.h b/hw/spapr_vio.h
index cc85d26..f98ec0a 100644
--- a/hw/spapr_vio.h
+++ b/hw/spapr_vio.h
@@ -21,7 +21,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dma.h"
+#include "sysemu/dma.h"
 
 #define TYPE_VIO_SPAPR_DEVICE "vio-spapr-device"
 #define VIO_SPAPR_DEVICE(obj) \
diff --git a/hw/spapr_vscsi.c b/hw/spapr_vscsi.c
index e3d4b23..7fc0e13 100644
--- a/hw/spapr_vscsi.c
+++ b/hw/spapr_vscsi.c
@@ -34,7 +34,6 @@
 #include "hw.h"
 #include "scsi.h"
 #include "scsi-defs.h"
-#include "net.h" /* Remove that when we can */
 #include "srp.h"
 #include "hw/qdev.h"
 #include "hw/spapr.h"
@@ -968,7 +967,7 @@ static void spapr_vscsi_class_init(ObjectClass *klass, void *data)
     k->rtce_window_size = 0x10000000;
 }
 
-static TypeInfo spapr_vscsi_info = {
+static const TypeInfo spapr_vscsi_info = {
     .name          = "spapr-vscsi",
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VSCSIState),
diff --git a/hw/spapr_vty.c b/hw/spapr_vty.c
index 14f862f..5c63eaa 100644
--- a/hw/spapr_vty.c
+++ b/hw/spapr_vty.c
@@ -1,5 +1,5 @@
 #include "qdev.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "hw/spapr.h"
 #include "hw/spapr_vio.h"
 
@@ -150,7 +150,7 @@ static void spapr_vty_class_init(ObjectClass *klass, void *data)
     dc->props = spapr_vty_properties;
 }
 
-static TypeInfo spapr_vty_info = {
+static const TypeInfo spapr_vty_info = {
     .name          = "spapr-vty",
     .parent        = TYPE_VIO_SPAPR_DEVICE,
     .instance_size = sizeof(VIOsPAPRVTYDevice),
diff --git a/hw/sparc32_dma.c b/hw/sparc32_dma.c
index d11a302..6d0df51 100644
--- a/hw/sparc32_dma.c
+++ b/hw/sparc32_dma.c
@@ -300,7 +300,7 @@ static void sparc32_dma_class_init(ObjectClass *klass, void *data)
     dc->props = sparc32_dma_properties;
 }
 
-static TypeInfo sparc32_dma_info = {
+static const TypeInfo sparc32_dma_info = {
     .name          = "sparc32_dma",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(DMAState),
diff --git a/hw/spitz.c b/hw/spitz.c
index 12e2815..5bc49fc 100644
--- a/hw/spitz.c
+++ b/hw/spitz.c
@@ -13,21 +13,21 @@
 #include "hw.h"
 #include "pxa.h"
 #include "arm-misc.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "pcmcia.h"
 #include "i2c.h"
 #include "ssi.h"
 #include "flash.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "devices.h"
 #include "sharpsl.h"
-#include "console.h"
-#include "block.h"
+#include "ui/console.h"
+#include "block/block.h"
 #include "audio/audio.h"
 #include "boards.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #undef REG_FMT
 #define REG_FMT			"0x%02lx"
@@ -156,7 +156,7 @@ static void sl_flash_register(PXA2xxState *cpu, int size)
         qdev_prop_set_uint8(dev, "chip_id", 0xf1);
 
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, FLASH_BASE);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, FLASH_BASE);
 }
 
 static int sl_nand_init(SysBusDevice *dev) {
@@ -459,7 +459,7 @@ static void spitz_keyboard_register(PXA2xxState *cpu)
     SpitzKeyboardState *s;
 
     dev = sysbus_create_simple("spitz-keyboard", -1, NULL);
-    s = FROM_SYSBUS(SpitzKeyboardState, sysbus_from_qdev(dev));
+    s = FROM_SYSBUS(SpitzKeyboardState, SYS_BUS_DEVICE(dev));
 
     for (i = 0; i < SPITZ_KEY_SENSE_NUM; i ++)
         qdev_connect_gpio_out(dev, i, qdev_get_gpio_in(cpu->gpio, spitz_gpio_key_sense[i]));
@@ -959,24 +959,28 @@ static QEMUMachine akitapda_machine = {
     .name = "akita",
     .desc = "Akita PDA (PXA270)",
     .init = akita_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine spitzpda_machine = {
     .name = "spitz",
     .desc = "Spitz PDA (PXA270)",
     .init = spitz_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine borzoipda_machine = {
     .name = "borzoi",
     .desc = "Borzoi PDA (PXA270)",
     .init = borzoi_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine terrierpda_machine = {
     .name = "terrier",
     .desc = "Terrier PDA (PXA270)",
     .init = terrier_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void spitz_machine_init(void)
@@ -1022,7 +1026,7 @@ static void sl_nand_class_init(ObjectClass *klass, void *data)
     dc->props = sl_nand_properties;
 }
 
-static TypeInfo sl_nand_info = {
+static const TypeInfo sl_nand_info = {
     .name          = "sl-nand",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLNANDState),
@@ -1057,7 +1061,7 @@ static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
     dc->props = spitz_keyboard_properties;
 }
 
-static TypeInfo spitz_keyboard_info = {
+static const TypeInfo spitz_keyboard_info = {
     .name          = "spitz-keyboard",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SpitzKeyboardState),
@@ -1086,7 +1090,7 @@ static void corgi_ssp_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_corgi_ssp_regs;
 }
 
-static TypeInfo corgi_ssp_info = {
+static const TypeInfo corgi_ssp_info = {
     .name          = "corgi-ssp",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(CorgiSSPState),
@@ -1116,7 +1120,7 @@ static void spitz_lcdtg_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_spitz_lcdtg_regs;
 }
 
-static TypeInfo spitz_lcdtg_info = {
+static const TypeInfo spitz_lcdtg_info = {
     .name          = "spitz-lcdtg",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(SpitzLCDTG),
diff --git a/hw/ssd0303.c b/hw/ssd0303.c
index d7fd828..8777b16 100644
--- a/hw/ssd0303.c
+++ b/hw/ssd0303.c
@@ -11,7 +11,7 @@
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
 #include "i2c.h"
-#include "console.h"
+#include "ui/console.h"
 
 //#define DEBUG_SSD0303 1
 
@@ -306,7 +306,7 @@ static void ssd0303_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_ssd0303;
 }
 
-static TypeInfo ssd0303_info = {
+static const TypeInfo ssd0303_info = {
     .name          = "ssd0303",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(ssd0303_state),
diff --git a/hw/ssd0323.c b/hw/ssd0323.c
index 4098830..84c86a5 100644
--- a/hw/ssd0323.c
+++ b/hw/ssd0323.c
@@ -11,7 +11,7 @@
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
 #include "ssi.h"
-#include "console.h"
+#include "ui/console.h"
 
 //#define DEBUG_SSD0323 1
 
@@ -357,7 +357,7 @@ static void ssd0323_class_init(ObjectClass *klass, void *data)
     k->cs_polarity = SSI_CS_HIGH;
 }
 
-static TypeInfo ssd0323_info = {
+static const TypeInfo ssd0323_info = {
     .name          = "ssd0323",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ssd0323_state),
diff --git a/hw/ssi-sd.c b/hw/ssi-sd.c
index fc9cd2d..271e82d 100644
--- a/hw/ssi-sd.c
+++ b/hw/ssi-sd.c
@@ -10,7 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "ssi.h"
 #include "sd.h"
 
@@ -259,7 +259,7 @@ static void ssi_sd_class_init(ObjectClass *klass, void *data)
     k->cs_polarity = SSI_CS_LOW;
 }
 
-static TypeInfo ssi_sd_info = {
+static const TypeInfo ssi_sd_info = {
     .name          = "ssi-sd",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ssi_sd_state),
diff --git a/hw/ssi.c b/hw/ssi.c
index 2b56357..0b18176 100644
--- a/hw/ssi.c
+++ b/hw/ssi.c
@@ -78,7 +78,7 @@ static void ssi_slave_class_init(ObjectClass *klass, void *data)
     }
 }
 
-static TypeInfo ssi_slave_info = {
+static const TypeInfo ssi_slave_info = {
     .name = TYPE_SSI_SLAVE,
     .parent = TYPE_DEVICE,
     .class_init = ssi_slave_class_init,
diff --git a/hw/stellaris.c b/hw/stellaris.c
index b038f10..9b8f203 100644
--- a/hw/stellaris.c
+++ b/hw/stellaris.c
@@ -11,11 +11,11 @@
 #include "ssi.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i2c.h"
-#include "net.h"
+#include "net/net.h"
 #include "boards.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define GPIO_A 0
 #define GPIO_B 1
@@ -1286,8 +1286,8 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         enet = qdev_create(NULL, "stellaris_enet");
         qdev_set_nic_properties(enet, &nd_table[0]);
         qdev_init_nofail(enet);
-        sysbus_mmio_map(sysbus_from_qdev(enet), 0, 0x40048000);
-        sysbus_connect_irq(sysbus_from_qdev(enet), 0, pic[42]);
+        sysbus_mmio_map(SYS_BUS_DEVICE(enet), 0, 0x40048000);
+        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, pic[42]);
     }
     if (board->peripherals & BP_GAMEPAD) {
         qemu_irq gpad_irq[5];
@@ -1331,12 +1331,14 @@ static QEMUMachine lm3s811evb_machine = {
     .name = "lm3s811evb",
     .desc = "Stellaris LM3S811EVB",
     .init = lm3s811evb_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine lm3s6965evb_machine = {
     .name = "lm3s6965evb",
     .desc = "Stellaris LM3S6965EVB",
     .init = lm3s6965evb_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void stellaris_machine_init(void)
@@ -1354,7 +1356,7 @@ static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
     sdc->init = stellaris_i2c_init;
 }
 
-static TypeInfo stellaris_i2c_info = {
+static const TypeInfo stellaris_i2c_info = {
     .name          = "stellaris-i2c",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_i2c_state),
@@ -1368,7 +1370,7 @@ static void stellaris_gptm_class_init(ObjectClass *klass, void *data)
     sdc->init = stellaris_gptm_init;
 }
 
-static TypeInfo stellaris_gptm_info = {
+static const TypeInfo stellaris_gptm_info = {
     .name          = "stellaris-gptm",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(gptm_state),
@@ -1382,7 +1384,7 @@ static void stellaris_adc_class_init(ObjectClass *klass, void *data)
     sdc->init = stellaris_adc_init;
 }
 
-static TypeInfo stellaris_adc_info = {
+static const TypeInfo stellaris_adc_info = {
     .name          = "stellaris-adc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_adc_state),
diff --git a/hw/stellaris_enet.c b/hw/stellaris_enet.c
index a530b10..6c701fb 100644
--- a/hw/stellaris_enet.c
+++ b/hw/stellaris_enet.c
@@ -7,7 +7,7 @@
  * This code is licensed under the GPL.
  */
 #include "sysbus.h"
-#include "net.h"
+#include "net/net.h"
 #include <zlib.h>
 
 //#define DEBUG_STELLARIS_ENET 1
@@ -80,7 +80,7 @@ static void stellaris_enet_update(stellaris_enet_state *s)
 /* TODO: Implement MAC address filtering.  */
 static ssize_t stellaris_enet_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    stellaris_enet_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    stellaris_enet_state *s = qemu_get_nic_opaque(nc);
     int n;
     uint8_t *p;
     uint32_t crc;
@@ -122,7 +122,7 @@ static ssize_t stellaris_enet_receive(NetClientState *nc, const uint8_t *buf, si
 
 static int stellaris_enet_can_receive(NetClientState *nc)
 {
-    stellaris_enet_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    stellaris_enet_state *s = qemu_get_nic_opaque(nc);
 
     if ((s->rctl & SE_RCTL_RXEN) == 0)
         return 1;
@@ -259,7 +259,8 @@ static void stellaris_enet_write(void *opaque, hwaddr offset,
                     memset(&s->tx_fifo[s->tx_frame_len], 0, 60 - s->tx_frame_len);
                     s->tx_fifo_len = 60;
                 }
-                qemu_send_packet(&s->nic->nc, s->tx_fifo, s->tx_frame_len);
+                qemu_send_packet(qemu_get_queue(s->nic), s->tx_fifo,
+                                 s->tx_frame_len);
                 s->tx_frame_len = -1;
                 s->ris |= SE_INT_TXEMP;
                 stellaris_enet_update(s);
@@ -383,7 +384,7 @@ static int stellaris_enet_load(QEMUFile *f, void *opaque, int version_id)
 
 static void stellaris_enet_cleanup(NetClientState *nc)
 {
-    stellaris_enet_state *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    stellaris_enet_state *s = qemu_get_nic_opaque(nc);
 
     unregister_savevm(&s->busdev.qdev, "stellaris_enet", s);
 
@@ -412,7 +413,7 @@ static int stellaris_enet_init(SysBusDevice *dev)
 
     s->nic = qemu_new_nic(&net_stellaris_enet_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     stellaris_enet_reset(s);
     register_savevm(&s->busdev.qdev, "stellaris_enet", -1, 1,
@@ -434,7 +435,7 @@ static void stellaris_enet_class_init(ObjectClass *klass, void *data)
     dc->props = stellaris_enet_properties;
 }
 
-static TypeInfo stellaris_enet_info = {
+static const TypeInfo stellaris_enet_info = {
     .name          = "stellaris_enet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_enet_state),
diff --git a/hw/stellaris_input.c b/hw/stellaris_input.c
index 68c600c..7a95c3f 100644
--- a/hw/stellaris_input.c
+++ b/hw/stellaris_input.c
@@ -8,7 +8,7 @@
  */
 #include "hw.h"
 #include "devices.h"
-#include "console.h"
+#include "ui/console.h"
 
 typedef struct {
     qemu_irq irq;
diff --git a/hw/stream.c b/hw/stream.c
index be57e8b..d4cf84d 100644
--- a/hw/stream.c
+++ b/hw/stream.c
@@ -8,7 +8,7 @@ stream_push(StreamSlave *sink, uint8_t *buf, size_t len, uint32_t *app)
     k->push(sink, buf, len, app);
 }
 
-static TypeInfo stream_slave_info = {
+static const TypeInfo stream_slave_info = {
     .name          = TYPE_STREAM_SLAVE,
     .parent        = TYPE_INTERFACE,
     .class_size = sizeof(StreamSlaveClass),
diff --git a/hw/stream.h b/hw/stream.h
index 21123a9..f6137d6 100644
--- a/hw/stream.h
+++ b/hw/stream.h
@@ -2,7 +2,7 @@
 #define STREAM_H 1
 
 #include "qemu-common.h"
-#include "qemu/object.h"
+#include "qom/object.h"
 
 /* stream slave. Used until qdev provides a generic way.  */
 #define TYPE_STREAM_SLAVE "stream-slave"
diff --git a/hw/strongarm.c b/hw/strongarm.c
index 4385515..ab736e3 100644
--- a/hw/strongarm.c
+++ b/hw/strongarm.c
@@ -28,9 +28,10 @@
  */
 #include "sysbus.h"
 #include "strongarm.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "arm-misc.h"
-#include "sysemu.h"
+#include "char/char.h"
+#include "sysemu/sysemu.h"
 #include "ssi.h"
 
 //#define DEBUG
@@ -211,7 +212,7 @@ static void strongarm_pic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_strongarm_pic_regs;
 }
 
-static TypeInfo strongarm_pic_info = {
+static const TypeInfo strongarm_pic_info = {
     .name          = "strongarm_pic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMPICState),
@@ -432,7 +433,7 @@ static void strongarm_rtc_sysbus_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_strongarm_rtc_regs;
 }
 
-static TypeInfo strongarm_rtc_sysbus_info = {
+static const TypeInfo strongarm_rtc_sysbus_info = {
     .name          = "strongarm-rtc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMRTCState),
@@ -618,9 +619,9 @@ static DeviceState *strongarm_gpio_init(hwaddr base,
     dev = qdev_create(NULL, "strongarm-gpio");
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     for (i = 0; i < 12; i++)
-        sysbus_connect_irq(sysbus_from_qdev(dev), i,
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i,
                     qdev_get_gpio_in(pic, SA_PIC_GPIO0_EDGE + i));
 
     return dev;
@@ -673,7 +674,7 @@ static void strongarm_gpio_class_init(ObjectClass *klass, void *data)
     dc->desc = "StrongARM GPIO controller";
 }
 
-static TypeInfo strongarm_gpio_info = {
+static const TypeInfo strongarm_gpio_info = {
     .name          = "strongarm-gpio",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMGPIOInfo),
@@ -839,7 +840,7 @@ static void strongarm_ppc_class_init(ObjectClass *klass, void *data)
     dc->desc = "StrongARM PPC controller";
 }
 
-static TypeInfo strongarm_ppc_info = {
+static const TypeInfo strongarm_ppc_info = {
     .name          = "strongarm-ppc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMPPCInfo),
@@ -1298,7 +1299,7 @@ static void strongarm_uart_class_init(ObjectClass *klass, void *data)
     dc->props = strongarm_uart_properties;
 }
 
-static TypeInfo strongarm_uart_info = {
+static const TypeInfo strongarm_uart_info = {
     .name          = "strongarm-uart",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMUARTState),
@@ -1537,7 +1538,7 @@ static void strongarm_ssp_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_strongarm_ssp_regs;
 }
 
-static TypeInfo strongarm_ssp_info = {
+static const TypeInfo strongarm_ssp_info = {
     .name          = "strongarm-ssp",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMSSPState),
@@ -1596,9 +1597,9 @@ StrongARMState *sa1110_init(MemoryRegion *sysmem,
         DeviceState *dev = qdev_create(NULL, "strongarm-uart");
         qdev_prop_set_chr(dev, "chardev", serial_hds[i]);
         qdev_init_nofail(dev);
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0,
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0,
                 sa_serial[i].io_base);
-        sysbus_connect_irq(sysbus_from_qdev(dev), 0,
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0,
                 qdev_get_gpio_in(s->pic, sa_serial[i].irq));
     }
 
diff --git a/hw/strongarm.h b/hw/strongarm.h
index d30dd6a..2893f94 100644
--- a/hw/strongarm.h
+++ b/hw/strongarm.h
@@ -1,7 +1,7 @@
 #ifndef _STRONGARM_H
 #define _STRONGARM_H
 
-#include "memory.h"
+#include "exec/memory.h"
 
 #define SA_CS0          0x00000000
 #define SA_CS1          0x08000000
diff --git a/hw/sun4c_intctl.c b/hw/sun4c_intctl.c
index 702e9f5..f8f4d02 100644
--- a/hw/sun4c_intctl.c
+++ b/hw/sun4c_intctl.c
@@ -24,7 +24,7 @@
 
 #include "hw.h"
 #include "sun4m.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "sysbus.h"
 
 //#define DEBUG_IRQ_COUNT
@@ -193,7 +193,7 @@ static void sun4c_intctl_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_sun4c_intctl;
 }
 
-static TypeInfo sun4c_intctl_info = {
+static const TypeInfo sun4c_intctl_info = {
     .name          = "sun4c_intctl",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Sun4c_INTCTLState),
diff --git a/hw/sun4m.c b/hw/sun4m.c
index 1a78676..9903f44 100644
--- a/hw/sun4m.c
+++ b/hw/sun4m.c
@@ -22,13 +22,13 @@
  * THE SOFTWARE.
  */
 #include "sysbus.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sun4m.h"
 #include "nvram.h"
 #include "sparc32_dma.h"
 #include "fdc.h"
-#include "sysemu.h"
-#include "net.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 #include "boards.h"
 #include "firmware_abi.h"
 #include "esp.h"
@@ -40,7 +40,7 @@
 #include "qdev-addr.h"
 #include "loader.h"
 #include "elf.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "trace.h"
 
 /*
@@ -216,13 +216,13 @@ static void nvram_init(M48t59State *nvram, uint8_t *macaddr,
 
 static DeviceState *slavio_intctl;
 
-void sun4m_pic_info(Monitor *mon)
+void sun4m_pic_info(Monitor *mon, const QDict *qdict)
 {
     if (slavio_intctl)
         slavio_pic_info(mon, slavio_intctl);
 }
 
-void sun4m_irq_info(Monitor *mon)
+void sun4m_irq_info(Monitor *mon, const QDict *qdict)
 {
     if (slavio_intctl)
         slavio_irq_info(mon, slavio_intctl);
@@ -381,7 +381,7 @@ static void *iommu_init(hwaddr addr, uint32_t version, qemu_irq irq)
     dev = qdev_create(NULL, "iommu");
     qdev_prop_set_uint32(dev, "version", version);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_mmio_map(s, 0, addr);
 
@@ -398,7 +398,7 @@ static void *sparc32_dma_init(hwaddr daddr, qemu_irq parent_irq,
     qdev_prop_set_ptr(dev, "iommu_opaque", iommu);
     qdev_prop_set_uint32(dev, "is_ledma", is_ledma);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, parent_irq);
     *dev_irq = qdev_get_gpio_in(dev, 0);
     sysbus_mmio_map(s, 0, daddr);
@@ -419,7 +419,7 @@ static void lance_init(NICInfo *nd, hwaddr leaddr,
     qdev_set_nic_properties(dev, nd);
     qdev_prop_set_ptr(dev, "dma", dma_opaque);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, leaddr);
     sysbus_connect_irq(s, 0, irq);
     reset = qdev_get_gpio_in(dev, 0);
@@ -437,7 +437,7 @@ static DeviceState *slavio_intctl_init(hwaddr addr,
     dev = qdev_create(NULL, "slavio_intctl");
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     for (i = 0; i < MAX_CPUS; i++) {
         for (j = 0; j < MAX_PILS; j++) {
@@ -465,7 +465,7 @@ static void slavio_timer_init_all(hwaddr addr, qemu_irq master_irq,
     dev = qdev_create(NULL, "slavio_timer");
     qdev_prop_set_uint32(dev, "num_cpus", num_cpus);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, master_irq);
     sysbus_mmio_map(s, 0, addr + SYS_TIMER_OFFSET);
 
@@ -502,7 +502,7 @@ static void slavio_misc_init(hwaddr base,
 
     dev = qdev_create(NULL, "slavio_misc");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     if (base) {
         /* 8 bit registers */
         /* Slavio control */
@@ -540,7 +540,7 @@ static void ecc_init(hwaddr base, qemu_irq irq, uint32_t version)
     dev = qdev_create(NULL, "eccmemctl");
     qdev_prop_set_uint32(dev, "version", version);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     sysbus_mmio_map(s, 0, base);
     if (version == 0) { // SS-600MP only
@@ -555,7 +555,7 @@ static void apc_init(hwaddr power_base, qemu_irq cpu_halt)
 
     dev = qdev_create(NULL, "apc");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     /* Power management (APC) XXX: not a Slavio device */
     sysbus_mmio_map(s, 0, power_base);
     sysbus_connect_irq(s, 0, cpu_halt);
@@ -574,7 +574,7 @@ static void tcx_init(hwaddr addr, int vram_size, int width,
     qdev_prop_set_uint16(dev, "height", height);
     qdev_prop_set_uint16(dev, "depth", depth);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     /* 8-bit plane */
     sysbus_mmio_map(s, 0, addr + 0x00800000ULL);
     /* DAC */
@@ -604,7 +604,7 @@ static void idreg_init(hwaddr addr)
 
     dev = qdev_create(NULL, "macio_idreg");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
     cpu_physical_memory_write_rom(addr, idreg_data, sizeof(idreg_data));
@@ -633,7 +633,7 @@ static void idreg_class_init(ObjectClass *klass, void *data)
     k->init = idreg_init1;
 }
 
-static TypeInfo idreg_info = {
+static const TypeInfo idreg_info = {
     .name          = "macio_idreg",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IDRegState),
@@ -653,7 +653,7 @@ static void afx_init(hwaddr addr)
 
     dev = qdev_create(NULL, "tcx_afx");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
 }
@@ -675,7 +675,7 @@ static void afx_class_init(ObjectClass *klass, void *data)
     k->init = afx_init1;
 }
 
-static TypeInfo afx_info = {
+static const TypeInfo afx_info = {
     .name          = "tcx_afx",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(AFXState),
@@ -703,7 +703,7 @@ static void prom_init(hwaddr addr, const char *bios_name)
 
     dev = qdev_create(NULL, "openprom");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
 
@@ -752,7 +752,7 @@ static void prom_class_init(ObjectClass *klass, void *data)
     dc->props = prom_properties;
 }
 
-static TypeInfo prom_info = {
+static const TypeInfo prom_info = {
     .name          = "openprom",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PROMState),
@@ -793,7 +793,7 @@ static void ram_init(hwaddr addr, ram_addr_t RAM_size,
         exit(1);
     }
     dev = qdev_create(NULL, "memory");
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     d = FROM_SYSBUS(RamDevice, s);
     d->size = RAM_size;
@@ -816,7 +816,7 @@ static void ram_class_init(ObjectClass *klass, void *data)
     dc->props = ram_properties;
 }
 
-static TypeInfo ram_info = {
+static const TypeInfo ram_info = {
     .name          = "memory",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RamDevice),
@@ -1021,6 +1021,7 @@ static void sun4m_hw_init(const struct sun4m_hwdef *hwdef, ram_addr_t RAM_size,
                  hwdef->ecc_version);
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -1030,9 +1031,7 @@ static void sun4m_hw_init(const struct sun4m_hwdef *hwdef, ram_addr_t RAM_size,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE, kernel_cmdline);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
                        strlen(kernel_cmdline) + 1);
     } else {
@@ -1426,67 +1425,76 @@ static QEMUMachine ss5_machine = {
     .name = "SS-5",
     .desc = "Sun4m platform, SPARCstation 5",
     .init = ss5_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss10_machine = {
     .name = "SS-10",
     .desc = "Sun4m platform, SPARCstation 10",
     .init = ss10_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss600mp_machine = {
     .name = "SS-600MP",
     .desc = "Sun4m platform, SPARCserver 600MP",
     .init = ss600mp_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss20_machine = {
     .name = "SS-20",
     .desc = "Sun4m platform, SPARCstation 20",
     .init = ss20_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine voyager_machine = {
     .name = "Voyager",
     .desc = "Sun4m platform, SPARCstation Voyager",
     .init = vger_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss_lx_machine = {
     .name = "LX",
     .desc = "Sun4m platform, SPARCstation LX",
     .init = ss_lx_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss4_machine = {
     .name = "SS-4",
     .desc = "Sun4m platform, SPARCstation 4",
     .init = ss4_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine scls_machine = {
     .name = "SPARCClassic",
     .desc = "Sun4m platform, SPARCClassic",
     .init = scls_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine sbook_machine = {
     .name = "SPARCbook",
     .desc = "Sun4m platform, SPARCbook",
     .init = sbook_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static const struct sun4d_hwdef sun4d_hwdefs[] = {
@@ -1553,7 +1561,7 @@ static DeviceState *sbi_init(hwaddr addr, qemu_irq **parent_irq)
     dev = qdev_create(NULL, "sbi");
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     for (i = 0; i < MAX_CPUS; i++) {
         sysbus_connect_irq(s, i, *parent_irq[i]);
@@ -1658,6 +1666,7 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
                "Sun4d");
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -1667,9 +1676,7 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE, kernel_cmdline);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, 0);
     }
@@ -1709,16 +1716,18 @@ static QEMUMachine ss1000_machine = {
     .name = "SS-1000",
     .desc = "Sun4d platform, SPARCserver 1000",
     .init = ss1000_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 8,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine ss2000_machine = {
     .name = "SS-2000",
     .desc = "Sun4d platform, SPARCcenter 2000",
     .init = ss2000_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 20,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static const struct sun4c_hwdef sun4c_hwdefs[] = {
@@ -1754,7 +1763,7 @@ static DeviceState *sun4c_intctl_init(hwaddr addr,
     dev = qdev_create(NULL, "sun4c_intctl");
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     for (i = 0; i < MAX_PILS; i++) {
         sysbus_connect_irq(s, i, parent_irq[i]);
@@ -1858,6 +1867,7 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
                "Sun4c");
 
     fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -1867,9 +1877,7 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE, kernel_cmdline);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, 0);
     }
@@ -1896,7 +1904,8 @@ static QEMUMachine ss2_machine = {
     .name = "SS-2",
     .desc = "Sun4c platform, SPARCstation 2",
     .init = ss2_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void sun4m_register_types(void)
diff --git a/hw/sun4m.h b/hw/sun4m.h
index 47eb945..0361eee 100644
--- a/hw/sun4m.h
+++ b/hw/sun4m.h
@@ -27,8 +27,8 @@ void slavio_pic_info(Monitor *mon, DeviceState *dev);
 void slavio_irq_info(Monitor *mon, DeviceState *dev);
 
 /* sun4m.c */
-void sun4m_pic_info(Monitor *mon);
-void sun4m_irq_info(Monitor *mon);
+void sun4m_pic_info(Monitor *mon, const QDict *qdict);
+void sun4m_irq_info(Monitor *mon, const QDict *qdict);
 
 /* sparc32_dma.c */
 #include "sparc32_dma.h"
diff --git a/hw/sun4m_iommu.c b/hw/sun4m_iommu.c
index ce6819e..8f9635f 100644
--- a/hw/sun4m_iommu.c
+++ b/hw/sun4m_iommu.c
@@ -373,7 +373,7 @@ static void iommu_class_init(ObjectClass *klass, void *data)
     dc->props = iommu_properties;
 }
 
-static TypeInfo iommu_info = {
+static const TypeInfo iommu_info = {
     .name          = "iommu",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IOMMUState),
diff --git a/hw/sun4u.c b/hw/sun4u.c
index b2b51e3..9fbda29 100644
--- a/hw/sun4u.c
+++ b/hw/sun4u.c
@@ -22,15 +22,15 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "apb_pci.h"
 #include "pc.h"
 #include "serial.h"
 #include "nvram.h"
 #include "fdc.h"
-#include "net.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "firmware_abi.h"
 #include "fw_cfg.h"
@@ -38,8 +38,8 @@
 #include "ide.h"
 #include "loader.h"
 #include "elf.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 
 //#define DEBUG_IRQ
 //#define DEBUG_EBUS
@@ -618,7 +618,7 @@ static void ebus_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_BRIDGE_OTHER;
 }
 
-static TypeInfo ebus_info = {
+static const TypeInfo ebus_info = {
     .name          = "ebus",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(EbusState),
@@ -646,7 +646,7 @@ static void prom_init(hwaddr addr, const char *bios_name)
 
     dev = qdev_create(NULL, "openprom");
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     sysbus_mmio_map(s, 0, addr);
 
@@ -695,7 +695,7 @@ static void prom_class_init(ObjectClass *klass, void *data)
     dc->props = prom_properties;
 }
 
-static TypeInfo prom_info = {
+static const TypeInfo prom_info = {
     .name          = "openprom",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PROMState),
@@ -729,7 +729,7 @@ static void ram_init(hwaddr addr, ram_addr_t RAM_size)
 
     /* allocate RAM */
     dev = qdev_create(NULL, "memory");
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
 
     d = FROM_SYSBUS(RamDevice, s);
     d->size = RAM_size;
@@ -752,7 +752,7 @@ static void ram_class_init(ObjectClass *klass, void *data)
     dc->props = ram_properties;
 }
 
-static TypeInfo ram_info = {
+static const TypeInfo ram_info = {
     .name          = "memory",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RamDevice),
@@ -878,6 +878,7 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
                            (uint8_t *)&nd_table[0].macaddr);
 
     fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
@@ -886,9 +887,7 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
     if (kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
                        strlen(kernel_cmdline) + 1);
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_CMDLINE_DATA,
-                         (uint8_t*)strdup(kernel_cmdline),
-                         strlen(kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, 0);
     }
@@ -978,6 +977,7 @@ static QEMUMachine sun4u_machine = {
     .init = sun4u_init,
     .max_cpus = 1, // XXX for now
     .is_default = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine sun4v_machine = {
@@ -985,6 +985,7 @@ static QEMUMachine sun4v_machine = {
     .desc = "Sun4v platform",
     .init = sun4v_init,
     .max_cpus = 1, // XXX for now
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine niagara_machine = {
@@ -992,6 +993,7 @@ static QEMUMachine niagara_machine = {
     .desc = "Sun4v platform, Niagara",
     .init = niagara_init,
     .max_cpus = 1, // XXX for now
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void sun4u_register_types(void)
diff --git a/hw/sysbus.c b/hw/sysbus.c
index ef8ffb6..6d9d1df 100644
--- a/hw/sysbus.c
+++ b/hw/sysbus.c
@@ -18,8 +18,8 @@
  */
 
 #include "sysbus.h"
-#include "monitor.h"
-#include "exec-memory.h"
+#include "monitor/monitor.h"
+#include "exec/address-spaces.h"
 
 static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent);
 static char *sysbus_get_fw_dev_path(DeviceState *dev);
@@ -131,7 +131,7 @@ DeviceState *sysbus_create_varargs(const char *name,
     int n;
 
     dev = qdev_create(NULL, name);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     if (addr != (hwaddr)-1) {
         sysbus_mmio_map(s, 0, addr);
@@ -163,7 +163,7 @@ DeviceState *sysbus_try_create_varargs(const char *name,
     if (!dev) {
         return NULL;
     }
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     if (addr != (hwaddr)-1) {
         sysbus_mmio_map(s, 0, addr);
@@ -184,7 +184,7 @@ DeviceState *sysbus_try_create_varargs(const char *name,
 
 static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 {
-    SysBusDevice *s = sysbus_from_qdev(dev);
+    SysBusDevice *s = SYS_BUS_DEVICE(dev);
     hwaddr size;
     int i;
 
@@ -198,7 +198,7 @@ static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 
 static char *sysbus_get_fw_dev_path(DeviceState *dev)
 {
-    SysBusDevice *s = sysbus_from_qdev(dev);
+    SysBusDevice *s = SYS_BUS_DEVICE(dev);
     char path[40];
     int off;
 
@@ -255,7 +255,7 @@ static void sysbus_device_class_init(ObjectClass *klass, void *data)
     k->bus_type = TYPE_SYSTEM_BUS;
 }
 
-static TypeInfo sysbus_device_type_info = {
+static const TypeInfo sysbus_device_type_info = {
     .name = TYPE_SYS_BUS_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(SysBusDevice),
diff --git a/hw/sysbus.h b/hw/sysbus.h
index e58baaa..a7fcded 100644
--- a/hw/sysbus.h
+++ b/hw/sysbus.h
@@ -4,7 +4,7 @@
 /* Devices attached directly to the main system bus.  */
 
 #include "qdev.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 #define QDEV_MAX_MMIO 32
 #define QDEV_MAX_PIO 32
@@ -44,7 +44,6 @@ struct SysBusDevice {
 };
 
 /* Macros to compensate for lack of type inheritance in C.  */
-#define sysbus_from_qdev(dev) ((SysBusDevice *)(dev))
 #define FROM_SYSBUS(type, dev) DO_UPCAST(type, busdev, dev)
 
 void *sysbus_new(void);
diff --git a/hw/tc6393xb.c b/hw/tc6393xb.c
index f032027..e815f83 100644
--- a/hw/tc6393xb.c
+++ b/hw/tc6393xb.c
@@ -13,9 +13,9 @@
 #include "hw.h"
 #include "devices.h"
 #include "flash.h"
-#include "console.h"
-#include "pixel_ops.h"
-#include "blockdev.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
+#include "sysemu/blockdev.h"
 
 #define IRQ_TC6393_NAND		0
 #define IRQ_TC6393_MMC		1
diff --git a/hw/tcx.c b/hw/tcx.c
index 7aee2a9..0ce2952 100644
--- a/hw/tcx.c
+++ b/hw/tcx.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "console.h"
-#include "pixel_ops.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/pixel_ops.h"
 #include "sysbus.h"
 #include "qdev-addr.h"
 
@@ -715,7 +716,7 @@ static void tcx_class_init(ObjectClass *klass, void *data)
     dc->props = tcx_properties;
 }
 
-static TypeInfo tcx_info = {
+static const TypeInfo tcx_info = {
     .name          = "SUNW,tcx",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(TCXState),
diff --git a/hw/tmp105.c b/hw/tmp105.c
index 8e8dbd9..3ad2d2f 100644
--- a/hw/tmp105.c
+++ b/hw/tmp105.c
@@ -20,20 +20,8 @@
 
 #include "hw.h"
 #include "i2c.h"
-
-typedef struct {
-    I2CSlave i2c;
-    uint8_t len;
-    uint8_t buf[2];
-    qemu_irq pin;
-
-    uint8_t pointer;
-    uint8_t config;
-    int16_t temperature;
-    int16_t limit[2];
-    int faults;
-    uint8_t alarm;
-} TMP105State;
+#include "tmp105.h"
+#include "qapi/visitor.h"
 
 static void tmp105_interrupt_update(TMP105State *s)
 {
@@ -64,15 +52,30 @@ static void tmp105_alarm_update(TMP105State *s)
     tmp105_interrupt_update(s);
 }
 
+static void tmp105_get_temperature(Object *obj, Visitor *v, void *opaque,
+                                   const char *name, Error **errp)
+{
+    TMP105State *s = TMP105(obj);
+    int64_t value = s->temperature;
+
+    visit_type_int(v, &value, name, errp);
+}
+
 /* Units are 0.001 centigrades relative to 0 C.  */
-void tmp105_set(I2CSlave *i2c, int temp)
+static void tmp105_set_temperature(Object *obj, Visitor *v, void *opaque,
+                                   const char *name, Error **errp)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(obj);
+    int64_t temp;
 
+    visit_type_int(v, &temp, name, errp);
+    if (error_is_set(errp)) {
+        return;
+    }
     if (temp >= 128000 || temp < -128000) {
-        fprintf(stderr, "%s: values is out of range (%i.%03i C)\n",
-                        __FUNCTION__, temp / 1000, temp % 1000);
-        exit(-1);
+        error_setg(errp, "value %" PRId64 ".%03" PRIu64 " °C is out of range",
+                   temp / 1000, temp % 1000);
+        return;
     }
 
     s->temperature = ((int16_t) (temp * 0x800 / 128000)) << 4;
@@ -92,22 +95,22 @@ static void tmp105_read(TMP105State *s)
     }
 
     switch (s->pointer & 3) {
-    case 0:	/* Temperature */
+    case TMP105_REG_TEMPERATURE:
         s->buf[s->len ++] = (((uint16_t) s->temperature) >> 8);
         s->buf[s->len ++] = (((uint16_t) s->temperature) >> 0) &
                 (0xf0 << ((~s->config >> 5) & 3));		/* R */
         break;
 
-    case 1:	/* Configuration */
+    case TMP105_REG_CONFIG:
         s->buf[s->len ++] = s->config;
         break;
 
-    case 2:	/* T_LOW */
+    case TMP105_REG_T_LOW:
         s->buf[s->len ++] = ((uint16_t) s->limit[0]) >> 8;
         s->buf[s->len ++] = ((uint16_t) s->limit[0]) >> 0;
         break;
 
-    case 3:	/* T_HIGH */
+    case TMP105_REG_T_HIGH:
         s->buf[s->len ++] = ((uint16_t) s->limit[1]) >> 8;
         s->buf[s->len ++] = ((uint16_t) s->limit[1]) >> 0;
         break;
@@ -117,10 +120,10 @@ static void tmp105_read(TMP105State *s)
 static void tmp105_write(TMP105State *s)
 {
     switch (s->pointer & 3) {
-    case 0:	/* Temperature */
+    case TMP105_REG_TEMPERATURE:
         break;
 
-    case 1:	/* Configuration */
+    case TMP105_REG_CONFIG:
         if (s->buf[0] & ~s->config & (1 << 0))			/* SD */
             printf("%s: TMP105 shutdown\n", __FUNCTION__);
         s->config = s->buf[0];
@@ -128,8 +131,8 @@ static void tmp105_write(TMP105State *s)
         tmp105_alarm_update(s);
         break;
 
-    case 2:	/* T_LOW */
-    case 3:	/* T_HIGH */
+    case TMP105_REG_T_LOW:
+    case TMP105_REG_T_HIGH:
         if (s->len >= 3)
             s->limit[s->pointer & 1] = (int16_t)
                     ((((uint16_t) s->buf[0]) << 8) | s->buf[1]);
@@ -140,23 +143,27 @@ static void tmp105_write(TMP105State *s)
 
 static int tmp105_rx(I2CSlave *i2c)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
-    if (s->len < 2)
+    if (s->len < 2) {
         return s->buf[s->len ++];
-    else
+    } else {
         return 0xff;
+    }
 }
 
 static int tmp105_tx(I2CSlave *i2c, uint8_t data)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
-    if (!s->len ++)
+    if (s->len == 0) {
         s->pointer = data;
-    else {
-        if (s->len <= 2)
+        s->len++;
+    } else {
+        if (s->len <= 2) {
             s->buf[s->len - 1] = data;
+        }
+        s->len++;
         tmp105_write(s);
     }
 
@@ -165,10 +172,11 @@ static int tmp105_tx(I2CSlave *i2c, uint8_t data)
 
 static void tmp105_event(I2CSlave *i2c, enum i2c_event event)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
-    if (event == I2C_START_RECV)
+    if (event == I2C_START_RECV) {
         tmp105_read(s);
+    }
 
     s->len = 0;
 }
@@ -204,7 +212,7 @@ static const VMStateDescription vmstate_tmp105 = {
 
 static void tmp105_reset(I2CSlave *i2c)
 {
-    TMP105State *s = (TMP105State *) i2c;
+    TMP105State *s = TMP105(i2c);
 
     s->temperature = 0;
     s->pointer = 0;
@@ -217,7 +225,7 @@ static void tmp105_reset(I2CSlave *i2c)
 
 static int tmp105_init(I2CSlave *i2c)
 {
-    TMP105State *s = FROM_I2C_SLAVE(TMP105State, i2c);
+    TMP105State *s = TMP105(i2c);
 
     qdev_init_gpio_out(&i2c->qdev, &s->pin, 1);
 
@@ -226,6 +234,13 @@ static int tmp105_init(I2CSlave *i2c)
     return 0;
 }
 
+static void tmp105_initfn(Object *obj)
+{
+    object_property_add(obj, "temperature", "int",
+                        tmp105_get_temperature,
+                        tmp105_set_temperature, NULL, NULL, NULL);
+}
+
 static void tmp105_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -238,10 +253,11 @@ static void tmp105_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_tmp105;
 }
 
-static TypeInfo tmp105_info = {
-    .name          = "tmp105",
+static const TypeInfo tmp105_info = {
+    .name          = TYPE_TMP105,
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(TMP105State),
+    .instance_init = tmp105_initfn,
     .class_init    = tmp105_class_init,
 };
 
diff --git a/hw/tmp105.h b/hw/tmp105.h
new file mode 100644
index 0000000..d218919
--- /dev/null
+++ b/hw/tmp105.h
@@ -0,0 +1,47 @@
+/*
+ * Texas Instruments TMP105 Temperature Sensor
+ *
+ * Browse the data sheet:
+ *
+ *    http://www.ti.com/lit/gpn/tmp105
+ *
+ * Copyright (C) 2012 Alex Horn <alex.horn@cs.ox.ac.uk>
+ * Copyright (C) 2008-2012 Andrzej Zaborowski <balrogg@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_TMP105_H
+#define QEMU_TMP105_H
+
+#include "i2c.h"
+#include "tmp105_regs.h"
+
+#define TYPE_TMP105 "tmp105"
+#define TMP105(obj) OBJECT_CHECK(TMP105State, (obj), TYPE_TMP105)
+
+/**
+ * TMP105State:
+ * @config: Bits 5 and 6 (value 32 and 64) determine the precision of the
+ * temperature. See Table 8 in the data sheet.
+ *
+ * @see_also: http://www.ti.com/lit/gpn/tmp105
+ */
+typedef struct TMP105State {
+    /*< private >*/
+    I2CSlave i2c;
+    /*< public >*/
+
+    uint8_t len;
+    uint8_t buf[2];
+    qemu_irq pin;
+
+    uint8_t pointer;
+    uint8_t config;
+    int16_t temperature;
+    int16_t limit[2];
+    int faults;
+    uint8_t alarm;
+} TMP105State;
+
+#endif
diff --git a/hw/tmp105_regs.h b/hw/tmp105_regs.h
new file mode 100644
index 0000000..9b55aba
--- /dev/null
+++ b/hw/tmp105_regs.h
@@ -0,0 +1,50 @@
+/*
+ * Texas Instruments TMP105 Temperature Sensor I2C messages
+ *
+ * Browse the data sheet:
+ *
+ *    http://www.ti.com/lit/gpn/tmp105
+ *
+ * Copyright (C) 2012 Alex Horn <alex.horn@cs.ox.ac.uk>
+ * Copyright (C) 2008-2012 Andrzej Zaborowski <balrogg@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_TMP105_MSGS_H
+#define QEMU_TMP105_MSGS_H
+
+/**
+ * TMP105Reg:
+ * @TMP105_REG_TEMPERATURE: Temperature register
+ * @TMP105_REG_CONFIG: Configuration register
+ * @TMP105_REG_T_LOW: Low temperature register (also known as T_hyst)
+ * @TMP105_REG_T_HIGH: High temperature register (also known as T_OS)
+ *
+ * The following temperature sensors are
+ * compatible with the TMP105 registers:
+ * - adt75
+ * - ds1775
+ * - ds75
+ * - lm75
+ * - lm75a
+ * - max6625
+ * - max6626
+ * - mcp980x
+ * - stds75
+ * - tcn75
+ * - tmp100
+ * - tmp101
+ * - tmp105
+ * - tmp175
+ * - tmp275
+ * - tmp75
+ **/
+typedef enum TMP105Reg {
+    TMP105_REG_TEMPERATURE = 0,
+    TMP105_REG_CONFIG,
+    TMP105_REG_T_LOW,
+    TMP105_REG_T_HIGH,
+} TMP105Reg;
+
+#endif
diff --git a/hw/tosa.c b/hw/tosa.c
index 512278c..efea109 100644
--- a/hw/tosa.c
+++ b/hw/tosa.c
@@ -17,13 +17,13 @@
 #include "devices.h"
 #include "sharpsl.h"
 #include "pcmcia.h"
-#include "block.h"
+#include "block/block.h"
 #include "boards.h"
 #include "i2c.h"
 #include "ssi.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "sysbus.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #define TOSA_RAM    0x04000000
 #define TOSA_ROM	0x00800000
@@ -251,6 +251,7 @@ static QEMUMachine tosapda_machine = {
     .name = "tosa",
     .desc = "Tosa PDA (PXA255)",
     .init = tosa_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void tosapda_machine_init(void)
@@ -270,7 +271,7 @@ static void tosa_dac_class_init(ObjectClass *klass, void *data)
     k->send = tosa_dac_send;
 }
 
-static TypeInfo tosa_dac_info = {
+static const TypeInfo tosa_dac_info = {
     .name          = "tosa_dac",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(TosaDACState),
@@ -285,7 +286,7 @@ static void tosa_ssp_class_init(ObjectClass *klass, void *data)
     k->transfer = tosa_ssp_tansfer;
 }
 
-static TypeInfo tosa_ssp_info = {
+static const TypeInfo tosa_ssp_info = {
     .name          = "tosa-ssp",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(SSISlave),
diff --git a/hw/tpci200.c b/hw/tpci200.c
new file mode 100644
index 0000000..a4823fb
--- /dev/null
+++ b/hw/tpci200.c
@@ -0,0 +1,671 @@
+/*
+ * QEMU TEWS TPCI200 IndustryPack carrier emulation
+ *
+ * Copyright (C) 2012 Igalia, S.L.
+ * Author: Alberto Garcia <agarcia@igalia.com>
+ *
+ * This code is licensed under the GNU GPL v2 or (at your option) any
+ * later version.
+ */
+
+#include "ipack.h"
+#include "pci/pci.h"
+#include "qemu/bitops.h"
+#include <stdio.h>
+
+/* #define DEBUG_TPCI */
+
+#ifdef DEBUG_TPCI
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "TPCI200: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) do { } while (0)
+#endif
+
+#define N_MODULES 4
+
+#define IP_ID_SPACE  2
+#define IP_INT_SPACE 3
+#define IP_IO_SPACE_ADDR_MASK  0x7F
+#define IP_ID_SPACE_ADDR_MASK  0x3F
+#define IP_INT_SPACE_ADDR_MASK 0x3F
+
+#define STATUS_INT(IP, INTNO) BIT((IP) * 2 + (INTNO))
+#define STATUS_TIME(IP)       BIT((IP) + 12)
+#define STATUS_ERR_ANY        0xF00
+
+#define CTRL_CLKRATE          BIT(0)
+#define CTRL_RECOVER          BIT(1)
+#define CTRL_TIME_INT         BIT(2)
+#define CTRL_ERR_INT          BIT(3)
+#define CTRL_INT_EDGE(INTNO)  BIT(4 + (INTNO))
+#define CTRL_INT(INTNO)       BIT(6 + (INTNO))
+
+#define REG_REV_ID    0x00
+#define REG_IP_A_CTRL 0x02
+#define REG_IP_B_CTRL 0x04
+#define REG_IP_C_CTRL 0x06
+#define REG_IP_D_CTRL 0x08
+#define REG_RESET     0x0A
+#define REG_STATUS    0x0C
+#define IP_N_FROM_REG(REG) ((REG) / 2 - 1)
+
+typedef struct {
+    PCIDevice dev;
+    IPackBus bus;
+    MemoryRegion mmio;
+    MemoryRegion io;
+    MemoryRegion las0;
+    MemoryRegion las1;
+    MemoryRegion las2;
+    MemoryRegion las3;
+    bool big_endian[3];
+    uint8_t ctrl[N_MODULES];
+    uint16_t status;
+    uint8_t int_set;
+} TPCI200State;
+
+#define TYPE_TPCI200 "tpci200"
+
+#define TPCI200(obj) \
+    OBJECT_CHECK(TPCI200State, (obj), TYPE_TPCI200)
+
+static const uint8_t local_config_regs[] = {
+    0x00, 0xFF, 0xFF, 0x0F, 0x00, 0xFC, 0xFF, 0x0F, 0x00, 0x00, 0x00,
+    0x0E, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+    0x00, 0x08, 0x01, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00, 0x00, 0x01,
+    0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0xA0, 0x60, 0x41, 0xD4,
+    0xA2, 0x20, 0x41, 0x14, 0xA2, 0x20, 0x41, 0x14, 0xA2, 0x20, 0x01,
+    0x14, 0x00, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x08, 0x01, 0x02,
+    0x00, 0x04, 0x01, 0x00, 0x00, 0x01, 0x01, 0x00, 0x80, 0x02, 0x41,
+    0x00, 0x00, 0x00, 0x00, 0x40, 0x7A, 0x00, 0x52, 0x92, 0x24, 0x02
+};
+
+static void adjust_addr(bool big_endian, hwaddr *addr, unsigned size)
+{
+    /* During 8 bit access in big endian mode,
+       odd and even addresses are swapped */
+    if (big_endian && size == 1) {
+        *addr ^= 1;
+    }
+}
+
+static uint64_t adjust_value(bool big_endian, uint64_t *val, unsigned size)
+{
+    /* Local spaces only support 8/16 bit access,
+     * so there's no need to care for sizes > 2 */
+    if (big_endian && size == 2) {
+        *val = bswap16(*val);
+    }
+    return *val;
+}
+
+static void tpci200_set_irq(void *opaque, int intno, int level)
+{
+    IPackDevice *ip = opaque;
+    IPackBus *bus = IPACK_BUS(qdev_get_parent_bus(DEVICE(ip)));
+    PCIDevice *pcidev = PCI_DEVICE(BUS(bus)->parent);
+    TPCI200State *dev = TPCI200(pcidev);
+    unsigned ip_n = ip->slot;
+    uint16_t prev_status = dev->status;
+
+    assert(ip->slot >= 0 && ip->slot < N_MODULES);
+
+    /* The requested interrupt must be enabled in the IP CONTROL
+     * register */
+    if (!(dev->ctrl[ip_n] & CTRL_INT(intno))) {
+        return;
+    }
+
+    /* Update the interrupt status in the IP STATUS register */
+    if (level) {
+        dev->status |=  STATUS_INT(ip_n, intno);
+    } else {
+        dev->status &= ~STATUS_INT(ip_n, intno);
+    }
+
+    /* Return if there are no changes */
+    if (dev->status == prev_status) {
+        return;
+    }
+
+    DPRINTF("IP %u INT%u#: %u\n", ip_n, intno, level);
+
+    /* Check if the interrupt is edge sensitive */
+    if (dev->ctrl[ip_n] & CTRL_INT_EDGE(intno)) {
+        if (level) {
+            qemu_set_irq(dev->dev.irq[0], !dev->int_set);
+            qemu_set_irq(dev->dev.irq[0],  dev->int_set);
+        }
+    } else {
+        unsigned i, j;
+        uint16_t level_status = dev->status;
+
+        /* Check if there are any level sensitive interrupts set by
+           removing the ones that are edge sensitive from the status
+           register */
+        for (i = 0; i < N_MODULES; i++) {
+            for (j = 0; j < 2; j++) {
+                if (dev->ctrl[i] & CTRL_INT_EDGE(j)) {
+                    level_status &= ~STATUS_INT(i, j);
+                }
+            }
+        }
+
+        if (level_status && !dev->int_set) {
+            qemu_irq_raise(dev->dev.irq[0]);
+            dev->int_set = 1;
+        } else if (!level_status && dev->int_set) {
+            qemu_irq_lower(dev->dev.irq[0]);
+            dev->int_set = 0;
+        }
+    }
+}
+
+static uint64_t tpci200_read_cfg(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    uint8_t ret = 0;
+    if (addr < ARRAY_SIZE(local_config_regs)) {
+        ret = local_config_regs[addr];
+    }
+    /* Endianness is stored in the first bit of these registers */
+    if ((addr == 0x2b && s->big_endian[0]) ||
+        (addr == 0x2f && s->big_endian[1]) ||
+        (addr == 0x33 && s->big_endian[2])) {
+        ret |= 1;
+    }
+    DPRINTF("Read from LCR 0x%x: 0x%x\n", (unsigned) addr, (unsigned) ret);
+    return ret;
+}
+
+static void tpci200_write_cfg(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
+{
+    TPCI200State *s = opaque;
+    /* Endianness is stored in the first bit of these registers */
+    if (addr == 0x2b || addr == 0x2f || addr == 0x33) {
+        unsigned las = (addr - 0x2b) / 4;
+        s->big_endian[las] = val & 1;
+        DPRINTF("LAS%u big endian mode: %u\n", las, (unsigned) val & 1);
+    } else {
+        DPRINTF("Write to LCR 0x%x: 0x%x\n", (unsigned) addr, (unsigned) val);
+    }
+}
+
+static uint64_t tpci200_read_las0(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    uint64_t ret = 0;
+
+    switch (addr) {
+
+    case REG_REV_ID:
+        DPRINTF("Read REVISION ID\n"); /* Current value is 0x00 */
+        break;
+
+    case REG_IP_A_CTRL:
+    case REG_IP_B_CTRL:
+    case REG_IP_C_CTRL:
+    case REG_IP_D_CTRL:
+        {
+            unsigned ip_n = IP_N_FROM_REG(addr);
+            ret = s->ctrl[ip_n];
+            DPRINTF("Read IP %c CONTROL: 0x%x\n", 'A' + ip_n, (unsigned) ret);
+        }
+        break;
+
+    case REG_RESET:
+        DPRINTF("Read RESET\n"); /* Not implemented */
+        break;
+
+    case REG_STATUS:
+        ret = s->status;
+        DPRINTF("Read STATUS: 0x%x\n", (unsigned) ret);
+        break;
+
+    /* Reserved */
+    default:
+        DPRINTF("Unsupported read from LAS0 0x%x\n", (unsigned) addr);
+        break;
+    }
+
+    return adjust_value(s->big_endian[0], &ret, size);
+}
+
+static void tpci200_write_las0(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+
+    adjust_value(s->big_endian[0], &val, size);
+
+    switch (addr) {
+
+    case REG_REV_ID:
+        DPRINTF("Write Revision ID: 0x%x\n", (unsigned) val); /* No effect */
+        break;
+
+    case REG_IP_A_CTRL:
+    case REG_IP_B_CTRL:
+    case REG_IP_C_CTRL:
+    case REG_IP_D_CTRL:
+        {
+            unsigned ip_n = IP_N_FROM_REG(addr);
+            s->ctrl[ip_n] = val;
+            DPRINTF("Write IP %c CONTROL: 0x%x\n", 'A' + ip_n, (unsigned) val);
+        }
+        break;
+
+    case REG_RESET:
+        DPRINTF("Write RESET: 0x%x\n", (unsigned) val); /* Not implemented */
+        break;
+
+    case REG_STATUS:
+        {
+            unsigned i;
+
+            for (i = 0; i < N_MODULES; i++) {
+                IPackDevice *ip = ipack_device_find(&s->bus, i);
+
+                if (ip != NULL) {
+                    if (val & STATUS_INT(i, 0)) {
+                        DPRINTF("Clear IP %c INT0# status\n", 'A' + i);
+                        qemu_irq_lower(ip->irq[0]);
+                    }
+                    if (val & STATUS_INT(i, 1)) {
+                        DPRINTF("Clear IP %c INT1# status\n", 'A' + i);
+                        qemu_irq_lower(ip->irq[1]);
+                    }
+                }
+
+                if (val & STATUS_TIME(i)) {
+                    DPRINTF("Clear IP %c timeout\n", 'A' + i);
+                    s->status &= ~STATUS_TIME(i);
+                }
+            }
+
+            if (val & STATUS_ERR_ANY) {
+                DPRINTF("Unexpected write to STATUS register: 0x%x\n",
+                        (unsigned) val);
+            }
+        }
+        break;
+
+    /* Reserved */
+    default:
+        DPRINTF("Unsupported write to LAS0 0x%x: 0x%x\n",
+                (unsigned) addr, (unsigned) val);
+        break;
+    }
+}
+
+static uint64_t tpci200_read_las1(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    uint64_t ret = 0;
+    unsigned ip_n, space;
+    uint8_t offset;
+
+    adjust_addr(s->big_endian[1], &addr, size);
+
+    /*
+     * The address is divided into the IP module number (0-4), the IP
+     * address space (I/O, ID, INT) and the offset within that space.
+     */
+    ip_n = addr >> 8;
+    space = (addr >> 6) & 3;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Read LAS1: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        switch (space) {
+
+        case IP_ID_SPACE:
+            offset = addr & IP_ID_SPACE_ADDR_MASK;
+            if (k->id_read) {
+                ret = k->id_read(ip, offset);
+            }
+            break;
+
+        case IP_INT_SPACE:
+            offset = addr & IP_INT_SPACE_ADDR_MASK;
+
+            /* Read address 0 to ACK IP INT0# and address 2 to ACK IP INT1# */
+            if (offset == 0 || offset == 2) {
+                unsigned intno = offset / 2;
+                bool int_set = s->status & STATUS_INT(ip_n, intno);
+                bool int_edge_sensitive = s->ctrl[ip_n] & CTRL_INT_EDGE(intno);
+                if (int_set && !int_edge_sensitive) {
+                    qemu_irq_lower(ip->irq[intno]);
+                }
+            }
+
+            if (k->int_read) {
+                ret = k->int_read(ip, offset);
+            }
+            break;
+
+        default:
+            offset = addr & IP_IO_SPACE_ADDR_MASK;
+            if (k->io_read) {
+                ret = k->io_read(ip, offset);
+            }
+            break;
+        }
+    }
+
+    return adjust_value(s->big_endian[1], &ret, size);
+}
+
+static void tpci200_write_las1(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    unsigned ip_n, space;
+    uint8_t offset;
+
+    adjust_addr(s->big_endian[1], &addr, size);
+    adjust_value(s->big_endian[1], &val, size);
+
+    /*
+     * The address is divided into the IP module number, the IP
+     * address space (I/O, ID, INT) and the offset within that space.
+     */
+    ip_n = addr >> 8;
+    space = (addr >> 6) & 3;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Write LAS1: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        switch (space) {
+
+        case IP_ID_SPACE:
+            offset = addr & IP_ID_SPACE_ADDR_MASK;
+            if (k->id_write) {
+                k->id_write(ip, offset, val);
+            }
+            break;
+
+        case IP_INT_SPACE:
+            offset = addr & IP_INT_SPACE_ADDR_MASK;
+            if (k->int_write) {
+                k->int_write(ip, offset, val);
+            }
+            break;
+
+        default:
+            offset = addr & IP_IO_SPACE_ADDR_MASK;
+            if (k->io_write) {
+                k->io_write(ip, offset, val);
+            }
+            break;
+        }
+    }
+}
+
+static uint64_t tpci200_read_las2(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    uint64_t ret = 0;
+    unsigned ip_n;
+    uint32_t offset;
+
+    adjust_addr(s->big_endian[2], &addr, size);
+
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    ip_n = addr >> 23;
+    offset = addr & 0x7fffff;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Read LAS2: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_read16) {
+            ret = k->mem_read16(ip, offset);
+        }
+    }
+
+    return adjust_value(s->big_endian[2], &ret, size);
+}
+
+static void tpci200_write_las2(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    unsigned ip_n;
+    uint32_t offset;
+
+    adjust_addr(s->big_endian[2], &addr, size);
+    adjust_value(s->big_endian[2], &val, size);
+
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    ip_n = addr >> 23;
+    offset = addr & 0x7fffff;
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Write LAS2: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_write16) {
+            k->mem_write16(ip, offset, val);
+        }
+    }
+}
+
+static uint64_t tpci200_read_las3(void *opaque, hwaddr addr, unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    uint64_t ret = 0;
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    unsigned ip_n = addr >> 22;
+    uint32_t offset = addr & 0x3fffff;
+
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Read LAS3: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_read8) {
+            ret = k->mem_read8(ip, offset);
+        }
+    }
+
+    return ret;
+}
+
+static void tpci200_write_las3(void *opaque, hwaddr addr, uint64_t val,
+                               unsigned size)
+{
+    TPCI200State *s = opaque;
+    IPackDevice *ip;
+    /*
+     * The address is divided into the IP module number and the offset
+     * within the IP module MEM space.
+     */
+    unsigned ip_n = addr >> 22;
+    uint32_t offset = addr & 0x3fffff;
+
+    ip = ipack_device_find(&s->bus, ip_n);
+
+    if (ip == NULL) {
+        DPRINTF("Write LAS3: IP module %u not installed\n", ip_n);
+    } else {
+        IPackDeviceClass *k = IPACK_DEVICE_GET_CLASS(ip);
+        if (k->mem_write8) {
+            k->mem_write8(ip, offset, val);
+        }
+    }
+}
+
+static const MemoryRegionOps tpci200_cfg_ops = {
+    .read = tpci200_read_cfg,
+    .write = tpci200_write_cfg,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 4
+    },
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1
+    }
+};
+
+static const MemoryRegionOps tpci200_las0_ops = {
+    .read = tpci200_read_las0,
+    .write = tpci200_write_las0,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 2,
+        .max_access_size = 2
+    }
+};
+
+static const MemoryRegionOps tpci200_las1_ops = {
+    .read = tpci200_read_las1,
+    .write = tpci200_write_las1,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 2
+    }
+};
+
+static const MemoryRegionOps tpci200_las2_ops = {
+    .read = tpci200_read_las2,
+    .write = tpci200_write_las2,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 2
+    }
+};
+
+static const MemoryRegionOps tpci200_las3_ops = {
+    .read = tpci200_read_las3,
+    .write = tpci200_write_las3,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid =  {
+        .min_access_size = 1,
+        .max_access_size = 1
+    }
+};
+
+static int tpci200_initfn(PCIDevice *pci_dev)
+{
+    TPCI200State *s = TPCI200(pci_dev);
+    uint8_t *c = s->dev.config;
+
+    pci_set_word(c + PCI_COMMAND, 0x0003);
+    pci_set_word(c + PCI_STATUS,  0x0280);
+
+    pci_set_byte(c + PCI_INTERRUPT_PIN, 0x01); /* Interrupt pin A */
+
+    pci_set_byte(c + PCI_CAPABILITY_LIST, 0x40);
+    pci_set_long(c + 0x40, 0x48014801);
+    pci_set_long(c + 0x48, 0x00024C06);
+    pci_set_long(c + 0x4C, 0x00000003);
+
+    memory_region_init_io(&s->mmio, &tpci200_cfg_ops,
+                          s, "tpci200_mmio", 128);
+    memory_region_init_io(&s->io,   &tpci200_cfg_ops,
+                          s, "tpci200_io",   128);
+    memory_region_init_io(&s->las0, &tpci200_las0_ops,
+                          s, "tpci200_las0", 256);
+    memory_region_init_io(&s->las1, &tpci200_las1_ops,
+                          s, "tpci200_las1", 1024);
+    memory_region_init_io(&s->las2, &tpci200_las2_ops,
+                          s, "tpci200_las2", 1024*1024*32);
+    memory_region_init_io(&s->las3, &tpci200_las3_ops,
+                          s, "tpci200_las3", 1024*1024*16);
+    pci_register_bar(&s->dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mmio);
+    pci_register_bar(&s->dev, 1, PCI_BASE_ADDRESS_SPACE_IO,     &s->io);
+    pci_register_bar(&s->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las0);
+    pci_register_bar(&s->dev, 3, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las1);
+    pci_register_bar(&s->dev, 4, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las2);
+    pci_register_bar(&s->dev, 5, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->las3);
+
+    ipack_bus_new_inplace(&s->bus, DEVICE(&s->dev), NULL,
+                          N_MODULES, tpci200_set_irq);
+
+    return 0;
+}
+
+static void tpci200_exitfn(PCIDevice *pci_dev)
+{
+    TPCI200State *s = TPCI200(pci_dev);
+
+    memory_region_destroy(&s->mmio);
+    memory_region_destroy(&s->io);
+    memory_region_destroy(&s->las0);
+    memory_region_destroy(&s->las1);
+    memory_region_destroy(&s->las2);
+    memory_region_destroy(&s->las3);
+}
+
+static const VMStateDescription vmstate_tpci200 = {
+    .name = "tpci200",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, TPCI200State),
+        VMSTATE_BOOL_ARRAY(big_endian, TPCI200State, 3),
+        VMSTATE_UINT8_ARRAY(ctrl, TPCI200State, N_MODULES),
+        VMSTATE_UINT16(status, TPCI200State),
+        VMSTATE_UINT8(int_set, TPCI200State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void tpci200_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = tpci200_initfn;
+    k->exit = tpci200_exitfn;
+    k->vendor_id = PCI_VENDOR_ID_TEWS;
+    k->device_id = PCI_DEVICE_ID_TEWS_TPCI200;
+    k->class_id = PCI_CLASS_BRIDGE_OTHER;
+    k->subsystem_vendor_id = PCI_VENDOR_ID_TEWS;
+    k->subsystem_id = 0x300A;
+    dc->desc = "TEWS TPCI200 IndustryPack carrier";
+    dc->vmsd = &vmstate_tpci200;
+}
+
+static const TypeInfo tpci200_info = {
+    .name          = TYPE_TPCI200,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(TPCI200State),
+    .class_init    = tpci200_class_init,
+};
+
+static void tpci200_register_types(void)
+{
+    type_register_static(&tpci200_info);
+}
+
+type_init(tpci200_register_types)
diff --git a/hw/tsc2005.c b/hw/tsc2005.c
index ce05652..8717dae 100644
--- a/hw/tsc2005.c
+++ b/hw/tsc2005.c
@@ -19,8 +19,8 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 #include "devices.h"
 #include "spi.h"
 
diff --git a/hw/tsc210x.c b/hw/tsc210x.c
index 710245d..7c32b00 100644
--- a/hw/tsc210x.c
+++ b/hw/tsc210x.c
@@ -21,8 +21,8 @@
 
 #include "hw.h"
 #include "audio/audio.h"
-#include "qemu-timer.h"
-#include "console.h"
+#include "qemu/timer.h"
+#include "ui/console.h"
 #include "omap.h"	/* For I2SCodec and uWireSlave */
 #include "devices.h"
 #include "spi.h"
diff --git a/hw/tusb6010.c b/hw/tusb6010.c
index 325200b..2c7d033 100644
--- a/hw/tusb6010.c
+++ b/hw/tusb6010.c
@@ -19,7 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "usb.h"
 #include "omap.h"
 #include "irq.h"
@@ -740,7 +740,7 @@ static void tusb6010_irq(void *opaque, int source, int level)
 
 static void tusb6010_reset(DeviceState *dev)
 {
-    TUSBState *s = FROM_SYSBUS(TUSBState, sysbus_from_qdev(dev));
+    TUSBState *s = FROM_SYSBUS(TUSBState, SYS_BUS_DEVICE(dev));
     int i;
 
     s->test_reset = TUSB_PROD_TEST_RESET_VAL;
@@ -798,7 +798,7 @@ static void tusb6010_class_init(ObjectClass *klass, void *data)
     dc->reset = tusb6010_reset;
 }
 
-static TypeInfo tusb6010_info = {
+static const TypeInfo tusb6010_info = {
     .name          = "tusb6010",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(TUSBState),
diff --git a/hw/twl4030.c b/hw/twl4030.c
index 355a59e..27e19e9 100644
--- a/hw/twl4030.c
+++ b/hw/twl4030.c
@@ -24,11 +24,11 @@
 
 #include <sys/time.h>
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i2c.h"
-#include "sysemu.h"
-#include "console.h"
-#include "cpu-all.h"
+#include "sysemu/sysemu.h"
+#include "ui/console.h"
+#include "exec/cpu-all.h"
 
 //#define DEBUG_GENERAL
 //#define DEBUG_RTC
@@ -1747,7 +1747,7 @@ void *twl4030_init(i2c_bus *bus, qemu_irq irq1, qemu_irq irq2,
     for (i = 0; i < ARRAY_SIZE(twl4030_info); i++) {
         DeviceState *ds = i2c_create_slave(bus, twl4030_info[i].name,
                                            0x48 + i);
-        s->i2c[i] = FROM_I2C_SLAVE(TWL4030NodeState, I2C_SLAVE_FROM_QDEV(ds));
+        s->i2c[i] = FROM_I2C_SLAVE(TWL4030NodeState, I2C_SLAVE(ds));
         s->i2c[i]->twl4030 = s;
     }
 
diff --git a/hw/twl92230.c b/hw/twl92230.c
index 0d70d84..70d9b03 100644
--- a/hw/twl92230.c
+++ b/hw/twl92230.c
@@ -20,10 +20,10 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "i2c.h"
-#include "sysemu.h"
-#include "console.h"
+#include "sysemu/sysemu.h"
+#include "ui/console.h"
 
 #define VERBOSE 1
 
@@ -867,7 +867,7 @@ static void twl92230_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_menelaus;
 }
 
-static TypeInfo twl92230_info = {
+static const TypeInfo twl92230_info = {
     .name          = "twl92230",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(MenelausState),
diff --git a/uboot_image.h b/hw/uboot_image.h
index 9fc2760..9fc2760 100644
--- a/uboot_image.h
+++ b/hw/uboot_image.h
diff --git a/hw/unin_pci.c b/hw/unin_pci.c
index 9981d94..f1c3c20 100644
--- a/hw/unin_pci.c
+++ b/hw/unin_pci.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "ppc_mac.h"
-#include "pci.h"
-#include "pci_host.h"
+#include "ppc/mac.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
 
 /* debug UniNorth */
 //#define DEBUG_UNIN
diff --git a/hw/usb.h b/hw/usb.h
index 7d6de69..bc42639 100644
--- a/hw/usb.h
+++ b/hw/usb.h
@@ -26,7 +26,7 @@
  */
 
 #include "qdev.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 /* Constants related to the USB / PCI interaction */
 #define USB_SBRN    0x60 /* Serial Bus Release Number Register */
@@ -197,6 +197,7 @@ struct USBEndpoint {
 
 enum USBDeviceFlags {
     USB_DEV_FLAG_FULL_PATH,
+    USB_DEV_FLAG_IS_HOST,
 };
 
 /* definition of a USB device */
@@ -229,6 +230,7 @@ struct USBDevice {
     USBEndpoint ep_out[USB_MAX_ENDPOINTS];
 
     QLIST_HEAD(, USBDescString) strings;
+    const USBDesc *usb_desc; /* Overrides class usb_desc if not NULL */
     const USBDescDevice *device;
 
     int configuration;
@@ -282,7 +284,7 @@ typedef struct USBDeviceClass {
      * Called from handle_packet().
      *
      * Status gets stored in p->status, and if p->status == USB_RET_SUCCESS
-     * then the number of bytes transfered is stored in p->actual_length
+     * then the number of bytes transferred is stored in p->actual_length
      */
     void (*handle_control)(USBDevice *dev, USBPacket *p, int request, int value,
                            int index, int length, uint8_t *data);
@@ -292,7 +294,7 @@ typedef struct USBDeviceClass {
      * Called from handle_packet().
      *
      * Status gets stored in p->status, and if p->status == USB_RET_SUCCESS
-     * then the number of bytes transfered is stored in p->actual_length
+     * then the number of bytes transferred is stored in p->actual_length
      */
     void (*handle_data)(USBDevice *dev, USBPacket *p);
 
@@ -305,6 +307,12 @@ typedef struct USBDeviceClass {
      */
     void (*flush_ep_queue)(USBDevice *dev, USBEndpoint *ep);
 
+    /*
+     * Called by the hcd to let the device know the queue for an endpoint
+     * has been unlinked / stopped. Optional may be NULL.
+     */
+    void (*ep_stopped)(USBDevice *dev, USBEndpoint *ep);
+
     const char *product_desc;
     const USBDesc *usb_desc;
 } USBDeviceClass;
@@ -358,7 +366,7 @@ struct USBPacket {
     bool short_not_ok;
     bool int_req;
     int status; /* USB_RET_* status code */
-    int actual_length; /* Number of bytes actually transfered */
+    int actual_length; /* Number of bytes actually transferred */
     /* Internal use by the USB layer.  */
     USBPacketState state;
     USBCombinedPacket *combined;
@@ -427,7 +435,7 @@ int set_usb_string(uint8_t *buf, const char *str);
 /* usb-linux.c */
 USBDevice *usb_host_device_open(USBBus *bus, const char *devname);
 int usb_host_device_close(const char *devname);
-void usb_host_info(Monitor *mon);
+void usb_host_info(Monitor *mon, const QDict *qdict);
 
 /* usb-bt.c */
 USBDevice *usb_bt_init(USBBus *bus, HCIInfo *hci);
@@ -537,11 +545,23 @@ void usb_device_set_interface(USBDevice *dev, int interface,
 
 void usb_device_flush_ep_queue(USBDevice *dev, USBEndpoint *ep);
 
+void usb_device_ep_stopped(USBDevice *dev, USBEndpoint *ep);
+
 const char *usb_device_get_product_desc(USBDevice *dev);
 
 const USBDesc *usb_device_get_usb_desc(USBDevice *dev);
 
 int ehci_create_ich9_with_companions(PCIBus *bus, int slot);
 
-#endif
+/* quirks.c */
+
+/* In bulk endpoints are streaming data sources (iow behave like isoc eps) */
+#define USB_QUIRK_BUFFER_BULK_IN	0x01
+/* Bulk pkts in FTDI format, need special handling when combining packets */
+#define USB_QUIRK_IS_FTDI		0x02
 
+int usb_get_quirks(uint16_t vendor_id, uint16_t product_id,
+                   uint8_t interface_class, uint8_t interface_subclass,
+                   uint8_t interface_protocol);
+
+#endif
diff --git a/hw/usb/Makefile.objs b/hw/usb/Makefile.objs
index 5a4eeb6..d1bbbc0 100644
--- a/hw/usb/Makefile.objs
+++ b/hw/usb/Makefile.objs
@@ -4,11 +4,11 @@ common-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o hcd-ehci-pci.o hcd-ehci-sysbus.o
 common-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
 common-obj-y += libhw.o
 
-common-obj-$(CONFIG_SMARTCARD) += dev-smartcard-reader.o
-common-obj-$(CONFIG_USB_REDIR) += redirect.o
+common-obj-$(CONFIG_USB_REDIR) += redirect.o quirks.o
 
 common-obj-y += core.o combined-packet.o bus.o desc.o dev-hub.o
 common-obj-y += host-$(HOST_USB).o dev-bluetooth.o
 common-obj-y += dev-hid.o dev-storage.o dev-wacom.o
 common-obj-y += dev-serial.o dev-network.o dev-audio.o
+common-obj-y += dev-smartcard-reader.o
 common-obj-y += dev-uas.o
diff --git a/hw/usb/bus.c b/hw/usb/bus.c
index 55d0edd..e58cd9a 100644
--- a/hw/usb/bus.c
+++ b/hw/usb/bus.c
@@ -1,8 +1,8 @@
 #include "hw/hw.h"
 #include "hw/usb.h"
 #include "hw/qdev.h"
-#include "sysemu.h"
-#include "monitor.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
 #include "trace.h"
 
 static void usb_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent);
@@ -166,6 +166,9 @@ const char *usb_device_get_product_desc(USBDevice *dev)
 const USBDesc *usb_device_get_usb_desc(USBDevice *dev)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+    if (dev->usb_desc) {
+        return dev->usb_desc;
+    }
     return klass->usb_desc;
 }
 
@@ -186,6 +189,14 @@ void usb_device_flush_ep_queue(USBDevice *dev, USBEndpoint *ep)
     }
 }
 
+void usb_device_ep_stopped(USBDevice *dev, USBEndpoint *ep)
+{
+    USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+    if (klass->ep_stopped) {
+        klass->ep_stopped(dev, ep);
+    }
+}
+
 static int usb_qdev_init(DeviceState *qdev)
 {
     USBDevice *dev = USB_DEVICE(qdev);
@@ -531,7 +542,7 @@ static char *usb_get_fw_dev_path(DeviceState *qdev)
     return fw_path;
 }
 
-void usb_info(Monitor *mon)
+void usb_info(Monitor *mon, const QDict *qdict)
 {
     USBBus *bus;
     USBDevice *dev;
@@ -617,7 +628,7 @@ static void usb_device_class_init(ObjectClass *klass, void *data)
     k->props    = usb_props;
 }
 
-static TypeInfo usb_device_type_info = {
+static const TypeInfo usb_device_type_info = {
     .name = TYPE_USB_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(USBDevice),
diff --git a/hw/usb/combined-packet.c b/hw/usb/combined-packet.c
index 4a0c299..13f6602 100644
--- a/hw/usb/combined-packet.c
+++ b/hw/usb/combined-packet.c
@@ -21,7 +21,7 @@
  */
 #include "qemu-common.h"
 #include "hw/usb.h"
-#include "iov.h"
+#include "qemu/iov.h"
 #include "trace.h"
 
 static void usb_combined_packet_add(USBCombinedPacket *combined, USBPacket *p)
diff --git a/hw/usb/core.c b/hw/usb/core.c
index 52b5310..d057aab 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -25,7 +25,7 @@
  */
 #include "qemu-common.h"
 #include "hw/usb.h"
-#include "iov.h"
+#include "qemu/iov.h"
 #include "trace.h"
 
 void usb_attach(USBPort *port)
@@ -406,7 +406,11 @@ void usb_handle_packet(USBDevice *dev, USBPacket *p)
     if (QTAILQ_EMPTY(&p->ep->queue) || p->ep->pipeline) {
         usb_process_one(p);
         if (p->status == USB_RET_ASYNC) {
+            /* hcd drivers cannot handle async for isoc */
             assert(p->ep->type != USB_ENDPOINT_XFER_ISOC);
+            /* using async for interrupt packets breaks migration */
+            assert(p->ep->type != USB_ENDPOINT_XFER_INT ||
+                   (dev->flags & USB_DEV_FLAG_IS_HOST));
             usb_packet_set_state(p, USB_PACKET_ASYNC);
             QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
         } else if (p->status == USB_RET_ADD_TO_QUEUE) {
@@ -757,7 +761,7 @@ USBPacket *usb_ep_find_packet_by_id(USBDevice *dev, int pid, int ep,
     struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
     USBPacket *p;
 
-    while ((p = QTAILQ_FIRST(&uep->queue)) != NULL) {
+    QTAILQ_FOREACH(p, &uep->queue, queue) {
         if (p->id == id) {
             return p;
         }
diff --git a/hw/usb/dev-audio.c b/hw/usb/dev-audio.c
index b669601..b8c79b8 100644
--- a/hw/usb/dev-audio.c
+++ b/hw/usb/dev-audio.c
@@ -684,7 +684,7 @@ static void usb_audio_class_init(ObjectClass *klass, void *data)
     k->set_interface  = usb_audio_set_interface;
 }
 
-static TypeInfo usb_audio_info = {
+static const TypeInfo usb_audio_info = {
     .name          = "usb-audio",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBAudioState),
diff --git a/hw/usb/dev-bluetooth.c b/hw/usb/dev-bluetooth.c
index 39984f5..adbf9d4 100644
--- a/hw/usb/dev-bluetooth.c
+++ b/hw/usb/dev-bluetooth.c
@@ -21,7 +21,7 @@
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "net.h"
+#include "bt/bt.h"
 #include "hw/bt.h"
 
 struct USBBtState {
@@ -555,7 +555,7 @@ static void usb_bt_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_bt;
 }
 
-static TypeInfo bt_info = {
+static const TypeInfo bt_info = {
     .name          = "usb-bt-dongle",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(struct USBBtState),
diff --git a/hw/usb/dev-hid.c b/hw/usb/dev-hid.c
index 55266b1..29b6481 100644
--- a/hw/usb/dev-hid.c
+++ b/hw/usb/dev-hid.c
@@ -23,10 +23,10 @@
  * THE SOFTWARE.
  */
 #include "hw/hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/hid.h"
 
 /* HID interface requests */
@@ -46,6 +46,7 @@ typedef struct USBHIDState {
     USBDevice dev;
     USBEndpoint *intr;
     HIDState hid;
+    uint32_t usb_version;
 } USBHIDState;
 
 enum {
@@ -131,6 +132,36 @@ static const USBDescIface desc_iface_tablet = {
     },
 };
 
+static const USBDescIface desc_iface_tablet2 = {
+    .bInterfaceNumber              = 0,
+    .bNumEndpoints                 = 1,
+    .bInterfaceClass               = USB_CLASS_HID,
+    .bInterfaceProtocol            = 0x02,
+    .ndesc                         = 1,
+    .descs = (USBDescOther[]) {
+        {
+            /* HID descriptor */
+            .data = (uint8_t[]) {
+                0x09,          /*  u8  bLength */
+                USB_DT_HID,    /*  u8  bDescriptorType */
+                0x01, 0x00,    /*  u16 HID_class */
+                0x00,          /*  u8  country_code */
+                0x01,          /*  u8  num_descriptors */
+                USB_DT_REPORT, /*  u8  type: Report */
+                74, 0,         /*  u16 len */
+            },
+        },
+    },
+    .eps = (USBDescEndpoint[]) {
+        {
+            .bEndpointAddress      = USB_DIR_IN | 0x01,
+            .bmAttributes          = USB_ENDPOINT_XFER_INT,
+            .wMaxPacketSize        = 8,
+            .bInterval             = 4, /* 2 ^ (4-1) * 125 usecs = 1 ms */
+        },
+    },
+};
+
 static const USBDescIface desc_iface_keyboard = {
     .bInterfaceNumber              = 0,
     .bNumEndpoints                 = 1,
@@ -196,6 +227,23 @@ static const USBDescDevice desc_device_tablet = {
     },
 };
 
+static const USBDescDevice desc_device_tablet2 = {
+    .bcdUSB                        = 0x0200,
+    .bMaxPacketSize0               = 64,
+    .bNumConfigurations            = 1,
+    .confs = (USBDescConfig[]) {
+        {
+            .bNumInterfaces        = 1,
+            .bConfigurationValue   = 1,
+            .iConfiguration        = STR_CONFIG_TABLET,
+            .bmAttributes          = 0xa0,
+            .bMaxPower             = 50,
+            .nif = 1,
+            .ifs = &desc_iface_tablet2,
+        },
+    },
+};
+
 static const USBDescDevice desc_device_keyboard = {
     .bcdUSB                        = 0x0100,
     .bMaxPacketSize0               = 8,
@@ -239,6 +287,20 @@ static const USBDesc desc_tablet = {
     .str  = desc_strings,
 };
 
+static const USBDesc desc_tablet2 = {
+    .id = {
+        .idVendor          = 0x0627,
+        .idProduct         = 0x0001,
+        .bcdDevice         = 0,
+        .iManufacturer     = STR_MANUFACTURER,
+        .iProduct          = STR_PRODUCT_TABLET,
+        .iSerialNumber     = STR_SERIALNUMBER,
+    },
+    .full = &desc_device_tablet,
+    .high = &desc_device_tablet2,
+    .str  = desc_strings,
+};
+
 static const USBDesc desc_keyboard = {
     .id = {
         .idVendor          = 0x0627,
@@ -439,7 +501,7 @@ static void usb_hid_handle_control(USBDevice *dev, USBPacket *p,
         break;
     case SET_IDLE:
         hs->idle = (uint8_t) (value >> 8);
-        hid_set_next_idle(hs, qemu_get_clock_ns(vm_clock));
+        hid_set_next_idle(hs);
         if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
             hid_pointer_activate(hs);
         }
@@ -461,16 +523,14 @@ static void usb_hid_handle_data(USBDevice *dev, USBPacket *p)
     switch (p->pid) {
     case USB_TOKEN_IN:
         if (p->ep->nr == 1) {
-            int64_t curtime = qemu_get_clock_ns(vm_clock);
             if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
                 hid_pointer_activate(hs);
             }
-            if (!hid_has_events(hs) &&
-                (!hs->idle || hs->next_idle_clock - curtime > 0)) {
+            if (!hid_has_events(hs)) {
                 p->status = USB_RET_NAK;
                 return;
             }
-            hid_set_next_idle(hs, curtime);
+            hid_set_next_idle(hs);
             if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
                 len = hid_pointer_poll(hs, buf, p->iov.size);
             } else if (hs->kind == HID_KEYBOARD) {
@@ -508,6 +568,21 @@ static int usb_hid_initfn(USBDevice *dev, int kind)
 
 static int usb_tablet_initfn(USBDevice *dev)
 {
+    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+
+    switch (us->usb_version) {
+    case 1:
+        dev->usb_desc = &desc_tablet;
+        break;
+    case 2:
+        dev->usb_desc = &desc_tablet2;
+        break;
+    default:
+        error_report("Invalid usb version %d for usb-tabler (must be 1 or 2)",
+                     us->usb_version);
+        return -1;
+    }
+
     return usb_hid_initfn(dev, HID_TABLET);
 }
 
@@ -562,8 +637,14 @@ static void usb_hid_class_initfn(ObjectClass *klass, void *data)
     uc->handle_control = usb_hid_handle_control;
     uc->handle_data    = usb_hid_handle_data;
     uc->handle_destroy = usb_hid_handle_destroy;
+    uc->handle_attach  = usb_desc_attach;
 }
 
+static Property usb_tablet_properties[] = {
+        DEFINE_PROP_UINT32("usb_version", USBHIDState, usb_version, 2),
+        DEFINE_PROP_END_OF_LIST(),
+};
+
 static void usb_tablet_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -572,11 +653,11 @@ static void usb_tablet_class_initfn(ObjectClass *klass, void *data)
     usb_hid_class_initfn(klass, data);
     uc->init           = usb_tablet_initfn;
     uc->product_desc   = "QEMU USB Tablet";
-    uc->usb_desc       = &desc_tablet;
     dc->vmsd = &vmstate_usb_ptr;
+    dc->props = usb_tablet_properties;
 }
 
-static TypeInfo usb_tablet_info = {
+static const TypeInfo usb_tablet_info = {
     .name          = "usb-tablet",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHIDState),
@@ -595,7 +676,7 @@ static void usb_mouse_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_ptr;
 }
 
-static TypeInfo usb_mouse_info = {
+static const TypeInfo usb_mouse_info = {
     .name          = "usb-mouse",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHIDState),
@@ -614,7 +695,7 @@ static void usb_keyboard_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_kbd;
 }
 
-static TypeInfo usb_keyboard_info = {
+static const TypeInfo usb_keyboard_info = {
     .name          = "usb-kbd",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHIDState),
diff --git a/hw/usb/dev-hub.c b/hw/usb/dev-hub.c
index 9ee60dd..79f2f46 100644
--- a/hw/usb/dev-hub.c
+++ b/hw/usb/dev-hub.c
@@ -184,6 +184,7 @@ static void usb_hub_detach(USBPort *port1)
         port->wPortStatus &= ~PORT_STAT_ENABLE;
         port->wPortChange |= PORT_STAT_C_ENABLE;
     }
+    usb_wakeup(s->intr);
 }
 
 static void usb_hub_child_detach(USBPort *port1, USBDevice *child)
@@ -363,6 +364,7 @@ static void usb_hub_handle_control(USBDevice *dev, USBPacket *p,
                     port->wPortChange |= PORT_STAT_C_RESET;
                     /* set enable bit */
                     port->wPortStatus |= PORT_STAT_ENABLE;
+                    usb_wakeup(s->intr);
                 }
                 break;
             case PORT_POWER:
@@ -566,7 +568,7 @@ static void usb_hub_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_hub;
 }
 
-static TypeInfo hub_info = {
+static const TypeInfo hub_info = {
     .name          = "usb-hub",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHubState),
diff --git a/hw/usb/dev-network.c b/hw/usb/dev-network.c
index 14d9e5a..a01a5e7 100644
--- a/hw/usb/dev-network.c
+++ b/hw/usb/dev-network.c
@@ -26,10 +26,11 @@
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "net.h"
-#include "qemu-queue.h"
-#include "sysemu.h"
-#include "iov.h"
+#include "net/net.h"
+#include "qemu/queue.h"
+#include "qemu/config-file.h"
+#include "sysemu/sysemu.h"
+#include "qemu/iov.h"
 
 /*#define TRAFFIC_DEBUG*/
 /* Thanks to NetChip Technologies for donating this product ID.
@@ -639,6 +640,8 @@ typedef struct USBNetState {
     unsigned int in_ptr, in_len;
     uint8_t in_buf[2048];
 
+    USBEndpoint *intr;
+
     char usbstring_mac[13];
     NICState *nic;
     NICConf conf;
@@ -851,6 +854,10 @@ static void *rndis_queue_response(USBNetState *s, unsigned int length)
     struct rndis_response *r =
             g_malloc0(sizeof(struct rndis_response) + length);
 
+    if (QTAILQ_EMPTY(&s->rndis_resp)) {
+        usb_wakeup(s->intr);
+    }
+
     QTAILQ_INSERT_TAIL(&s->rndis_resp, r, entries);
     r->length = length;
 
@@ -1005,7 +1012,7 @@ static int rndis_keepalive_response(USBNetState *s,
 static void usb_net_reset_in_buf(USBNetState *s)
 {
     s->in_ptr = s->in_len = 0;
-    qemu_flush_queued_packets(&s->nic->nc);
+    qemu_flush_queued_packets(qemu_get_queue(s->nic));
 }
 
 static int rndis_parse(USBNetState *s, uint8_t *data, int length)
@@ -1189,7 +1196,7 @@ static void usb_net_handle_dataout(USBNetState *s, USBPacket *p)
 
     if (!is_rndis(s)) {
         if (p->iov.size < 64) {
-            qemu_send_packet(&s->nic->nc, s->out_buf, s->out_ptr);
+            qemu_send_packet(qemu_get_queue(s->nic), s->out_buf, s->out_ptr);
             s->out_ptr = 0;
         }
         return;
@@ -1202,7 +1209,7 @@ static void usb_net_handle_dataout(USBNetState *s, USBPacket *p)
         uint32_t offs = 8 + le32_to_cpu(msg->DataOffset);
         uint32_t size = le32_to_cpu(msg->DataLength);
         if (offs + size <= len)
-            qemu_send_packet(&s->nic->nc, s->out_buf + offs, size);
+            qemu_send_packet(qemu_get_queue(s->nic), s->out_buf + offs, size);
     }
     s->out_ptr -= len;
     memmove(s->out_buf, &s->out_buf[len], s->out_ptr);
@@ -1254,7 +1261,7 @@ static void usb_net_handle_data(USBDevice *dev, USBPacket *p)
 
 static ssize_t usbnet_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    USBNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    USBNetState *s = qemu_get_nic_opaque(nc);
     uint8_t *in_buf = s->in_buf;
     size_t total_size = size;
 
@@ -1301,7 +1308,7 @@ static ssize_t usbnet_receive(NetClientState *nc, const uint8_t *buf, size_t siz
 
 static int usbnet_can_receive(NetClientState *nc)
 {
-    USBNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    USBNetState *s = qemu_get_nic_opaque(nc);
 
     if (is_rndis(s) && s->rndis_state != RNDIS_DATA_INITIALIZED) {
         return 1;
@@ -1312,7 +1319,7 @@ static int usbnet_can_receive(NetClientState *nc)
 
 static void usbnet_cleanup(NetClientState *nc)
 {
-    USBNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    USBNetState *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -1323,7 +1330,7 @@ static void usb_net_handle_destroy(USBDevice *dev)
 
     /* TODO: remove the nd_table[] entry */
     rndis_clear_responsequeue(s);
-    qemu_del_net_client(&s->nic->nc);
+    qemu_del_nic(s->nic);
 }
 
 static NetClientInfo net_usbnet_info = {
@@ -1349,11 +1356,12 @@ static int usb_net_initfn(USBDevice *dev)
     s->media_state = 0;	/* NDIS_MEDIA_STATE_CONNECTED */;
     s->filter = 0;
     s->vendorid = 0x1234;
+    s->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
 
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_usbnet_info, &s->conf,
                           object_get_typename(OBJECT(s)), s->dev.qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
     snprintf(s->usbstring_mac, sizeof(s->usbstring_mac),
              "%02x%02x%02x%02x%02x%02x",
              0x40,
@@ -1425,7 +1433,7 @@ static void usb_net_class_initfn(ObjectClass *klass, void *data)
     dc->props = net_properties;
 }
 
-static TypeInfo net_info = {
+static const TypeInfo net_info = {
     .name          = "usb-net",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBNetState),
diff --git a/hw/usb/dev-serial.c b/hw/usb/dev-serial.c
index 99b19df..47ac8c9 100644
--- a/hw/usb/dev-serial.c
+++ b/hw/usb/dev-serial.c
@@ -9,10 +9,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 //#define DEBUG_Serial
 
@@ -600,7 +600,7 @@ static void usb_serial_class_initfn(ObjectClass *klass, void *data)
     dc->props = serial_properties;
 }
 
-static TypeInfo serial_info = {
+static const TypeInfo serial_info = {
     .name          = "usb-serial",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBSerialState),
@@ -628,7 +628,7 @@ static void usb_braille_class_initfn(ObjectClass *klass, void *data)
     dc->props = braille_properties;
 }
 
-static TypeInfo braille_info = {
+static const TypeInfo braille_info = {
     .name          = "usb-braille",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBSerialState),
diff --git a/hw/usb/dev-smartcard-reader.c b/hw/usb/dev-smartcard-reader.c
index de955b7..979a473 100644
--- a/hw/usb/dev-smartcard-reader.c
+++ b/hw/usb/dev-smartcard-reader.c
@@ -35,10 +35,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 #include "hw/ccid.h"
 
@@ -1322,7 +1322,7 @@ static void ccid_class_initfn(ObjectClass *klass, void *data)
     dc->props = ccid_properties;
 }
 
-static TypeInfo ccid_info = {
+static const TypeInfo ccid_info = {
     .name          = CCID_DEV_NAME,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBCCIDState),
@@ -1338,7 +1338,7 @@ static void ccid_card_class_init(ObjectClass *klass, void *data)
     k->props = ccid_props;
 }
 
-static TypeInfo ccid_card_type_info = {
+static const TypeInfo ccid_card_type_info = {
     .name = TYPE_CCID_CARD,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(CCIDCardState),
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index 50af971..b89d00f 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -8,15 +8,15 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 #include "hw/scsi.h"
-#include "console.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "blockdev.h"
+#include "ui/console.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/blockdev.h"
 
 //#define DEBUG_MSD
 
@@ -54,12 +54,12 @@ typedef struct {
     struct usb_msd_csw csw;
     SCSIRequest *req;
     SCSIBus bus;
+    /* For async completion.  */
+    USBPacket *packet;
+    /* usb-storage only */
     BlockConf conf;
     char *serial;
-    SCSIDevice *scsi_dev;
     uint32_t removable;
-    /* For async completion.  */
-    USBPacket *packet;
 } MSDState;
 
 struct usb_msd_cbw {
@@ -343,7 +343,8 @@ static void usb_msd_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     MSDState *s = (MSDState *)dev;
-    int ret;
+    SCSIDevice *scsi_dev;
+    int ret, maxlun;
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
@@ -359,7 +360,19 @@ static void usb_msd_handle_control(USBDevice *dev, USBPacket *p,
         s->mode = USB_MSDM_CBW;
         break;
     case ClassInterfaceRequest | GetMaxLun:
-        data[0] = 0;
+        maxlun = 0;
+        for (;;) {
+            scsi_dev = scsi_device_find(&s->bus, 0, 0, maxlun+1);
+            if (scsi_dev == NULL) {
+                break;
+            }
+            if (scsi_dev->lun != maxlun+1) {
+                break;
+            }
+            maxlun++;
+        }
+        DPRINTF("MaxLun %d\n", maxlun);
+        data[0] = maxlun;
         p->actual_length = 1;
         break;
     default:
@@ -386,6 +399,7 @@ static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
     uint32_t tag;
     struct usb_msd_cbw cbw;
     uint8_t devep = p->ep->nr;
+    SCSIDevice *scsi_dev;
 
     switch (p->pid) {
     case USB_TOKEN_OUT:
@@ -405,7 +419,8 @@ static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 goto fail;
             }
             DPRINTF("Command on LUN %d\n", cbw.lun);
-            if (cbw.lun != 0) {
+            scsi_dev = scsi_device_find(&s->bus, 0, 0, cbw.lun);
+            if (scsi_dev == NULL) {
                 fprintf(stderr, "usb-msd: Bad LUN %d\n", cbw.lun);
                 goto fail;
             }
@@ -422,7 +437,7 @@ static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                     tag, cbw.flags, cbw.cmd_len, s->data_len);
             assert(le32_to_cpu(s->csw.residue) == 0);
             s->scsi_len = 0;
-            s->req = scsi_req_new(s->scsi_dev, tag, 0, cbw.cmd, NULL);
+            s->req = scsi_req_new(scsi_dev, tag, cbw.lun, cbw.cmd, NULL);
 #ifdef DEBUG_MSD
             scsi_req_print(s->req);
 #endif
@@ -553,7 +568,7 @@ static void *usb_msd_load_request(QEMUFile *f, SCSIRequest *req)
     return NULL;
 }
 
-static const struct SCSIBusInfo usb_msd_scsi_info = {
+static const struct SCSIBusInfo usb_msd_scsi_info_storage = {
     .tcq = false,
     .max_target = 0,
     .max_lun = 0,
@@ -564,10 +579,22 @@ static const struct SCSIBusInfo usb_msd_scsi_info = {
     .load_request = usb_msd_load_request,
 };
 
-static int usb_msd_initfn(USBDevice *dev)
+static const struct SCSIBusInfo usb_msd_scsi_info_bot = {
+    .tcq = false,
+    .max_target = 0,
+    .max_lun = 15,
+
+    .transfer_data = usb_msd_transfer_data,
+    .complete = usb_msd_command_complete,
+    .cancel = usb_msd_request_cancelled,
+    .load_request = usb_msd_load_request,
+};
+
+static int usb_msd_initfn_storage(USBDevice *dev)
 {
     MSDState *s = DO_UPCAST(MSDState, dev, dev);
     BlockDriverState *bs = s->conf.bs;
+    SCSIDevice *scsi_dev;
 
     if (!bs) {
         error_report("drive property not set");
@@ -595,10 +622,10 @@ static int usb_msd_initfn(USBDevice *dev)
     }
 
     usb_desc_init(dev);
-    scsi_bus_new(&s->bus, &s->dev.qdev, &usb_msd_scsi_info);
-    s->scsi_dev = scsi_bus_legacy_add_drive(&s->bus, bs, 0, !!s->removable,
+    scsi_bus_new(&s->bus, &s->dev.qdev, &usb_msd_scsi_info_storage);
+    scsi_dev = scsi_bus_legacy_add_drive(&s->bus, bs, 0, !!s->removable,
                                             s->conf.bootindex);
-    if (!s->scsi_dev) {
+    if (!scsi_dev) {
         return -1;
     }
     s->bus.qbus.allow_hotplug = 0;
@@ -616,6 +643,19 @@ static int usb_msd_initfn(USBDevice *dev)
     return 0;
 }
 
+static int usb_msd_initfn_bot(USBDevice *dev)
+{
+    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+
+    usb_desc_create_serial(dev);
+    usb_desc_init(dev);
+    scsi_bus_new(&s->bus, &s->dev.qdev, &usb_msd_scsi_info_bot);
+    s->bus.qbus.allow_hotplug = 0;
+    usb_msd_handle_reset(dev);
+
+    return 0;
+}
+
 static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
 {
     static int nr=0;
@@ -698,12 +738,11 @@ static Property msd_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void usb_msd_class_initfn(ObjectClass *klass, void *data)
+static void usb_msd_class_initfn_common(ObjectClass *klass)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->init           = usb_msd_initfn;
     uc->product_desc   = "QEMU USB MSD";
     uc->usb_desc       = &desc;
     uc->cancel_packet  = usb_msd_cancel_io;
@@ -713,19 +752,44 @@ static void usb_msd_class_initfn(ObjectClass *klass, void *data)
     uc->handle_data    = usb_msd_handle_data;
     dc->fw_name = "storage";
     dc->vmsd = &vmstate_usb_msd;
+}
+
+static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+
+    uc->init = usb_msd_initfn_storage;
     dc->props = msd_properties;
+    usb_msd_class_initfn_common(klass);
 }
 
-static TypeInfo msd_info = {
+static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data)
+{
+    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+
+    uc->init = usb_msd_initfn_bot;
+    usb_msd_class_initfn_common(klass);
+}
+
+static const TypeInfo msd_info = {
     .name          = "usb-storage",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(MSDState),
-    .class_init    = usb_msd_class_initfn,
+    .class_init    = usb_msd_class_initfn_storage,
+};
+
+static const TypeInfo bot_info = {
+    .name          = "usb-bot",
+    .parent        = TYPE_USB_DEVICE,
+    .instance_size = sizeof(MSDState),
+    .class_init    = usb_msd_class_initfn_bot,
 };
 
 static void usb_msd_register_types(void)
 {
     type_register_static(&msd_info);
+    type_register_static(&bot_info);
     usb_legacy_register("usb-storage", "disk", usb_msd_init);
 }
 
diff --git a/hw/usb/dev-uas.c b/hw/usb/dev-uas.c
index a21b2ba..d904d1a 100644
--- a/hw/usb/dev-uas.c
+++ b/hw/usb/dev-uas.c
@@ -10,8 +10,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "trace.h"
 
 #include "hw/usb.h"
@@ -757,7 +757,7 @@ static void usb_uas_class_initfn(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_uas;
 }
 
-static TypeInfo uas_info = {
+static const TypeInfo uas_info = {
     .name          = "usb-uas",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(UASDevice),
diff --git a/hw/usb/dev-wacom.c b/hw/usb/dev-wacom.c
index 08b416d..ab9fa2e 100644
--- a/hw/usb/dev-wacom.c
+++ b/hw/usb/dev-wacom.c
@@ -26,7 +26,7 @@
  * THE SOFTWARE.
  */
 #include "hw/hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 
@@ -43,6 +43,7 @@
 
 typedef struct USBWacomState {
     USBDevice dev;
+    USBEndpoint *intr;
     QEMUPutMouseEntry *eh_entry;
     int dx, dy, dz, buttons_state;
     int x, y;
@@ -137,6 +138,7 @@ static void usb_mouse_event(void *opaque,
     s->dz += dz1;
     s->buttons_state = buttons_state;
     s->changed = 1;
+    usb_wakeup(s->intr);
 }
 
 static void usb_wacom_event(void *opaque,
@@ -150,6 +152,7 @@ static void usb_wacom_event(void *opaque,
     s->dz += dz;
     s->buttons_state = buttons_state;
     s->changed = 1;
+    usb_wakeup(s->intr);
 }
 
 static inline int int_clamp(int val, int vmin, int vmax)
@@ -337,6 +340,7 @@ static int usb_wacom_initfn(USBDevice *dev)
     USBWacomState *s = DO_UPCAST(USBWacomState, dev, dev);
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    s->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
     s->changed = 1;
     return 0;
 }
@@ -362,7 +366,7 @@ static void usb_wacom_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_wacom;
 }
 
-static TypeInfo wacom_info = {
+static const TypeInfo wacom_info = {
     .name          = "usb-wacom-tablet",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBWacomState),
diff --git a/hw/usb/hcd-ehci-pci.c b/hw/usb/hcd-ehci-pci.c
index 41dbb53..0eb7826 100644
--- a/hw/usb/hcd-ehci-pci.c
+++ b/hw/usb/hcd-ehci-pci.c
@@ -16,13 +16,7 @@
  */
 
 #include "hw/usb/hcd-ehci.h"
-#include "hw/pci.h"
-#include "range.h"
-
-typedef struct EHCIPCIState {
-    PCIDevice pcidev;
-    EHCIState ehci;
-} EHCIPCIState;
+#include "qemu/range.h"
 
 typedef struct EHCIPCIInfo {
     const char *name;
@@ -33,7 +27,7 @@ typedef struct EHCIPCIInfo {
 
 static int usb_ehci_pci_initfn(PCIDevice *dev)
 {
-    EHCIPCIState *i = DO_UPCAST(EHCIPCIState, pcidev, dev);
+    EHCIPCIState *i = PCI_EHCI(dev);
     EHCIState *s = &i->ehci;
     uint8_t *pci_conf = dev->config;
 
@@ -83,7 +77,7 @@ static int usb_ehci_pci_initfn(PCIDevice *dev)
 static void usb_ehci_pci_write_config(PCIDevice *dev, uint32_t addr,
                                       uint32_t val, int l)
 {
-    EHCIPCIState *i = DO_UPCAST(EHCIPCIState, pcidev, dev);
+    EHCIPCIState *i = PCI_EHCI(dev);
     bool busmaster;
 
     pci_default_write_config(dev, addr, val, l);
@@ -115,12 +109,8 @@ static void ehci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-    EHCIPCIInfo *i = data;
 
     k->init = usb_ehci_pci_initfn;
-    k->vendor_id = i->vendor_id;
-    k->device_id = i->device_id;
-    k->revision = i->revision;
     k->class_id = PCI_CLASS_SERIAL_USB;
     k->config_write = usb_ehci_pci_write_config;
     k->no_hotplug = 1;
@@ -128,6 +118,24 @@ static void ehci_class_init(ObjectClass *klass, void *data)
     dc->props = ehci_pci_properties;
 }
 
+static const TypeInfo ehci_pci_type_info = {
+    .name = TYPE_PCI_EHCI,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(EHCIPCIState),
+    .abstract = true,
+    .class_init = ehci_class_init,
+};
+
+static void ehci_data_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    EHCIPCIInfo *i = data;
+
+    k->vendor_id = i->vendor_id;
+    k->device_id = i->device_id;
+    k->revision = i->revision;
+}
+
 static struct EHCIPCIInfo ehci_pci_info[] = {
     {
         .name      = "usb-ehci",
@@ -150,12 +158,13 @@ static struct EHCIPCIInfo ehci_pci_info[] = {
 static void ehci_pci_register_types(void)
 {
     TypeInfo ehci_type_info = {
-        .parent        = TYPE_PCI_DEVICE,
-        .instance_size = sizeof(EHCIPCIState),
-        .class_init    = ehci_class_init,
+        .parent        = TYPE_PCI_EHCI,
+        .class_init    = ehci_data_class_init,
     };
     int i;
 
+    type_register_static(&ehci_pci_type_info);
+
     for (i = 0; i < ARRAY_SIZE(ehci_pci_info); i++) {
         ehci_type_info.name = ehci_pci_info[i].name;
         ehci_type_info.class_data = ehci_pci_info + i;
diff --git a/hw/usb/hcd-ehci-sysbus.c b/hw/usb/hcd-ehci-sysbus.c
index 803df92..b68a66a 100644
--- a/hw/usb/hcd-ehci-sysbus.c
+++ b/hw/usb/hcd-ehci-sysbus.c
@@ -16,12 +16,6 @@
  */
 
 #include "hw/usb/hcd-ehci.h"
-#include "hw/sysbus.h"
-
-typedef struct EHCISysBusState {
-    SysBusDevice busdev;
-    EHCIState ehci;
-} EHCISysBusState;
 
 static const VMStateDescription vmstate_ehci_sysbus = {
     .name        = "ehci-sysbus",
@@ -40,11 +34,12 @@ static Property ehci_sysbus_properties[] = {
 
 static int usb_ehci_sysbus_initfn(SysBusDevice *dev)
 {
-    EHCISysBusState *i = FROM_SYSBUS(EHCISysBusState, dev);
+    EHCISysBusState *i = SYS_BUS_EHCI(dev);
+    SysBusEHCIClass *sec = SYS_BUS_EHCI_GET_CLASS(dev);
     EHCIState *s = &i->ehci;
 
-    s->capsbase = 0x100;
-    s->opregbase = 0x140;
+    s->capsbase = sec->capsbase;
+    s->opregbase = sec->opregbase;
     s->dma = &dma_context_memory;
 
     usb_ehci_initfn(s, DEVICE(dev));
@@ -63,16 +58,48 @@ static void ehci_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = ehci_sysbus_properties;
 }
 
-TypeInfo ehci_xlnx_type_info = {
-    .name          = "xlnx,ps7-usb",
+static const TypeInfo ehci_type_info = {
+    .name          = TYPE_SYS_BUS_EHCI,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(EHCISysBusState),
+    .abstract      = true,
     .class_init    = ehci_sysbus_class_init,
+    .class_size    = sizeof(SysBusEHCIClass),
+};
+
+static void ehci_xlnx_class_init(ObjectClass *oc, void *data)
+{
+    SysBusEHCIClass *sec = SYS_BUS_EHCI_CLASS(oc);
+
+    sec->capsbase = 0x100;
+    sec->opregbase = 0x140;
+}
+
+static const TypeInfo ehci_xlnx_type_info = {
+    .name          = "xlnx,ps7-usb",
+    .parent        = TYPE_SYS_BUS_EHCI,
+    .class_init    = ehci_xlnx_class_init,
+};
+
+static void ehci_exynos4210_class_init(ObjectClass *oc, void *data)
+{
+    SysBusEHCIClass *sec = SYS_BUS_EHCI_CLASS(oc);
+
+    sec->capsbase = 0x0;
+    sec->opregbase = 0x10;
+}
+
+static const TypeInfo ehci_exynos4210_type_info = {
+    .name          = TYPE_EXYNOS4210_EHCI,
+    .parent        = TYPE_SYS_BUS_EHCI,
+    .class_init    = ehci_exynos4210_class_init,
 };
 
 static void ehci_sysbus_register_types(void)
 {
+    type_register_static(&ehci_type_info);
     type_register_static(&ehci_xlnx_type_info);
+    type_register_static(&ehci_exynos4210_type_info);
 }
 
 type_init(ehci_sysbus_register_types)
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index 7df8e21..7040659 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -109,11 +109,13 @@
 
 #define FRAME_TIMER_FREQ 1000
 #define FRAME_TIMER_NS   (1000000000 / FRAME_TIMER_FREQ)
+#define UFRAME_TIMER_NS  (FRAME_TIMER_NS / 8)
 
 #define NB_MAXINTRATE    8        // Max rate at which controller issues ints
 #define BUFF_SIZE        5*4096   // Max bytes to transfer per transaction
 #define MAX_QH           100      // Max allowable queue heads in a chain
-#define MIN_FR_PER_TICK  3        // Min frames to process when catching up
+#define MIN_UFR_PER_TICK 24       /* Min frames to process when catching up */
+#define PERIODIC_ACTIVE  512      /* Micro-frames */
 
 /*  Internal periodic / asynchronous schedule state machine states
  */
@@ -191,6 +193,7 @@ static int ehci_state_executing(EHCIQueue *q);
 static int ehci_state_writeback(EHCIQueue *q);
 static int ehci_state_advqueue(EHCIQueue *q);
 static int ehci_fill_queue(EHCIPacket *p);
+static void ehci_free_packet(EHCIPacket *p);
 
 static const char *nr2str(const char **n, size_t len, uint32_t nr)
 {
@@ -437,6 +440,136 @@ static inline bool ehci_periodic_enabled(EHCIState *s)
     return ehci_enabled(s) && (s->usbcmd & USBCMD_PSE);
 }
 
+/* Get an array of dwords from main memory */
+static inline int get_dwords(EHCIState *ehci, uint32_t addr,
+                             uint32_t *buf, int num)
+{
+    int i;
+
+    if (!ehci->dma) {
+        ehci_raise_irq(ehci, USBSTS_HSE);
+        ehci->usbcmd &= ~USBCMD_RUNSTOP;
+        trace_usb_ehci_dma_error();
+        return -1;
+    }
+
+    for (i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
+        dma_memory_read(ehci->dma, addr, buf, sizeof(*buf));
+        *buf = le32_to_cpu(*buf);
+    }
+
+    return num;
+}
+
+/* Put an array of dwords in to main memory */
+static inline int put_dwords(EHCIState *ehci, uint32_t addr,
+                             uint32_t *buf, int num)
+{
+    int i;
+
+    if (!ehci->dma) {
+        ehci_raise_irq(ehci, USBSTS_HSE);
+        ehci->usbcmd &= ~USBCMD_RUNSTOP;
+        trace_usb_ehci_dma_error();
+        return -1;
+    }
+
+    for (i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
+        uint32_t tmp = cpu_to_le32(*buf);
+        dma_memory_write(ehci->dma, addr, &tmp, sizeof(tmp));
+    }
+
+    return num;
+}
+
+static int ehci_get_pid(EHCIqtd *qtd)
+{
+    switch (get_field(qtd->token, QTD_TOKEN_PID)) {
+    case 0:
+        return USB_TOKEN_OUT;
+    case 1:
+        return USB_TOKEN_IN;
+    case 2:
+        return USB_TOKEN_SETUP;
+    default:
+        fprintf(stderr, "bad token\n");
+        return 0;
+    }
+}
+
+static bool ehci_verify_qh(EHCIQueue *q, EHCIqh *qh)
+{
+    uint32_t devaddr = get_field(qh->epchar, QH_EPCHAR_DEVADDR);
+    uint32_t endp    = get_field(qh->epchar, QH_EPCHAR_EP);
+    if ((devaddr != get_field(q->qh.epchar, QH_EPCHAR_DEVADDR)) ||
+        (endp    != get_field(q->qh.epchar, QH_EPCHAR_EP)) ||
+        (qh->current_qtd != q->qh.current_qtd) ||
+        (q->async && qh->next_qtd != q->qh.next_qtd) ||
+        (memcmp(&qh->altnext_qtd, &q->qh.altnext_qtd,
+                                 7 * sizeof(uint32_t)) != 0) ||
+        (q->dev != NULL && q->dev->addr != devaddr)) {
+        return false;
+    } else {
+        return true;
+    }
+}
+
+static bool ehci_verify_qtd(EHCIPacket *p, EHCIqtd *qtd)
+{
+    if (p->qtdaddr != p->queue->qtdaddr ||
+        (p->queue->async && !NLPTR_TBIT(p->qtd.next) &&
+            (p->qtd.next != qtd->next)) ||
+        (!NLPTR_TBIT(p->qtd.altnext) && (p->qtd.altnext != qtd->altnext)) ||
+        p->qtd.token != qtd->token ||
+        p->qtd.bufptr[0] != qtd->bufptr[0]) {
+        return false;
+    } else {
+        return true;
+    }
+}
+
+static bool ehci_verify_pid(EHCIQueue *q, EHCIqtd *qtd)
+{
+    int ep  = get_field(q->qh.epchar, QH_EPCHAR_EP);
+    int pid = ehci_get_pid(qtd);
+
+    /* Note the pid changing is normal for ep 0 (the control ep) */
+    if (q->last_pid && ep != 0 && pid != q->last_pid) {
+        return false;
+    } else {
+        return true;
+    }
+}
+
+/* Finish executing and writeback a packet outside of the regular
+   fetchqh -> fetchqtd -> execute -> writeback cycle */
+static void ehci_writeback_async_complete_packet(EHCIPacket *p)
+{
+    EHCIQueue *q = p->queue;
+    EHCIqtd qtd;
+    EHCIqh qh;
+    int state;
+
+    /* Verify the qh + qtd, like we do when going through fetchqh & fetchqtd */
+    get_dwords(q->ehci, NLPTR_GET(q->qhaddr),
+               (uint32_t *) &qh, sizeof(EHCIqh) >> 2);
+    get_dwords(q->ehci, NLPTR_GET(q->qtdaddr),
+               (uint32_t *) &qtd, sizeof(EHCIqtd) >> 2);
+    if (!ehci_verify_qh(q, &qh) || !ehci_verify_qtd(p, &qtd)) {
+        p->async = EHCI_ASYNC_INITIALIZED;
+        ehci_free_packet(p);
+        return;
+    }
+
+    state = ehci_get_state(q->ehci, q->async);
+    ehci_state_executing(q);
+    ehci_state_writeback(q); /* Frees the packet! */
+    if (!(q->qh.token & QTD_TOKEN_HALT)) {
+        ehci_state_advqueue(q);
+    }
+    ehci_set_state(q->ehci, q->async, state);
+}
+
 /* packet management */
 
 static EHCIPacket *ehci_alloc_packet(EHCIQueue *q)
@@ -454,17 +587,7 @@ static EHCIPacket *ehci_alloc_packet(EHCIQueue *q)
 static void ehci_free_packet(EHCIPacket *p)
 {
     if (p->async == EHCI_ASYNC_FINISHED) {
-        EHCIQueue *q = p->queue;
-        int state = ehci_get_state(q->ehci, q->async);
-        /* This is a normal, but rare condition (cancel racing completion) */
-        fprintf(stderr, "EHCI: Warning packet completed but not processed\n");
-        ehci_state_executing(q);
-        ehci_state_writeback(q);
-        if (!(q->qh.token & QTD_TOKEN_HALT)) {
-            ehci_state_advqueue(q);
-        }
-        ehci_set_state(q->ehci, q->async, state);
-        /* state_writeback recurses into us with async == EHCI_ASYNC_NONE!! */
+        ehci_writeback_async_complete_packet(p);
         return;
     }
     trace_usb_ehci_packet_action(p->queue, p, "free");
@@ -499,6 +622,17 @@ static EHCIQueue *ehci_alloc_queue(EHCIState *ehci, uint32_t addr, int async)
     return q;
 }
 
+static void ehci_queue_stopped(EHCIQueue *q)
+{
+    int endp  = get_field(q->qh.epchar, QH_EPCHAR_EP);
+
+    if (!q->last_pid || !q->dev) {
+        return;
+    }
+
+    usb_device_ep_stopped(q->dev, usb_ep_get(q->dev, q->last_pid, endp));
+}
+
 static int ehci_cancel_queue(EHCIQueue *q)
 {
     EHCIPacket *p;
@@ -506,7 +640,7 @@ static int ehci_cancel_queue(EHCIQueue *q)
 
     p = QTAILQ_FIRST(&q->packets);
     if (p == NULL) {
-        return 0;
+        goto leave;
     }
 
     trace_usb_ehci_queue_action(q, "cancel");
@@ -514,6 +648,9 @@ static int ehci_cancel_queue(EHCIQueue *q)
         ehci_free_packet(p);
         packets++;
     } while ((p = QTAILQ_FIRST(&q->packets)) != NULL);
+
+leave:
+    ehci_queue_stopped(q);
     return packets;
 }
 
@@ -525,6 +662,7 @@ static int ehci_reset_queue(EHCIQueue *q)
     packets = ehci_cancel_queue(q);
     q->dev = NULL;
     q->qtdaddr = 0;
+    q->last_pid = 0;
     return packets;
 }
 
@@ -633,7 +771,6 @@ static void ehci_attach(USBPort *port)
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
-    ehci_commit_irq(s);
 }
 
 static void ehci_detach(USBPort *port)
@@ -663,7 +800,6 @@ static void ehci_detach(USBPort *port)
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
-    ehci_commit_irq(s);
 }
 
 static void ehci_child_detach(USBPort *port, USBDevice *child)
@@ -738,6 +874,19 @@ static int ehci_register_companion(USBBus *bus, USBPort *ports[],
     return 0;
 }
 
+static void ehci_wakeup_endpoint(USBBus *bus, USBEndpoint *ep)
+{
+    EHCIState *s = container_of(bus, EHCIState, bus);
+    uint32_t portsc = s->portsc[ep->dev->port->index];
+
+    if (portsc & PORTSC_POWNER) {
+        return;
+    }
+
+    s->periodic_sched_active = PERIODIC_ACTIVE;
+    qemu_bh_schedule(s->async_bh);
+}
+
 static USBDevice *ehci_find_device(EHCIState *ehci, uint8_t addr)
 {
     USBDevice *dev;
@@ -819,7 +968,15 @@ static uint64_t ehci_opreg_read(void *ptr, hwaddr addr,
     EHCIState *s = ptr;
     uint32_t val;
 
-    val = s->opreg[addr >> 2];
+    switch (addr) {
+    case FRINDEX:
+        /* Round down to mult of 8, else it can go backwards on migration */
+        val = s->frindex & ~7;
+        break;
+    default:
+        val = s->opreg[addr >> 2];
+    }
+
     trace_usb_ehci_opreg_read(addr + s->opregbase, addr2str(addr), val);
     return val;
 }
@@ -970,7 +1127,8 @@ static void ehci_opreg_write(void *ptr, hwaddr addr,
         break;
 
     case FRINDEX:
-        val &= 0x00003ff8; /* frindex is 14bits and always a multiple of 8 */
+        val &= 0x00003fff; /* frindex is 14bits */
+        s->usbsts_frindex = val;
         break;
 
     case CONFIGFLAG:
@@ -1003,48 +1161,6 @@ static void ehci_opreg_write(void *ptr, hwaddr addr,
                                 *mmio, old);
 }
 
-/* Get an array of dwords from main memory */
-static inline int get_dwords(EHCIState *ehci, uint32_t addr,
-                             uint32_t *buf, int num)
-{
-    int i;
-
-    if (!ehci->dma) {
-        ehci_raise_irq(ehci, USBSTS_HSE);
-        ehci->usbcmd &= ~USBCMD_RUNSTOP;
-        trace_usb_ehci_dma_error();
-        return -1;
-    }
-
-    for(i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
-        dma_memory_read(ehci->dma, addr, buf, sizeof(*buf));
-        *buf = le32_to_cpu(*buf);
-    }
-
-    return num;
-}
-
-/* Put an array of dwords in to main memory */
-static inline int put_dwords(EHCIState *ehci, uint32_t addr,
-                             uint32_t *buf, int num)
-{
-    int i;
-
-    if (!ehci->dma) {
-        ehci_raise_irq(ehci, USBSTS_HSE);
-        ehci->usbcmd &= ~USBCMD_RUNSTOP;
-        trace_usb_ehci_dma_error();
-        return -1;
-    }
-
-    for(i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
-        uint32_t tmp = cpu_to_le32(*buf);
-        dma_memory_write(ehci->dma, addr, &tmp, sizeof(tmp));
-    }
-
-    return num;
-}
-
 /*
  *  Write the qh back to guest physical memory.  This step isn't
  *  in the EHCI spec but we need to do it since we don't share
@@ -1188,9 +1304,10 @@ static void ehci_async_complete_packet(USBPort *port, USBPacket *packet)
     trace_usb_ehci_packet_action(p->queue, p, "wakeup");
     p->async = EHCI_ASYNC_FINISHED;
 
-    if (p->queue->async) {
-        qemu_bh_schedule(p->queue->ehci->async_bh);
+    if (!p->queue->async) {
+        s->periodic_sched_active = PERIODIC_ACTIVE;
     }
+    qemu_bh_schedule(s->async_bh);
 }
 
 static void ehci_execute_complete(EHCIQueue *q)
@@ -1242,6 +1359,9 @@ static void ehci_execute_complete(EHCIQueue *q)
         if (tbytes) {
             /* 4.15.1.2 must raise int on a short input packet */
             ehci_raise_irq(q->ehci, USBSTS_INT);
+            if (q->async) {
+                q->ehci->int_req_by_async = true;
+            }
         }
     } else {
         tbytes = 0;
@@ -1286,22 +1406,11 @@ static int ehci_execute(EHCIPacket *p, const char *action)
         return -1;
     }
 
-    p->pid = (p->qtd.token & QTD_TOKEN_PID_MASK) >> QTD_TOKEN_PID_SH;
-    switch (p->pid) {
-    case 0:
-        p->pid = USB_TOKEN_OUT;
-        break;
-    case 1:
-        p->pid = USB_TOKEN_IN;
-        break;
-    case 2:
-        p->pid = USB_TOKEN_SETUP;
-        break;
-    default:
-        fprintf(stderr, "bad token\n");
-        break;
+    if (!ehci_verify_pid(p->queue, &p->qtd)) {
+        ehci_queue_stopped(p->queue); /* Mark the ep in the prev dir stopped */
     }
-
+    p->pid = ehci_get_pid(&p->qtd);
+    p->queue->last_pid = p->pid;
     endp = get_field(p->queue->qh.epchar, QH_EPCHAR_EP);
     ep = usb_ep_get(p->queue->dev, p->pid, endp);
 
@@ -1344,6 +1453,8 @@ static int ehci_process_itd(EHCIState *ehci,
     uint32_t i, len, pid, dir, devaddr, endp;
     uint32_t pg, off, ptr1, ptr2, max, mult;
 
+    ehci->periodic_sched_active = PERIODIC_ACTIVE;
+
     dir =(itd->bufptr[1] & ITD_BUFPTR_DIRECTION);
     devaddr = get_field(itd->bufptr[0], ITD_BUFPTR_DEVADDR);
     endp = get_field(itd->bufptr[0], ITD_BUFPTR_EP);
@@ -1534,8 +1645,7 @@ out:
 
 static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
 {
-    EHCIPacket *p;
-    uint32_t entry, devaddr, endp;
+    uint32_t entry;
     EHCIQueue *q;
     EHCIqh qh;
 
@@ -1544,7 +1654,6 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
     if (NULL == q) {
         q = ehci_alloc_queue(ehci, entry, async);
     }
-    p = QTAILQ_FIRST(&q->packets);
 
     q->seen++;
     if (q->seen > 1) {
@@ -1565,19 +1674,10 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
      * The overlay area of the qh should never be changed by the guest,
      * except when idle, in which case the reset is a nop.
      */
-    devaddr = get_field(qh.epchar, QH_EPCHAR_DEVADDR);
-    endp    = get_field(qh.epchar, QH_EPCHAR_EP);
-    if ((devaddr != get_field(q->qh.epchar, QH_EPCHAR_DEVADDR)) ||
-        (endp    != get_field(q->qh.epchar, QH_EPCHAR_EP)) ||
-        (qh.current_qtd != q->qh.current_qtd) ||
-        (q->async && qh.next_qtd != q->qh.next_qtd) ||
-        (memcmp(&qh.altnext_qtd, &q->qh.altnext_qtd,
-                                 7 * sizeof(uint32_t)) != 0) ||
-        (q->dev != NULL && q->dev->addr != devaddr)) {
+    if (!ehci_verify_qh(q, &qh)) {
         if (ehci_reset_queue(q) > 0) {
             ehci_trace_guest_bug(ehci, "guest updated active QH");
         }
-        p = NULL;
     }
     q->qh = qh;
 
@@ -1587,14 +1687,8 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
     }
 
     if (q->dev == NULL) {
-        q->dev = ehci_find_device(q->ehci, devaddr);
-    }
-
-    if (p && p->async == EHCI_ASYNC_FINISHED) {
-        /* I/O finished -- continue processing queue */
-        trace_usb_ehci_packet_action(p->queue, p, "complete");
-        ehci_set_state(ehci, async, EST_EXECUTING);
-        goto out;
+        q->dev = ehci_find_device(q->ehci,
+                                  get_field(q->qh.epchar, QH_EPCHAR_DEVADDR));
     }
 
     if (async && (q->qh.epchar & QH_EPCHAR_H)) {
@@ -1745,13 +1839,11 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
 
     p = QTAILQ_FIRST(&q->packets);
     if (p != NULL) {
-        if (p->qtdaddr != q->qtdaddr ||
-            (q->async && !NLPTR_TBIT(p->qtd.next) &&
-                (p->qtd.next != qtd.next)) ||
-            (!NLPTR_TBIT(p->qtd.altnext) && (p->qtd.altnext != qtd.altnext)) ||
-            p->qtd.bufptr[0] != qtd.bufptr[0]) {
+        if (!ehci_verify_qtd(p, &qtd)) {
             ehci_cancel_queue(q);
-            ehci_trace_guest_bug(q->ehci, "guest updated active QH or qTD");
+            if (qtd.token & QTD_TOKEN_ACTIVE) {
+                ehci_trace_guest_bug(q->ehci, "guest updated active qTD");
+            }
             p = NULL;
         } else {
             p->qtd = qtd;
@@ -1760,11 +1852,6 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
     }
 
     if (!(qtd.token & QTD_TOKEN_ACTIVE)) {
-        if (p != NULL) {
-            /* transfer canceled by guest (clear active) */
-            ehci_cancel_queue(q);
-            p = NULL;
-        }
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
     } else if (p != NULL) {
         switch (p->async) {
@@ -1780,10 +1867,7 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
             ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
             break;
         case EHCI_ASYNC_FINISHED:
-            /*
-             * We get here when advqueue moves to a packet which is already
-             * finished, which can happen with packets queued up by fill_queue
-             */
+            /* Complete executing of the packet */
             ehci_set_state(q->ehci, q->async, EST_EXECUTING);
             break;
         }
@@ -1842,6 +1926,10 @@ static int ehci_fill_queue(EHCIPacket *p)
         if (!(qtd.token & QTD_TOKEN_ACTIVE)) {
             break;
         }
+        if (!ehci_verify_pid(q, &qtd)) {
+            ehci_trace_guest_bug(q->ehci, "guest queued token with wrong pid");
+            break;
+        }
         p = ehci_alloc_packet(q);
         p->qtdaddr = qtdaddr;
         p->qtd = qtd;
@@ -2004,18 +2092,22 @@ static void ehci_advance_state(EHCIState *ehci, int async)
             break;
 
         case EST_ADVANCEQUEUE:
+            assert(q != NULL);
             again = ehci_state_advqueue(q);
             break;
 
         case EST_FETCHQTD:
+            assert(q != NULL);
             again = ehci_state_fetchqtd(q);
             break;
 
         case EST_HORIZONTALQH:
+            assert(q != NULL);
             again = ehci_state_horizqh(q);
             break;
 
         case EST_EXECUTE:
+            assert(q != NULL);
             again = ehci_state_execute(q);
             if (async) {
                 ehci->async_stepdown = 0;
@@ -2033,6 +2125,9 @@ static void ehci_advance_state(EHCIState *ehci, int async)
         case EST_WRITEBACK:
             assert(q != NULL);
             again = ehci_state_writeback(q);
+            if (!async) {
+                ehci->periodic_sched_active = PERIODIC_ACTIVE;
+            }
             break;
 
         default:
@@ -2156,16 +2251,16 @@ static void ehci_advance_periodic_state(EHCIState *ehci)
     }
 }
 
-static void ehci_update_frindex(EHCIState *ehci, int frames)
+static void ehci_update_frindex(EHCIState *ehci, int uframes)
 {
     int i;
 
-    if (!ehci_enabled(ehci)) {
+    if (!ehci_enabled(ehci) && ehci->pstate == EST_INACTIVE) {
         return;
     }
 
-    for (i = 0; i < frames; i++) {
-        ehci->frindex += 8;
+    for (i = 0; i < uframes; i++) {
+        ehci->frindex++;
 
         if (ehci->frindex == 0x00002000) {
             ehci_raise_irq(ehci, USBSTS_FLR);
@@ -2189,49 +2284,57 @@ static void ehci_frame_timer(void *opaque)
     int need_timer = 0;
     int64_t expire_time, t_now;
     uint64_t ns_elapsed;
-    int frames, skipped_frames;
+    int uframes, skipped_uframes;
     int i;
 
     t_now = qemu_get_clock_ns(vm_clock);
     ns_elapsed = t_now - ehci->last_run_ns;
-    frames = ns_elapsed / FRAME_TIMER_NS;
+    uframes = ns_elapsed / UFRAME_TIMER_NS;
 
     if (ehci_periodic_enabled(ehci) || ehci->pstate != EST_INACTIVE) {
         need_timer++;
-        ehci->async_stepdown = 0;
 
-        if (frames > ehci->maxframes) {
-            skipped_frames = frames - ehci->maxframes;
-            ehci_update_frindex(ehci, skipped_frames);
-            ehci->last_run_ns += FRAME_TIMER_NS * skipped_frames;
-            frames -= skipped_frames;
-            DPRINTF("WARNING - EHCI skipped %d frames\n", skipped_frames);
+        if (uframes > (ehci->maxframes * 8)) {
+            skipped_uframes = uframes - (ehci->maxframes * 8);
+            ehci_update_frindex(ehci, skipped_uframes);
+            ehci->last_run_ns += UFRAME_TIMER_NS * skipped_uframes;
+            uframes -= skipped_uframes;
+            DPRINTF("WARNING - EHCI skipped %d uframes\n", skipped_uframes);
         }
 
-        for (i = 0; i < frames; i++) {
+        for (i = 0; i < uframes; i++) {
             /*
              * If we're running behind schedule, we should not catch up
              * too fast, as that will make some guests unhappy:
-             * 1) We must process a minimum of MIN_FR_PER_TICK frames,
+             * 1) We must process a minimum of MIN_UFR_PER_TICK frames,
              *    otherwise we will never catch up
              * 2) Process frames until the guest has requested an irq (IOC)
              */
-            if (i >= MIN_FR_PER_TICK) {
+            if (i >= MIN_UFR_PER_TICK) {
                 ehci_commit_irq(ehci);
                 if ((ehci->usbsts & USBINTR_MASK) & ehci->usbintr) {
                     break;
                 }
             }
+            if (ehci->periodic_sched_active) {
+                ehci->periodic_sched_active--;
+            }
             ehci_update_frindex(ehci, 1);
-            ehci_advance_periodic_state(ehci);
-            ehci->last_run_ns += FRAME_TIMER_NS;
+            if ((ehci->frindex & 7) == 0) {
+                ehci_advance_periodic_state(ehci);
+            }
+            ehci->last_run_ns += UFRAME_TIMER_NS;
         }
     } else {
-        if (ehci->async_stepdown < ehci->maxframes / 2) {
-            ehci->async_stepdown++;
-        }
-        ehci_update_frindex(ehci, frames);
-        ehci->last_run_ns += FRAME_TIMER_NS * frames;
+        ehci->periodic_sched_active = 0;
+        ehci_update_frindex(ehci, uframes);
+        ehci->last_run_ns += UFRAME_TIMER_NS * uframes;
+    }
+
+    if (ehci->periodic_sched_active) {
+        ehci->async_stepdown = 0;
+    } else if (ehci->async_stepdown < ehci->maxframes / 2) {
+        ehci->async_stepdown++;
     }
 
     /*  Async is not inside loop since it executes everything it can once
@@ -2256,7 +2359,7 @@ static void ehci_frame_timer(void *opaque)
         /* If we've raised int, we speed up the timer, so that we quickly
          * notice any new packets queued up in response */
         if (ehci->int_req_by_async && (ehci->usbsts & USBSTS_INT)) {
-            expire_time = t_now + get_ticks_per_sec() / (FRAME_TIMER_FREQ * 2);
+            expire_time = t_now + get_ticks_per_sec() / (FRAME_TIMER_FREQ * 4);
             ehci->int_req_by_async = false;
         } else {
             expire_time = t_now + (get_ticks_per_sec()
@@ -2301,8 +2404,20 @@ static USBPortOps ehci_port_ops = {
 
 static USBBusOps ehci_bus_ops = {
     .register_companion = ehci_register_companion,
+    .wakeup_endpoint = ehci_wakeup_endpoint,
 };
 
+static void usb_ehci_pre_save(void *opaque)
+{
+    EHCIState *ehci = opaque;
+    uint32_t new_frindex;
+
+    /* Round down frindex to a multiple of 8 for migration compatibility */
+    new_frindex = ehci->frindex & ~7;
+    ehci->last_run_ns -= (ehci->frindex - new_frindex) * UFRAME_TIMER_NS;
+    ehci->frindex = new_frindex;
+}
+
 static int usb_ehci_post_load(void *opaque, int version_id)
 {
     EHCIState *s = opaque;
@@ -2353,6 +2468,7 @@ const VMStateDescription vmstate_ehci = {
     .name        = "ehci-core",
     .version_id  = 2,
     .minimum_version_id  = 1,
+    .pre_save    = usb_ehci_pre_save,
     .post_load   = usb_ehci_post_load,
     .fields      = (VMStateField[]) {
         /* mmio registers */
diff --git a/hw/usb/hcd-ehci.h b/hw/usb/hcd-ehci.h
index d8078f4..e95bb7e 100644
--- a/hw/usb/hcd-ehci.h
+++ b/hw/usb/hcd-ehci.h
@@ -14,14 +14,18 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef HW_USB_EHCI_H
+#define HW_USB_EHCI_H 1
 
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "trace.h"
-#include "dma.h"
-#include "sysemu.h"
+#include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/sysbus.h"
 
 #ifndef EHCI_DEBUG
 #define EHCI_DEBUG   0
@@ -246,6 +250,7 @@ struct EHCIQueue {
     EHCIqh qh;             /* copy of current QH (being worked on) */
     uint32_t qhaddr;       /* address QH read from                 */
     uint32_t qtdaddr;      /* address QTD read from                */
+    int last_pid;          /* pid of last packet executed          */
     USBDevice *dev;
     QTAILQ_HEAD(pkts_head, EHCIPacket) packets;
 };
@@ -311,9 +316,51 @@ struct EHCIState {
 
     uint64_t last_run_ns;
     uint32_t async_stepdown;
+    uint32_t periodic_sched_active;
     bool int_req_by_async;
 };
 
 extern const VMStateDescription vmstate_ehci;
 
 void usb_ehci_initfn(EHCIState *s, DeviceState *dev);
+
+#define TYPE_PCI_EHCI "pci-ehci-usb"
+#define PCI_EHCI(obj) OBJECT_CHECK(EHCIPCIState, (obj), TYPE_PCI_EHCI)
+
+typedef struct EHCIPCIState {
+    /*< private >*/
+    PCIDevice pcidev;
+    /*< public >*/
+
+    EHCIState ehci;
+} EHCIPCIState;
+
+
+#define TYPE_SYS_BUS_EHCI "sysbus-ehci-usb"
+#define TYPE_EXYNOS4210_EHCI "exynos4210-ehci-usb"
+
+#define SYS_BUS_EHCI(obj) \
+    OBJECT_CHECK(EHCISysBusState, (obj), TYPE_SYS_BUS_EHCI)
+#define SYS_BUS_EHCI_CLASS(class) \
+    OBJECT_CLASS_CHECK(SysBusEHCIClass, (class), TYPE_SYS_BUS_EHCI)
+#define SYS_BUS_EHCI_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(SysBusEHCIClass, (obj), TYPE_SYS_BUS_EHCI)
+
+typedef struct EHCISysBusState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    EHCIState ehci;
+} EHCISysBusState;
+
+typedef struct SysBusEHCIClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    uint16_t capsbase;
+    uint16_t opregbase;
+} SysBusEHCIClass;
+
+#endif
diff --git a/hw/usb/hcd-musb.c b/hw/usb/hcd-musb.c
index 7856f53..20fd5b1 100644
--- a/hw/usb/hcd-musb.c
+++ b/hw/usb/hcd-musb.c
@@ -21,7 +21,7 @@
  * Only host-mode and non-DMA accesses are currently supported.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
 #include "hw/irq.h"
 #include "hw/hw.h"
diff --git a/hw/usb/hcd-ohci.c b/hw/usb/hcd-ohci.c
index e16a2ec..dd9967b 100644
--- a/hw/usb/hcd-ohci.c
+++ b/hw/usb/hcd-ohci.c
@@ -27,9 +27,9 @@
 */
 
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/sysbus.h"
 #include "hw/qdev-dma.h"
 
@@ -430,6 +430,23 @@ static USBDevice *ohci_find_device(OHCIState *ohci, uint8_t addr)
     return NULL;
 }
 
+static void ohci_stop_endpoints(OHCIState *ohci)
+{
+    USBDevice *dev;
+    int i, j;
+
+    for (i = 0; i < ohci->num_ports; i++) {
+        dev = ohci->rhport[i].port.dev;
+        if (dev && dev->attached) {
+            usb_device_ep_stopped(dev, &dev->ep_ctl);
+            for (j = 0; j < USB_MAX_ENDPOINTS; j++) {
+                usb_device_ep_stopped(dev, &dev->ep_in[j]);
+                usb_device_ep_stopped(dev, &dev->ep_out[j]);
+            }
+        }
+    }
+}
+
 /* Reset the controller */
 static void ohci_reset(void *opaque)
 {
@@ -478,6 +495,7 @@ static void ohci_reset(void *opaque)
         usb_cancel_packet(&ohci->usb_packet);
         ohci->async_td = 0;
     }
+    ohci_stop_endpoints(ohci);
     DPRINTF("usb-ohci: Reset %s\n", ohci->name);
 }
 
@@ -1147,6 +1165,8 @@ static int ohci_service_ed_list(OHCIState *ohci, uint32_t head, int completion)
             if (ohci->async_td && addr == ohci->async_td) {
                 usb_cancel_packet(&ohci->usb_packet);
                 ohci->async_td = 0;
+                usb_device_ep_stopped(ohci->usb_packet.ep->dev,
+                                      ohci->usb_packet.ep);
             }
             continue;
         }
@@ -1227,10 +1247,12 @@ static void ohci_frame_boundary(void *opaque)
     }
 
     /* Cancel all pending packets if either of the lists has been disabled.  */
-    if (ohci->async_td &&
-        ohci->old_ctl & (~ohci->ctl) & (OHCI_CTL_BLE | OHCI_CTL_CLE)) {
-        usb_cancel_packet(&ohci->usb_packet);
-        ohci->async_td = 0;
+    if (ohci->old_ctl & (~ohci->ctl) & (OHCI_CTL_BLE | OHCI_CTL_CLE)) {
+        if (ohci->async_td) {
+            usb_cancel_packet(&ohci->usb_packet);
+            ohci->async_td = 0;
+        }
+        ohci_stop_endpoints(ohci);
     }
     ohci->old_ctl = ohci->ctl;
     ohci_process_lists(ohci, 0);
@@ -1714,6 +1736,7 @@ static void ohci_mem_write(void *opaque,
     /* PXA27x specific registers */
     case 24: /* HcStatus */
         ohci->hstatus &= ~(val & ohci->hmask);
+        break;
 
     case 25: /* HcHReset */
         ohci->hreset = val & ~OHCI_HRESET_FSBIR;
@@ -1887,7 +1910,7 @@ static void ohci_pci_class_init(ObjectClass *klass, void *data)
     dc->props = ohci_pci_properties;
 }
 
-static TypeInfo ohci_pci_info = {
+static const TypeInfo ohci_pci_info = {
     .name          = "pci-ohci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(OHCIPCIState),
@@ -1910,7 +1933,7 @@ static void ohci_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = ohci_sysbus_properties;
 }
 
-static TypeInfo ohci_sysbus_info = {
+static const TypeInfo ohci_sysbus_info = {
     .name          = "sysbus-ohci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OHCISysBusState),
diff --git a/hw/usb/hcd-uhci.c b/hw/usb/hcd-uhci.c
index d053791..60645aa 100644
--- a/hw/usb/hcd-uhci.c
+++ b/hw/usb/hcd-uhci.c
@@ -27,10 +27,10 @@
  */
 #include "hw/hw.h"
 #include "hw/usb.h"
-#include "hw/pci.h"
-#include "qemu-timer.h"
-#include "iov.h"
-#include "dma.h"
+#include "hw/pci/pci.h"
+#include "qemu/timer.h"
+#include "qemu/iov.h"
+#include "sysemu/dma.h"
 #include "trace.h"
 
 //#define DEBUG
@@ -75,6 +75,11 @@
 
 #define FRAME_MAX_LOOPS  256
 
+/* Must be large enough to handle 10 frame delay for initial isoc requests */
+#define QH_VALID         32
+
+#define MAX_FRAMES_PER_TICK    (QH_VALID / 2)
+
 #define NB_PORTS 2
 
 enum {
@@ -166,6 +171,7 @@ struct UHCIState {
     /* Properties */
     char *masterbus;
     uint32_t firstport;
+    uint32_t maxframes;
 };
 
 typedef struct UHCI_TD {
@@ -206,9 +212,7 @@ static UHCIQueue *uhci_queue_new(UHCIState *s, uint32_t qh_addr, UHCI_TD *td,
     queue->ep = ep;
     QTAILQ_INIT(&queue->asyncs);
     QTAILQ_INSERT_HEAD(&s->queues, queue, next);
-    /* valid needs to be large enough to handle 10 frame delay
-     * for initial isochronous requests */
-    queue->valid = 32;
+    queue->valid = QH_VALID;
     trace_usb_uhci_queue_add(queue->token);
     return queue;
 }
@@ -222,6 +226,7 @@ static void uhci_queue_free(UHCIQueue *queue, const char *reason)
         async = QTAILQ_FIRST(&queue->asyncs);
         uhci_async_cancel(async);
     }
+    usb_device_ep_stopped(queue->ep->dev, queue->ep);
 
     trace_usb_uhci_queue_del(queue->token, reason);
     QTAILQ_REMOVE(&s->queues, queue, next);
@@ -433,7 +438,7 @@ static int uhci_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_uhci = {
     .name = "uhci",
-    .version_id = 2,
+    .version_id = 3,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .post_load = uhci_post_load,
@@ -451,44 +456,16 @@ static const VMStateDescription vmstate_uhci = {
         VMSTATE_UINT8(status2, UHCIState),
         VMSTATE_TIMER(frame_timer, UHCIState),
         VMSTATE_INT64_V(expire_time, UHCIState, 2),
+        VMSTATE_UINT32_V(pending_int_mask, UHCIState, 3),
         VMSTATE_END_OF_LIST()
     }
 };
 
-static void uhci_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
-{
-    UHCIState *s = opaque;
-
-    addr &= 0x1f;
-    switch(addr) {
-    case 0x0c:
-        s->sof_timing = val;
-        break;
-    }
-}
-
-static uint32_t uhci_ioport_readb(void *opaque, uint32_t addr)
-{
-    UHCIState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x1f;
-    switch(addr) {
-    case 0x0c:
-        val = s->sof_timing;
-        break;
-    default:
-        val = 0xff;
-        break;
-    }
-    return val;
-}
-
-static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
+static void uhci_port_write(void *opaque, hwaddr addr,
+                            uint64_t val, unsigned size)
 {
     UHCIState *s = opaque;
 
-    addr &= 0x1f;
     trace_usb_uhci_mmio_writew(addr, val);
 
     switch(addr) {
@@ -498,7 +475,7 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
             trace_usb_uhci_schedule_start();
             s->expire_time = qemu_get_clock_ns(vm_clock) +
                 (get_ticks_per_sec() / FRAME_TIMER_FREQ);
-            qemu_mod_timer(s->frame_timer, qemu_get_clock_ns(vm_clock));
+            qemu_mod_timer(s->frame_timer, s->expire_time);
             s->status &= ~UHCI_STS_HCHALTED;
         } else if (!(val & UHCI_CMD_RS)) {
             s->status |= UHCI_STS_HCHALTED;
@@ -537,6 +514,17 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
         if (s->status & UHCI_STS_HCHALTED)
             s->frnum = val & 0x7ff;
         break;
+    case 0x08:
+        s->fl_base_addr &= 0xffff0000;
+        s->fl_base_addr |= val & ~0xfff;
+        break;
+    case 0x0a:
+        s->fl_base_addr &= 0x0000ffff;
+        s->fl_base_addr |= (val << 16);
+        break;
+    case 0x0c:
+        s->sof_timing = val & 0xff;
+        break;
     case 0x10 ... 0x1f:
         {
             UHCIPort *port;
@@ -568,12 +556,11 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t uhci_ioport_readw(void *opaque, uint32_t addr)
+static uint64_t uhci_port_read(void *opaque, hwaddr addr, unsigned size)
 {
     UHCIState *s = opaque;
     uint32_t val;
 
-    addr &= 0x1f;
     switch(addr) {
     case 0x00:
         val = s->cmd;
@@ -587,6 +574,15 @@ static uint32_t uhci_ioport_readw(void *opaque, uint32_t addr)
     case 0x06:
         val = s->frnum;
         break;
+    case 0x08:
+        val = s->fl_base_addr & 0xffff;
+        break;
+    case 0x0a:
+        val = (s->fl_base_addr >> 16) & 0xffff;
+        break;
+    case 0x0c:
+        val = s->sof_timing;
+        break;
     case 0x10 ... 0x1f:
         {
             UHCIPort *port;
@@ -609,38 +605,6 @@ static uint32_t uhci_ioport_readw(void *opaque, uint32_t addr)
     return val;
 }
 
-static void uhci_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
-{
-    UHCIState *s = opaque;
-
-    addr &= 0x1f;
-    trace_usb_uhci_mmio_writel(addr, val);
-
-    switch(addr) {
-    case 0x08:
-        s->fl_base_addr = val & ~0xfff;
-        break;
-    }
-}
-
-static uint32_t uhci_ioport_readl(void *opaque, uint32_t addr)
-{
-    UHCIState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x1f;
-    switch(addr) {
-    case 0x08:
-        val = s->fl_base_addr;
-        break;
-    default:
-        val = 0xffffffff;
-        break;
-    }
-    trace_usb_uhci_mmio_readl(addr, val);
-    return val;
-}
-
 /* signal resume if controller suspended */
 static void uhci_resume (void *opaque)
 {
@@ -853,7 +817,7 @@ static int uhci_handle_td(UHCIState *s, UHCIQueue *q, uint32_t qh_addr,
     }
 
     if (q) {
-        q->valid = 32;
+        q->valid = QH_VALID;
     }
 
     /* Is active ? */
@@ -1174,10 +1138,10 @@ static void uhci_bh(void *opaque)
 static void uhci_frame_timer(void *opaque)
 {
     UHCIState *s = opaque;
+    uint64_t t_now, t_last_run;
+    int i, frames;
+    const uint64_t frame_t = get_ticks_per_sec() / FRAME_TIMER_FREQ;
 
-    /* prepare the timer for the next frame */
-    s->expire_time += (get_ticks_per_sec() / FRAME_TIMER_FREQ);
-    s->frame_bytes = 0;
     s->completions_only = false;
     qemu_bh_cancel(s->bh);
 
@@ -1191,7 +1155,35 @@ static void uhci_frame_timer(void *opaque)
         return;
     }
 
-    /* Complete the previous frame */
+    /* We still store expire_time in our state, for migration */
+    t_last_run = s->expire_time - frame_t;
+    t_now = qemu_get_clock_ns(vm_clock);
+
+    /* Process up to MAX_FRAMES_PER_TICK frames */
+    frames = (t_now - t_last_run) / frame_t;
+    if (frames > s->maxframes) {
+        int skipped = frames - s->maxframes;
+        s->expire_time += skipped * frame_t;
+        s->frnum = (s->frnum + skipped) & 0x7ff;
+        frames -= skipped;
+    }
+    if (frames > MAX_FRAMES_PER_TICK) {
+        frames = MAX_FRAMES_PER_TICK;
+    }
+
+    for (i = 0; i < frames; i++) {
+        s->frame_bytes = 0;
+        trace_usb_uhci_frame_start(s->frnum);
+        uhci_async_validate_begin(s);
+        uhci_process_frame(s);
+        uhci_async_validate_end(s);
+        /* The spec says frnum is the frame currently being processed, and
+         * the guest must look at frnum - 1 on interrupt, so inc frnum now */
+        s->frnum = (s->frnum + 1) & 0x7ff;
+        s->expire_time += frame_t;
+    }
+
+    /* Complete the previous frame(s) */
     if (s->pending_int_mask) {
         s->status2 |= s->pending_int_mask;
         s->status  |= UHCI_STS_USBINT;
@@ -1199,32 +1191,17 @@ static void uhci_frame_timer(void *opaque)
     }
     s->pending_int_mask = 0;
 
-    /* Start new frame */
-    s->frnum = (s->frnum + 1) & 0x7ff;
-
-    trace_usb_uhci_frame_start(s->frnum);
-
-    uhci_async_validate_begin(s);
-
-    uhci_process_frame(s);
-
-    uhci_async_validate_end(s);
-
-    qemu_mod_timer(s->frame_timer, s->expire_time);
+    qemu_mod_timer(s->frame_timer, t_now + frame_t);
 }
 
-static const MemoryRegionPortio uhci_portio[] = {
-    { 0, 32, 2, .write = uhci_ioport_writew, },
-    { 0, 32, 2, .read = uhci_ioport_readw, },
-    { 0, 32, 4, .write = uhci_ioport_writel, },
-    { 0, 32, 4, .read = uhci_ioport_readl, },
-    { 0, 32, 1, .write = uhci_ioport_writeb, },
-    { 0, 32, 1, .read = uhci_ioport_readb, },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps uhci_ioport_ops = {
-    .old_portio = uhci_portio,
+    .read  = uhci_port_read,
+    .write = uhci_port_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static USBPortOps uhci_port_ops = {
@@ -1311,6 +1288,7 @@ static Property uhci_properties[] = {
     DEFINE_PROP_STRING("masterbus", UHCIState, masterbus),
     DEFINE_PROP_UINT32("firstport", UHCIState, firstport, 0),
     DEFINE_PROP_UINT32("bandwidth", UHCIState, frame_bandwidth, 1280),
+    DEFINE_PROP_UINT32("maxframes", UHCIState, maxframes, 128),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index efb509e..5fb0c48 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -19,11 +19,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "hw/hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "hw/usb.h"
-#include "hw/pci.h"
-#include "hw/msi.h"
-#include "hw/msix.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
 #include "trace.h"
 
 //#define DEBUG_XHCI
@@ -1177,6 +1177,7 @@ static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
     XHCISlot *slot;
     XHCIEPContext *epctx;
     int i, xferi, killed = 0;
+    USBEndpoint *ep = NULL;
     assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
@@ -1192,9 +1193,16 @@ static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
 
     xferi = epctx->next_xfer;
     for (i = 0; i < TD_QUEUE; i++) {
+        if (epctx->transfers[xferi].packet.ep) {
+            ep = epctx->transfers[xferi].packet.ep;
+        }
         killed += xhci_ep_nuke_one_xfer(&epctx->transfers[xferi]);
+        epctx->transfers[xferi].packet.ep = NULL;
         xferi = (xferi + 1) % TD_QUEUE;
     }
+    if (ep) {
+        usb_device_ep_stopped(ep->dev, ep);
+    }
     return killed;
 }
 
@@ -1963,13 +1971,18 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
     if (bsr) {
         slot_ctx[3] = SLOT_DEFAULT << SLOT_STATE_SHIFT;
     } else {
+        USBPacket p;
         slot->devaddr = xhci->devaddr++;
         slot_ctx[3] = (SLOT_ADDRESSED << SLOT_STATE_SHIFT) | slot->devaddr;
         DPRINTF("xhci: device address is %d\n", slot->devaddr);
         usb_device_reset(dev);
-        usb_device_handle_control(dev, NULL,
+        usb_packet_setup(&p, USB_TOKEN_OUT,
+                         usb_ep_get(dev, USB_TOKEN_OUT, 0),
+                         0, false, false);
+        usb_device_handle_control(dev, &p,
                                   DeviceOutRequest | USB_REQ_SET_ADDRESS,
                                   slot->devaddr, 0, 0, NULL);
+        assert(p.status != USB_RET_ASYNC);
     }
 
     res = xhci_enable_ep(xhci, slotid, 1, octx+32, ep0_ctx);
@@ -2186,6 +2199,28 @@ static unsigned int xhci_get_slot(XHCIState *xhci, XHCIEvent *event, XHCITRB *tr
     return slotid;
 }
 
+/* cleanup slot state on usb device detach */
+static void xhci_detach_slot(XHCIState *xhci, USBPort *uport)
+{
+    int slot, ep;
+
+    for (slot = 0; slot < xhci->numslots; slot++) {
+        if (xhci->slots[slot].uport == uport) {
+            break;
+        }
+    }
+    if (slot == xhci->numslots) {
+        return;
+    }
+
+    for (ep = 0; ep < 31; ep++) {
+        if (xhci->slots[slot].eps[ep]) {
+            xhci_ep_nuke_xfers(xhci, slot+1, ep+1);
+        }
+    }
+    xhci->slots[slot].uport = NULL;
+}
+
 static TRBCCode xhci_get_port_bandwidth(XHCIState *xhci, uint64_t pctx)
 {
     dma_addr_t ctx;
@@ -2928,6 +2963,7 @@ static void xhci_detach(USBPort *usbport)
     XHCIState *xhci = usbport->opaque;
     XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
+    xhci_detach_slot(xhci, usbport);
     xhci_port_update(port, 1);
 }
 
@@ -2959,13 +2995,8 @@ static void xhci_child_detach(USBPort *uport, USBDevice *child)
 {
     USBBus *bus = usb_bus_from_device(child);
     XHCIState *xhci = container_of(bus, XHCIState, bus);
-    int i;
 
-    for (i = 0; i < xhci->numslots; i++) {
-        if (xhci->slots[i].uport == uport) {
-            xhci->slots[i].uport = NULL;
-        }
-    }
+    xhci_detach_slot(xhci, uport);
 }
 
 static USBPortOps xhci_uport_ops = {
@@ -3170,7 +3201,7 @@ static void xhci_class_init(ObjectClass *klass, void *data)
     k->no_hotplug   = 1;
 }
 
-static TypeInfo xhci_info = {
+static const TypeInfo xhci_info = {
     .name          = "nec-usb-xhci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XHCIState),
diff --git a/hw/usb/host-bsd.c b/hw/usb/host-bsd.c
index 6473e8b..07f0e01 100644
--- a/hw/usb/host-bsd.c
+++ b/hw/usb/host-bsd.c
@@ -25,7 +25,7 @@
  */
 
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "hw/usb.h"
 
 /* usb.h declares these */
@@ -292,6 +292,7 @@ static void usb_host_handle_destroy(USBDevice *opaque)
 
 static int usb_host_initfn(USBDevice *dev)
 {
+    dev->flags |= (1 << USB_DEV_FLAG_IS_HOST);
     return 0;
 }
 
@@ -406,7 +407,7 @@ static void usb_host_class_initfn(ObjectClass *klass, void *data)
     uc->handle_destroy = usb_host_handle_destroy;
 }
 
-static TypeInfo usb_host_dev_info = {
+static const TypeInfo usb_host_dev_info = {
     .name          = "usb-host",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHostDevice),
@@ -632,7 +633,7 @@ static int usb_host_info_device(void *opaque,
     return 0;
 }
 
-void usb_host_info(Monitor *mon)
+void usb_host_info(Monitor *mon, const QDict *qdict)
 {
     usb_host_scan(mon, usb_host_info_device);
 }
diff --git a/hw/usb/host-linux.c b/hw/usb/host-linux.c
index aa77b77..a2cff8a 100644
--- a/hw/usb/host-linux.c
+++ b/hw/usb/host-linux.c
@@ -31,9 +31,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/timer.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
 #include <dirent.h>
@@ -1314,7 +1314,7 @@ static int usb_host_open(USBHostDevice *dev, int bus_num,
 
     dev->bus_num = bus_num;
     dev->addr = addr;
-    strcpy(dev->port, port);
+    pstrcpy(dev->port, sizeof(dev->port), port);
     dev->fd = fd;
 
     /* read the device description */
@@ -1476,6 +1476,7 @@ static int usb_host_initfn(USBDevice *dev)
 {
     USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
 
+    dev->flags |= (1 << USB_DEV_FLAG_IS_HOST);
     dev->auto_attach = 0;
     s->fd = -1;
     s->hub_fd = -1;
@@ -1533,7 +1534,7 @@ static void usb_host_class_initfn(ObjectClass *klass, void *data)
     dc->props = usb_host_dev_properties;
 }
 
-static TypeInfo usb_host_dev_info = {
+static const TypeInfo usb_host_dev_info = {
     .name          = "usb-host",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHostDevice),
@@ -1759,7 +1760,7 @@ static int usb_host_auto_scan(void *opaque, int bus_num,
         if (f->addr > 0 && f->addr != addr) {
             continue;
         }
-        if (f->port != NULL && (port == NULL || strcmp(f->port, port) != 0)) {
+        if (f->port != NULL && strcmp(f->port, port) != 0) {
             continue;
         }
 
@@ -1997,7 +1998,7 @@ static void hex2str(int val, char *str, size_t size)
     }
 }
 
-void usb_host_info(Monitor *mon)
+void usb_host_info(Monitor *mon, const QDict *qdict)
 {
     struct USBAutoFilter *f;
     struct USBHostDevice *s;
diff --git a/hw/usb/host-stub.c b/hw/usb/host-stub.c
index b4e10c1..8affba7 100644
--- a/hw/usb/host-stub.c
+++ b/hw/usb/host-stub.c
@@ -31,11 +31,11 @@
  */
 
 #include "qemu-common.h"
-#include "console.h"
+#include "ui/console.h"
 #include "hw/usb.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
-void usb_host_info(Monitor *mon)
+void usb_host_info(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "USB host devices not supported\n");
 }
diff --git a/hw/usb/libhw.c b/hw/usb/libhw.c
index 24d3cad..75f022f 100644
--- a/hw/usb/libhw.c
+++ b/hw/usb/libhw.c
@@ -20,9 +20,9 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "cpu-common.h"
+#include "exec/cpu-common.h"
 #include "hw/usb.h"
-#include "dma.h"
+#include "sysemu/dma.h"
 
 int usb_packet_map(USBPacket *p, QEMUSGList *sgl)
 {
diff --git a/hw/usb/quirks-ftdi-ids.h b/hw/usb/quirks-ftdi-ids.h
new file mode 100644
index 0000000..57c12ef
--- /dev/null
+++ b/hw/usb/quirks-ftdi-ids.h
@@ -0,0 +1,1255 @@
+/*
+ * vendor/product IDs (VID/PID) of devices using FTDI USB serial converters.
+ * Please keep numerically sorted within individual areas, thanks!
+ *
+ * Philipp Gühring - pg@futureware.at - added the Device ID of the USB relais
+ * from Rudolf Gugler
+ *
+ */
+
+
+/**********************************/
+/***** devices using FTDI VID *****/
+/**********************************/
+
+
+#define FTDI_VID	0x0403	/* Vendor Id */
+
+
+/*** "original" FTDI device PIDs ***/
+
+#define FTDI_8U232AM_PID 0x6001 /* Similar device to SIO above */
+#define FTDI_8U232AM_ALT_PID 0x6006 /* FTDI's alternate PID for above */
+#define FTDI_8U2232C_PID 0x6010 /* Dual channel device */
+#define FTDI_4232H_PID 0x6011 /* Quad channel hi-speed device */
+#define FTDI_232H_PID  0x6014 /* Single channel hi-speed device */
+#define FTDI_FTX_PID   0x6015 /* FT-X series (FT201X, FT230X, FT231X, etc) */
+#define FTDI_SIO_PID	0x8372	/* Product Id SIO application of 8U100AX */
+#define FTDI_232RL_PID  0xFBFA  /* Product ID for FT232RL */
+
+
+/*** third-party PIDs (using FTDI_VID) ***/
+
+#define FTDI_LUMEL_PD12_PID	0x6002
+
+/*
+ * Marvell OpenRD Base, Client
+ * http://www.open-rd.org
+ * OpenRD Base, Client use VID 0x0403
+ */
+#define MARVELL_OPENRD_PID	0x9e90
+
+/* www.candapter.com Ewert Energy Systems CANdapter device */
+#define FTDI_CANDAPTER_PID 0x9F80 /* Product Id */
+
+/*
+ * Texas Instruments XDS100v2 JTAG / BeagleBone A3
+ * http://processors.wiki.ti.com/index.php/XDS100
+ * http://beagleboard.org/bone
+ */
+#define TI_XDS100V2_PID		0xa6d0
+
+#define FTDI_NXTCAM_PID		0xABB8 /* NXTCam for Mindstorms NXT */
+
+/* US Interface Navigator (http://www.usinterface.com/) */
+#define FTDI_USINT_CAT_PID	0xb810	/* Navigator CAT and 2nd PTT lines */
+#define FTDI_USINT_WKEY_PID	0xb811	/* Navigator WKEY and FSK lines */
+#define FTDI_USINT_RS232_PID	0xb812	/* Navigator RS232 and CONFIG lines */
+
+/* OOCDlink by Joern Kaipf <joernk@web.de>
+ * (http://www.joernonline.de/) */
+#define FTDI_OOCDLINK_PID	0xbaf8	/* Amontec JTAGkey */
+
+/* Luminary Micro Stellaris Boards, VID = FTDI_VID */
+/* FTDI 2332C Dual channel device, side A=245 FIFO (JTAG), Side B=RS232 UART */
+#define LMI_LM3S_DEVEL_BOARD_PID	0xbcd8
+#define LMI_LM3S_EVAL_BOARD_PID		0xbcd9
+#define LMI_LM3S_ICDI_BOARD_PID		0xbcda
+
+#define FTDI_TURTELIZER_PID	0xBDC8 /* JTAG/RS-232 adapter by egnite GmbH */
+
+/* OpenDCC (www.opendcc.de) product id */
+#define FTDI_OPENDCC_PID	0xBFD8
+#define FTDI_OPENDCC_SNIFFER_PID	0xBFD9
+#define FTDI_OPENDCC_THROTTLE_PID	0xBFDA
+#define FTDI_OPENDCC_GATEWAY_PID	0xBFDB
+#define FTDI_OPENDCC_GBM_PID	0xBFDC
+
+/* NZR SEM 16+ USB (http://www.nzr.de) */
+#define FTDI_NZR_SEM_USB_PID	0xC1E0	/* NZR SEM-LOG16+ */
+
+/*
+ * RR-CirKits LocoBuffer USB (http://www.rr-cirkits.com)
+ */
+#define FTDI_RRCIRKITS_LOCOBUFFER_PID	0xc7d0	/* LocoBuffer USB */
+
+/* DMX4ALL DMX Interfaces */
+#define FTDI_DMX4ALL 0xC850
+
+/*
+ * ASK.fr devices
+ */
+#define FTDI_ASK_RDR400_PID	0xC991	/* ASK RDR 400 series card reader */
+
+/* www.starting-point-systems.com µChameleon device */
+#define FTDI_MICRO_CHAMELEON_PID	0xCAA0	/* Product Id */
+
+/*
+ * Tactrix OpenPort (ECU) devices.
+ * OpenPort 1.3M submitted by Donour Sizemore.
+ * OpenPort 1.3S and 1.3U submitted by Ian Abbott.
+ */
+#define FTDI_TACTRIX_OPENPORT_13M_PID	0xCC48	/* OpenPort 1.3 Mitsubishi */
+#define FTDI_TACTRIX_OPENPORT_13S_PID	0xCC49	/* OpenPort 1.3 Subaru */
+#define FTDI_TACTRIX_OPENPORT_13U_PID	0xCC4A	/* OpenPort 1.3 Universal */
+
+#define FTDI_DISTORTEC_JTAG_LOCK_PICK_PID	0xCFF8
+
+/* SCS HF Radio Modems PID's (http://www.scs-ptc.com) */
+/* the VID is the standard ftdi vid (FTDI_VID) */
+#define FTDI_SCS_DEVICE_0_PID 0xD010    /* SCS PTC-IIusb */
+#define FTDI_SCS_DEVICE_1_PID 0xD011    /* SCS Tracker / DSP TNC */
+#define FTDI_SCS_DEVICE_2_PID 0xD012
+#define FTDI_SCS_DEVICE_3_PID 0xD013
+#define FTDI_SCS_DEVICE_4_PID 0xD014
+#define FTDI_SCS_DEVICE_5_PID 0xD015
+#define FTDI_SCS_DEVICE_6_PID 0xD016
+#define FTDI_SCS_DEVICE_7_PID 0xD017
+
+/* iPlus device */
+#define FTDI_IPLUS_PID 0xD070 /* Product Id */
+#define FTDI_IPLUS2_PID 0xD071 /* Product Id */
+
+/*
+ * Gamma Scout (http://gamma-scout.com/). Submitted by rsc@runtux.com.
+ */
+#define FTDI_GAMMA_SCOUT_PID		0xD678	/* Gamma Scout online */
+
+/* Propox devices */
+#define FTDI_PROPOX_JTAGCABLEII_PID	0xD738
+#define FTDI_PROPOX_ISPCABLEIII_PID	0xD739
+
+/* Lenz LI-USB Computer Interface. */
+#define FTDI_LENZ_LIUSB_PID	0xD780
+
+/* Vardaan Enterprises Serial Interface VEUSB422R3 */
+#define FTDI_VARDAAN_PID	0xF070
+
+/*
+ * Xsens Technologies BV products (http://www.xsens.com).
+ */
+#define XSENS_CONVERTER_0_PID	0xD388
+#define XSENS_CONVERTER_1_PID	0xD389
+#define XSENS_CONVERTER_2_PID	0xD38A
+#define XSENS_CONVERTER_3_PID	0xD38B
+#define XSENS_CONVERTER_4_PID	0xD38C
+#define XSENS_CONVERTER_5_PID	0xD38D
+#define XSENS_CONVERTER_6_PID	0xD38E
+#define XSENS_CONVERTER_7_PID	0xD38F
+
+/*
+ * NDI (www.ndigital.com) product ids
+ */
+#define FTDI_NDI_HUC_PID		0xDA70	/* NDI Host USB Converter */
+#define FTDI_NDI_SPECTRA_SCU_PID	0xDA71	/* NDI Spectra SCU */
+#define FTDI_NDI_FUTURE_2_PID		0xDA72	/* NDI future device #2 */
+#define FTDI_NDI_FUTURE_3_PID		0xDA73	/* NDI future device #3 */
+#define FTDI_NDI_AURORA_SCU_PID		0xDA74	/* NDI Aurora SCU */
+
+/*
+ * ChamSys Limited (www.chamsys.co.uk) USB wing/interface product IDs
+ */
+#define FTDI_CHAMSYS_24_MASTER_WING_PID        0xDAF8
+#define FTDI_CHAMSYS_PC_WING_PID       0xDAF9
+#define FTDI_CHAMSYS_USB_DMX_PID       0xDAFA
+#define FTDI_CHAMSYS_MIDI_TIMECODE_PID 0xDAFB
+#define FTDI_CHAMSYS_MINI_WING_PID     0xDAFC
+#define FTDI_CHAMSYS_MAXI_WING_PID     0xDAFD
+#define FTDI_CHAMSYS_MEDIA_WING_PID    0xDAFE
+#define FTDI_CHAMSYS_WING_PID  0xDAFF
+
+/*
+ * Westrex International devices submitted by Cory Lee
+ */
+#define FTDI_WESTREX_MODEL_777_PID	0xDC00	/* Model 777 */
+#define FTDI_WESTREX_MODEL_8900F_PID	0xDC01	/* Model 8900F */
+
+/*
+ * ACG Identification Technologies GmbH products (http://www.acg.de/).
+ * Submitted by anton -at- goto10 -dot- org.
+ */
+#define FTDI_ACG_HFDUAL_PID		0xDD20	/* HF Dual ISO Reader (RFID) */
+
+/*
+ * Definitions for Artemis astronomical USB based cameras
+ * Check it at http://www.artemisccd.co.uk/
+ */
+#define FTDI_ARTEMIS_PID	0xDF28	/* All Artemis Cameras */
+
+/*
+ * Definitions for ATIK Instruments astronomical USB based cameras
+ * Check it at http://www.atik-instruments.com/
+ */
+#define FTDI_ATIK_ATK16_PID	0xDF30	/* ATIK ATK-16 Grayscale Camera */
+#define FTDI_ATIK_ATK16C_PID	0xDF32	/* ATIK ATK-16C Colour Camera */
+#define FTDI_ATIK_ATK16HR_PID	0xDF31	/* ATIK ATK-16HR Grayscale Camera */
+#define FTDI_ATIK_ATK16HRC_PID	0xDF33	/* ATIK ATK-16HRC Colour Camera */
+#define FTDI_ATIK_ATK16IC_PID   0xDF35  /* ATIK ATK-16IC Grayscale Camera */
+
+/*
+ * Yost Engineering, Inc. products (www.yostengineering.com).
+ * PID 0xE050 submitted by Aaron Prose.
+ */
+#define FTDI_YEI_SERVOCENTER31_PID	0xE050	/* YEI ServoCenter3.1 USB */
+
+/*
+ * ELV USB devices submitted by Christian Abt of ELV (www.elv.de).
+ * All of these devices use FTDI's vendor ID (0x0403).
+ * Further IDs taken from ELV Windows .inf file.
+ *
+ * The previously included PID for the UO 100 module was incorrect.
+ * In fact, that PID was for ELV's UR 100 USB-RS232 converter (0xFB58).
+ *
+ * Armin Laeuger originally sent the PID for the UM 100 module.
+ */
+#define FTDI_ELV_USR_PID	0xE000	/* ELV Universal-Sound-Recorder */
+#define FTDI_ELV_MSM1_PID	0xE001	/* ELV Mini-Sound-Modul */
+#define FTDI_ELV_KL100_PID	0xE002	/* ELV Kfz-Leistungsmesser KL 100 */
+#define FTDI_ELV_WS550_PID	0xE004	/* WS 550 */
+#define FTDI_ELV_EC3000_PID	0xE006	/* ENERGY CONTROL 3000 USB */
+#define FTDI_ELV_WS888_PID	0xE008	/* WS 888 */
+#define FTDI_ELV_TWS550_PID	0xE009	/* Technoline WS 550 */
+#define FTDI_ELV_FEM_PID	0xE00A	/* Funk Energie Monitor */
+#define FTDI_ELV_FHZ1300PC_PID	0xE0E8	/* FHZ 1300 PC */
+#define FTDI_ELV_WS500_PID	0xE0E9	/* PC-Wetterstation (WS 500) */
+#define FTDI_ELV_HS485_PID	0xE0EA	/* USB to RS-485 adapter */
+#define FTDI_ELV_UMS100_PID	0xE0EB	/* ELV USB Master-Slave Schaltsteckdose UMS 100 */
+#define FTDI_ELV_TFD128_PID	0xE0EC	/* ELV Temperatur-Feuchte-Datenlogger TFD 128 */
+#define FTDI_ELV_FM3RX_PID	0xE0ED	/* ELV Messwertuebertragung FM3 RX */
+#define FTDI_ELV_WS777_PID	0xE0EE	/* Conrad WS 777 */
+#define FTDI_ELV_EM1010PC_PID	0xE0EF	/* Energy monitor EM 1010 PC */
+#define FTDI_ELV_CSI8_PID	0xE0F0	/* Computer-Schalt-Interface (CSI 8) */
+#define FTDI_ELV_EM1000DL_PID	0xE0F1	/* PC-Datenlogger fuer Energiemonitor (EM 1000 DL) */
+#define FTDI_ELV_PCK100_PID	0xE0F2	/* PC-Kabeltester (PCK 100) */
+#define FTDI_ELV_RFP500_PID	0xE0F3	/* HF-Leistungsmesser (RFP 500) */
+#define FTDI_ELV_FS20SIG_PID	0xE0F4	/* Signalgeber (FS 20 SIG) */
+#define FTDI_ELV_UTP8_PID	0xE0F5	/* ELV UTP 8 */
+#define FTDI_ELV_WS300PC_PID	0xE0F6	/* PC-Wetterstation (WS 300 PC) */
+#define FTDI_ELV_WS444PC_PID	0xE0F7	/* Conrad WS 444 PC */
+#define FTDI_PHI_FISCO_PID      0xE40B  /* PHI Fisco USB to Serial cable */
+#define FTDI_ELV_UAD8_PID	0xF068	/* USB-AD-Wandler (UAD 8) */
+#define FTDI_ELV_UDA7_PID	0xF069	/* USB-DA-Wandler (UDA 7) */
+#define FTDI_ELV_USI2_PID	0xF06A	/* USB-Schrittmotoren-Interface (USI 2) */
+#define FTDI_ELV_T1100_PID	0xF06B	/* Thermometer (T 1100) */
+#define FTDI_ELV_PCD200_PID	0xF06C	/* PC-Datenlogger (PCD 200) */
+#define FTDI_ELV_ULA200_PID	0xF06D	/* USB-LCD-Ansteuerung (ULA 200) */
+#define FTDI_ELV_ALC8500_PID	0xF06E	/* ALC 8500 Expert */
+#define FTDI_ELV_FHZ1000PC_PID	0xF06F	/* FHZ 1000 PC */
+#define FTDI_ELV_UR100_PID	0xFB58	/* USB-RS232-Umsetzer (UR 100) */
+#define FTDI_ELV_UM100_PID	0xFB5A	/* USB-Modul UM 100 */
+#define FTDI_ELV_UO100_PID	0xFB5B	/* USB-Modul UO 100 */
+/* Additional ELV PIDs that default to using the FTDI D2XX drivers on
+ * MS Windows, rather than the FTDI Virtual Com Port drivers.
+ * Maybe these will be easier to use with the libftdi/libusb user-space
+ * drivers, or possibly the Comedi drivers in some cases. */
+#define FTDI_ELV_CLI7000_PID	0xFB59	/* Computer-Light-Interface (CLI 7000) */
+#define FTDI_ELV_PPS7330_PID	0xFB5C	/* Processor-Power-Supply (PPS 7330) */
+#define FTDI_ELV_TFM100_PID	0xFB5D	/* Temperatur-Feuchte-Messgeraet (TFM 100) */
+#define FTDI_ELV_UDF77_PID	0xFB5E	/* USB DCF Funkuhr (UDF 77) */
+#define FTDI_ELV_UIO88_PID	0xFB5F	/* USB-I/O Interface (UIO 88) */
+
+/*
+ * EVER Eco Pro UPS (http://www.ever.com.pl/)
+ */
+
+#define	EVER_ECO_PRO_CDS	0xe520	/* RS-232 converter */
+
+/*
+ * Active Robots product ids.
+ */
+#define FTDI_ACTIVE_ROBOTS_PID	0xE548	/* USB comms board */
+
+/* Pyramid Computer GmbH */
+#define FTDI_PYRAMID_PID	0xE6C8	/* Pyramid Appliance Display */
+
+/* www.elsterelectricity.com Elster Unicom III Optical Probe */
+#define FTDI_ELSTER_UNICOM_PID		0xE700 /* Product Id */
+
+/*
+ * Gude Analog- und Digitalsysteme GmbH
+ */
+#define FTDI_GUDEADS_E808_PID    0xE808
+#define FTDI_GUDEADS_E809_PID    0xE809
+#define FTDI_GUDEADS_E80A_PID    0xE80A
+#define FTDI_GUDEADS_E80B_PID    0xE80B
+#define FTDI_GUDEADS_E80C_PID    0xE80C
+#define FTDI_GUDEADS_E80D_PID    0xE80D
+#define FTDI_GUDEADS_E80E_PID    0xE80E
+#define FTDI_GUDEADS_E80F_PID    0xE80F
+#define FTDI_GUDEADS_E888_PID    0xE888  /* Expert ISDN Control USB */
+#define FTDI_GUDEADS_E889_PID    0xE889  /* USB RS-232 OptoBridge */
+#define FTDI_GUDEADS_E88A_PID    0xE88A
+#define FTDI_GUDEADS_E88B_PID    0xE88B
+#define FTDI_GUDEADS_E88C_PID    0xE88C
+#define FTDI_GUDEADS_E88D_PID    0xE88D
+#define FTDI_GUDEADS_E88E_PID    0xE88E
+#define FTDI_GUDEADS_E88F_PID    0xE88F
+
+/*
+ * Eclo (http://www.eclo.pt/) product IDs.
+ * PID 0xEA90 submitted by Martin Grill.
+ */
+#define FTDI_ECLO_COM_1WIRE_PID	0xEA90	/* COM to 1-Wire USB adaptor */
+
+/* TNC-X USB-to-packet-radio adapter, versions prior to 3.0 (DLP module) */
+#define FTDI_TNC_X_PID		0xEBE0
+
+/*
+ * Teratronik product ids.
+ * Submitted by O. Wölfelschneider.
+ */
+#define FTDI_TERATRONIK_VCP_PID	 0xEC88	/* Teratronik device (preferring VCP driver on windows) */
+#define FTDI_TERATRONIK_D2XX_PID 0xEC89	/* Teratronik device (preferring D2XX driver on windows) */
+
+/* Rig Expert Ukraine devices */
+#define FTDI_REU_TINY_PID		0xED22	/* RigExpert Tiny */
+
+/*
+ * Hameg HO820 and HO870 interface (using VID 0x0403)
+ */
+#define HAMEG_HO820_PID			0xed74
+#define HAMEG_HO730_PID			0xed73
+#define HAMEG_HO720_PID			0xed72
+#define HAMEG_HO870_PID			0xed71
+
+/*
+ *  MaxStream devices	www.maxstream.net
+ */
+#define FTDI_MAXSTREAM_PID	0xEE18	/* Xbee PKG-U Module */
+
+/*
+ * microHAM product IDs (http://www.microham.com).
+ * Submitted by Justin Burket (KL1RL) <zorton@jtan.com>
+ * and Mike Studer (K6EEP) <k6eep@hamsoftware.org>.
+ * Ian Abbott <abbotti@mev.co.uk> added a few more from the driver INF file.
+ */
+#define FTDI_MHAM_KW_PID	0xEEE8	/* USB-KW interface */
+#define FTDI_MHAM_YS_PID	0xEEE9	/* USB-YS interface */
+#define FTDI_MHAM_Y6_PID	0xEEEA	/* USB-Y6 interface */
+#define FTDI_MHAM_Y8_PID	0xEEEB	/* USB-Y8 interface */
+#define FTDI_MHAM_IC_PID	0xEEEC	/* USB-IC interface */
+#define FTDI_MHAM_DB9_PID	0xEEED	/* USB-DB9 interface */
+#define FTDI_MHAM_RS232_PID	0xEEEE	/* USB-RS232 interface */
+#define FTDI_MHAM_Y9_PID	0xEEEF	/* USB-Y9 interface */
+
+/* Domintell products  http://www.domintell.com */
+#define FTDI_DOMINTELL_DGQG_PID	0xEF50	/* Master */
+#define FTDI_DOMINTELL_DUSB_PID	0xEF51	/* DUSB01 module */
+
+/*
+ * The following are the values for the Perle Systems
+ * UltraPort USB serial converters
+ */
+#define FTDI_PERLE_ULTRAPORT_PID 0xF0C0	/* Perle UltraPort Product Id */
+
+/* Sprog II (Andrew Crosland's SprogII DCC interface) */
+#define FTDI_SPROG_II		0xF0C8
+
+/* an infrared receiver for user access control with IR tags */
+#define FTDI_PIEGROUP_PID	0xF208	/* Product Id */
+
+/* ACT Solutions HomePro ZWave interface
+   (http://www.act-solutions.com/HomePro-Product-Matrix.html) */
+#define FTDI_ACTZWAVE_PID	0xF2D0
+
+/*
+ * 4N-GALAXY.DE PIDs for CAN-USB, USB-RS232, USB-RS422, USB-RS485,
+ * USB-TTY aktiv, USB-TTY passiv.  Some PIDs are used by several devices
+ * and I'm not entirely sure which are used by which.
+ */
+#define FTDI_4N_GALAXY_DE_1_PID	0xF3C0
+#define FTDI_4N_GALAXY_DE_2_PID	0xF3C1
+#define FTDI_4N_GALAXY_DE_3_PID	0xF3C2
+
+/*
+ * Linx Technologies product ids
+ */
+#define LINX_SDMUSBQSS_PID	0xF448	/* Linx SDM-USB-QS-S */
+#define LINX_MASTERDEVEL2_PID   0xF449	/* Linx Master Development 2.0 */
+#define LINX_FUTURE_0_PID   0xF44A	/* Linx future device */
+#define LINX_FUTURE_1_PID   0xF44B	/* Linx future device */
+#define LINX_FUTURE_2_PID   0xF44C	/* Linx future device */
+
+/*
+ * Oceanic product ids
+ */
+#define FTDI_OCEANIC_PID	0xF460  /* Oceanic dive instrument */
+
+/*
+ * SUUNTO product ids
+ */
+#define FTDI_SUUNTO_SPORTS_PID	0xF680	/* Suunto Sports instrument */
+
+/* USB-UIRT - An infrared receiver and transmitter using the 8U232AM chip */
+/* http://www.usbuirt.com/ */
+#define FTDI_USB_UIRT_PID	0xF850	/* Product Id */
+
+/* CCS Inc. ICDU/ICDU40 product ID -
+ * the FT232BM is used in an in-circuit-debugger unit for PIC16's/PIC18's */
+#define FTDI_CCSICDU20_0_PID    0xF9D0
+#define FTDI_CCSICDU40_1_PID    0xF9D1
+#define FTDI_CCSMACHX_2_PID     0xF9D2
+#define FTDI_CCSLOAD_N_GO_3_PID 0xF9D3
+#define FTDI_CCSICDU64_4_PID    0xF9D4
+#define FTDI_CCSPRIME8_5_PID    0xF9D5
+
+/*
+ * The following are the values for the Matrix Orbital LCD displays,
+ * which are the FT232BM ( similar to the 8U232AM )
+ */
+#define FTDI_MTXORB_0_PID      0xFA00  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_1_PID      0xFA01  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_2_PID      0xFA02  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_3_PID      0xFA03  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_4_PID      0xFA04  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_5_PID      0xFA05  /* Matrix Orbital Product Id */
+#define FTDI_MTXORB_6_PID      0xFA06  /* Matrix Orbital Product Id */
+
+/*
+ * Home Electronics (www.home-electro.com) USB gadgets
+ */
+#define FTDI_HE_TIRA1_PID	0xFA78	/* Tira-1 IR transceiver */
+
+/* Inside Accesso contactless reader (http://www.insidecontactless.com/) */
+#define INSIDE_ACCESSO		0xFAD0
+
+/*
+ * ThorLabs USB motor drivers
+ */
+#define FTDI_THORLABS_PID		0xfaf0 /* ThorLabs USB motor drivers */
+
+/*
+ * Protego product ids
+ */
+#define PROTEGO_SPECIAL_1	0xFC70	/* special/unknown device */
+#define PROTEGO_R2X0		0xFC71	/* R200-USB TRNG unit (R210, R220, and R230) */
+#define PROTEGO_SPECIAL_3	0xFC72	/* special/unknown device */
+#define PROTEGO_SPECIAL_4	0xFC73	/* special/unknown device */
+
+/*
+ * Sony Ericsson product ids
+ */
+#define FTDI_DSS20_PID		0xFC82	/* DSS-20 Sync Station for Sony Ericsson P800 */
+#define FTDI_URBAN_0_PID	0xFC8A	/* Sony Ericsson Urban, uart #0 */
+#define FTDI_URBAN_1_PID	0xFC8B	/* Sony Ericsson Urban, uart #1 */
+
+/* www.irtrans.de device */
+#define FTDI_IRTRANS_PID 0xFC60 /* Product Id */
+
+/*
+ * RM Michaelides CANview USB (http://www.rmcan.com) (FTDI_VID)
+ * CAN fieldbus interface adapter, added by port GmbH www.port.de)
+ * Ian Abbott changed the macro names for consistency.
+ */
+#define FTDI_RM_CANVIEW_PID	0xfd60	/* Product Id */
+/* www.thoughttechnology.com/ TT-USB provide with procomp use ftdi_sio */
+#define FTDI_TTUSB_PID 0xFF20 /* Product Id */
+
+#define FTDI_USBX_707_PID 0xF857	/* ADSTech IR Blaster USBX-707 (FTDI_VID) */
+
+#define FTDI_RELAIS_PID	0xFA10  /* Relais device from Rudolf Gugler */
+
+/*
+ * PCDJ use ftdi based dj-controllers. The following PID is
+ * for their DAC-2 device http://www.pcdjhardware.com/DAC2.asp
+ * (the VID is the standard ftdi vid (FTDI_VID), PID sent by Wouter Paesen)
+ */
+#define FTDI_PCDJ_DAC2_PID 0xFA88
+
+#define FTDI_R2000KU_TRUE_RNG	0xFB80  /* R2000KU TRUE RNG (FTDI_VID) */
+
+/*
+ * DIEBOLD BCS SE923 (FTDI_VID)
+ */
+#define DIEBOLD_BCS_SE923_PID	0xfb99
+
+/* www.crystalfontz.com devices
+ * - thanx for providing free devices for evaluation !
+ * they use the ftdi chipset for the USB interface
+ * and the vendor id is the same
+ */
+#define FTDI_XF_632_PID 0xFC08	/* 632: 16x2 Character Display */
+#define FTDI_XF_634_PID 0xFC09	/* 634: 20x4 Character Display */
+#define FTDI_XF_547_PID 0xFC0A	/* 547: Two line Display */
+#define FTDI_XF_633_PID 0xFC0B	/* 633: 16x2 Character Display with Keys */
+#define FTDI_XF_631_PID 0xFC0C	/* 631: 20x2 Character Display */
+#define FTDI_XF_635_PID 0xFC0D	/* 635: 20x4 Character Display */
+#define FTDI_XF_640_PID 0xFC0E	/* 640: Two line Display */
+#define FTDI_XF_642_PID 0xFC0F	/* 642: Two line Display */
+
+/*
+ * Video Networks Limited / Homechoice in the UK use an ftdi-based device
+ * for their 1Mb broadband internet service.  The following PID is exhibited
+ * by the usb device supplied (the VID is the standard ftdi vid (FTDI_VID)
+ */
+#define FTDI_VNHCPCUSB_D_PID 0xfe38 /* Product Id */
+
+/* AlphaMicro Components AMC-232USB01 device (FTDI_VID) */
+#define FTDI_AMC232_PID 0xFF00 /* Product Id */
+
+/*
+ * IBS elektronik product ids (FTDI_VID)
+ * Submitted by Thomas Schleusener
+ */
+#define FTDI_IBS_US485_PID	0xff38  /* IBS US485 (USB<-->RS422/485 interface) */
+#define FTDI_IBS_PICPRO_PID	0xff39  /* IBS PIC-Programmer */
+#define FTDI_IBS_PCMCIA_PID	0xff3a  /* IBS Card reader for PCMCIA SRAM-cards */
+#define FTDI_IBS_PK1_PID	0xff3b  /* IBS PK1 - Particel counter */
+#define FTDI_IBS_RS232MON_PID	0xff3c  /* IBS RS232 - Monitor */
+#define FTDI_IBS_APP70_PID	0xff3d  /* APP 70 (dust monitoring system) */
+#define FTDI_IBS_PEDO_PID	0xff3e  /* IBS PEDO-Modem (RF modem 868.35 MHz) */
+#define FTDI_IBS_PROD_PID	0xff3f  /* future device */
+/* www.canusb.com Lawicel CANUSB device (FTDI_VID) */
+#define FTDI_CANUSB_PID 0xFFA8 /* Product Id */
+
+/*
+ * TavIR AVR product ids (FTDI_VID)
+ */
+#define FTDI_TAVIR_STK500_PID	0xFA33	/* STK500 AVR programmer */
+
+/*
+ * TIAO product ids (FTDI_VID)
+ * http://www.tiaowiki.com/w/Main_Page
+ */
+#define FTDI_TIAO_UMPA_PID	0x8a98	/* TIAO/DIYGADGET USB Multi-Protocol Adapter */
+
+
+/********************************/
+/** third-party VID/PID combos **/
+/********************************/
+
+
+
+/*
+ * Atmel STK541
+ */
+#define ATMEL_VID		0x03eb /* Vendor ID */
+#define STK541_PID		0x2109 /* Zigbee Controller */
+
+/*
+ * Blackfin gnICE JTAG
+ * http://docs.blackfin.uclinux.org/doku.php?id=hw:jtag:gnice
+ */
+#define ADI_VID			0x0456
+#define ADI_GNICE_PID		0xF000
+#define ADI_GNICEPLUS_PID	0xF001
+
+/*
+ * Microchip Technology, Inc.
+ *
+ * MICROCHIP_VID (0x04D8) and MICROCHIP_USB_BOARD_PID (0x000A) are
+ * used by single function CDC ACM class based firmware demo
+ * applications.  The VID/PID has also been used in firmware
+ * emulating FTDI serial chips by:
+ * Hornby Elite - Digital Command Control Console
+ * http://www.hornby.com/hornby-dcc/controllers/
+ */
+#define MICROCHIP_VID		0x04D8
+#define MICROCHIP_USB_BOARD_PID	0x000A /* CDC RS-232 Emulation Demo */
+
+/*
+ * RATOC REX-USB60F
+ */
+#define RATOC_VENDOR_ID		0x0584
+#define RATOC_PRODUCT_ID_USB60F	0xb020
+
+/*
+ * Acton Research Corp.
+ */
+#define ACTON_VID		0x0647	/* Vendor ID */
+#define ACTON_SPECTRAPRO_PID	0x0100
+
+/*
+ * Contec products (http://www.contec.com)
+ * Submitted by Daniel Sangorrin
+ */
+#define CONTEC_VID		0x06CE	/* Vendor ID */
+#define CONTEC_COM1USBH_PID	0x8311	/* COM-1(USB)H */
+
+/*
+ * Definitions for B&B Electronics products.
+ */
+#define BANDB_VID		0x0856	/* B&B Electronics Vendor ID */
+#define BANDB_USOTL4_PID	0xAC01	/* USOTL4 Isolated RS-485 Converter */
+#define BANDB_USTL4_PID		0xAC02	/* USTL4 RS-485 Converter */
+#define BANDB_USO9ML2_PID	0xAC03	/* USO9ML2 Isolated RS-232 Converter */
+#define BANDB_USOPTL4_PID	0xAC11
+#define BANDB_USPTL4_PID	0xAC12
+#define BANDB_USO9ML2DR_2_PID	0xAC16
+#define BANDB_USO9ML2DR_PID	0xAC17
+#define BANDB_USOPTL4DR2_PID	0xAC18	/* USOPTL4R-2 2-port Isolated RS-232 Converter */
+#define BANDB_USOPTL4DR_PID	0xAC19
+#define BANDB_485USB9F_2W_PID	0xAC25
+#define BANDB_485USB9F_4W_PID	0xAC26
+#define BANDB_232USB9M_PID	0xAC27
+#define BANDB_485USBTB_2W_PID	0xAC33
+#define BANDB_485USBTB_4W_PID	0xAC34
+#define BANDB_TTL5USB9M_PID	0xAC49
+#define BANDB_TTL3USB9M_PID	0xAC50
+#define BANDB_ZZ_PROG1_USB_PID	0xBA02
+
+/*
+ * Intrepid Control Systems (http://www.intrepidcs.com/) ValueCAN and NeoVI
+ */
+#define INTREPID_VID		0x093C
+#define INTREPID_VALUECAN_PID	0x0601
+#define INTREPID_NEOVI_PID	0x0701
+
+/*
+ * Definitions for ID TECH (www.idt-net.com) devices
+ */
+#define IDTECH_VID		0x0ACD	/* ID TECH Vendor ID */
+#define IDTECH_IDT1221U_PID	0x0300	/* IDT1221U USB to RS-232 adapter */
+
+/*
+ * Definitions for Omnidirectional Control Technology, Inc. devices
+ */
+#define OCT_VID			0x0B39	/* OCT vendor ID */
+/* Note: OCT US101 is also rebadged as Dick Smith Electronics (NZ) XH6381 */
+/* Also rebadged as Dick Smith Electronics (Aus) XH6451 */
+/* Also rebadged as SIIG Inc. model US2308 hardware version 1 */
+#define OCT_DK201_PID		0x0103	/* OCT DK201 USB docking station */
+#define OCT_US101_PID		0x0421	/* OCT US101 USB to RS-232 */
+
+/*
+ * Definitions for Icom Inc. devices
+ */
+#define ICOM_VID		0x0C26 /* Icom vendor ID */
+/* Note: ID-1 is a communications tranceiver for HAM-radio operators */
+#define ICOM_ID_1_PID		0x0004 /* ID-1 USB to RS-232 */
+/* Note: OPC is an Optional cable to connect an Icom Tranceiver */
+#define ICOM_OPC_U_UC_PID	0x0018 /* OPC-478UC, OPC-1122U cloning cable */
+/* Note: ID-RP* devices are Icom Repeater Devices for HAM-radio */
+#define ICOM_ID_RP2C1_PID	0x0009 /* ID-RP2C Asset 1 to RS-232 */
+#define ICOM_ID_RP2C2_PID	0x000A /* ID-RP2C Asset 2 to RS-232 */
+#define ICOM_ID_RP2D_PID	0x000B /* ID-RP2D configuration port*/
+#define ICOM_ID_RP2VT_PID	0x000C /* ID-RP2V Transmit config port */
+#define ICOM_ID_RP2VR_PID	0x000D /* ID-RP2V Receive config port */
+#define ICOM_ID_RP4KVT_PID	0x0010 /* ID-RP4000V Transmit config port */
+#define ICOM_ID_RP4KVR_PID	0x0011 /* ID-RP4000V Receive config port */
+#define ICOM_ID_RP2KVT_PID	0x0012 /* ID-RP2000V Transmit config port */
+#define ICOM_ID_RP2KVR_PID	0x0013 /* ID-RP2000V Receive config port */
+
+/*
+ * GN Otometrics (http://www.otometrics.com)
+ * Submitted by Ville Sundberg.
+ */
+#define GN_OTOMETRICS_VID	0x0c33	/* Vendor ID */
+#define AURICAL_USB_PID		0x0010	/* Aurical USB Audiometer */
+
+/*
+ * The following are the values for the Sealevel SeaLINK+ adapters.
+ * (Original list sent by Tuan Hoang.  Ian Abbott renamed the macros and
+ * removed some PIDs that don't seem to match any existing products.)
+ */
+#define SEALEVEL_VID		0x0c52	/* Sealevel Vendor ID */
+#define SEALEVEL_2101_PID	0x2101	/* SeaLINK+232 (2101/2105) */
+#define SEALEVEL_2102_PID	0x2102	/* SeaLINK+485 (2102) */
+#define SEALEVEL_2103_PID	0x2103	/* SeaLINK+232I (2103) */
+#define SEALEVEL_2104_PID	0x2104	/* SeaLINK+485I (2104) */
+#define SEALEVEL_2106_PID	0x9020	/* SeaLINK+422 (2106) */
+#define SEALEVEL_2201_1_PID	0x2211	/* SeaPORT+2/232 (2201) Port 1 */
+#define SEALEVEL_2201_2_PID	0x2221	/* SeaPORT+2/232 (2201) Port 2 */
+#define SEALEVEL_2202_1_PID	0x2212	/* SeaPORT+2/485 (2202) Port 1 */
+#define SEALEVEL_2202_2_PID	0x2222	/* SeaPORT+2/485 (2202) Port 2 */
+#define SEALEVEL_2203_1_PID	0x2213	/* SeaPORT+2 (2203) Port 1 */
+#define SEALEVEL_2203_2_PID	0x2223	/* SeaPORT+2 (2203) Port 2 */
+#define SEALEVEL_2401_1_PID	0x2411	/* SeaPORT+4/232 (2401) Port 1 */
+#define SEALEVEL_2401_2_PID	0x2421	/* SeaPORT+4/232 (2401) Port 2 */
+#define SEALEVEL_2401_3_PID	0x2431	/* SeaPORT+4/232 (2401) Port 3 */
+#define SEALEVEL_2401_4_PID	0x2441	/* SeaPORT+4/232 (2401) Port 4 */
+#define SEALEVEL_2402_1_PID	0x2412	/* SeaPORT+4/485 (2402) Port 1 */
+#define SEALEVEL_2402_2_PID	0x2422	/* SeaPORT+4/485 (2402) Port 2 */
+#define SEALEVEL_2402_3_PID	0x2432	/* SeaPORT+4/485 (2402) Port 3 */
+#define SEALEVEL_2402_4_PID	0x2442	/* SeaPORT+4/485 (2402) Port 4 */
+#define SEALEVEL_2403_1_PID	0x2413	/* SeaPORT+4 (2403) Port 1 */
+#define SEALEVEL_2403_2_PID	0x2423	/* SeaPORT+4 (2403) Port 2 */
+#define SEALEVEL_2403_3_PID	0x2433	/* SeaPORT+4 (2403) Port 3 */
+#define SEALEVEL_2403_4_PID	0x2443	/* SeaPORT+4 (2403) Port 4 */
+#define SEALEVEL_2801_1_PID	0X2811	/* SeaLINK+8/232 (2801) Port 1 */
+#define SEALEVEL_2801_2_PID	0X2821	/* SeaLINK+8/232 (2801) Port 2 */
+#define SEALEVEL_2801_3_PID	0X2831	/* SeaLINK+8/232 (2801) Port 3 */
+#define SEALEVEL_2801_4_PID	0X2841	/* SeaLINK+8/232 (2801) Port 4 */
+#define SEALEVEL_2801_5_PID	0X2851	/* SeaLINK+8/232 (2801) Port 5 */
+#define SEALEVEL_2801_6_PID	0X2861	/* SeaLINK+8/232 (2801) Port 6 */
+#define SEALEVEL_2801_7_PID	0X2871	/* SeaLINK+8/232 (2801) Port 7 */
+#define SEALEVEL_2801_8_PID	0X2881	/* SeaLINK+8/232 (2801) Port 8 */
+#define SEALEVEL_2802_1_PID	0X2812	/* SeaLINK+8/485 (2802) Port 1 */
+#define SEALEVEL_2802_2_PID	0X2822	/* SeaLINK+8/485 (2802) Port 2 */
+#define SEALEVEL_2802_3_PID	0X2832	/* SeaLINK+8/485 (2802) Port 3 */
+#define SEALEVEL_2802_4_PID	0X2842	/* SeaLINK+8/485 (2802) Port 4 */
+#define SEALEVEL_2802_5_PID	0X2852	/* SeaLINK+8/485 (2802) Port 5 */
+#define SEALEVEL_2802_6_PID	0X2862	/* SeaLINK+8/485 (2802) Port 6 */
+#define SEALEVEL_2802_7_PID	0X2872	/* SeaLINK+8/485 (2802) Port 7 */
+#define SEALEVEL_2802_8_PID	0X2882	/* SeaLINK+8/485 (2802) Port 8 */
+#define SEALEVEL_2803_1_PID	0X2813	/* SeaLINK+8 (2803) Port 1 */
+#define SEALEVEL_2803_2_PID	0X2823	/* SeaLINK+8 (2803) Port 2 */
+#define SEALEVEL_2803_3_PID	0X2833	/* SeaLINK+8 (2803) Port 3 */
+#define SEALEVEL_2803_4_PID	0X2843	/* SeaLINK+8 (2803) Port 4 */
+#define SEALEVEL_2803_5_PID	0X2853	/* SeaLINK+8 (2803) Port 5 */
+#define SEALEVEL_2803_6_PID	0X2863	/* SeaLINK+8 (2803) Port 6 */
+#define SEALEVEL_2803_7_PID	0X2873	/* SeaLINK+8 (2803) Port 7 */
+#define SEALEVEL_2803_8_PID	0X2883	/* SeaLINK+8 (2803) Port 8 */
+#define SEALEVEL_2803R_1_PID	0Xa02a	/* SeaLINK+8 (2803-ROHS) Port 1+2 */
+#define SEALEVEL_2803R_2_PID	0Xa02b	/* SeaLINK+8 (2803-ROHS) Port 3+4 */
+#define SEALEVEL_2803R_3_PID	0Xa02c	/* SeaLINK+8 (2803-ROHS) Port 5+6 */
+#define SEALEVEL_2803R_4_PID	0Xa02d	/* SeaLINK+8 (2803-ROHS) Port 7+8 */
+
+/*
+ * JETI SPECTROMETER SPECBOS 1201
+ * http://www.jeti.com/cms/index.php/instruments/other-instruments/specbos-2101
+ */
+#define JETI_VID		0x0c6c
+#define JETI_SPC1201_PID	0x04b2
+
+/*
+ * FTDI USB UART chips used in construction projects from the
+ * Elektor Electronics magazine (http://www.elektor.com/)
+ */
+#define ELEKTOR_VID		0x0C7D
+#define ELEKTOR_FT323R_PID	0x0005	/* RFID-Reader, issue 09-2006 */
+
+/*
+ * Posiflex inc retail equipment (http://www.posiflex.com.tw)
+ */
+#define POSIFLEX_VID		0x0d3a  /* Vendor ID */
+#define POSIFLEX_PP7000_PID	0x0300  /* PP-7000II thermal printer */
+
+/*
+ * The following are the values for two KOBIL chipcard terminals.
+ */
+#define KOBIL_VID		0x0d46	/* KOBIL Vendor ID */
+#define KOBIL_CONV_B1_PID	0x2020	/* KOBIL Konverter for B1 */
+#define KOBIL_CONV_KAAN_PID	0x2021	/* KOBIL_Konverter for KAAN */
+
+#define FTDI_NF_RIC_VID	0x0DCD	/* Vendor Id */
+#define FTDI_NF_RIC_PID	0x0001	/* Product Id */
+
+/*
+ * Falcom Wireless Communications GmbH
+ */
+#define FALCOM_VID		0x0F94	/* Vendor Id */
+#define FALCOM_TWIST_PID	0x0001	/* Falcom Twist USB GPRS modem */
+#define FALCOM_SAMBA_PID	0x0005	/* Falcom Samba USB GPRS modem */
+
+/* Larsen and Brusgaard AltiTrack/USBtrack */
+#define LARSENBRUSGAARD_VID		0x0FD8
+#define LB_ALTITRACK_PID		0x0001
+
+/*
+ * TTi (Thurlby Thandar Instruments)
+ */
+#define TTI_VID			0x103E	/* Vendor Id */
+#define TTI_QL355P_PID		0x03E8	/* TTi QL355P power supply */
+
+/* Interbiometrics USB I/O Board */
+/* Developed for Interbiometrics by Rudolf Gugler */
+#define INTERBIOMETRICS_VID              0x1209
+#define INTERBIOMETRICS_IOBOARD_PID      0x1002
+#define INTERBIOMETRICS_MINI_IOBOARD_PID 0x1006
+
+/*
+ * Testo products (http://www.testo.com/)
+ * Submitted by Colin Leroy
+ */
+#define TESTO_VID			0x128D
+#define TESTO_USB_INTERFACE_PID		0x0001
+
+/*
+ * Mobility Electronics products.
+ */
+#define MOBILITY_VID			0x1342
+#define MOBILITY_USB_SERIAL_PID		0x0202	/* EasiDock USB 200 serial */
+
+/*
+ * FIC / OpenMoko, Inc. http://wiki.openmoko.org/wiki/Neo1973_Debug_Board_v3
+ * Submitted by Harald Welte <laforge@openmoko.org>
+ */
+#define	FIC_VID			0x1457
+#define	FIC_NEO1973_DEBUG_PID	0x5118
+
+/* Olimex */
+#define OLIMEX_VID			0x15BA
+#define OLIMEX_ARM_USB_OCD_PID		0x0003
+#define OLIMEX_ARM_USB_OCD_H_PID	0x002b
+
+/*
+ * Telldus Technologies
+ */
+#define TELLDUS_VID			0x1781	/* Vendor ID */
+#define TELLDUS_TELLSTICK_PID		0x0C30	/* RF control dongle 433 MHz using FT232RL */
+
+/*
+ * RT Systems programming cables for various ham radios
+ */
+#define RTSYSTEMS_VID			0x2100	/* Vendor ID */
+#define RTSYSTEMS_SERIAL_VX7_PID	0x9e52	/* Serial converter for VX-7 Radios using FT232RL */
+#define RTSYSTEMS_CT29B_PID		0x9e54	/* CT29B Radio Cable */
+#define RTSYSTEMS_RTS01_PID		0x9e57	/* USB-RTS01 Radio Cable */
+
+
+/*
+ * Physik Instrumente
+ * http://www.physikinstrumente.com/en/products/
+ */
+/* These two devices use the VID of FTDI */
+#define PI_C865_PID	0xe0a0  /* PI C-865 Piezomotor Controller */
+#define PI_C857_PID	0xe0a1  /* PI Encoder Trigger Box */
+
+#define PI_VID              0x1a72  /* Vendor ID */
+#define PI_C866_PID	0x1000  /* PI C-866 Piezomotor Controller */
+#define PI_C663_PID	0x1001  /* PI C-663 Mercury-Step */
+#define PI_C725_PID	0x1002  /* PI C-725 Piezomotor Controller */
+#define PI_E517_PID	0x1005  /* PI E-517 Digital Piezo Controller Operation Module */
+#define PI_C863_PID	0x1007  /* PI C-863 */
+#define PI_E861_PID	0x1008  /* PI E-861 Piezomotor Controller */
+#define PI_C867_PID	0x1009  /* PI C-867 Piezomotor Controller */
+#define PI_E609_PID	0x100D  /* PI E-609 Digital Piezo Controller */
+#define PI_E709_PID	0x100E  /* PI E-709 Digital Piezo Controller */
+#define PI_100F_PID	0x100F  /* PI Digital Piezo Controller */
+#define PI_1011_PID	0x1011  /* PI Digital Piezo Controller */
+#define PI_1012_PID	0x1012  /* PI Motion Controller */
+#define PI_1013_PID	0x1013  /* PI Motion Controller */
+#define PI_1014_PID	0x1014  /* PI Device */
+#define PI_1015_PID	0x1015  /* PI Device */
+#define PI_1016_PID	0x1016  /* PI Digital Servo Module */
+
+/*
+ * Kondo Kagaku Co.Ltd.
+ * http://www.kondo-robot.com/EN
+ */
+#define KONDO_VID 		0x165c
+#define KONDO_USB_SERIAL_PID	0x0002
+
+/*
+ * Bayer Ascensia Contour blood glucose meter USB-converter cable.
+ * http://winglucofacts.com/cables/
+ */
+#define BAYER_VID                      0x1A79
+#define BAYER_CONTOUR_CABLE_PID        0x6001
+
+/*
+ * The following are the values for the Matrix Orbital FTDI Range
+ * Anything in this range will use an FT232RL.
+ */
+#define MTXORB_VID			0x1B3D
+#define MTXORB_FTDI_RANGE_0100_PID	0x0100
+#define MTXORB_FTDI_RANGE_0101_PID	0x0101
+#define MTXORB_FTDI_RANGE_0102_PID	0x0102
+#define MTXORB_FTDI_RANGE_0103_PID	0x0103
+#define MTXORB_FTDI_RANGE_0104_PID	0x0104
+#define MTXORB_FTDI_RANGE_0105_PID	0x0105
+#define MTXORB_FTDI_RANGE_0106_PID	0x0106
+#define MTXORB_FTDI_RANGE_0107_PID	0x0107
+#define MTXORB_FTDI_RANGE_0108_PID	0x0108
+#define MTXORB_FTDI_RANGE_0109_PID	0x0109
+#define MTXORB_FTDI_RANGE_010A_PID	0x010A
+#define MTXORB_FTDI_RANGE_010B_PID	0x010B
+#define MTXORB_FTDI_RANGE_010C_PID	0x010C
+#define MTXORB_FTDI_RANGE_010D_PID	0x010D
+#define MTXORB_FTDI_RANGE_010E_PID	0x010E
+#define MTXORB_FTDI_RANGE_010F_PID	0x010F
+#define MTXORB_FTDI_RANGE_0110_PID	0x0110
+#define MTXORB_FTDI_RANGE_0111_PID	0x0111
+#define MTXORB_FTDI_RANGE_0112_PID	0x0112
+#define MTXORB_FTDI_RANGE_0113_PID	0x0113
+#define MTXORB_FTDI_RANGE_0114_PID	0x0114
+#define MTXORB_FTDI_RANGE_0115_PID	0x0115
+#define MTXORB_FTDI_RANGE_0116_PID	0x0116
+#define MTXORB_FTDI_RANGE_0117_PID	0x0117
+#define MTXORB_FTDI_RANGE_0118_PID	0x0118
+#define MTXORB_FTDI_RANGE_0119_PID	0x0119
+#define MTXORB_FTDI_RANGE_011A_PID	0x011A
+#define MTXORB_FTDI_RANGE_011B_PID	0x011B
+#define MTXORB_FTDI_RANGE_011C_PID	0x011C
+#define MTXORB_FTDI_RANGE_011D_PID	0x011D
+#define MTXORB_FTDI_RANGE_011E_PID	0x011E
+#define MTXORB_FTDI_RANGE_011F_PID	0x011F
+#define MTXORB_FTDI_RANGE_0120_PID	0x0120
+#define MTXORB_FTDI_RANGE_0121_PID	0x0121
+#define MTXORB_FTDI_RANGE_0122_PID	0x0122
+#define MTXORB_FTDI_RANGE_0123_PID	0x0123
+#define MTXORB_FTDI_RANGE_0124_PID	0x0124
+#define MTXORB_FTDI_RANGE_0125_PID	0x0125
+#define MTXORB_FTDI_RANGE_0126_PID	0x0126
+#define MTXORB_FTDI_RANGE_0127_PID	0x0127
+#define MTXORB_FTDI_RANGE_0128_PID	0x0128
+#define MTXORB_FTDI_RANGE_0129_PID	0x0129
+#define MTXORB_FTDI_RANGE_012A_PID	0x012A
+#define MTXORB_FTDI_RANGE_012B_PID	0x012B
+#define MTXORB_FTDI_RANGE_012C_PID	0x012C
+#define MTXORB_FTDI_RANGE_012D_PID	0x012D
+#define MTXORB_FTDI_RANGE_012E_PID	0x012E
+#define MTXORB_FTDI_RANGE_012F_PID	0x012F
+#define MTXORB_FTDI_RANGE_0130_PID	0x0130
+#define MTXORB_FTDI_RANGE_0131_PID	0x0131
+#define MTXORB_FTDI_RANGE_0132_PID	0x0132
+#define MTXORB_FTDI_RANGE_0133_PID	0x0133
+#define MTXORB_FTDI_RANGE_0134_PID	0x0134
+#define MTXORB_FTDI_RANGE_0135_PID	0x0135
+#define MTXORB_FTDI_RANGE_0136_PID	0x0136
+#define MTXORB_FTDI_RANGE_0137_PID	0x0137
+#define MTXORB_FTDI_RANGE_0138_PID	0x0138
+#define MTXORB_FTDI_RANGE_0139_PID	0x0139
+#define MTXORB_FTDI_RANGE_013A_PID	0x013A
+#define MTXORB_FTDI_RANGE_013B_PID	0x013B
+#define MTXORB_FTDI_RANGE_013C_PID	0x013C
+#define MTXORB_FTDI_RANGE_013D_PID	0x013D
+#define MTXORB_FTDI_RANGE_013E_PID	0x013E
+#define MTXORB_FTDI_RANGE_013F_PID	0x013F
+#define MTXORB_FTDI_RANGE_0140_PID	0x0140
+#define MTXORB_FTDI_RANGE_0141_PID	0x0141
+#define MTXORB_FTDI_RANGE_0142_PID	0x0142
+#define MTXORB_FTDI_RANGE_0143_PID	0x0143
+#define MTXORB_FTDI_RANGE_0144_PID	0x0144
+#define MTXORB_FTDI_RANGE_0145_PID	0x0145
+#define MTXORB_FTDI_RANGE_0146_PID	0x0146
+#define MTXORB_FTDI_RANGE_0147_PID	0x0147
+#define MTXORB_FTDI_RANGE_0148_PID	0x0148
+#define MTXORB_FTDI_RANGE_0149_PID	0x0149
+#define MTXORB_FTDI_RANGE_014A_PID	0x014A
+#define MTXORB_FTDI_RANGE_014B_PID	0x014B
+#define MTXORB_FTDI_RANGE_014C_PID	0x014C
+#define MTXORB_FTDI_RANGE_014D_PID	0x014D
+#define MTXORB_FTDI_RANGE_014E_PID	0x014E
+#define MTXORB_FTDI_RANGE_014F_PID	0x014F
+#define MTXORB_FTDI_RANGE_0150_PID	0x0150
+#define MTXORB_FTDI_RANGE_0151_PID	0x0151
+#define MTXORB_FTDI_RANGE_0152_PID	0x0152
+#define MTXORB_FTDI_RANGE_0153_PID	0x0153
+#define MTXORB_FTDI_RANGE_0154_PID	0x0154
+#define MTXORB_FTDI_RANGE_0155_PID	0x0155
+#define MTXORB_FTDI_RANGE_0156_PID	0x0156
+#define MTXORB_FTDI_RANGE_0157_PID	0x0157
+#define MTXORB_FTDI_RANGE_0158_PID	0x0158
+#define MTXORB_FTDI_RANGE_0159_PID	0x0159
+#define MTXORB_FTDI_RANGE_015A_PID	0x015A
+#define MTXORB_FTDI_RANGE_015B_PID	0x015B
+#define MTXORB_FTDI_RANGE_015C_PID	0x015C
+#define MTXORB_FTDI_RANGE_015D_PID	0x015D
+#define MTXORB_FTDI_RANGE_015E_PID	0x015E
+#define MTXORB_FTDI_RANGE_015F_PID	0x015F
+#define MTXORB_FTDI_RANGE_0160_PID	0x0160
+#define MTXORB_FTDI_RANGE_0161_PID	0x0161
+#define MTXORB_FTDI_RANGE_0162_PID	0x0162
+#define MTXORB_FTDI_RANGE_0163_PID	0x0163
+#define MTXORB_FTDI_RANGE_0164_PID	0x0164
+#define MTXORB_FTDI_RANGE_0165_PID	0x0165
+#define MTXORB_FTDI_RANGE_0166_PID	0x0166
+#define MTXORB_FTDI_RANGE_0167_PID	0x0167
+#define MTXORB_FTDI_RANGE_0168_PID	0x0168
+#define MTXORB_FTDI_RANGE_0169_PID	0x0169
+#define MTXORB_FTDI_RANGE_016A_PID	0x016A
+#define MTXORB_FTDI_RANGE_016B_PID	0x016B
+#define MTXORB_FTDI_RANGE_016C_PID	0x016C
+#define MTXORB_FTDI_RANGE_016D_PID	0x016D
+#define MTXORB_FTDI_RANGE_016E_PID	0x016E
+#define MTXORB_FTDI_RANGE_016F_PID	0x016F
+#define MTXORB_FTDI_RANGE_0170_PID	0x0170
+#define MTXORB_FTDI_RANGE_0171_PID	0x0171
+#define MTXORB_FTDI_RANGE_0172_PID	0x0172
+#define MTXORB_FTDI_RANGE_0173_PID	0x0173
+#define MTXORB_FTDI_RANGE_0174_PID	0x0174
+#define MTXORB_FTDI_RANGE_0175_PID	0x0175
+#define MTXORB_FTDI_RANGE_0176_PID	0x0176
+#define MTXORB_FTDI_RANGE_0177_PID	0x0177
+#define MTXORB_FTDI_RANGE_0178_PID	0x0178
+#define MTXORB_FTDI_RANGE_0179_PID	0x0179
+#define MTXORB_FTDI_RANGE_017A_PID	0x017A
+#define MTXORB_FTDI_RANGE_017B_PID	0x017B
+#define MTXORB_FTDI_RANGE_017C_PID	0x017C
+#define MTXORB_FTDI_RANGE_017D_PID	0x017D
+#define MTXORB_FTDI_RANGE_017E_PID	0x017E
+#define MTXORB_FTDI_RANGE_017F_PID	0x017F
+#define MTXORB_FTDI_RANGE_0180_PID	0x0180
+#define MTXORB_FTDI_RANGE_0181_PID	0x0181
+#define MTXORB_FTDI_RANGE_0182_PID	0x0182
+#define MTXORB_FTDI_RANGE_0183_PID	0x0183
+#define MTXORB_FTDI_RANGE_0184_PID	0x0184
+#define MTXORB_FTDI_RANGE_0185_PID	0x0185
+#define MTXORB_FTDI_RANGE_0186_PID	0x0186
+#define MTXORB_FTDI_RANGE_0187_PID	0x0187
+#define MTXORB_FTDI_RANGE_0188_PID	0x0188
+#define MTXORB_FTDI_RANGE_0189_PID	0x0189
+#define MTXORB_FTDI_RANGE_018A_PID	0x018A
+#define MTXORB_FTDI_RANGE_018B_PID	0x018B
+#define MTXORB_FTDI_RANGE_018C_PID	0x018C
+#define MTXORB_FTDI_RANGE_018D_PID	0x018D
+#define MTXORB_FTDI_RANGE_018E_PID	0x018E
+#define MTXORB_FTDI_RANGE_018F_PID	0x018F
+#define MTXORB_FTDI_RANGE_0190_PID	0x0190
+#define MTXORB_FTDI_RANGE_0191_PID	0x0191
+#define MTXORB_FTDI_RANGE_0192_PID	0x0192
+#define MTXORB_FTDI_RANGE_0193_PID	0x0193
+#define MTXORB_FTDI_RANGE_0194_PID	0x0194
+#define MTXORB_FTDI_RANGE_0195_PID	0x0195
+#define MTXORB_FTDI_RANGE_0196_PID	0x0196
+#define MTXORB_FTDI_RANGE_0197_PID	0x0197
+#define MTXORB_FTDI_RANGE_0198_PID	0x0198
+#define MTXORB_FTDI_RANGE_0199_PID	0x0199
+#define MTXORB_FTDI_RANGE_019A_PID	0x019A
+#define MTXORB_FTDI_RANGE_019B_PID	0x019B
+#define MTXORB_FTDI_RANGE_019C_PID	0x019C
+#define MTXORB_FTDI_RANGE_019D_PID	0x019D
+#define MTXORB_FTDI_RANGE_019E_PID	0x019E
+#define MTXORB_FTDI_RANGE_019F_PID	0x019F
+#define MTXORB_FTDI_RANGE_01A0_PID	0x01A0
+#define MTXORB_FTDI_RANGE_01A1_PID	0x01A1
+#define MTXORB_FTDI_RANGE_01A2_PID	0x01A2
+#define MTXORB_FTDI_RANGE_01A3_PID	0x01A3
+#define MTXORB_FTDI_RANGE_01A4_PID	0x01A4
+#define MTXORB_FTDI_RANGE_01A5_PID	0x01A5
+#define MTXORB_FTDI_RANGE_01A6_PID	0x01A6
+#define MTXORB_FTDI_RANGE_01A7_PID	0x01A7
+#define MTXORB_FTDI_RANGE_01A8_PID	0x01A8
+#define MTXORB_FTDI_RANGE_01A9_PID	0x01A9
+#define MTXORB_FTDI_RANGE_01AA_PID	0x01AA
+#define MTXORB_FTDI_RANGE_01AB_PID	0x01AB
+#define MTXORB_FTDI_RANGE_01AC_PID	0x01AC
+#define MTXORB_FTDI_RANGE_01AD_PID	0x01AD
+#define MTXORB_FTDI_RANGE_01AE_PID	0x01AE
+#define MTXORB_FTDI_RANGE_01AF_PID	0x01AF
+#define MTXORB_FTDI_RANGE_01B0_PID	0x01B0
+#define MTXORB_FTDI_RANGE_01B1_PID	0x01B1
+#define MTXORB_FTDI_RANGE_01B2_PID	0x01B2
+#define MTXORB_FTDI_RANGE_01B3_PID	0x01B3
+#define MTXORB_FTDI_RANGE_01B4_PID	0x01B4
+#define MTXORB_FTDI_RANGE_01B5_PID	0x01B5
+#define MTXORB_FTDI_RANGE_01B6_PID	0x01B6
+#define MTXORB_FTDI_RANGE_01B7_PID	0x01B7
+#define MTXORB_FTDI_RANGE_01B8_PID	0x01B8
+#define MTXORB_FTDI_RANGE_01B9_PID	0x01B9
+#define MTXORB_FTDI_RANGE_01BA_PID	0x01BA
+#define MTXORB_FTDI_RANGE_01BB_PID	0x01BB
+#define MTXORB_FTDI_RANGE_01BC_PID	0x01BC
+#define MTXORB_FTDI_RANGE_01BD_PID	0x01BD
+#define MTXORB_FTDI_RANGE_01BE_PID	0x01BE
+#define MTXORB_FTDI_RANGE_01BF_PID	0x01BF
+#define MTXORB_FTDI_RANGE_01C0_PID	0x01C0
+#define MTXORB_FTDI_RANGE_01C1_PID	0x01C1
+#define MTXORB_FTDI_RANGE_01C2_PID	0x01C2
+#define MTXORB_FTDI_RANGE_01C3_PID	0x01C3
+#define MTXORB_FTDI_RANGE_01C4_PID	0x01C4
+#define MTXORB_FTDI_RANGE_01C5_PID	0x01C5
+#define MTXORB_FTDI_RANGE_01C6_PID	0x01C6
+#define MTXORB_FTDI_RANGE_01C7_PID	0x01C7
+#define MTXORB_FTDI_RANGE_01C8_PID	0x01C8
+#define MTXORB_FTDI_RANGE_01C9_PID	0x01C9
+#define MTXORB_FTDI_RANGE_01CA_PID	0x01CA
+#define MTXORB_FTDI_RANGE_01CB_PID	0x01CB
+#define MTXORB_FTDI_RANGE_01CC_PID	0x01CC
+#define MTXORB_FTDI_RANGE_01CD_PID	0x01CD
+#define MTXORB_FTDI_RANGE_01CE_PID	0x01CE
+#define MTXORB_FTDI_RANGE_01CF_PID	0x01CF
+#define MTXORB_FTDI_RANGE_01D0_PID	0x01D0
+#define MTXORB_FTDI_RANGE_01D1_PID	0x01D1
+#define MTXORB_FTDI_RANGE_01D2_PID	0x01D2
+#define MTXORB_FTDI_RANGE_01D3_PID	0x01D3
+#define MTXORB_FTDI_RANGE_01D4_PID	0x01D4
+#define MTXORB_FTDI_RANGE_01D5_PID	0x01D5
+#define MTXORB_FTDI_RANGE_01D6_PID	0x01D6
+#define MTXORB_FTDI_RANGE_01D7_PID	0x01D7
+#define MTXORB_FTDI_RANGE_01D8_PID	0x01D8
+#define MTXORB_FTDI_RANGE_01D9_PID	0x01D9
+#define MTXORB_FTDI_RANGE_01DA_PID	0x01DA
+#define MTXORB_FTDI_RANGE_01DB_PID	0x01DB
+#define MTXORB_FTDI_RANGE_01DC_PID	0x01DC
+#define MTXORB_FTDI_RANGE_01DD_PID	0x01DD
+#define MTXORB_FTDI_RANGE_01DE_PID	0x01DE
+#define MTXORB_FTDI_RANGE_01DF_PID	0x01DF
+#define MTXORB_FTDI_RANGE_01E0_PID	0x01E0
+#define MTXORB_FTDI_RANGE_01E1_PID	0x01E1
+#define MTXORB_FTDI_RANGE_01E2_PID	0x01E2
+#define MTXORB_FTDI_RANGE_01E3_PID	0x01E3
+#define MTXORB_FTDI_RANGE_01E4_PID	0x01E4
+#define MTXORB_FTDI_RANGE_01E5_PID	0x01E5
+#define MTXORB_FTDI_RANGE_01E6_PID	0x01E6
+#define MTXORB_FTDI_RANGE_01E7_PID	0x01E7
+#define MTXORB_FTDI_RANGE_01E8_PID	0x01E8
+#define MTXORB_FTDI_RANGE_01E9_PID	0x01E9
+#define MTXORB_FTDI_RANGE_01EA_PID	0x01EA
+#define MTXORB_FTDI_RANGE_01EB_PID	0x01EB
+#define MTXORB_FTDI_RANGE_01EC_PID	0x01EC
+#define MTXORB_FTDI_RANGE_01ED_PID	0x01ED
+#define MTXORB_FTDI_RANGE_01EE_PID	0x01EE
+#define MTXORB_FTDI_RANGE_01EF_PID	0x01EF
+#define MTXORB_FTDI_RANGE_01F0_PID	0x01F0
+#define MTXORB_FTDI_RANGE_01F1_PID	0x01F1
+#define MTXORB_FTDI_RANGE_01F2_PID	0x01F2
+#define MTXORB_FTDI_RANGE_01F3_PID	0x01F3
+#define MTXORB_FTDI_RANGE_01F4_PID	0x01F4
+#define MTXORB_FTDI_RANGE_01F5_PID	0x01F5
+#define MTXORB_FTDI_RANGE_01F6_PID	0x01F6
+#define MTXORB_FTDI_RANGE_01F7_PID	0x01F7
+#define MTXORB_FTDI_RANGE_01F8_PID	0x01F8
+#define MTXORB_FTDI_RANGE_01F9_PID	0x01F9
+#define MTXORB_FTDI_RANGE_01FA_PID	0x01FA
+#define MTXORB_FTDI_RANGE_01FB_PID	0x01FB
+#define MTXORB_FTDI_RANGE_01FC_PID	0x01FC
+#define MTXORB_FTDI_RANGE_01FD_PID	0x01FD
+#define MTXORB_FTDI_RANGE_01FE_PID	0x01FE
+#define MTXORB_FTDI_RANGE_01FF_PID	0x01FF
+
+
+
+/*
+ * The Mobility Lab (TML)
+ * Submitted by Pierre Castella
+ */
+#define TML_VID			0x1B91	/* Vendor ID */
+#define TML_USB_SERIAL_PID	0x0064	/* USB - Serial Converter */
+
+/* Alti-2 products  http://www.alti-2.com */
+#define ALTI2_VID	0x1BC9
+#define ALTI2_N3_PID	0x6001	/* Neptune 3 */
+
+/*
+ * Ionics PlugComputer
+ */
+#define IONICS_VID			0x1c0c
+#define IONICS_PLUGCOMPUTER_PID		0x0102
+
+/*
+ * Dresden Elektronik Sensor Terminal Board
+ */
+#define DE_VID			0x1cf1 /* Vendor ID */
+#define STB_PID			0x0001 /* Sensor Terminal Board */
+#define WHT_PID			0x0004 /* Wireless Handheld Terminal */
+
+/*
+ * STMicroelectonics
+ */
+#define ST_VID			0x0483
+#define ST_STMCLT1030_PID	0x3747 /* ST Micro Connect Lite STMCLT1030 */
+
+/*
+ * Papouch products (http://www.papouch.com/)
+ * Submitted by Folkert van Heusden
+ */
+
+#define PAPOUCH_VID			0x5050	/* Vendor ID */
+#define PAPOUCH_SB485_PID		0x0100	/* Papouch SB485 USB-485/422 Converter */
+#define PAPOUCH_AP485_PID		0x0101	/* AP485 USB-RS485 Converter */
+#define PAPOUCH_SB422_PID		0x0102	/* Papouch SB422 USB-RS422 Converter  */
+#define PAPOUCH_SB485_2_PID		0x0103	/* Papouch SB485 USB-485/422 Converter */
+#define PAPOUCH_AP485_2_PID		0x0104	/* AP485 USB-RS485 Converter */
+#define PAPOUCH_SB422_2_PID		0x0105	/* Papouch SB422 USB-RS422 Converter  */
+#define PAPOUCH_SB485S_PID		0x0106	/* Papouch SB485S USB-485/422 Converter */
+#define PAPOUCH_SB485C_PID		0x0107	/* Papouch SB485C USB-485/422 Converter */
+#define PAPOUCH_LEC_PID			0x0300	/* LEC USB Converter */
+#define PAPOUCH_SB232_PID		0x0301	/* Papouch SB232 USB-RS232 Converter */
+#define PAPOUCH_TMU_PID			0x0400	/* TMU USB Thermometer */
+#define PAPOUCH_IRAMP_PID		0x0500	/* Papouch IRAmp Duplex */
+#define PAPOUCH_DRAK5_PID		0x0700	/* Papouch DRAK5 */
+#define PAPOUCH_QUIDO8x8_PID		0x0800	/* Papouch Quido 8/8 Module */
+#define PAPOUCH_QUIDO4x4_PID		0x0900	/* Papouch Quido 4/4 Module */
+#define PAPOUCH_QUIDO2x2_PID		0x0a00	/* Papouch Quido 2/2 Module */
+#define PAPOUCH_QUIDO10x1_PID		0x0b00	/* Papouch Quido 10/1 Module */
+#define PAPOUCH_QUIDO30x3_PID		0x0c00	/* Papouch Quido 30/3 Module */
+#define PAPOUCH_QUIDO60x3_PID		0x0d00	/* Papouch Quido 60(100)/3 Module */
+#define PAPOUCH_QUIDO2x16_PID		0x0e00	/* Papouch Quido 2/16 Module */
+#define PAPOUCH_QUIDO3x32_PID		0x0f00	/* Papouch Quido 3/32 Module */
+#define PAPOUCH_DRAK6_PID		0x1000	/* Papouch DRAK6 */
+#define PAPOUCH_UPSUSB_PID		0x8000	/* Papouch UPS-USB adapter */
+#define PAPOUCH_MU_PID			0x8001	/* MU controller */
+#define PAPOUCH_SIMUKEY_PID		0x8002	/* Papouch SimuKey */
+#define PAPOUCH_AD4USB_PID		0x8003	/* AD4USB Measurement Module */
+#define PAPOUCH_GMUX_PID		0x8004	/* Papouch GOLIATH MUX */
+#define PAPOUCH_GMSR_PID		0x8005	/* Papouch GOLIATH MSR */
+
+/*
+ * Marvell SheevaPlug
+ */
+#define MARVELL_VID		0x9e88
+#define MARVELL_SHEEVAPLUG_PID	0x9e8f
+
+/*
+ * Evolution Robotics products (http://www.evolution.com/).
+ * Submitted by Shawn M. Lavelle.
+ */
+#define EVOLUTION_VID		0xDEEE	/* Vendor ID */
+#define EVOLUTION_ER1_PID	0x0300	/* ER1 Control Module */
+#define EVO_8U232AM_PID		0x02FF	/* Evolution robotics RCM2 (FT232AM)*/
+#define EVO_HYBRID_PID		0x0302	/* Evolution robotics RCM4 PID (FT232BM)*/
+#define EVO_RCM4_PID		0x0303	/* Evolution robotics RCM4 PID */
+
+/*
+ * MJS Gadgets HD Radio / XM Radio / Sirius Radio interfaces (using VID 0x0403)
+ */
+#define MJSG_GENERIC_PID	0x9378
+#define MJSG_SR_RADIO_PID	0x9379
+#define MJSG_XM_RADIO_PID	0x937A
+#define MJSG_HD_RADIO_PID	0x937C
+
+/*
+ * D.O.Tec products (http://www.directout.eu)
+ */
+#define FTDI_DOTEC_PID 0x9868
+
+/*
+ * Xverve Signalyzer tools (http://www.signalyzer.com/)
+ */
+#define XVERVE_SIGNALYZER_ST_PID	0xBCA0
+#define XVERVE_SIGNALYZER_SLITE_PID	0xBCA1
+#define XVERVE_SIGNALYZER_SH2_PID	0xBCA2
+#define XVERVE_SIGNALYZER_SH4_PID	0xBCA4
+
+/*
+ * Segway Robotic Mobility Platform USB interface (using VID 0x0403)
+ * Submitted by John G. Rogers
+ */
+#define SEGWAY_RMP200_PID	0xe729
+
+
+/*
+ * Accesio USB Data Acquisition products (http://www.accesio.com/)
+ */
+#define ACCESIO_COM4SM_PID 	0xD578
+
+/* www.sciencescope.co.uk educational dataloggers */
+#define FTDI_SCIENCESCOPE_LOGBOOKML_PID		0xFF18
+#define FTDI_SCIENCESCOPE_LS_LOGBOOK_PID	0xFF1C
+#define FTDI_SCIENCESCOPE_HS_LOGBOOK_PID	0xFF1D
+
+/*
+ * Milkymist One JTAG/Serial
+ */
+#define QIHARDWARE_VID			0x20B7
+#define MILKYMISTONE_JTAGSERIAL_PID	0x0713
+
+/*
+ * CTI GmbH RS485 Converter http://www.cti-lean.com/
+ */
+/* USB-485-Mini*/
+#define FTDI_CTI_MINI_PID	0xF608
+/* USB-Nano-485*/
+#define FTDI_CTI_NANO_PID	0xF60B
+
+/*
+ * ZeitControl cardsystems GmbH rfid-readers http://zeitconrol.de
+ */
+/* TagTracer MIFARE*/
+#define FTDI_ZEITCONTROL_TAGTRACE_MIFARE_PID	0xF7C0
+
+/*
+ * Rainforest Automation
+ */
+/* ZigBee controller */
+#define FTDI_RF_R106		0x8A28
+
+/*
+ * Product: HCP HIT GPRS modem
+ * Manufacturer: HCP d.o.o.
+ * ATI command output: Cinterion MC55i
+ */
+#define FTDI_CINTERION_MC55I_PID	0xA951
diff --git a/hw/usb/quirks-pl2303-ids.h b/hw/usb/quirks-pl2303-ids.h
new file mode 100644
index 0000000..8dbdb46
--- /dev/null
+++ b/hw/usb/quirks-pl2303-ids.h
@@ -0,0 +1,150 @@
+/*
+ * Prolific PL2303 USB to serial adaptor driver header file
+ *
+ *	This program is free software; you can redistribute it and/or modify
+ *	it under the terms of the GNU General Public License as published by
+ *	the Free Software Foundation; either version 2 of the License, or
+ *	(at your option) any later version.
+ *
+ */
+
+#define BENQ_VENDOR_ID			0x04a5
+#define BENQ_PRODUCT_ID_S81		0x4027
+
+#define PL2303_VENDOR_ID	0x067b
+#define PL2303_PRODUCT_ID	0x2303
+#define PL2303_PRODUCT_ID_RSAQ2		0x04bb
+#define PL2303_PRODUCT_ID_DCU11		0x1234
+#define PL2303_PRODUCT_ID_PHAROS	0xaaa0
+#define PL2303_PRODUCT_ID_RSAQ3		0xaaa2
+#define PL2303_PRODUCT_ID_ALDIGA	0x0611
+#define PL2303_PRODUCT_ID_MMX		0x0612
+#define PL2303_PRODUCT_ID_GPRS		0x0609
+#define PL2303_PRODUCT_ID_HCR331	0x331a
+#define PL2303_PRODUCT_ID_MOTOROLA	0x0307
+
+#define ATEN_VENDOR_ID		0x0557
+#define ATEN_VENDOR_ID2		0x0547
+#define ATEN_PRODUCT_ID		0x2008
+
+#define IODATA_VENDOR_ID	0x04bb
+#define IODATA_PRODUCT_ID	0x0a03
+#define IODATA_PRODUCT_ID_RSAQ5	0x0a0e
+
+#define ELCOM_VENDOR_ID		0x056e
+#define ELCOM_PRODUCT_ID	0x5003
+#define ELCOM_PRODUCT_ID_UCSGT	0x5004
+
+#define ITEGNO_VENDOR_ID	0x0eba
+#define ITEGNO_PRODUCT_ID	0x1080
+#define ITEGNO_PRODUCT_ID_2080	0x2080
+
+#define MA620_VENDOR_ID		0x0df7
+#define MA620_PRODUCT_ID	0x0620
+
+#define RATOC_VENDOR_ID		0x0584
+#define RATOC_PRODUCT_ID	0xb000
+
+#define TRIPP_VENDOR_ID		0x2478
+#define TRIPP_PRODUCT_ID	0x2008
+
+#define RADIOSHACK_VENDOR_ID	0x1453
+#define RADIOSHACK_PRODUCT_ID	0x4026
+
+#define DCU10_VENDOR_ID		0x0731
+#define DCU10_PRODUCT_ID	0x0528
+
+#define SITECOM_VENDOR_ID	0x6189
+#define SITECOM_PRODUCT_ID	0x2068
+
+/* Alcatel OT535/735 USB cable */
+#define ALCATEL_VENDOR_ID	0x11f7
+#define ALCATEL_PRODUCT_ID	0x02df
+
+/* Samsung I330 phone cradle */
+#define SAMSUNG_VENDOR_ID	0x04e8
+#define SAMSUNG_PRODUCT_ID	0x8001
+
+#define SIEMENS_VENDOR_ID	0x11f5
+#define SIEMENS_PRODUCT_ID_SX1	0x0001
+#define SIEMENS_PRODUCT_ID_X65	0x0003
+#define SIEMENS_PRODUCT_ID_X75	0x0004
+#define SIEMENS_PRODUCT_ID_EF81	0x0005
+
+#define SYNTECH_VENDOR_ID	0x0745
+#define SYNTECH_PRODUCT_ID	0x0001
+
+/* Nokia CA-42 Cable */
+#define NOKIA_CA42_VENDOR_ID	0x078b
+#define NOKIA_CA42_PRODUCT_ID	0x1234
+
+/* CA-42 CLONE Cable www.ca-42.com chipset: Prolific Technology Inc */
+#define CA_42_CA42_VENDOR_ID	0x10b5
+#define CA_42_CA42_PRODUCT_ID	0xac70
+
+#define SAGEM_VENDOR_ID		0x079b
+#define SAGEM_PRODUCT_ID	0x0027
+
+/* Leadtek GPS 9531 (ID 0413:2101) */
+#define LEADTEK_VENDOR_ID	0x0413
+#define LEADTEK_9531_PRODUCT_ID	0x2101
+
+/* USB GSM cable from Speed Dragon Multimedia, Ltd */
+#define SPEEDDRAGON_VENDOR_ID	0x0e55
+#define SPEEDDRAGON_PRODUCT_ID	0x110b
+
+/* DATAPILOT Universal-2 Phone Cable */
+#define DATAPILOT_U2_VENDOR_ID	0x0731
+#define DATAPILOT_U2_PRODUCT_ID	0x2003
+
+/* Belkin "F5U257" Serial Adapter */
+#define BELKIN_VENDOR_ID	0x050d
+#define BELKIN_PRODUCT_ID	0x0257
+
+/* Alcor Micro Corp. USB 2.0 TO RS-232 */
+#define ALCOR_VENDOR_ID		0x058F
+#define ALCOR_PRODUCT_ID	0x9720
+
+/* Willcom WS002IN Data Driver (by NetIndex Inc.) */
+#define WS002IN_VENDOR_ID	0x11f6
+#define WS002IN_PRODUCT_ID	0x2001
+
+/* Corega CG-USBRS232R Serial Adapter */
+#define COREGA_VENDOR_ID	0x07aa
+#define COREGA_PRODUCT_ID	0x002a
+
+/* Y.C. Cable U.S.A., Inc - USB to RS-232 */
+#define YCCABLE_VENDOR_ID	0x05ad
+#define YCCABLE_PRODUCT_ID	0x0fba
+
+/* "Superial" USB - Serial */
+#define SUPERIAL_VENDOR_ID	0x5372
+#define SUPERIAL_PRODUCT_ID	0x2303
+
+/* Hewlett-Packard LD220-HP POS Pole Display */
+#define HP_VENDOR_ID		0x03f0
+#define HP_LD220_PRODUCT_ID	0x3524
+
+/* Cressi Edy (diving computer) PC interface */
+#define CRESSI_VENDOR_ID	0x04b8
+#define CRESSI_EDY_PRODUCT_ID	0x0521
+
+/* Zeagle dive computer interface */
+#define ZEAGLE_VENDOR_ID	0x04b8
+#define ZEAGLE_N2ITION3_PRODUCT_ID	0x0522
+
+/* Sony, USB data cable for CMD-Jxx mobile phones */
+#define SONY_VENDOR_ID		0x054c
+#define SONY_QN3USB_PRODUCT_ID	0x0437
+
+/* Sanwa KB-USB2 multimeter cable (ID: 11ad:0001) */
+#define SANWA_VENDOR_ID		0x11ad
+#define SANWA_PRODUCT_ID	0x0001
+
+/* ADLINK ND-6530 RS232,RS485 and RS422 adapter */
+#define ADLINK_VENDOR_ID		0x0b63
+#define ADLINK_ND6530_PRODUCT_ID	0x6530
+
+/* SMART USB Serial Adapter */
+#define SMART_VENDOR_ID	0x0b8c
+#define SMART_PRODUCT_ID	0x2303
diff --git a/hw/usb/quirks.c b/hw/usb/quirks.c
new file mode 100644
index 0000000..a761a96
--- /dev/null
+++ b/hw/usb/quirks.c
@@ -0,0 +1,53 @@
+/*
+ * USB quirk handling
+ *
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include "quirks.h"
+#include "hw/usb.h"
+
+static bool usb_id_match(const struct usb_device_id *ids,
+                         uint16_t vendor_id, uint16_t product_id,
+                         uint8_t interface_class, uint8_t interface_subclass,
+                         uint8_t interface_protocol) {
+    int i;
+
+    for (i = 0; ids[i].vendor_id != -1; i++) {
+        if (ids[i].vendor_id  == vendor_id &&
+            ids[i].product_id == product_id &&
+            (ids[i].interface_class == -1 ||
+             (ids[i].interface_class == interface_class &&
+              ids[i].interface_subclass == interface_subclass &&
+              ids[i].interface_protocol == interface_protocol))) {
+            return true;
+        }
+    }
+    return false;
+}
+
+int usb_get_quirks(uint16_t vendor_id, uint16_t product_id,
+                   uint8_t interface_class, uint8_t interface_subclass,
+                   uint8_t interface_protocol)
+{
+    int quirks = 0;
+
+    if (usb_id_match(usbredir_raw_serial_ids, vendor_id, product_id,
+                   interface_class, interface_subclass, interface_protocol)) {
+        quirks |= USB_QUIRK_BUFFER_BULK_IN;
+    }
+    if (usb_id_match(usbredir_ftdi_serial_ids, vendor_id, product_id,
+                   interface_class, interface_subclass, interface_protocol)) {
+        quirks |= USB_QUIRK_BUFFER_BULK_IN | USB_QUIRK_IS_FTDI;
+    }
+
+    return quirks;
+}
diff --git a/hw/usb/quirks.h b/hw/usb/quirks.h
new file mode 100644
index 0000000..8dc6065
--- /dev/null
+++ b/hw/usb/quirks.h
@@ -0,0 +1,910 @@
+/*
+ * USB quirk handling
+ *
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+/* 1 on 1 copy of linux/drivers/usb/serial/ftdi_sio_ids.h */
+#include "quirks-ftdi-ids.h"
+/* 1 on 1 copy of linux/drivers/usb/serial/pl2303.h */
+#include "quirks-pl2303-ids.h"
+
+struct usb_device_id {
+    int vendor_id;
+    int product_id;
+    int interface_class;
+    int interface_subclass;
+    int interface_protocol;
+};
+
+#define USB_DEVICE(vendor, product) \
+    .vendor_id = vendor, .product_id = product, .interface_class = -1,
+
+#define USB_DEVICE_AND_INTERFACE_INFO(vend, prod, iclass, isubclass, iproto) \
+    .vendor_id = vend, .product_id = prod, .interface_class = iclass, \
+    .interface_subclass = isubclass, .interface_protocol = iproto
+
+static const struct usb_device_id usbredir_raw_serial_ids[] = {
+    /*
+     * Silicon Laboratories CP210x USB to RS232 serial adapter ids
+     * copied from linux/drivers/usb/serial/cp210x.c
+     *
+     * Copyright (C) 2005 Craig Shelley (craig@microtron.org.uk)
+     */
+    { USB_DEVICE(0x045B, 0x0053) }, /* Renesas RX610 RX-Stick */
+    { USB_DEVICE(0x0471, 0x066A) }, /* AKTAKOM ACE-1001 cable */
+    { USB_DEVICE(0x0489, 0xE000) }, /* Pirelli Broadband S.p.A, DP-L10 SIP/GSM Mobile */
+    { USB_DEVICE(0x0489, 0xE003) }, /* Pirelli Broadband S.p.A, DP-L10 SIP/GSM Mobile */
+    { USB_DEVICE(0x0745, 0x1000) }, /* CipherLab USB CCD Barcode Scanner 1000 */
+    { USB_DEVICE(0x08e6, 0x5501) }, /* Gemalto Prox-PU/CU contactless smartcard reader */
+    { USB_DEVICE(0x08FD, 0x000A) }, /* Digianswer A/S , ZigBee/802.15.4 MAC Device */
+    { USB_DEVICE(0x0BED, 0x1100) }, /* MEI (TM) Cashflow-SC Bill/Voucher Acceptor */
+    { USB_DEVICE(0x0BED, 0x1101) }, /* MEI series 2000 Combo Acceptor */
+    { USB_DEVICE(0x0FCF, 0x1003) }, /* Dynastream ANT development board */
+    { USB_DEVICE(0x0FCF, 0x1004) }, /* Dynastream ANT2USB */
+    { USB_DEVICE(0x0FCF, 0x1006) }, /* Dynastream ANT development board */
+    { USB_DEVICE(0x10A6, 0xAA26) }, /* Knock-off DCU-11 cable */
+    { USB_DEVICE(0x10AB, 0x10C5) }, /* Siemens MC60 Cable */
+    { USB_DEVICE(0x10B5, 0xAC70) }, /* Nokia CA-42 USB */
+    { USB_DEVICE(0x10C4, 0x0F91) }, /* Vstabi */
+    { USB_DEVICE(0x10C4, 0x1101) }, /* Arkham Technology DS101 Bus Monitor */
+    { USB_DEVICE(0x10C4, 0x1601) }, /* Arkham Technology DS101 Adapter */
+    { USB_DEVICE(0x10C4, 0x800A) }, /* SPORTident BSM7-D-USB main station */
+    { USB_DEVICE(0x10C4, 0x803B) }, /* Pololu USB-serial converter */
+    { USB_DEVICE(0x10C4, 0x8044) }, /* Cygnal Debug Adapter */
+    { USB_DEVICE(0x10C4, 0x804E) }, /* Software Bisque Paramount ME build-in converter */
+    { USB_DEVICE(0x10C4, 0x8053) }, /* Enfora EDG1228 */
+    { USB_DEVICE(0x10C4, 0x8054) }, /* Enfora GSM2228 */
+    { USB_DEVICE(0x10C4, 0x8066) }, /* Argussoft In-System Programmer */
+    { USB_DEVICE(0x10C4, 0x806F) }, /* IMS USB to RS422 Converter Cable */
+    { USB_DEVICE(0x10C4, 0x807A) }, /* Crumb128 board */
+    { USB_DEVICE(0x10C4, 0x80C4) }, /* Cygnal Integrated Products, Inc., Optris infrared thermometer */
+    { USB_DEVICE(0x10C4, 0x80CA) }, /* Degree Controls Inc */
+    { USB_DEVICE(0x10C4, 0x80DD) }, /* Tracient RFID */
+    { USB_DEVICE(0x10C4, 0x80F6) }, /* Suunto sports instrument */
+    { USB_DEVICE(0x10C4, 0x8115) }, /* Arygon NFC/Mifare Reader */
+    { USB_DEVICE(0x10C4, 0x813D) }, /* Burnside Telecom Deskmobile */
+    { USB_DEVICE(0x10C4, 0x813F) }, /* Tams Master Easy Control */
+    { USB_DEVICE(0x10C4, 0x814A) }, /* West Mountain Radio RIGblaster P&P */
+    { USB_DEVICE(0x10C4, 0x814B) }, /* West Mountain Radio RIGtalk */
+    { USB_DEVICE(0x10C4, 0x8156) }, /* B&G H3000 link cable */
+    { USB_DEVICE(0x10C4, 0x815E) }, /* Helicomm IP-Link 1220-DVM */
+    { USB_DEVICE(0x10C4, 0x815F) }, /* Timewave HamLinkUSB */
+    { USB_DEVICE(0x10C4, 0x818B) }, /* AVIT Research USB to TTL */
+    { USB_DEVICE(0x10C4, 0x819F) }, /* MJS USB Toslink Switcher */
+    { USB_DEVICE(0x10C4, 0x81A6) }, /* ThinkOptics WavIt */
+    { USB_DEVICE(0x10C4, 0x81A9) }, /* Multiplex RC Interface */
+    { USB_DEVICE(0x10C4, 0x81AC) }, /* MSD Dash Hawk */
+    { USB_DEVICE(0x10C4, 0x81AD) }, /* INSYS USB Modem */
+    { USB_DEVICE(0x10C4, 0x81C8) }, /* Lipowsky Industrie Elektronik GmbH, Baby-JTAG */
+    { USB_DEVICE(0x10C4, 0x81E2) }, /* Lipowsky Industrie Elektronik GmbH, Baby-LIN */
+    { USB_DEVICE(0x10C4, 0x81E7) }, /* Aerocomm Radio */
+    { USB_DEVICE(0x10C4, 0x81E8) }, /* Zephyr Bioharness */
+    { USB_DEVICE(0x10C4, 0x81F2) }, /* C1007 HF band RFID controller */
+    { USB_DEVICE(0x10C4, 0x8218) }, /* Lipowsky Industrie Elektronik GmbH, HARP-1 */
+    { USB_DEVICE(0x10C4, 0x822B) }, /* Modem EDGE(GSM) Comander 2 */
+    { USB_DEVICE(0x10C4, 0x826B) }, /* Cygnal Integrated Products, Inc., Fasttrax GPS demonstration module */
+    { USB_DEVICE(0x10C4, 0x8293) }, /* Telegesis ETRX2USB */
+    { USB_DEVICE(0x10C4, 0x82F9) }, /* Procyon AVS */
+    { USB_DEVICE(0x10C4, 0x8341) }, /* Siemens MC35PU GPRS Modem */
+    { USB_DEVICE(0x10C4, 0x8382) }, /* Cygnal Integrated Products, Inc. */
+    { USB_DEVICE(0x10C4, 0x83A8) }, /* Amber Wireless AMB2560 */
+    { USB_DEVICE(0x10C4, 0x83D8) }, /* DekTec DTA Plus VHF/UHF Booster/Attenuator */
+    { USB_DEVICE(0x10C4, 0x8411) }, /* Kyocera GPS Module */
+    { USB_DEVICE(0x10C4, 0x8418) }, /* IRZ Automation Teleport SG-10 GSM/GPRS Modem */
+    { USB_DEVICE(0x10C4, 0x846E) }, /* BEI USB Sensor Interface (VCP) */
+    { USB_DEVICE(0x10C4, 0x8477) }, /* Balluff RFID */
+    { USB_DEVICE(0x10C4, 0x85EA) }, /* AC-Services IBUS-IF */
+    { USB_DEVICE(0x10C4, 0x85EB) }, /* AC-Services CIS-IBUS */
+    { USB_DEVICE(0x10C4, 0x8664) }, /* AC-Services CAN-IF */
+    { USB_DEVICE(0x10C4, 0x8665) }, /* AC-Services OBD-IF */
+    { USB_DEVICE(0x10C4, 0xEA60) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA61) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA70) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA80) }, /* Silicon Labs factory default */
+    { USB_DEVICE(0x10C4, 0xEA71) }, /* Infinity GPS-MIC-1 Radio Monophone */
+    { USB_DEVICE(0x10C4, 0xF001) }, /* Elan Digital Systems USBscope50 */
+    { USB_DEVICE(0x10C4, 0xF002) }, /* Elan Digital Systems USBwave12 */
+    { USB_DEVICE(0x10C4, 0xF003) }, /* Elan Digital Systems USBpulse100 */
+    { USB_DEVICE(0x10C4, 0xF004) }, /* Elan Digital Systems USBcount50 */
+    { USB_DEVICE(0x10C5, 0xEA61) }, /* Silicon Labs MobiData GPRS USB Modem */
+    { USB_DEVICE(0x10CE, 0xEA6A) }, /* Silicon Labs MobiData GPRS USB Modem 100EU */
+    { USB_DEVICE(0x13AD, 0x9999) }, /* Baltech card reader */
+    { USB_DEVICE(0x1555, 0x0004) }, /* Owen AC4 USB-RS485 Converter */
+    { USB_DEVICE(0x166A, 0x0201) }, /* Clipsal 5500PACA C-Bus Pascal Automation Controller */
+    { USB_DEVICE(0x166A, 0x0301) }, /* Clipsal 5800PC C-Bus Wireless PC Interface */
+    { USB_DEVICE(0x166A, 0x0303) }, /* Clipsal 5500PCU C-Bus USB interface */
+    { USB_DEVICE(0x166A, 0x0304) }, /* Clipsal 5000CT2 C-Bus Black and White Touchscreen */
+    { USB_DEVICE(0x166A, 0x0305) }, /* Clipsal C-5000CT2 C-Bus Spectrum Colour Touchscreen */
+    { USB_DEVICE(0x166A, 0x0401) }, /* Clipsal L51xx C-Bus Architectural Dimmer */
+    { USB_DEVICE(0x166A, 0x0101) }, /* Clipsal 5560884 C-Bus Multi-room Audio Matrix Switcher */
+    { USB_DEVICE(0x16D6, 0x0001) }, /* Jablotron serial interface */
+    { USB_DEVICE(0x16DC, 0x0010) }, /* W-IE-NE-R Plein & Baus GmbH PL512 Power Supply */
+    { USB_DEVICE(0x16DC, 0x0011) }, /* W-IE-NE-R Plein & Baus GmbH RCM Remote Control for MARATON Power Supply */
+    { USB_DEVICE(0x16DC, 0x0012) }, /* W-IE-NE-R Plein & Baus GmbH MPOD Multi Channel Power Supply */
+    { USB_DEVICE(0x16DC, 0x0015) }, /* W-IE-NE-R Plein & Baus GmbH CML Control, Monitoring and Data Logger */
+    { USB_DEVICE(0x17A8, 0x0001) }, /* Kamstrup Optical Eye/3-wire */
+    { USB_DEVICE(0x17A8, 0x0005) }, /* Kamstrup M-Bus Master MultiPort 250D */
+    { USB_DEVICE(0x17F4, 0xAAAA) }, /* Wavesense Jazz blood glucose meter */
+    { USB_DEVICE(0x1843, 0x0200) }, /* Vaisala USB Instrument Cable */
+    { USB_DEVICE(0x18EF, 0xE00F) }, /* ELV USB-I2C-Interface */
+    { USB_DEVICE(0x1BE3, 0x07A6) }, /* WAGO 750-923 USB Service Cable */
+    { USB_DEVICE(0x1E29, 0x0102) }, /* Festo CPX-USB */
+    { USB_DEVICE(0x1E29, 0x0501) }, /* Festo CMSP */
+    { USB_DEVICE(0x3195, 0xF190) }, /* Link Instruments MSO-19 */
+    { USB_DEVICE(0x3195, 0xF280) }, /* Link Instruments MSO-28 */
+    { USB_DEVICE(0x3195, 0xF281) }, /* Link Instruments MSO-28 */
+    { USB_DEVICE(0x413C, 0x9500) }, /* DW700 GPS USB interface */
+
+    /*
+     * Prolific pl2303 USB to RS232 serial adapter ids
+     * copied from linux/drivers/usb/serial/pl2303.c
+     *
+     * Copyright (C) 2001-2007 Greg Kroah-Hartman (greg@kroah.com)
+     * Copyright (C) 2003 IBM Corp.
+     */
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_RSAQ2) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_DCU11) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_RSAQ3) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_PHAROS) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_ALDIGA) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_MMX) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_GPRS) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_HCR331) },
+    { USB_DEVICE(PL2303_VENDOR_ID, PL2303_PRODUCT_ID_MOTOROLA) },
+    { USB_DEVICE(IODATA_VENDOR_ID, IODATA_PRODUCT_ID) },
+    { USB_DEVICE(IODATA_VENDOR_ID, IODATA_PRODUCT_ID_RSAQ5) },
+    { USB_DEVICE(ATEN_VENDOR_ID, ATEN_PRODUCT_ID) },
+    { USB_DEVICE(ATEN_VENDOR_ID2, ATEN_PRODUCT_ID) },
+    { USB_DEVICE(ELCOM_VENDOR_ID, ELCOM_PRODUCT_ID) },
+    { USB_DEVICE(ELCOM_VENDOR_ID, ELCOM_PRODUCT_ID_UCSGT) },
+    { USB_DEVICE(ITEGNO_VENDOR_ID, ITEGNO_PRODUCT_ID) },
+    { USB_DEVICE(ITEGNO_VENDOR_ID, ITEGNO_PRODUCT_ID_2080) },
+    { USB_DEVICE(MA620_VENDOR_ID, MA620_PRODUCT_ID) },
+    { USB_DEVICE(RATOC_VENDOR_ID, RATOC_PRODUCT_ID) },
+    { USB_DEVICE(TRIPP_VENDOR_ID, TRIPP_PRODUCT_ID) },
+    { USB_DEVICE(RADIOSHACK_VENDOR_ID, RADIOSHACK_PRODUCT_ID) },
+    { USB_DEVICE(DCU10_VENDOR_ID, DCU10_PRODUCT_ID) },
+    { USB_DEVICE(SITECOM_VENDOR_ID, SITECOM_PRODUCT_ID) },
+    { USB_DEVICE(ALCATEL_VENDOR_ID, ALCATEL_PRODUCT_ID) },
+    { USB_DEVICE(SAMSUNG_VENDOR_ID, SAMSUNG_PRODUCT_ID) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_SX1) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_X65) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_X75) },
+    { USB_DEVICE(SIEMENS_VENDOR_ID, SIEMENS_PRODUCT_ID_EF81) },
+    { USB_DEVICE(BENQ_VENDOR_ID, BENQ_PRODUCT_ID_S81) }, /* Benq/Siemens S81 */
+    { USB_DEVICE(SYNTECH_VENDOR_ID, SYNTECH_PRODUCT_ID) },
+    { USB_DEVICE(NOKIA_CA42_VENDOR_ID, NOKIA_CA42_PRODUCT_ID) },
+    { USB_DEVICE(CA_42_CA42_VENDOR_ID, CA_42_CA42_PRODUCT_ID) },
+    { USB_DEVICE(SAGEM_VENDOR_ID, SAGEM_PRODUCT_ID) },
+    { USB_DEVICE(LEADTEK_VENDOR_ID, LEADTEK_9531_PRODUCT_ID) },
+    { USB_DEVICE(SPEEDDRAGON_VENDOR_ID, SPEEDDRAGON_PRODUCT_ID) },
+    { USB_DEVICE(DATAPILOT_U2_VENDOR_ID, DATAPILOT_U2_PRODUCT_ID) },
+    { USB_DEVICE(BELKIN_VENDOR_ID, BELKIN_PRODUCT_ID) },
+    { USB_DEVICE(ALCOR_VENDOR_ID, ALCOR_PRODUCT_ID) },
+    { USB_DEVICE(WS002IN_VENDOR_ID, WS002IN_PRODUCT_ID) },
+    { USB_DEVICE(COREGA_VENDOR_ID, COREGA_PRODUCT_ID) },
+    { USB_DEVICE(YCCABLE_VENDOR_ID, YCCABLE_PRODUCT_ID) },
+    { USB_DEVICE(SUPERIAL_VENDOR_ID, SUPERIAL_PRODUCT_ID) },
+    { USB_DEVICE(HP_VENDOR_ID, HP_LD220_PRODUCT_ID) },
+    { USB_DEVICE(CRESSI_VENDOR_ID, CRESSI_EDY_PRODUCT_ID) },
+    { USB_DEVICE(ZEAGLE_VENDOR_ID, ZEAGLE_N2ITION3_PRODUCT_ID) },
+    { USB_DEVICE(SONY_VENDOR_ID, SONY_QN3USB_PRODUCT_ID) },
+    { USB_DEVICE(SANWA_VENDOR_ID, SANWA_PRODUCT_ID) },
+    { USB_DEVICE(ADLINK_VENDOR_ID, ADLINK_ND6530_PRODUCT_ID) },
+    { USB_DEVICE(SMART_VENDOR_ID, SMART_PRODUCT_ID) },
+
+    { USB_DEVICE(-1, -1) } /* Terminating Entry */
+};
+
+static const struct usb_device_id usbredir_ftdi_serial_ids[] = {
+    /*
+     * FTDI USB to RS232 serial adapter ids
+     * copied from linux/drivers/usb/serial/ftdi_sio.c
+     *
+     * Copyright (C) 2009 - 2010
+     *    Johan Hovold (jhovold@gmail.com)
+     * Copyright (C) 1999 - 2001
+     *    Greg Kroah-Hartman (greg@kroah.com)
+     *    Bill Ryder (bryder@sgi.com)
+     * Copyright (C) 2002
+     *    Kuba Ober (kuba@mareimbrium.org)
+     */
+    { USB_DEVICE(FTDI_VID, FTDI_ZEITCONTROL_TAGTRACE_MIFARE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CTI_MINI_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CTI_NANO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_AMC232_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CANUSB_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CANDAPTER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NXTCAM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_5_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_6_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCS_DEVICE_7_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USINT_CAT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USINT_WKEY_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USINT_RS232_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ACTZWAVE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IRTRANS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IPLUS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IPLUS2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DMX4ALL) },
+    { USB_DEVICE(FTDI_VID, FTDI_SIO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_8U232AM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_8U232AM_ALT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_232RL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_8U2232C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_4232H_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_232H_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_FTX_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MICRO_CHAMELEON_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RELAIS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_SNIFFER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_THROTTLE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_GATEWAY_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_GBM_PID) },
+    { USB_DEVICE(INTERBIOMETRICS_VID, INTERBIOMETRICS_IOBOARD_PID) },
+    { USB_DEVICE(INTERBIOMETRICS_VID, INTERBIOMETRICS_MINI_IOBOARD_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SPROG_II) },
+    { USB_DEVICE(FTDI_VID, FTDI_LENZ_LIUSB_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_632_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_634_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_547_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_633_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_631_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_635_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_640_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_XF_642_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DSS20_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_URBAN_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_URBAN_1_PID) },
+    { USB_DEVICE(FTDI_NF_RIC_VID, FTDI_NF_RIC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_VNHCPCUSB_D_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_5_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MTXORB_6_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_R2000KU_TRUE_RNG) },
+    { USB_DEVICE(FTDI_VID, FTDI_VARDAAN_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0100_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0101_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0102_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0103_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0104_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0105_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0106_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0107_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0108_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0109_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_010F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0110_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0111_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0112_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0113_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0114_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0115_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0116_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0117_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0118_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0119_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_011F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0120_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0121_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0122_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0123_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0124_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0125_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0126_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0127_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0128_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0129_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_012F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0130_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0131_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0132_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0133_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0134_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0135_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0136_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0137_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0138_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0139_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_013F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0140_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0141_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0142_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0143_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0144_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0145_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0146_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0147_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0148_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0149_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_014F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0150_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0151_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0152_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0153_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0154_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0155_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0156_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0157_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0158_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0159_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_015F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0160_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0161_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0162_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0163_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0164_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0165_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0166_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0167_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0168_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0169_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_016F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0170_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0171_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0172_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0173_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0174_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0175_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0176_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0177_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0178_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0179_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_017F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0180_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0181_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0182_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0183_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0184_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0185_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0186_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0187_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0188_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0189_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_018F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0190_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0191_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0192_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0193_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0194_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0195_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0196_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0197_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0198_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_0199_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019A_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019B_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019C_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019D_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019E_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_019F_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01A9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01AF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01B9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01BF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01C9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01CF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01D9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01DF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01E9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01ED_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01EF_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F0_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F1_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F2_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F3_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F4_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F5_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F6_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F7_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F8_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01F9_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FA_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FB_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FC_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FD_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FE_PID) },
+    { USB_DEVICE(MTXORB_VID, MTXORB_FTDI_RANGE_01FF_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PERLE_ULTRAPORT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PIEGROUP_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TNC_X_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USBX_707_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2101_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2102_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2103_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2104_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2106_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2201_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2201_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2202_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2202_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2203_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2203_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2401_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2402_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2403_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_5_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_6_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_7_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2801_8_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_5_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_6_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_7_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2802_8_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_4_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_5_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_6_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_7_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803_8_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_1_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_2_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_3_PID) },
+    { USB_DEVICE(SEALEVEL_VID, SEALEVEL_2803R_4_PID) },
+    { USB_DEVICE(IDTECH_VID, IDTECH_IDT1221U_PID) },
+    { USB_DEVICE(OCT_VID, OCT_US101_PID) },
+    { USB_DEVICE(OCT_VID, OCT_DK201_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_HE_TIRA1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_USB_UIRT_PID) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_SPECIAL_1) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_R2X0) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_SPECIAL_3) },
+    { USB_DEVICE(FTDI_VID, PROTEGO_SPECIAL_4) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E808_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E809_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80A_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80B_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80D_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80E_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E80F_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E888_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E889_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88A_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88B_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88D_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88E_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GUDEADS_E88F_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UO100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UM100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UR100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_ALC8500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PYRAMID_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FHZ1000PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_US485_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PICPRO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PCMCIA_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PK1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_RS232MON_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_APP70_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PEDO_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_IBS_PROD_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TAVIR_STK500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TIAO_UMPA_PID) },
+    /*
+     * ELV devices:
+     */
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_USR_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_MSM1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_KL100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS550_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_EC3000_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS888_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_TWS550_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FEM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_CLI7000_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_PPS7330_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_TFM100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UDF77_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UIO88_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UAD8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UDA7_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_USI2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_T1100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_PCD200_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_ULA200_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_CSI8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_EM1000DL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_PCK100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_RFP500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FS20SIG_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UTP8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS300PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS444PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FHZ1300PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_EM1010PC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS500_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_HS485_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_UMS100_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_TFD128_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_FM3RX_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELV_WS777_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_SDMUSBQSS_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_MASTERDEVEL2_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_FUTURE_0_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_FUTURE_1_PID) },
+    { USB_DEVICE(FTDI_VID, LINX_FUTURE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSICDU20_0_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSICDU40_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSMACHX_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSLOAD_N_GO_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSICDU64_4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CCSPRIME8_5_PID) },
+    { USB_DEVICE(FTDI_VID, INSIDE_ACCESSO) },
+    { USB_DEVICE(INTREPID_VID, INTREPID_VALUECAN_PID) },
+    { USB_DEVICE(INTREPID_VID, INTREPID_NEOVI_PID) },
+    { USB_DEVICE(FALCOM_VID, FALCOM_TWIST_PID) },
+    { USB_DEVICE(FALCOM_VID, FALCOM_SAMBA_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SUUNTO_SPORTS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OCEANIC_PID) },
+    { USB_DEVICE(TTI_VID, TTI_QL355P_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RM_CANVIEW_PID) },
+    { USB_DEVICE(ACTON_VID, ACTON_SPECTRAPRO_PID) },
+    { USB_DEVICE(CONTEC_VID, CONTEC_COM1USBH_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USO9ML2_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOPTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USPTL4_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USO9ML2DR_2_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USO9ML2DR_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOPTL4DR2_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_USOPTL4DR_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USB9F_2W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USB9F_4W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_232USB9M_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USBTB_2W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_485USBTB_4W_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_TTL5USB9M_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_TTL3USB9M_PID) },
+    { USB_DEVICE(BANDB_VID, BANDB_ZZ_PROG1_USB_PID) },
+    { USB_DEVICE(FTDI_VID, EVER_ECO_PRO_CDS) },
+    { USB_DEVICE(FTDI_VID, FTDI_4N_GALAXY_DE_1_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_4N_GALAXY_DE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_4N_GALAXY_DE_3_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_0_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_1_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_2_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_3_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_4_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_5_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_6_PID) },
+    { USB_DEVICE(FTDI_VID, XSENS_CONVERTER_7_PID) },
+    { USB_DEVICE(MOBILITY_VID, MOBILITY_USB_SERIAL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ACTIVE_ROBOTS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_KW_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_YS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_Y6_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_Y8_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_IC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_DB9_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_RS232_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MHAM_Y9_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TERATRONIK_VCP_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TERATRONIK_D2XX_PID) },
+    { USB_DEVICE(EVOLUTION_VID, EVOLUTION_ER1_PID) },
+    { USB_DEVICE(EVOLUTION_VID, EVO_HYBRID_PID) },
+    { USB_DEVICE(EVOLUTION_VID, EVO_RCM4_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ARTEMIS_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16C_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16HR_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16HRC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ATIK_ATK16IC_PID) },
+    { USB_DEVICE(KOBIL_VID, KOBIL_CONV_B1_PID) },
+    { USB_DEVICE(KOBIL_VID, KOBIL_CONV_KAAN_PID) },
+    { USB_DEVICE(POSIFLEX_VID, POSIFLEX_PP7000_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TTUSB_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ECLO_COM_1WIRE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_WESTREX_MODEL_777_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_WESTREX_MODEL_8900F_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PCDJ_DAC2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RRCIRKITS_LOCOBUFFER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ASK_RDR400_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NZR_SEM_USB_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_1_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_OPC_U_UC_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2C1_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2C2_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2D_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2VT_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2VR_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP4KVT_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP4KVR_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2KVT_PID) },
+    { USB_DEVICE(ICOM_VID, ICOM_ID_RP2KVR_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ACG_HFDUAL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_YEI_SERVOCENTER31_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_THORLABS_PID) },
+    { USB_DEVICE(TESTO_VID, TESTO_USB_INTERFACE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_GAMMA_SCOUT_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TACTRIX_OPENPORT_13M_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TACTRIX_OPENPORT_13S_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TACTRIX_OPENPORT_13U_PID) },
+    { USB_DEVICE(ELEKTOR_VID, ELEKTOR_FT323R_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_HUC_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_SPECTRA_SCU_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_FUTURE_2_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_FUTURE_3_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_NDI_AURORA_SCU_PID) },
+    { USB_DEVICE(TELLDUS_VID, TELLDUS_TELLSTICK_PID) },
+    { USB_DEVICE(RTSYSTEMS_VID, RTSYSTEMS_SERIAL_VX7_PID) },
+    { USB_DEVICE(RTSYSTEMS_VID, RTSYSTEMS_CT29B_PID) },
+    { USB_DEVICE(RTSYSTEMS_VID, RTSYSTEMS_RTS01_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_MAXSTREAM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PHI_FISCO_PID) },
+    { USB_DEVICE(TML_VID, TML_USB_SERIAL_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_ELSTER_UNICOM_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PROPOX_JTAGCABLEII_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_PROPOX_ISPCABLEIII_PID) },
+    { USB_DEVICE(OLIMEX_VID, OLIMEX_ARM_USB_OCD_PID) },
+    { USB_DEVICE(OLIMEX_VID, OLIMEX_ARM_USB_OCD_H_PID) },
+    { USB_DEVICE(FIC_VID, FIC_NEO1973_DEBUG_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_OOCDLINK_PID) },
+    { USB_DEVICE(FTDI_VID, LMI_LM3S_DEVEL_BOARD_PID) },
+    { USB_DEVICE(FTDI_VID, LMI_LM3S_EVAL_BOARD_PID) },
+    { USB_DEVICE(FTDI_VID, LMI_LM3S_ICDI_BOARD_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_TURTELIZER_PID) },
+    { USB_DEVICE(RATOC_VENDOR_ID, RATOC_PRODUCT_ID_USB60F) },
+    { USB_DEVICE(FTDI_VID, FTDI_REU_TINY_PID) },
+
+    /* Papouch devices based on FTDI chip */
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_AP485_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB422_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485_2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_AP485_2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB422_2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485S_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB485C_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_LEC_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SB232_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_TMU_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_IRAMP_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_DRAK5_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO8x8_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO4x4_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO2x2_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO10x1_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO30x3_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO60x3_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO2x16_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_QUIDO3x32_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_DRAK6_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_UPSUSB_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_MU_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_SIMUKEY_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_AD4USB_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_GMUX_PID) },
+    { USB_DEVICE(PAPOUCH_VID, PAPOUCH_GMSR_PID) },
+
+    { USB_DEVICE(FTDI_VID, FTDI_DOMINTELL_DGQG_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DOMINTELL_DUSB_PID) },
+    { USB_DEVICE(ALTI2_VID, ALTI2_N3_PID) },
+    { USB_DEVICE(FTDI_VID, DIEBOLD_BCS_SE923_PID) },
+    { USB_DEVICE(ATMEL_VID, STK541_PID) },
+    { USB_DEVICE(DE_VID, STB_PID) },
+    { USB_DEVICE(DE_VID, WHT_PID) },
+    { USB_DEVICE(ADI_VID, ADI_GNICE_PID) },
+    { USB_DEVICE(ADI_VID, ADI_GNICEPLUS_PID) },
+    { USB_DEVICE_AND_INTERFACE_INFO(MICROCHIP_VID, MICROCHIP_USB_BOARD_PID,
+                                    0xff, 0xff, 0x00) },
+    { USB_DEVICE(JETI_VID, JETI_SPC1201_PID) },
+    { USB_DEVICE(MARVELL_VID, MARVELL_SHEEVAPLUG_PID) },
+    { USB_DEVICE(LARSENBRUSGAARD_VID, LB_ALTITRACK_PID) },
+    { USB_DEVICE(GN_OTOMETRICS_VID, AURICAL_USB_PID) },
+    { USB_DEVICE(FTDI_VID, PI_C865_PID) },
+    { USB_DEVICE(FTDI_VID, PI_C857_PID) },
+    { USB_DEVICE(PI_VID, PI_C866_PID) },
+    { USB_DEVICE(PI_VID, PI_C663_PID) },
+    { USB_DEVICE(PI_VID, PI_C725_PID) },
+    { USB_DEVICE(PI_VID, PI_E517_PID) },
+    { USB_DEVICE(PI_VID, PI_C863_PID) },
+    { USB_DEVICE(PI_VID, PI_E861_PID) },
+    { USB_DEVICE(PI_VID, PI_C867_PID) },
+    { USB_DEVICE(PI_VID, PI_E609_PID) },
+    { USB_DEVICE(PI_VID, PI_E709_PID) },
+    { USB_DEVICE(PI_VID, PI_100F_PID) },
+    { USB_DEVICE(PI_VID, PI_1011_PID) },
+    { USB_DEVICE(PI_VID, PI_1012_PID) },
+    { USB_DEVICE(PI_VID, PI_1013_PID) },
+    { USB_DEVICE(PI_VID, PI_1014_PID) },
+    { USB_DEVICE(PI_VID, PI_1015_PID) },
+    { USB_DEVICE(PI_VID, PI_1016_PID) },
+    { USB_DEVICE(KONDO_VID, KONDO_USB_SERIAL_PID) },
+    { USB_DEVICE(BAYER_VID, BAYER_CONTOUR_CABLE_PID) },
+    { USB_DEVICE(FTDI_VID, MARVELL_OPENRD_PID) },
+    { USB_DEVICE(FTDI_VID, TI_XDS100V2_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO820_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO720_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO730_PID) },
+    { USB_DEVICE(FTDI_VID, HAMEG_HO870_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_GENERIC_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_SR_RADIO_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_HD_RADIO_PID) },
+    { USB_DEVICE(FTDI_VID, MJSG_XM_RADIO_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_ST_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SLITE_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SH2_PID) },
+    { USB_DEVICE(FTDI_VID, XVERVE_SIGNALYZER_SH4_PID) },
+    { USB_DEVICE(FTDI_VID, SEGWAY_RMP200_PID) },
+    { USB_DEVICE(FTDI_VID, ACCESIO_COM4SM_PID) },
+    { USB_DEVICE(IONICS_VID, IONICS_PLUGCOMPUTER_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_24_MASTER_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_PC_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_USB_DMX_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MIDI_TIMECODE_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MINI_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MAXI_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_MEDIA_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CHAMSYS_WING_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCIENCESCOPE_LOGBOOKML_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCIENCESCOPE_LS_LOGBOOK_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_SCIENCESCOPE_HS_LOGBOOK_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_CINTERION_MC55I_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_DOTEC_PID) },
+    { USB_DEVICE(QIHARDWARE_VID, MILKYMISTONE_JTAGSERIAL_PID) },
+    { USB_DEVICE(ST_VID, ST_STMCLT1030_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_RF_R106) },
+    { USB_DEVICE(FTDI_VID, FTDI_DISTORTEC_JTAG_LOCK_PICK_PID) },
+    { USB_DEVICE(FTDI_VID, FTDI_LUMEL_PD12_PID) },
+
+    { USB_DEVICE(-1, -1) } /* Terminating Entry */
+};
+
+#undef USB_DEVICE
+#undef USB_DEVICE_AND_INTERFACE_INFO
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 490c90f..8c0ead0 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -26,10 +26,11 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "iov.h"
+#include "qemu/timer.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/iov.h"
+#include "char/char.h"
 
 #include <dirent.h>
 #include <sys/ioctl.h>
@@ -43,18 +44,26 @@
 #define NO_INTERFACE_INFO 255 /* Valid interface_count always <= 32 */
 #define EP2I(ep_address) (((ep_address & 0x80) >> 3) | (ep_address & 0x0f))
 #define I2EP(i) (((i & 0x10) << 3) | (i & 0x0f))
+#define USBEP2I(usb_ep) (((usb_ep)->pid == USB_TOKEN_IN) ? \
+                         ((usb_ep)->nr | 0x10) : ((usb_ep)->nr))
+#define I2USBEP(d, i) (usb_ep_get(&(d)->dev, \
+                       ((i) & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT, \
+                       (i) & 0x0f))
 
 typedef struct USBRedirDevice USBRedirDevice;
 
-/* Struct to hold buffered packets (iso or int input packets) */
+/* Struct to hold buffered packets */
 struct buf_packet {
     uint8_t *data;
-    int len;
-    int status;
+    void *free_on_destroy;
+    uint16_t len;
+    uint16_t offset;
+    uint8_t status;
     QTAILQ_ENTRY(buf_packet)next;
 };
 
 struct endp_data {
+    USBRedirDevice *dev;
     uint8_t type;
     uint8_t interval;
     uint8_t interface; /* bInterfaceNumber this ep belongs to */
@@ -63,11 +72,14 @@ struct endp_data {
     uint8_t iso_error; /* For reporting iso errors to the HC */
     uint8_t interrupt_started;
     uint8_t interrupt_error;
+    uint8_t bulk_receiving_enabled;
+    uint8_t bulk_receiving_started;
     uint8_t bufpq_prefilled;
     uint8_t bufpq_dropping_packets;
     QTAILQ_HEAD(, buf_packet) bufpq;
     int32_t bufpq_size;
     int32_t bufpq_target_size;
+    USBPacket *pending_async_packet;
 };
 
 struct PacketIdQueueEntry {
@@ -101,6 +113,7 @@ struct USBRedirDevice {
     struct endp_data endpoint[MAX_ENDPOINTS];
     struct PacketIdQueue cancelled;
     struct PacketIdQueue already_in_flight;
+    void (*buffered_bulk_in_complete)(USBRedirDevice *, USBPacket *, uint8_t);
     /* Data for device filtering */
     struct usb_redir_device_connect_header device_info;
     struct usb_redir_interface_info_header interface_info;
@@ -128,6 +141,8 @@ static void usbredir_interrupt_receiving_status(void *priv, uint64_t id,
     *interrupt_receiving_status);
 static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status);
+static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
+    struct usb_redir_bulk_receiving_status_header *bulk_receiving_status);
 static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len);
@@ -140,6 +155,9 @@ static void usbredir_iso_packet(void *priv, uint64_t id,
 static void usbredir_interrupt_packet(void *priv, uint64_t id,
     struct usb_redir_interrupt_packet_header *interrupt_header,
     uint8_t *data, int data_len);
+static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
+    struct usb_redir_buffered_bulk_packet_header *buffered_bulk_packet,
+    uint8_t *data, int data_len);
 
 static void usbredir_handle_status(USBRedirDevice *dev, USBPacket *p,
     int status);
@@ -313,12 +331,19 @@ static void packet_id_queue_empty(struct PacketIdQueue *q)
 static void usbredir_cancel_packet(USBDevice *udev, USBPacket *p)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    int i = USBEP2I(p->ep);
 
     if (p->combined) {
         usb_combined_packet_cancel(udev, p);
         return;
     }
 
+    if (dev->endpoint[i].pending_async_packet) {
+        assert(dev->endpoint[i].pending_async_packet == p);
+        dev->endpoint[i].pending_async_packet = NULL;
+        return;
+    }
+
     packet_id_queue_add(&dev->cancelled, p->id);
     usbredirparser_send_cancel_data_packet(dev->parser, p->id);
     usbredirparser_do_write(dev->parser);
@@ -337,6 +362,11 @@ static void usbredir_fill_already_in_flight_from_ep(USBRedirDevice *dev,
 {
     static USBPacket *p;
 
+    /* async handled packets for bulk receiving eps do not count as inflight */
+    if (dev->endpoint[USBEP2I(ep)].bulk_receiving_started) {
+        return;
+    }
+
     QTAILQ_FOREACH(p, &ep->queue, queue) {
         /* Skip combined packets, except for the first */
         if (p->combined && p != p->combined->first) {
@@ -384,8 +414,8 @@ static USBPacket *usbredir_find_packet_by_id(USBRedirDevice *dev,
     return p;
 }
 
-static void bufp_alloc(USBRedirDevice *dev,
-    uint8_t *data, int len, int status, uint8_t ep)
+static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
+    uint8_t status, uint8_t ep, void *free_on_destroy)
 {
     struct buf_packet *bufp;
 
@@ -409,7 +439,9 @@ static void bufp_alloc(USBRedirDevice *dev,
     bufp = g_malloc(sizeof(struct buf_packet));
     bufp->data   = data;
     bufp->len    = len;
+    bufp->offset = 0;
     bufp->status = status;
+    bufp->free_on_destroy = free_on_destroy;
     QTAILQ_INSERT_TAIL(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size++;
 }
@@ -419,7 +451,7 @@ static void bufp_free(USBRedirDevice *dev, struct buf_packet *bufp,
 {
     QTAILQ_REMOVE(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size--;
-    free(bufp->data);
+    free(bufp->free_on_destroy);
     g_free(bufp);
 }
 
@@ -570,19 +602,162 @@ static void usbredir_stop_iso_stream(USBRedirDevice *dev, uint8_t ep)
     usbredir_free_bufpq(dev, ep);
 }
 
+/*
+ * The usb-host may poll the endpoint faster then our guest, resulting in lots
+ * of smaller bulkp-s. The below buffered_bulk_in_complete* functions combine
+ * data from multiple bulkp-s into a single packet, avoiding bufpq overflows.
+ */
+static void usbredir_buffered_bulk_add_data_to_packet(USBRedirDevice *dev,
+    struct buf_packet *bulkp, int count, USBPacket *p, uint8_t ep)
+{
+    usb_packet_copy(p, bulkp->data + bulkp->offset, count);
+    bulkp->offset += count;
+    if (bulkp->offset == bulkp->len) {
+        /* Store status in the last packet with data from this bulkp */
+        usbredir_handle_status(dev, p, bulkp->status);
+        bufp_free(dev, bulkp, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete_raw(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    struct buf_packet *bulkp;
+    int count;
+
+    while ((bulkp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq)) &&
+           p->actual_length < p->iov.size && p->status == USB_RET_SUCCESS) {
+        count = bulkp->len - bulkp->offset;
+        if (count > (p->iov.size - p->actual_length)) {
+            count = p->iov.size - p->actual_length;
+        }
+        usbredir_buffered_bulk_add_data_to_packet(dev, bulkp, count, p, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete_ftdi(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    const int maxp = dev->endpoint[EP2I(ep)].max_packet_size;
+    uint8_t header[2] = { 0, 0 };
+    struct buf_packet *bulkp;
+    int count;
+
+    while ((bulkp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq)) &&
+           p->actual_length < p->iov.size && p->status == USB_RET_SUCCESS) {
+        if (bulkp->len < 2) {
+            WARNING("malformed ftdi bulk in packet\n");
+            bufp_free(dev, bulkp, ep);
+            continue;
+        }
+
+        if ((p->actual_length % maxp) == 0) {
+            usb_packet_copy(p, bulkp->data, 2);
+            memcpy(header, bulkp->data, 2);
+        } else {
+            if (bulkp->data[0] != header[0] || bulkp->data[1] != header[1]) {
+                break; /* Different header, add to next packet */
+            }
+        }
+
+        if (bulkp->offset == 0) {
+            bulkp->offset = 2; /* Skip header */
+        }
+        count = bulkp->len - bulkp->offset;
+        /* Must repeat the header at maxp interval */
+        if (count > (maxp - (p->actual_length % maxp))) {
+            count = maxp - (p->actual_length % maxp);
+        }
+        usbredir_buffered_bulk_add_data_to_packet(dev, bulkp, count, p, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
+    dev->buffered_bulk_in_complete(dev, p, ep);
+    DPRINTF("bulk-token-in ep %02X status %d len %d id %"PRIu64"\n",
+            ep, p->status, p->actual_length, p->id);
+}
+
+static void usbredir_handle_buffered_bulk_in_data(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    /* Input bulk endpoint, buffered packet input */
+    if (!dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        int bpt;
+        struct usb_redir_start_bulk_receiving_header start = {
+            .endpoint = ep,
+            .stream_id = 0,
+            .no_transfers = 5,
+        };
+        /* Round bytes_per_transfer up to a multiple of max_packet_size */
+        bpt = 512 + dev->endpoint[EP2I(ep)].max_packet_size - 1;
+        bpt /= dev->endpoint[EP2I(ep)].max_packet_size;
+        bpt *= dev->endpoint[EP2I(ep)].max_packet_size;
+        start.bytes_per_transfer = bpt;
+        /* No id, we look at the ep when receiving a status back */
+        usbredirparser_send_start_bulk_receiving(dev->parser, 0, &start);
+        usbredirparser_do_write(dev->parser);
+        DPRINTF("bulk receiving started bytes/transfer %u count %d ep %02X\n",
+                start.bytes_per_transfer, start.no_transfers, ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 1;
+        /* We don't really want to drop bulk packets ever, but
+           having some upper limit to how much we buffer is good. */
+        dev->endpoint[EP2I(ep)].bufpq_target_size = 5000;
+        dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
+    }
+
+    if (QTAILQ_EMPTY(&dev->endpoint[EP2I(ep)].bufpq)) {
+        DPRINTF("bulk-token-in ep %02X, no bulkp\n", ep);
+        assert(dev->endpoint[EP2I(ep)].pending_async_packet == NULL);
+        dev->endpoint[EP2I(ep)].pending_async_packet = p;
+        p->status = USB_RET_ASYNC;
+        return;
+    }
+    usbredir_buffered_bulk_in_complete(dev, p, ep);
+}
+
+static void usbredir_stop_bulk_receiving(USBRedirDevice *dev, uint8_t ep)
+{
+    struct usb_redir_stop_bulk_receiving_header stop_bulk = {
+        .endpoint = ep,
+        .stream_id = 0,
+    };
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        usbredirparser_send_stop_bulk_receiving(dev->parser, 0, &stop_bulk);
+        DPRINTF("bulk receiving stopped ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 0;
+    }
+    usbredir_free_bufpq(dev, ep);
+}
+
 static void usbredir_handle_bulk_data(USBRedirDevice *dev, USBPacket *p,
                                       uint8_t ep)
 {
     struct usb_redir_bulk_packet_header bulk_packet;
     size_t size = (p->combined) ? p->combined->iov.size : p->iov.size;
-
-    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
+    const int maxp = dev->endpoint[EP2I(ep)].max_packet_size;
 
     if (usbredir_already_in_flight(dev, p->id)) {
         p->status = USB_RET_ASYNC;
         return;
     }
 
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_enabled) {
+        if (size != 0 && (size % maxp) == 0) {
+            usbredir_handle_buffered_bulk_in_data(dev, p, ep);
+            return;
+        }
+        WARNING("bulk recv invalid size %zd ep %02x, disabling\n", size, ep);
+        assert(dev->endpoint[EP2I(ep)].pending_async_packet == NULL);
+        usbredir_stop_bulk_receiving(dev, ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_enabled = 0;
+    }
+
+    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
+
     bulk_packet.endpoint  = ep;
     bulk_packet.length    = size;
     bulk_packet.stream_id = 0;
@@ -719,9 +894,6 @@ static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
         ERROR("handle_data called for control transfer on ep %02X\n", ep);
         p->status = USB_RET_NAK;
         break;
-    case USB_ENDPOINT_XFER_ISOC:
-        usbredir_handle_iso_data(dev, p, ep);
-        break;
     case USB_ENDPOINT_XFER_BULK:
         if (p->state == USB_PACKET_SETUP && p->pid == USB_TOKEN_IN &&
                 p->ep->pipeline) {
@@ -730,6 +902,9 @@ static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
         }
         usbredir_handle_bulk_data(dev, p, ep);
         break;
+    case USB_ENDPOINT_XFER_ISOC:
+        usbredir_handle_iso_data(dev, p, ep);
+        break;
     case USB_ENDPOINT_XFER_INT:
         if (ep & USB_DIR_IN) {
             usbredir_handle_interrupt_in_data(dev, p, ep);
@@ -751,6 +926,36 @@ static void usbredir_flush_ep_queue(USBDevice *dev, USBEndpoint *ep)
     }
 }
 
+static void usbredir_stop_ep(USBRedirDevice *dev, int i)
+{
+    uint8_t ep = I2EP(i);
+
+    switch (dev->endpoint[i].type) {
+    case USB_ENDPOINT_XFER_BULK:
+        if (ep & USB_DIR_IN) {
+            usbredir_stop_bulk_receiving(dev, ep);
+        }
+        break;
+    case USB_ENDPOINT_XFER_ISOC:
+        usbredir_stop_iso_stream(dev, ep);
+        break;
+    case USB_ENDPOINT_XFER_INT:
+        if (ep & USB_DIR_IN) {
+            usbredir_stop_interrupt_receiving(dev, ep);
+        }
+        break;
+    }
+    usbredir_free_bufpq(dev, ep);
+}
+
+static void usbredir_ep_stopped(USBDevice *udev, USBEndpoint *uep)
+{
+    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+
+    usbredir_stop_ep(dev, USBEP2I(uep));
+    usbredirparser_do_write(dev->parser);
+}
+
 static void usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
                                 int config)
 {
@@ -760,17 +965,7 @@ static void usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
     DPRINTF("set config %d id %"PRIu64"\n", config, p->id);
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
-        switch (dev->endpoint[i].type) {
-        case USB_ENDPOINT_XFER_ISOC:
-            usbredir_stop_iso_stream(dev, I2EP(i));
-            break;
-        case USB_ENDPOINT_XFER_INT:
-            if (i & 0x10) {
-                usbredir_stop_interrupt_receiving(dev, I2EP(i));
-            }
-            break;
-        }
-        usbredir_free_bufpq(dev, I2EP(i));
+        usbredir_stop_ep(dev, i);
     }
 
     set_config.configuration = config;
@@ -798,17 +993,7 @@ static void usbredir_set_interface(USBRedirDevice *dev, USBPacket *p,
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         if (dev->endpoint[i].interface == interface) {
-            switch (dev->endpoint[i].type) {
-            case USB_ENDPOINT_XFER_ISOC:
-                usbredir_stop_iso_stream(dev, I2EP(i));
-                break;
-            case USB_ENDPOINT_XFER_INT:
-                if (i & 0x10) {
-                    usbredir_stop_interrupt_receiving(dev, I2EP(i));
-                }
-                break;
-            }
-            usbredir_free_bufpq(dev, I2EP(i));
+            usbredir_stop_ep(dev, i);
         }
     }
 
@@ -930,10 +1115,12 @@ static void usbredir_create_parser(USBRedirDevice *dev)
     dev->parser->interrupt_receiving_status_func =
         usbredir_interrupt_receiving_status;
     dev->parser->bulk_streams_status_func = usbredir_bulk_streams_status;
+    dev->parser->bulk_receiving_status_func = usbredir_bulk_receiving_status;
     dev->parser->control_packet_func = usbredir_control_packet;
     dev->parser->bulk_packet_func = usbredir_bulk_packet;
     dev->parser->iso_packet_func = usbredir_iso_packet;
     dev->parser->interrupt_packet_func = usbredir_interrupt_packet;
+    dev->parser->buffered_bulk_packet_func = usbredir_buffered_bulk_packet;
     dev->read_buf = NULL;
     dev->read_buf_size = 0;
 
@@ -942,6 +1129,7 @@ static void usbredir_create_parser(USBRedirDevice *dev)
     usbredirparser_caps_set_cap(caps, usb_redir_cap_ep_info_max_packet_size);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_64bits_ids);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_32bits_bulk_length);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_receiving);
 
     if (runstate_check(RUN_STATE_INMIGRATE)) {
         flags |= usbredirparser_fl_no_hello;
@@ -969,6 +1157,8 @@ static void usbredir_do_attach(void *opaque)
         usbredirparser_peer_has_cap(dev->parser,
                                     usb_redir_cap_ep_info_max_packet_size) &&
         usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_32bits_bulk_length) &&
+        usbredirparser_peer_has_cap(dev->parser,
                                     usb_redir_cap_64bits_ids))) {
         ERROR("usb-redir-host lacks capabilities needed for use with XHCI\n");
         usbredir_reject_device(dev);
@@ -1050,6 +1240,18 @@ static void usbredir_vm_state_change(void *priv, int running, RunState state)
     }
 }
 
+static void usbredir_init_endpoints(USBRedirDevice *dev)
+{
+    int i;
+
+    usb_ep_init(&dev->dev);
+    memset(dev->endpoint, 0, sizeof(dev->endpoint));
+    for (i = 0; i < MAX_ENDPOINTS; i++) {
+        dev->endpoint[i].dev = dev;
+        QTAILQ_INIT(&dev->endpoint[i].bufpq);
+    }
+}
+
 static int usbredir_initfn(USBDevice *udev)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
@@ -1076,9 +1278,7 @@ static int usbredir_initfn(USBDevice *udev)
 
     packet_id_queue_init(&dev->cancelled, dev, "cancelled");
     packet_id_queue_init(&dev->already_in_flight, dev, "already-in-flight");
-    for (i = 0; i < MAX_ENDPOINTS; i++) {
-        QTAILQ_INIT(&dev->endpoint[i].bufpq);
-    }
+    usbredir_init_endpoints(dev);
 
     /* We'll do the attach once we receive the speed from the usb-host */
     udev->auto_attach = 0;
@@ -1168,6 +1368,52 @@ error:
     return -1;
 }
 
+static void usbredir_check_bulk_receiving(USBRedirDevice *dev)
+{
+    int i, j, quirks;
+
+    if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_bulk_receiving)) {
+        return;
+    }
+
+    for (i = EP2I(USB_DIR_IN); i < MAX_ENDPOINTS; i++) {
+        dev->endpoint[i].bulk_receiving_enabled = 0;
+    }
+    for (i = 0; i < dev->interface_info.interface_count; i++) {
+        quirks = usb_get_quirks(dev->device_info.vendor_id,
+                                dev->device_info.product_id,
+                                dev->interface_info.interface_class[i],
+                                dev->interface_info.interface_subclass[i],
+                                dev->interface_info.interface_protocol[i]);
+        if (!(quirks & USB_QUIRK_BUFFER_BULK_IN)) {
+            continue;
+        }
+        if (quirks & USB_QUIRK_IS_FTDI) {
+            dev->buffered_bulk_in_complete =
+                usbredir_buffered_bulk_in_complete_ftdi;
+        } else {
+            dev->buffered_bulk_in_complete =
+                usbredir_buffered_bulk_in_complete_raw;
+        }
+
+        for (j = EP2I(USB_DIR_IN); j < MAX_ENDPOINTS; j++) {
+            if (dev->endpoint[j].interface ==
+                                    dev->interface_info.interface[i] &&
+                    dev->endpoint[j].type == USB_ENDPOINT_XFER_BULK &&
+                    dev->endpoint[j].max_packet_size != 0) {
+                dev->endpoint[j].bulk_receiving_enabled = 1;
+                /*
+                 * With buffering pipelining is not necessary. Also packet
+                 * combining and bulk in buffering don't play nice together!
+                 */
+                I2USBEP(dev, j)->pipeline = false;
+                break; /* Only buffer for the first ep of each intf */
+            }
+        }
+    }
+}
+
 /*
  * usbredirparser packet complete callbacks
  */
@@ -1276,13 +1522,13 @@ static void usbredir_device_connect(void *priv,
         return;
     }
 
+    usbredir_check_bulk_receiving(dev);
     qemu_mod_timer(dev->attach_timer, dev->next_attach_time);
 }
 
 static void usbredir_device_disconnect(void *priv)
 {
     USBRedirDevice *dev = priv;
-    int i;
 
     /* Stop any pending attaches */
     qemu_del_timer(dev->attach_timer);
@@ -1299,11 +1545,7 @@ static void usbredir_device_disconnect(void *priv)
 
     /* Reset state so that the next dev connected starts with a clean slate */
     usbredir_cleanup_device_queues(dev);
-    memset(dev->endpoint, 0, sizeof(dev->endpoint));
-    for (i = 0; i < MAX_ENDPOINTS; i++) {
-        QTAILQ_INIT(&dev->endpoint[i].bufpq);
-    }
-    usb_ep_init(&dev->dev);
+    usbredir_init_endpoints(dev);
     dev->interface_info.interface_count = NO_INTERFACE_INFO;
     dev->dev.addr = 0;
     dev->dev.speed = 0;
@@ -1319,9 +1561,10 @@ static void usbredir_interface_info(void *priv,
 
     /*
      * If we receive interface info after the device has already been
-     * connected (ie on a set_config), re-check the filter.
+     * connected (ie on a set_config), re-check interface dependent things.
      */
     if (qemu_timer_pending(dev->attach_timer) || dev->dev.attached) {
+        usbredir_check_bulk_receiving(dev);
         if (usbredir_check_filter(dev)) {
             ERROR("Device no longer matches filter after interface info "
                   "change, disconnecting!\n");
@@ -1353,11 +1596,10 @@ static void usbredir_set_pipeline(USBRedirDevice *dev, struct USBEndpoint *uep)
 static void usbredir_setup_usb_eps(USBRedirDevice *dev)
 {
     struct USBEndpoint *usb_ep;
-    int i, pid;
+    int i;
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
-        pid = (i & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT;
-        usb_ep = usb_ep_get(&dev->dev, pid, i & 0x0f);
+        usb_ep = I2USBEP(dev, i);
         usb_ep->type = dev->endpoint[i].type;
         usb_ep->ifnum = dev->endpoint[i].interface;
         usb_ep->max_packet_size = dev->endpoint[i].max_packet_size;
@@ -1423,6 +1665,7 @@ static void usbredir_ep_info(void *priv,
         return;
     }
     usbredir_setup_usb_eps(dev);
+    usbredir_check_bulk_receiving(dev);
 }
 
 static void usbredir_configuration_status(void *priv, uint64_t id,
@@ -1513,6 +1756,25 @@ static void usbredir_bulk_streams_status(void *priv, uint64_t id,
 {
 }
 
+static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
+    struct usb_redir_bulk_receiving_status_header *bulk_receiving_status)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t ep = bulk_receiving_status->endpoint;
+
+    DPRINTF("bulk recv status %d ep %02X id %"PRIu64"\n",
+            bulk_receiving_status->status, ep, id);
+
+    if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        return;
+    }
+
+    if (bulk_receiving_status->status == usb_redir_stall) {
+        DPRINTF("bulk receiving stopped by peer ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 0;
+    }
+}
+
 static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len)
@@ -1618,7 +1880,7 @@ static void usbredir_iso_packet(void *priv, uint64_t id,
     }
 
     /* bufp_alloc also adds the packet to the ep queue */
-    bufp_alloc(dev, data, data_len, iso_packet->status, ep);
+    bufp_alloc(dev, data, data_len, iso_packet->status, ep, data);
 }
 
 static void usbredir_interrupt_packet(void *priv, uint64_t id,
@@ -1644,8 +1906,12 @@ static void usbredir_interrupt_packet(void *priv, uint64_t id,
             return;
         }
 
+        if (QTAILQ_EMPTY(&dev->endpoint[EP2I(ep)].bufpq)) {
+            usb_wakeup(usb_ep_get(&dev->dev, USB_TOKEN_IN, ep & 0x0f));
+        }
+
         /* bufp_alloc also adds the packet to the ep queue */
-        bufp_alloc(dev, data, data_len, interrupt_packet->status, ep);
+        bufp_alloc(dev, data, data_len, interrupt_packet->status, ep, data);
     } else {
         /*
          * We report output interrupt packets as completed directly upon
@@ -1658,6 +1924,52 @@ static void usbredir_interrupt_packet(void *priv, uint64_t id,
     }
 }
 
+static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
+    struct usb_redir_buffered_bulk_packet_header *buffered_bulk_packet,
+    uint8_t *data, int data_len)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t status, ep = buffered_bulk_packet->endpoint;
+    void *free_on_destroy;
+    int i, len;
+
+    DPRINTF("buffered-bulk-in status %d ep %02X len %d id %"PRIu64"\n",
+            buffered_bulk_packet->status, ep, data_len, id);
+
+    if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_BULK) {
+        ERROR("received buffered-bulk packet for non bulk ep %02X\n", ep);
+        free(data);
+        return;
+    }
+
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_started == 0) {
+        DPRINTF("received buffered-bulk packet on not started ep %02X\n", ep);
+        free(data);
+        return;
+    }
+
+    /* Data must be in maxp chunks for buffered_bulk_add_*_data_to_packet */
+    len = dev->endpoint[EP2I(ep)].max_packet_size;
+    status = usb_redir_success;
+    free_on_destroy = NULL;
+    for (i = 0; i < data_len; i += len) {
+        if (len >= (data_len - i)) {
+            len = data_len - i;
+            status = buffered_bulk_packet->status;
+            free_on_destroy = data;
+        }
+        /* bufp_alloc also adds the packet to the ep queue */
+        bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+    }
+
+    if (dev->endpoint[EP2I(ep)].pending_async_packet) {
+        USBPacket *p = dev->endpoint[EP2I(ep)].pending_async_packet;
+        dev->endpoint[EP2I(ep)].pending_async_packet = NULL;
+        usbredir_buffered_bulk_in_complete(dev, p, ep);
+        usb_packet_complete(&dev->dev, p);
+    }
+}
+
 /*
  * Migration code
  */
@@ -1692,6 +2004,7 @@ static int usbredir_post_load(void *priv, int version_id)
     dev->dev.speedmask = (1 << dev->dev.speed);
 
     usbredir_setup_usb_eps(dev);
+    usbredir_check_bulk_receiving(dev);
 
     return 0;
 }
@@ -1763,22 +2076,27 @@ static const VMStateInfo usbredir_parser_vmstate_info = {
 static void usbredir_put_bufpq(QEMUFile *f, void *priv, size_t unused)
 {
     struct endp_data *endp = priv;
+    USBRedirDevice *dev = endp->dev;
     struct buf_packet *bufp;
-    int remain = endp->bufpq_size;
+    int len, i = 0;
 
     qemu_put_be32(f, endp->bufpq_size);
     QTAILQ_FOREACH(bufp, &endp->bufpq, next) {
-        qemu_put_be32(f, bufp->len);
+        len = bufp->len - bufp->offset;
+        DPRINTF("put_bufpq %d/%d len %d status %d\n", i + 1, endp->bufpq_size,
+                len, bufp->status);
+        qemu_put_be32(f, len);
         qemu_put_be32(f, bufp->status);
-        qemu_put_buffer(f, bufp->data, bufp->len);
-        remain--;
+        qemu_put_buffer(f, bufp->data + bufp->offset, len);
+        i++;
     }
-    assert(remain == 0);
+    assert(i == endp->bufpq_size);
 }
 
 static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
 {
     struct endp_data *endp = priv;
+    USBRedirDevice *dev = endp->dev;
     struct buf_packet *bufp;
     int i;
 
@@ -1787,9 +2105,13 @@ static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
         bufp = g_malloc(sizeof(struct buf_packet));
         bufp->len = qemu_get_be32(f);
         bufp->status = qemu_get_be32(f);
+        bufp->offset = 0;
         bufp->data = qemu_oom_check(malloc(bufp->len)); /* regular malloc! */
+        bufp->free_on_destroy = bufp->data;
         qemu_get_buffer(f, bufp->data, bufp->len);
         QTAILQ_INSERT_TAIL(&endp->bufpq, bufp, next);
+        DPRINTF("get_bufpq %d/%d len %d status %d\n", i + 1, endp->bufpq_size,
+                bufp->len, bufp->status);
     }
     return 0;
 }
@@ -1802,6 +2124,23 @@ static const VMStateInfo usbredir_ep_bufpq_vmstate_info = {
 
 
 /* For endp_data migration */
+static const VMStateDescription usbredir_bulk_receiving_vmstate = {
+    .name = "usb-redir-ep/bulk-receiving",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(bulk_receiving_started, struct endp_data),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool usbredir_bulk_receiving_needed(void *priv)
+{
+    struct endp_data *endp = priv;
+
+    return endp->bulk_receiving_started;
+}
+
 static const VMStateDescription usbredir_ep_vmstate = {
     .name = "usb-redir-ep",
     .version_id = 1,
@@ -1828,6 +2167,14 @@ static const VMStateDescription usbredir_ep_vmstate = {
         },
         VMSTATE_INT32(bufpq_target_size, struct endp_data),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &usbredir_bulk_receiving_vmstate,
+            .needed = usbredir_bulk_receiving_needed,
+        }, {
+            /* empty */
+        }
     }
 };
 
@@ -1989,11 +2336,12 @@ static void usbredir_class_initfn(ObjectClass *klass, void *data)
     uc->handle_data    = usbredir_handle_data;
     uc->handle_control = usbredir_handle_control;
     uc->flush_ep_queue = usbredir_flush_ep_queue;
+    uc->ep_stopped     = usbredir_ep_stopped;
     dc->vmsd           = &usbredir_vmstate;
     dc->props          = usbredir_properties;
 }
 
-static TypeInfo usbredir_dev_info = {
+static const TypeInfo usbredir_dev_info = {
     .name          = "usb-redir",
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBRedirDevice),
diff --git a/hw/versatile_pci.c b/hw/versatile_pci.c
index e0c3ee3..9d99159 100644
--- a/hw/versatile_pci.c
+++ b/hw/versatile_pci.c
@@ -8,9 +8,9 @@
  */
 
 #include "sysbus.h"
-#include "pci.h"
-#include "pci_host.h"
-#include "exec-memory.h"
+#include "pci/pci.h"
+#include "pci/pci_host.h"
+#include "exec/address-spaces.h"
 
 typedef struct {
     SysBusDevice busdev;
@@ -119,7 +119,7 @@ static void versatile_pci_host_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_PROCESSOR_CO;
 }
 
-static TypeInfo versatile_pci_host_info = {
+static const TypeInfo versatile_pci_host_info = {
     .name          = "versatile_pci_host",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIDevice),
@@ -133,7 +133,7 @@ static void pci_vpb_class_init(ObjectClass *klass, void *data)
     sdc->init = pci_vpb_init;
 }
 
-static TypeInfo pci_vpb_info = {
+static const TypeInfo pci_vpb_info = {
     .name          = "versatile_pci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PCIVPBState),
@@ -147,7 +147,7 @@ static void pci_realview_class_init(ObjectClass *klass, void *data)
     sdc->init = pci_realview_init;
 }
 
-static TypeInfo pci_realview_info = {
+static const TypeInfo pci_realview_info = {
     .name          = "realview_pci",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PCIVPBState),
diff --git a/hw/versatilepb.c b/hw/versatilepb.c
index 25e652b..e0a28f0 100644
--- a/hw/versatilepb.c
+++ b/hw/versatilepb.c
@@ -10,13 +10,13 @@
 #include "sysbus.h"
 #include "arm-misc.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
-#include "pci.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
+#include "pci/pci.h"
 #include "i2c.h"
 #include "boards.h"
-#include "blockdev.h"
-#include "exec-memory.h"
+#include "sysemu/blockdev.h"
+#include "exec/address-spaces.h"
 #include "flash.h"
 
 #define VERSATILE_FLASH_ADDR 0x34000000
@@ -203,7 +203,7 @@ static void versatile_init(QEMUMachineInitArgs *args, int board_id)
     qdev_prop_set_uint32(sysctl, "sys_id", 0x41007004);
     qdev_prop_set_uint32(sysctl, "proc_id", 0x02000000);
     qdev_init_nofail(sysctl);
-    sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, 0x10000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(sysctl), 0, 0x10000000);
 
     cpu_pic = arm_pic_init_cpu(cpu);
     dev = sysbus_create_varargs("pl190", 0x10140000,
@@ -214,7 +214,7 @@ static void versatile_init(QEMUMachineInitArgs *args, int board_id)
     }
     dev = sysbus_create_simple("versatilepb_sic", 0x10003000, NULL);
     for (n = 0; n < 32; n++) {
-        sysbus_connect_irq(sysbus_from_qdev(dev), n, pic[n]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), n, pic[n]);
         sic[n] = qdev_get_gpio_in(dev, n);
     }
 
@@ -222,7 +222,7 @@ static void versatile_init(QEMUMachineInitArgs *args, int board_id)
     sysbus_create_simple("pl050_mouse", 0x10007000, sic[4]);
 
     dev = qdev_create(NULL, "versatile_pci");
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     qdev_init_nofail(dev);
     sysbus_mmio_map(busdev, 0, 0x41000000); /* PCI self-config */
     sysbus_mmio_map(busdev, 1, 0x42000000); /* PCI config */
@@ -287,8 +287,8 @@ static void versatile_init(QEMUMachineInitArgs *args, int board_id)
     pl041 = qdev_create(NULL, "pl041");
     qdev_prop_set_uint32(pl041, "nc_fifo_depth", 512);
     qdev_init_nofail(pl041);
-    sysbus_mmio_map(sysbus_from_qdev(pl041), 0, 0x10004000);
-    sysbus_connect_irq(sysbus_from_qdev(pl041), 0, sic[24]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(pl041), 0, 0x10004000);
+    sysbus_connect_irq(SYS_BUS_DEVICE(pl041), 0, sic[24]);
 
     /* Memory map for Versatile/PB:  */
     /* 0x10000000 System registers.  */
@@ -358,14 +358,16 @@ static QEMUMachine versatilepb_machine = {
     .name = "versatilepb",
     .desc = "ARM Versatile/PB (ARM926EJ-S)",
     .init = vpb_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine versatileab_machine = {
     .name = "versatileab",
     .desc = "ARM Versatile/AB (ARM926EJ-S)",
     .init = vab_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void versatile_machine_init(void)
@@ -386,7 +388,7 @@ static void vpb_sic_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_vpb_sic;
 }
 
-static TypeInfo vpb_sic_info = {
+static const TypeInfo vpb_sic_info = {
     .name          = "versatilepb_sic",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(vpb_sic_state),
diff --git a/hw/vexpress.c b/hw/vexpress.c
index d93f057..741b044 100644
--- a/hw/vexpress.c
+++ b/hw/vexpress.c
@@ -25,11 +25,11 @@
 #include "arm-misc.h"
 #include "primecell.h"
 #include "devices.h"
-#include "net.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
-#include "exec-memory.h"
-#include "blockdev.h"
+#include "exec/address-spaces.h"
+#include "sysemu/blockdev.h"
 #include "flash.h"
 
 #define VEXPRESS_BOARD_ID 0x8e0
@@ -211,7 +211,7 @@ static void a9_daughterboard_init(const VEDBoardInfo *daughterboard,
     dev = qdev_create(NULL, "a9mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0x1e000000);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
@@ -271,7 +271,7 @@ static void a15_daughterboard_init(const VEDBoardInfo *daughterboard,
         cpu_model = "cortex-a15";
     }
 
-    *proc_id = 0x14000217;
+    *proc_id = 0x14000237;
 
     for (n = 0; n < smp_cpus; n++) {
         ARMCPU *cpu;
@@ -307,7 +307,7 @@ static void a15_daughterboard_init(const VEDBoardInfo *daughterboard,
     dev = qdev_create(NULL, "a15mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0x2c000000);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
@@ -374,7 +374,7 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
     qdev_prop_set_uint32(sysctl, "sys_id", sys_id);
     qdev_prop_set_uint32(sysctl, "proc_id", proc_id);
     qdev_init_nofail(sysctl);
-    sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, map[VE_SYSREGS]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(sysctl), 0, map[VE_SYSREGS]);
 
     /* VE_SP810: not modelled */
     /* VE_SERIALPCI: not modelled */
@@ -382,8 +382,8 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
     pl041 = qdev_create(NULL, "pl041");
     qdev_prop_set_uint32(pl041, "nc_fifo_depth", 512);
     qdev_init_nofail(pl041);
-    sysbus_mmio_map(sysbus_from_qdev(pl041), 0, map[VE_PL041]);
-    sysbus_connect_irq(sysbus_from_qdev(pl041), 0, pic[11]);
+    sysbus_mmio_map(SYS_BUS_DEVICE(pl041), 0, map[VE_PL041]);
+    sysbus_connect_irq(SYS_BUS_DEVICE(pl041), 0, pic[11]);
 
     dev = sysbus_create_varargs("pl181", map[VE_MMCI], pic[9], pic[10], NULL);
     /* Wire up MMC card detect and read-only signals */
@@ -477,16 +477,18 @@ static QEMUMachine vexpress_a9_machine = {
     .name = "vexpress-a9",
     .desc = "ARM Versatile Express for Cortex-A9",
     .init = vexpress_a9_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine vexpress_a15_machine = {
     .name = "vexpress-a15",
     .desc = "ARM Versatile Express for Cortex-A15",
     .init = vexpress_a15_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void vexpress_machine_init(void)
diff --git a/hw/vfio_pci.c b/hw/vfio_pci.c
index 7c27834..ad9ae36 100644
--- a/hw/vfio_pci.c
+++ b/hw/vfio_pci.c
@@ -27,17 +27,17 @@
 #include <linux/vfio.h>
 
 #include "config.h"
-#include "event_notifier.h"
-#include "exec-memory.h"
-#include "kvm.h"
-#include "memory.h"
-#include "msi.h"
-#include "msix.h"
-#include "pci.h"
+#include "qemu/event_notifier.h"
+#include "exec/address-spaces.h"
+#include "sysemu/kvm.h"
+#include "exec/memory.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
+#include "pci/pci.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-queue.h"
-#include "range.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/range.h"
 
 /* #define DEBUG_VFIO */
 #ifdef DEBUG_VFIO
@@ -275,7 +275,7 @@ static void vfio_enable_intx_kvm(VFIODevice *vdev)
     int ret, argsz;
     int32_t *pfd;
 
-    if (!kvm_irqchip_in_kernel() ||
+    if (!kvm_irqfds_enabled() ||
         vdev->intx.route.mode != PCI_INTX_ENABLED ||
         !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
         return;
@@ -289,7 +289,7 @@ static void vfio_enable_intx_kvm(VFIODevice *vdev)
 
     /* Get an eventfd for resample/unmask */
     if (event_notifier_init(&vdev->intx.unmask, 0)) {
-        error_report("vfio: Error: event_notifier_init failed eoi\n");
+        error_report("vfio: Error: event_notifier_init failed eoi");
         goto fail;
     }
 
@@ -297,7 +297,7 @@ static void vfio_enable_intx_kvm(VFIODevice *vdev)
     irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
 
     if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
-        error_report("vfio: Error: Failed to setup resample irqfd: %m\n");
+        error_report("vfio: Error: Failed to setup resample irqfd: %m");
         goto fail_irqfd;
     }
 
@@ -316,7 +316,7 @@ static void vfio_enable_intx_kvm(VFIODevice *vdev)
     ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     g_free(irq_set);
     if (ret) {
-        error_report("vfio: Error: Failed to setup INTx unmask fd: %m\n");
+        error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
         goto fail_vfio;
     }
 
@@ -365,7 +365,7 @@ static void vfio_disable_intx_kvm(VFIODevice *vdev)
 
     /* Tell KVM to stop listening for an INTx irqfd */
     if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
-        error_report("vfio: Error: Failed to disable INTx irqfd: %m\n");
+        error_report("vfio: Error: Failed to disable INTx irqfd: %m");
     }
 
     /* We only need to close the eventfd for VFIO to cleanup the kernel side */
@@ -438,7 +438,8 @@ static int vfio_enable_intx(VFIODevice *vdev)
      * Only conditional to avoid generating error messages on platforms
      * where we won't actually use the result anyway.
      */
-    if (kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+    if (kvm_irqfds_enabled() &&
+        kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
         vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
                                                         vdev->intx.pin);
     }
@@ -446,7 +447,7 @@ static int vfio_enable_intx(VFIODevice *vdev)
 
     ret = event_notifier_init(&vdev->intx.interrupt, 0);
     if (ret) {
-        error_report("vfio: Error: event_notifier_init failed\n");
+        error_report("vfio: Error: event_notifier_init failed");
         return ret;
     }
 
@@ -466,7 +467,7 @@ static int vfio_enable_intx(VFIODevice *vdev)
     ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     g_free(irq_set);
     if (ret) {
-        error_report("vfio: Error: Failed to setup INTx fd: %m\n");
+        error_report("vfio: Error: Failed to setup INTx fd: %m");
         qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
         event_notifier_cleanup(&vdev->intx.interrupt);
         return -errno;
@@ -525,7 +526,7 @@ static void vfio_msi_interrupt(void *opaque)
     } else if (vdev->interrupt == VFIO_INT_MSI) {
         msi_notify(&vdev->pdev, nr);
     } else {
-        error_report("vfio: MSI interrupt receieved, but not enabled?\n");
+        error_report("vfio: MSI interrupt receieved, but not enabled?");
     }
 }
 
@@ -561,8 +562,8 @@ static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
     return ret;
 }
 
-static int vfio_msix_vector_use(PCIDevice *pdev,
-                                unsigned int nr, MSIMessage msg)
+static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
+                                   MSIMessage *msg, IOHandler *handler)
 {
     VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
     VFIOMSIVector *vector;
@@ -579,14 +580,14 @@ static int vfio_msix_vector_use(PCIDevice *pdev,
     msix_vector_use(pdev, nr);
 
     if (event_notifier_init(&vector->interrupt, 0)) {
-        error_report("vfio: Error: event_notifier_init failed\n");
+        error_report("vfio: Error: event_notifier_init failed");
     }
 
     /*
      * Attempt to enable route through KVM irqchip,
      * default to userspace handling if unavailable.
      */
-    vector->virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+    vector->virq = msg ? kvm_irqchip_add_msi_route(kvm_state, *msg) : -1;
     if (vector->virq < 0 ||
         kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
                                        vector->virq) < 0) {
@@ -595,7 +596,7 @@ static int vfio_msix_vector_use(PCIDevice *pdev,
             vector->virq = -1;
         }
         qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
-                            vfio_msi_interrupt, NULL, vector);
+                            handler, NULL, vector);
     }
 
     /*
@@ -608,7 +609,7 @@ static int vfio_msix_vector_use(PCIDevice *pdev,
         vdev->nr_vectors = nr + 1;
         ret = vfio_enable_vectors(vdev, true);
         if (ret) {
-            error_report("vfio: failed to enable vectors, %d\n", ret);
+            error_report("vfio: failed to enable vectors, %d", ret);
         }
     } else {
         int argsz;
@@ -631,13 +632,19 @@ static int vfio_msix_vector_use(PCIDevice *pdev,
         ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
         g_free(irq_set);
         if (ret) {
-            error_report("vfio: failed to modify vector, %d\n", ret);
+            error_report("vfio: failed to modify vector, %d", ret);
         }
     }
 
     return 0;
 }
 
+static int vfio_msix_vector_use(PCIDevice *pdev,
+                                unsigned int nr, MSIMessage msg)
+{
+    return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
+}
+
 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
 {
     VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
@@ -696,9 +703,25 @@ static void vfio_enable_msix(VFIODevice *vdev)
 
     vdev->interrupt = VFIO_INT_MSIX;
 
+    /*
+     * Some communication channels between VF & PF or PF & fw rely on the
+     * physical state of the device and expect that enabling MSI-X from the
+     * guest enables the same on the host.  When our guest is Linux, the
+     * guest driver call to pci_enable_msix() sets the enabling bit in the
+     * MSI-X capability, but leaves the vector table masked.  We therefore
+     * can't rely on a vector_use callback (from request_irq() in the guest)
+     * to switch the physical device into MSI-X mode because that may come a
+     * long time after pci_enable_msix().  This code enables vector 0 with
+     * triggering to userspace, then immediately release the vector, leaving
+     * the physical device with no vectors enabled, but MSI-X enabled, just
+     * like the guest view.
+     */
+    vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
+    vfio_msix_vector_release(&vdev->pdev, 0);
+
     if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
-                                  vfio_msix_vector_release)) {
-        error_report("vfio: msix_set_vector_notifiers failed\n");
+                                  vfio_msix_vector_release, NULL)) {
+        error_report("vfio: msix_set_vector_notifiers failed");
     }
 
     DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
@@ -723,7 +746,7 @@ retry:
         vector->use = true;
 
         if (event_notifier_init(&vector->interrupt, 0)) {
-            error_report("vfio: Error: event_notifier_init failed\n");
+            error_report("vfio: Error: event_notifier_init failed");
         }
 
         msg = msi_get_message(&vdev->pdev, i);
@@ -744,10 +767,10 @@ retry:
     ret = vfio_enable_vectors(vdev, false);
     if (ret) {
         if (ret < 0) {
-            error_report("vfio: Error: Failed to setup MSI fds: %m\n");
+            error_report("vfio: Error: Failed to setup MSI fds: %m");
         } else if (ret != vdev->nr_vectors) {
             error_report("vfio: Error: Failed to enable %d "
-                         "MSI vectors, retry with %d\n", vdev->nr_vectors, ret);
+                         "MSI vectors, retry with %d", vdev->nr_vectors, ret);
         }
 
         for (i = 0; i < vdev->nr_vectors; i++) {
@@ -868,7 +891,7 @@ static void vfio_bar_write(void *opaque, hwaddr addr,
     }
 
     if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
-        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m\n",
+        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
                      __func__, addr, data, size);
     }
 
@@ -899,7 +922,7 @@ static uint64_t vfio_bar_read(void *opaque,
     uint64_t data = 0;
 
     if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
-        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m\n",
+        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
                      __func__, addr, size);
         return (uint64_t)-1;
     }
@@ -956,7 +979,7 @@ static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
         val = pci_default_read_config(pdev, addr, len);
     } else {
         if (pread(vdev->fd, &val, len, vdev->config_offset + addr) != len) {
-            error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m\n",
+            error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
                          __func__, vdev->host.domain, vdev->host.bus,
                          vdev->host.slot, vdev->host.function, addr, len);
             return -errno;
@@ -998,7 +1021,7 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
 
     /* Write everything to VFIO, let it filter out what we can't write */
     if (pwrite(vdev->fd, &val_le, len, vdev->config_offset + addr) != len) {
-        error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m\n",
+        error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
                      __func__, vdev->host.domain, vdev->host.bus,
                      vdev->host.slot, vdev->host.function, addr, val, len);
     }
@@ -1115,7 +1138,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
 
     if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
                  (section->offset_within_region & ~TARGET_PAGE_MASK))) {
-        error_report("%s received unaligned region\n", __func__);
+        error_report("%s received unaligned region", __func__);
         return;
     }
 
@@ -1137,7 +1160,7 @@ static void vfio_listener_region_add(MemoryListener *listener,
     ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
     if (ret) {
         error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
-                     "0x%"HWADDR_PRIx", %p) = %d (%m)\n",
+                     "0x%"HWADDR_PRIx", %p) = %d (%m)",
                      container, iova, end - iova, vaddr, ret);
     }
 }
@@ -1159,7 +1182,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
 
     if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
                  (section->offset_within_region & ~TARGET_PAGE_MASK))) {
-        error_report("%s received unaligned region\n", __func__);
+        error_report("%s received unaligned region", __func__);
         return;
     }
 
@@ -1177,7 +1200,7 @@ static void vfio_listener_region_del(MemoryListener *listener,
     ret = vfio_dma_unmap(container, iova, end - iova);
     if (ret) {
         error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
-                     "0x%"HWADDR_PRIx") = %d (%m)\n",
+                     "0x%"HWADDR_PRIx") = %d (%m)",
                      container, iova, end - iova, ret);
     }
 }
@@ -1234,7 +1257,7 @@ static int vfio_setup_msi(VFIODevice *vdev, int pos)
         if (ret == -ENOTSUP) {
             return 0;
         }
-        error_report("vfio: msi_init failed\n");
+        error_report("vfio: msi_init failed");
         return ret;
     }
     vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
@@ -1309,7 +1332,7 @@ static int vfio_setup_msix(VFIODevice *vdev, int pos)
         if (ret == -ENOTSUP) {
             return 0;
         }
-        error_report("vfio: msix_init failed\n");
+        error_report("vfio: msix_init failed");
         return ret;
     }
 
@@ -1425,7 +1448,7 @@ static void vfio_map_bar(VFIODevice *vdev, int nr)
     ret = pread(vdev->fd, &pci_bar, sizeof(pci_bar),
                 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
     if (ret != sizeof(pci_bar)) {
-        error_report("vfio: Failed to read BAR %d (%m)\n", nr);
+        error_report("vfio: Failed to read BAR %d (%m)", nr);
         return;
     }
 
@@ -1448,7 +1471,7 @@ static void vfio_map_bar(VFIODevice *vdev, int nr)
     strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
     if (vfio_mmap_bar(bar, &bar->mem,
                       &bar->mmap_mem, &bar->mmap, size, 0, name)) {
-        error_report("%s unsupported. Performance may be slow\n", name);
+        error_report("%s unsupported. Performance may be slow", name);
     }
 
     if (vdev->msix && vdev->msix->table_bar == nr) {
@@ -1462,7 +1485,7 @@ static void vfio_map_bar(VFIODevice *vdev, int nr)
         /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
         if (vfio_mmap_bar(bar, &bar->mem, &vdev->msix->mmap_mem,
                           &vdev->msix->mmap, size, start, name)) {
-            error_report("%s unsupported. Performance may be slow\n", name);
+            error_report("%s unsupported. Performance may be slow", name);
         }
     }
 }
@@ -1549,7 +1572,7 @@ static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
 
     if (ret < 0) {
         error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
-                     "0x%x[0x%x]@0x%x: %d\n", vdev->host.domain,
+                     "0x%x[0x%x]@0x%x: %d", vdev->host.domain,
                      vdev->host.bus, vdev->host.slot, vdev->host.function,
                      cap_id, size, pos, ret);
         return ret;
@@ -1604,7 +1627,7 @@ static int vfio_load_rom(VFIODevice *vdev)
             if (errno == EINTR || errno == EAGAIN) {
                 continue;
             }
-            error_report("vfio: Error reading device ROM: %m\n");
+            error_report("vfio: Error reading device ROM: %m");
             memory_region_destroy(&vdev->pdev.rom);
             return -errno;
         }
@@ -1634,14 +1657,14 @@ static int vfio_connect_container(VFIOGroup *group)
 
     fd = qemu_open("/dev/vfio/vfio", O_RDWR);
     if (fd < 0) {
-        error_report("vfio: failed to open /dev/vfio/vfio: %m\n");
+        error_report("vfio: failed to open /dev/vfio/vfio: %m");
         return -errno;
     }
 
     ret = ioctl(fd, VFIO_GET_API_VERSION);
     if (ret != VFIO_API_VERSION) {
         error_report("vfio: supported vfio version: %d, "
-                     "reported version: %d\n", VFIO_API_VERSION, ret);
+                     "reported version: %d", VFIO_API_VERSION, ret);
         close(fd);
         return -EINVAL;
     }
@@ -1652,7 +1675,7 @@ static int vfio_connect_container(VFIOGroup *group)
     if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
-            error_report("vfio: failed to set group container: %m\n");
+            error_report("vfio: failed to set group container: %m");
             g_free(container);
             close(fd);
             return -errno;
@@ -1660,7 +1683,7 @@ static int vfio_connect_container(VFIOGroup *group)
 
         ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
         if (ret) {
-            error_report("vfio: failed to set iommu for container: %m\n");
+            error_report("vfio: failed to set iommu for container: %m");
             g_free(container);
             close(fd);
             return -errno;
@@ -1671,7 +1694,7 @@ static int vfio_connect_container(VFIOGroup *group)
 
         memory_listener_register(&container->iommu_data.listener, &address_space_memory);
     } else {
-        error_report("vfio: No available IOMMU models\n");
+        error_report("vfio: No available IOMMU models");
         g_free(container);
         close(fd);
         return -EINVAL;
@@ -1691,7 +1714,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
     VFIOContainer *container = group->container;
 
     if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
-        error_report("vfio: error disconnecting group %d from container\n",
+        error_report("vfio: error disconnecting group %d from container",
                      group->groupid);
     }
 
@@ -1726,13 +1749,13 @@ static VFIOGroup *vfio_get_group(int groupid)
     snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
     group->fd = qemu_open(path, O_RDWR);
     if (group->fd < 0) {
-        error_report("vfio: error opening %s: %m\n", path);
+        error_report("vfio: error opening %s: %m", path);
         g_free(group);
         return NULL;
     }
 
     if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
-        error_report("vfio: error getting group status: %m\n");
+        error_report("vfio: error getting group status: %m");
         close(group->fd);
         g_free(group);
         return NULL;
@@ -1741,7 +1764,7 @@ static VFIOGroup *vfio_get_group(int groupid)
     if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
         error_report("vfio: error, group %d is not viable, please ensure "
                      "all devices within the iommu_group are bound to their "
-                     "vfio bus driver.\n", groupid);
+                     "vfio bus driver.", groupid);
         close(group->fd);
         g_free(group);
         return NULL;
@@ -1751,7 +1774,7 @@ static VFIOGroup *vfio_get_group(int groupid)
     QLIST_INIT(&group->device_list);
 
     if (vfio_connect_container(group)) {
-        error_report("vfio: failed to setup container for group %d\n", groupid);
+        error_report("vfio: failed to setup container for group %d", groupid);
         close(group->fd);
         g_free(group);
         return NULL;
@@ -1783,9 +1806,9 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
 
     ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
     if (ret < 0) {
-        error_report("vfio: error getting device %s from group %d: %m\n",
+        error_report("vfio: error getting device %s from group %d: %m",
                      name, group->groupid);
-        error_report("Verify all devices in group %d are bound to vfio-pci "
+        error_printf("Verify all devices in group %d are bound to vfio-pci "
                      "or pci-stub and not already in use\n", group->groupid);
         return ret;
     }
@@ -1797,7 +1820,7 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
     /* Sanity check device */
     ret = ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &dev_info);
     if (ret) {
-        error_report("vfio: error getting device info: %m\n");
+        error_report("vfio: error getting device info: %m");
         goto error;
     }
 
@@ -1805,23 +1828,23 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
             dev_info.flags, dev_info.num_regions, dev_info.num_irqs);
 
     if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PCI)) {
-        error_report("vfio: Um, this isn't a PCI device\n");
+        error_report("vfio: Um, this isn't a PCI device");
         goto error;
     }
 
     vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
     if (!vdev->reset_works) {
-        error_report("Warning, device %s does not support reset\n", name);
+        error_report("Warning, device %s does not support reset", name);
     }
 
-    if (dev_info.num_regions != VFIO_PCI_NUM_REGIONS) {
-        error_report("vfio: unexpected number of io regions %u\n",
+    if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
+        error_report("vfio: unexpected number of io regions %u",
                      dev_info.num_regions);
         goto error;
     }
 
-    if (dev_info.num_irqs != VFIO_PCI_NUM_IRQS) {
-        error_report("vfio: unexpected number of irqs %u\n", dev_info.num_irqs);
+    if (dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
+        error_report("vfio: unexpected number of irqs %u", dev_info.num_irqs);
         goto error;
     }
 
@@ -1830,7 +1853,7 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
 
         ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
         if (ret) {
-            error_report("vfio: Error getting region %d info: %m\n", i);
+            error_report("vfio: Error getting region %d info: %m", i);
             goto error;
         }
 
@@ -1850,7 +1873,7 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
 
     ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
     if (ret) {
-        error_report("vfio: Error getting ROM info: %m\n");
+        error_report("vfio: Error getting ROM info: %m");
         goto error;
     }
 
@@ -1866,7 +1889,7 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
 
     ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
     if (ret) {
-        error_report("vfio: Error getting config info: %m\n");
+        error_report("vfio: Error getting config info: %m");
         goto error;
     }
 
@@ -1876,6 +1899,9 @@ static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
             (unsigned long)reg_info.flags);
 
     vdev->config_size = reg_info.size;
+    if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
+        vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
+    }
     vdev->config_offset = reg_info.offset;
 
 error:
@@ -1915,7 +1941,7 @@ static int vfio_initfn(PCIDevice *pdev)
              vdev->host.domain, vdev->host.bus, vdev->host.slot,
              vdev->host.function);
     if (stat(path, &st) < 0) {
-        error_report("vfio: error: no such host device: %s\n", path);
+        error_report("vfio: error: no such host device: %s", path);
         return -errno;
     }
 
@@ -1923,7 +1949,7 @@ static int vfio_initfn(PCIDevice *pdev)
 
     len = readlink(path, iommu_group_path, PATH_MAX);
     if (len <= 0) {
-        error_report("vfio: error no iommu_group for device\n");
+        error_report("vfio: error no iommu_group for device");
         return -errno;
     }
 
@@ -1931,7 +1957,7 @@ static int vfio_initfn(PCIDevice *pdev)
     group_name = basename(iommu_group_path);
 
     if (sscanf(group_name, "%d", &groupid) != 1) {
-        error_report("vfio: error reading %s: %m\n", path);
+        error_report("vfio: error reading %s: %m", path);
         return -errno;
     }
 
@@ -1940,7 +1966,7 @@ static int vfio_initfn(PCIDevice *pdev)
 
     group = vfio_get_group(groupid);
     if (!group) {
-        error_report("vfio: failed to get group %d\n", groupid);
+        error_report("vfio: failed to get group %d", groupid);
         return -ENOENT;
     }
 
@@ -1954,7 +1980,7 @@ static int vfio_initfn(PCIDevice *pdev)
             pvdev->host.slot == vdev->host.slot &&
             pvdev->host.function == vdev->host.function) {
 
-            error_report("vfio: error: device %s is already attached\n", path);
+            error_report("vfio: error: device %s is already attached", path);
             vfio_put_group(group);
             return -EBUSY;
         }
@@ -1962,7 +1988,7 @@ static int vfio_initfn(PCIDevice *pdev)
 
     ret = vfio_get_device(group, path, vdev);
     if (ret) {
-        error_report("vfio: failed to get device %s\n", path);
+        error_report("vfio: failed to get device %s", path);
         vfio_put_group(group);
         return ret;
     }
@@ -1973,7 +1999,7 @@ static int vfio_initfn(PCIDevice *pdev)
                 vdev->config_offset);
     if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
         ret = ret < 0 ? -errno : -EFAULT;
-        error_report("vfio: Failed to read device config space\n");
+        error_report("vfio: Failed to read device config space");
         goto out_put;
     }
 
@@ -2060,7 +2086,7 @@ static void vfio_pci_reset(DeviceState *dev)
     if (vdev->reset_works) {
         if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
             error_report("vfio: Error unable to reset physical device "
-                         "(%04x:%02x:%02x.%x): %m\n", vdev->host.domain,
+                         "(%04x:%02x:%02x.%x): %m", vdev->host.domain,
                          vdev->host.bus, vdev->host.slot, vdev->host.function);
         }
     }
@@ -2098,6 +2124,7 @@ static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
     pdc->exit = vfio_exitfn;
     pdc->config_read = vfio_pci_read_config;
     pdc->config_write = vfio_pci_write_config;
+    pdc->is_express = 1; /* We might be */
 }
 
 static const TypeInfo vfio_pci_dev_info = {
diff --git a/hw/vga-isa-mm.c b/hw/vga-isa-mm.c
index 8ef4320..311c966 100644
--- a/hw/vga-isa-mm.c
+++ b/hw/vga-isa-mm.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pc.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 
 #define VGA_RAM_SIZE (8192 * 1024)
 
diff --git a/hw/vga-isa.c b/hw/vga-isa.c
index 046602b..762e45a 100644
--- a/hw/vga-isa.c
+++ b/hw/vga-isa.c
@@ -24,11 +24,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pc.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 #include "loader.h"
 
 typedef struct ISAVGAState {
@@ -86,7 +86,7 @@ static void vga_class_initfn(ObjectClass *klass, void *data)
     dc->props = vga_isa_properties;
 }
 
-static TypeInfo vga_info = {
+static const TypeInfo vga_info = {
     .name          = "isa-vga",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(ISAVGAState),
diff --git a/hw/vga-pci.c b/hw/vga-pci.c
index 947e35c..c491af2 100644
--- a/hw/vga-pci.c
+++ b/hw/vga-pci.c
@@ -24,11 +24,11 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
-#include "pci.h"
+#include "ui/console.h"
+#include "pci/pci.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 #include "loader.h"
 
 #define PCI_VGA_IOPORT_OFFSET 0x400
@@ -200,7 +200,7 @@ static void vga_class_init(ObjectClass *klass, void *data)
     dc->props = vga_pci_properties;
 }
 
-static TypeInfo vga_info = {
+static const TypeInfo vga_info = {
     .name          = "VGA",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIVGAState),
diff --git a/hw/vga.c b/hw/vga.c
index 2b0200a..e2ba7f2 100644
--- a/hw/vga.c
+++ b/hw/vga.c
@@ -23,12 +23,12 @@
  */
 #include "hw.h"
 #include "vga.h"
-#include "console.h"
+#include "ui/console.h"
 #include "pc.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "vga_int.h"
-#include "pixel_ops.h"
-#include "qemu-timer.h"
+#include "ui/pixel_ops.h"
+#include "qemu/timer.h"
 #include "xen.h"
 #include "trace.h"
 
@@ -2413,7 +2413,7 @@ void ppm_save(const char *filename, struct DisplaySurface *ds, Error **errp)
     }
     linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, width);
     for (y = 0; y < height; y++) {
-        qemu_pixman_linebuf_fill(linebuf, ds->image, width, y);
+        qemu_pixman_linebuf_fill(linebuf, ds->image, width, 0, y);
         clearerr(f);
         ret = fwrite(pixman_image_get_data(linebuf), 1,
                      pixman_image_get_stride(linebuf), f);
diff --git a/hw/vga_int.h b/hw/vga_int.h
index bcb738d..8d496ea 100644
--- a/hw/vga_int.h
+++ b/hw/vga_int.h
@@ -21,10 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HW_VGA_INT_H
+#define HW_VGA_INT_H 1
 
 #include <hw/hw.h>
-#include "error.h"
-#include "memory.h"
+#include "qapi/error.h"
+#include "exec/memory.h"
 
 #define ST01_V_RETRACE      0x08
 #define ST01_DISP_ENABLE    0x01
@@ -212,3 +214,5 @@ extern const uint8_t gr_mask[16];
 #define VGABIOS_CIRRUS_FILENAME "vgabios-cirrus.bin"
 
 extern const MemoryRegionOps vga_mem_ops;
+
+#endif
diff --git a/hw/vhost.c b/hw/vhost.c
index 16322a1..8d41fdb 100644
--- a/hw/vhost.c
+++ b/hw/vhost.c
@@ -16,9 +16,9 @@
 #include <sys/ioctl.h>
 #include "vhost.h"
 #include "hw/hw.h"
-#include "range.h"
+#include "qemu/range.h"
 #include <linux/vhost.h>
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 static void vhost_dev_sync_region(struct vhost_dev *dev,
                                   MemoryRegionSection *section,
@@ -269,11 +269,8 @@ static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
     vhost_log_chunk_t *log;
     uint64_t log_base;
     int r, i;
-    if (size) {
-        log = g_malloc0(size * sizeof *log);
-    } else {
-        log = NULL;
-    }
+
+    log = g_malloc0(size * sizeof *log);
     log_base = (uint64_t)(unsigned long)log;
     r = ioctl(dev->control, VHOST_SET_LOG_BASE, &log_base);
     assert(r >= 0);
@@ -612,21 +609,24 @@ static void vhost_log_stop(MemoryListener *listener,
     /* FIXME: implement */
 }
 
-static int vhost_virtqueue_init(struct vhost_dev *dev,
+static int vhost_virtqueue_start(struct vhost_dev *dev,
                                 struct VirtIODevice *vdev,
                                 struct vhost_virtqueue *vq,
                                 unsigned idx)
 {
     hwaddr s, l, a;
     int r;
+    int vhost_vq_index = idx - dev->vq_index;
     struct vhost_vring_file file = {
-        .index = idx,
+        .index = vhost_vq_index
     };
     struct vhost_vring_state state = {
-        .index = idx,
+        .index = vhost_vq_index
     };
     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
     vq->num = state.num = virtio_queue_get_num(vdev, idx);
     r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
     if (r) {
@@ -669,11 +669,12 @@ static int vhost_virtqueue_init(struct vhost_dev *dev,
         goto fail_alloc_ring;
     }
 
-    r = vhost_virtqueue_set_addr(dev, vq, idx, dev->log_enabled);
+    r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
     if (r < 0) {
         r = -errno;
         goto fail_alloc;
     }
+
     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
     r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
     if (r) {
@@ -681,16 +682,11 @@ static int vhost_virtqueue_init(struct vhost_dev *dev,
         goto fail_kick;
     }
 
-    file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
-    r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
-    if (r) {
-        r = -errno;
-        goto fail_call;
-    }
+    /* Clear and discard previous events if any. */
+    event_notifier_test_and_clear(&vq->masked_notifier);
 
     return 0;
 
-fail_call:
 fail_kick:
 fail_alloc:
     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
@@ -708,15 +704,16 @@ fail_alloc_desc:
     return r;
 }
 
-static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
+static void vhost_virtqueue_stop(struct vhost_dev *dev,
                                     struct VirtIODevice *vdev,
                                     struct vhost_virtqueue *vq,
                                     unsigned idx)
 {
     struct vhost_vring_state state = {
-        .index = idx,
+        .index = idx - dev->vq_index
     };
     int r;
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
     r = ioctl(dev->control, VHOST_GET_VRING_BASE, &state);
     if (r < 0) {
         fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
@@ -746,11 +743,39 @@ static void vhost_eventfd_del(MemoryListener *listener,
 {
 }
 
+static int vhost_virtqueue_init(struct vhost_dev *dev,
+                                struct vhost_virtqueue *vq, int n)
+{
+    struct vhost_vring_file file = {
+        .index = n,
+    };
+    int r = event_notifier_init(&vq->masked_notifier, 0);
+    if (r < 0) {
+        return r;
+    }
+
+    file.fd = event_notifier_get_fd(&vq->masked_notifier);
+    r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
+    if (r) {
+        r = -errno;
+        goto fail_call;
+    }
+    return 0;
+fail_call:
+    event_notifier_cleanup(&vq->masked_notifier);
+    return r;
+}
+
+static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
+{
+    event_notifier_cleanup(&vq->masked_notifier);
+}
+
 int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
                    bool force)
 {
     uint64_t features;
-    int r;
+    int i, r;
     if (devfd >= 0) {
         hdev->control = devfd;
     } else {
@@ -768,6 +793,13 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
     if (r < 0) {
         goto fail;
     }
+
+    for (i = 0; i < hdev->nvqs; ++i) {
+        r = vhost_virtqueue_init(hdev, hdev->vqs + i, i);
+        if (r < 0) {
+            goto fail_vq;
+        }
+    }
     hdev->features = features;
 
     hdev->memory_listener = (MemoryListener) {
@@ -795,6 +827,10 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
     memory_listener_register(&hdev->memory_listener, &address_space_memory);
     hdev->force = force;
     return 0;
+fail_vq:
+    while (--i >= 0) {
+        vhost_virtqueue_cleanup(hdev->vqs + i);
+    }
 fail:
     r = -errno;
     close(hdev->control);
@@ -803,6 +839,10 @@ fail:
 
 void vhost_dev_cleanup(struct vhost_dev *hdev)
 {
+    int i;
+    for (i = 0; i < hdev->nvqs; ++i) {
+        vhost_virtqueue_cleanup(hdev->vqs + i);
+    }
     memory_listener_unregister(&hdev->memory_listener);
     g_free(hdev->mem);
     g_free(hdev->mem_sections);
@@ -829,7 +869,9 @@ int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
     }
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, true);
+        r = vdev->binding->set_host_notifier(vdev->binding_opaque,
+                                             hdev->vq_index + i,
+                                             true);
         if (r < 0) {
             fprintf(stderr, "vhost VQ %d notifier binding failed: %d\n", i, -r);
             goto fail_vq;
@@ -839,7 +881,9 @@ int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
     return 0;
 fail_vq:
     while (--i >= 0) {
-        r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, false);
+        r = vdev->binding->set_host_notifier(vdev->binding_opaque,
+                                             hdev->vq_index + i,
+                                             false);
         if (r < 0) {
             fprintf(stderr, "vhost VQ %d notifier cleanup error: %d\n", i, -r);
             fflush(stderr);
@@ -860,7 +904,9 @@ void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
     int i, r;
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = vdev->binding->set_host_notifier(vdev->binding_opaque, i, false);
+        r = vdev->binding->set_host_notifier(vdev->binding_opaque,
+                                             hdev->vq_index + i,
+                                             false);
         if (r < 0) {
             fprintf(stderr, "vhost VQ %d notifier cleanup failed: %d\n", i, -r);
             fflush(stderr);
@@ -869,21 +915,45 @@ void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
     }
 }
 
+/* Test and clear event pending status.
+ * Should be called after unmask to avoid losing events.
+ */
+bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
+{
+    struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
+    assert(hdev->started);
+    assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
+    return event_notifier_test_and_clear(&vq->masked_notifier);
+}
+
+/* Mask/unmask events from this vq. */
+void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
+                         bool mask)
+{
+    struct VirtQueue *vvq = virtio_get_queue(vdev, n);
+    int r, index = n - hdev->vq_index;
+
+    assert(hdev->started);
+    assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
+
+    struct vhost_vring_file file = {
+        .index = index
+    };
+    if (mask) {
+        file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
+    } else {
+        file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
+    }
+    r = ioctl(hdev->control, VHOST_SET_VRING_CALL, &file);
+    assert(r >= 0);
+}
+
 /* Host notifiers must be enabled at this point. */
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
 {
     int i, r;
-    if (!vdev->binding->set_guest_notifiers) {
-        fprintf(stderr, "binding does not support guest notifiers\n");
-        r = -ENOSYS;
-        goto fail;
-    }
 
-    r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, true);
-    if (r < 0) {
-        fprintf(stderr, "Error binding guest notifier: %d\n", -r);
-        goto fail_notifiers;
-    }
+    hdev->started = true;
 
     r = vhost_dev_set_features(hdev, hdev->log_enabled);
     if (r < 0) {
@@ -895,10 +965,10 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         goto fail_mem;
     }
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = vhost_virtqueue_init(hdev,
-                                 vdev,
-                                 hdev->vqs + i,
-                                 i);
+        r = vhost_virtqueue_start(hdev,
+                                  vdev,
+                                  hdev->vqs + i,
+                                  hdev->vq_index + i);
         if (r < 0) {
             goto fail_vq;
         }
@@ -916,49 +986,42 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         }
     }
 
-    hdev->started = true;
-
     return 0;
 fail_log:
 fail_vq:
     while (--i >= 0) {
-        vhost_virtqueue_cleanup(hdev,
-                                vdev,
-                                hdev->vqs + i,
-                                i);
+        vhost_virtqueue_stop(hdev,
+                             vdev,
+                             hdev->vqs + i,
+                             hdev->vq_index + i);
     }
+    i = hdev->nvqs;
 fail_mem:
 fail_features:
-    vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
-fail_notifiers:
-fail:
+
+    hdev->started = false;
     return r;
 }
 
 /* Host notifiers must be enabled at this point. */
 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
 {
-    int i, r;
+    int i;
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        vhost_virtqueue_cleanup(hdev,
-                                vdev,
-                                hdev->vqs + i,
-                                i);
+        vhost_virtqueue_stop(hdev,
+                             vdev,
+                             hdev->vqs + i,
+                             hdev->vq_index + i);
     }
     for (i = 0; i < hdev->n_mem_sections; ++i) {
         vhost_sync_dirty_bitmap(hdev, &hdev->mem_sections[i],
                                 0, (hwaddr)~0x0ull);
     }
-    r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
-    if (r < 0) {
-        fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
-        fflush(stderr);
-    }
-    assert (r >= 0);
 
     hdev->started = false;
     g_free(hdev->log);
     hdev->log = NULL;
     hdev->log_size = 0;
 }
+
diff --git a/hw/vhost.h b/hw/vhost.h
index 0c47229..f062d48 100644
--- a/hw/vhost.h
+++ b/hw/vhost.h
@@ -3,7 +3,7 @@
 
 #include "hw/hw.h"
 #include "hw/virtio.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 /* Generic structures common for any vhost based device. */
 struct vhost_virtqueue {
@@ -18,6 +18,7 @@ struct vhost_virtqueue {
     void *ring;
     unsigned long long ring_phys;
     unsigned ring_size;
+    EventNotifier masked_notifier;
 };
 
 typedef unsigned long vhost_log_chunk_t;
@@ -34,6 +35,8 @@ struct vhost_dev {
     MemoryRegionSection *mem_sections;
     struct vhost_virtqueue *vqs;
     int nvqs;
+    /* the first virtuque which would be used by this vhost dev */
+    int vq_index;
     unsigned long long features;
     unsigned long long acked_features;
     unsigned long long backend_features;
@@ -53,4 +56,13 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
 
+/* Test and clear masked event pending status.
+ * Should be called after unmask to avoid losing events.
+ */
+bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n);
+
+/* Mask/unmask events from this vq.
+ */
+void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
+                          bool mask);
 #endif
diff --git a/hw/vhost_net.c b/hw/vhost_net.c
index 8241601..d1df0e2 100644
--- a/hw/vhost_net.c
+++ b/hw/vhost_net.c
@@ -13,12 +13,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "net.h"
+#include "net/net.h"
 #include "net/tap.h"
 
 #include "virtio-net.h"
 #include "vhost_net.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #include "config.h"
 
@@ -109,6 +109,9 @@ struct vhost_net *vhost_net_init(NetClientState *backend, int devfd,
         (1 << VHOST_NET_F_VIRTIO_NET_HDR);
     net->backend = r;
 
+    net->dev.nvqs = 2;
+    net->dev.vqs = net->vqs;
+
     r = vhost_dev_init(&net->dev, devfd, "/dev/vhost-net", force);
     if (r < 0) {
         goto fail;
@@ -137,14 +140,20 @@ bool vhost_net_query(VHostNetState *net, VirtIODevice *dev)
     return vhost_dev_query(&net->dev, dev);
 }
 
-int vhost_net_start(struct vhost_net *net,
-                    VirtIODevice *dev)
+static int vhost_net_start_one(struct vhost_net *net,
+                               VirtIODevice *dev,
+                               int vq_index)
 {
     struct vhost_vring_file file = { };
     int r;
 
+    if (net->dev.started) {
+        return 0;
+    }
+
     net->dev.nvqs = 2;
     net->dev.vqs = net->vqs;
+    net->dev.vq_index = vq_index;
 
     r = vhost_dev_enable_notifiers(&net->dev, dev);
     if (r < 0) {
@@ -181,11 +190,15 @@ fail_notifiers:
     return r;
 }
 
-void vhost_net_stop(struct vhost_net *net,
-                    VirtIODevice *dev)
+static void vhost_net_stop_one(struct vhost_net *net,
+                               VirtIODevice *dev)
 {
     struct vhost_vring_file file = { .fd = -1 };
 
+    if (!net->dev.started) {
+        return;
+    }
+
     for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
         int r = ioctl(net->dev.control, VHOST_NET_SET_BACKEND, &file);
         assert(r >= 0);
@@ -195,11 +208,77 @@ void vhost_net_stop(struct vhost_net *net,
     vhost_dev_disable_notifiers(&net->dev, dev);
 }
 
+int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
+                    int total_queues)
+{
+    int r, i = 0;
+
+    if (!dev->binding->set_guest_notifiers) {
+        error_report("binding does not support guest notifiers");
+        r = -ENOSYS;
+        goto err;
+    }
+
+    for (i = 0; i < total_queues; i++) {
+        r = vhost_net_start_one(tap_get_vhost_net(ncs[i].peer), dev, i * 2);
+
+        if (r < 0) {
+            goto err;
+        }
+    }
+
+    r = dev->binding->set_guest_notifiers(dev->binding_opaque,
+                                          total_queues * 2,
+                                          true);
+    if (r < 0) {
+        error_report("Error binding guest notifier: %d", -r);
+        goto err;
+    }
+
+    return 0;
+
+err:
+    while (--i >= 0) {
+        vhost_net_stop_one(tap_get_vhost_net(ncs[i].peer), dev);
+    }
+    return r;
+}
+
+void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
+                    int total_queues)
+{
+    int i, r;
+
+    r = dev->binding->set_guest_notifiers(dev->binding_opaque,
+                                          total_queues * 2,
+                                          false);
+    if (r < 0) {
+        fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r);
+        fflush(stderr);
+    }
+    assert(r >= 0);
+
+    for (i = 0; i < total_queues; i++) {
+        vhost_net_stop_one(tap_get_vhost_net(ncs[i].peer), dev);
+    }
+}
+
 void vhost_net_cleanup(struct vhost_net *net)
 {
     vhost_dev_cleanup(&net->dev);
     g_free(net);
 }
+
+bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
+{
+    return vhost_virtqueue_pending(&net->dev, idx);
+}
+
+void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
+                              int idx, bool mask)
+{
+    vhost_virtqueue_mask(&net->dev, dev, idx, mask);
+}
 #else
 struct vhost_net *vhost_net_init(NetClientState *backend, int devfd,
                                  bool force)
@@ -213,13 +292,15 @@ bool vhost_net_query(VHostNetState *net, VirtIODevice *dev)
     return false;
 }
 
-int vhost_net_start(struct vhost_net *net,
-		    VirtIODevice *dev)
+int vhost_net_start(VirtIODevice *dev,
+                    NetClientState *ncs,
+                    int total_queues)
 {
     return -ENOSYS;
 }
-void vhost_net_stop(struct vhost_net *net,
-		    VirtIODevice *dev)
+void vhost_net_stop(VirtIODevice *dev,
+                    NetClientState *ncs,
+                    int total_queues)
 {
 }
 
@@ -234,4 +315,14 @@ unsigned vhost_net_get_features(struct vhost_net *net, unsigned features)
 void vhost_net_ack_features(struct vhost_net *net, unsigned features)
 {
 }
+
+bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
+{
+    return -ENOSYS;
+}
+
+void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
+                              int idx, bool mask)
+{
+}
 #endif
diff --git a/hw/vhost_net.h b/hw/vhost_net.h
index a9db234..2d936bb 100644
--- a/hw/vhost_net.h
+++ b/hw/vhost_net.h
@@ -1,7 +1,7 @@
 #ifndef VHOST_NET_H
 #define VHOST_NET_H
 
-#include "net.h"
+#include "net/net.h"
 
 struct vhost_net;
 typedef struct vhost_net VHostNetState;
@@ -9,12 +9,15 @@ typedef struct vhost_net VHostNetState;
 VHostNetState *vhost_net_init(NetClientState *backend, int devfd, bool force);
 
 bool vhost_net_query(VHostNetState *net, VirtIODevice *dev);
-int vhost_net_start(VHostNetState *net, VirtIODevice *dev);
-void vhost_net_stop(VHostNetState *net, VirtIODevice *dev);
+int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, int total_queues);
+void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs, int total_queues);
 
 void vhost_net_cleanup(VHostNetState *net);
 
 unsigned vhost_net_get_features(VHostNetState *net, unsigned features);
 void vhost_net_ack_features(VHostNetState *net, unsigned features);
 
+bool vhost_net_virtqueue_pending(VHostNetState *net, int n);
+void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
+                              int idx, bool mask);
 #endif
diff --git a/hw/virtex_ml507.c b/hw/virtex_ml507.c
index 6ab8fee..8c4e8e4 100644
--- a/hw/virtex_ml507.c
+++ b/hw/virtex_ml507.c
@@ -25,22 +25,21 @@
 #include "sysbus.h"
 #include "hw.h"
 #include "serial.h"
-#include "net.h"
 #include "flash.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "devices.h"
 #include "boards.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 #include "loader.h"
 #include "elf.h"
-#include "qemu-log.h"
-#include "exec-memory.h"
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
 
 #include "ppc.h"
 #include "ppc4xx.h"
 #include "ppc405.h"
 
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "xilinx.h"
 
 #define EPAPR_MAGIC    (0x45504150)
@@ -94,7 +93,7 @@ static PowerPCCPU *ppc440_init_xilinx(ram_addr_t *ram_size,
     }
     env = &cpu->env;
 
-    ppc_booke_timers_init(env, sysclk, 0/* no flags */);
+    ppc_booke_timers_init(cpu, sysclk, 0/* no flags */);
 
     ppc_dcr_init(env, NULL, NULL);
 
@@ -264,6 +263,7 @@ static QEMUMachine virtex_machine = {
     .name = "virtex-ml507",
     .desc = "Xilinx Virtex ML507 reference design",
     .init = virtex_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void virtex_machine_init(void)
diff --git a/hw/virtio-balloon.c b/hw/virtio-balloon.c
index dd1a650..c0a7902 100644
--- a/hw/virtio-balloon.c
+++ b/hw/virtio-balloon.c
@@ -13,15 +13,17 @@
  *
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
+#include "qemu/timer.h"
 #include "qemu-common.h"
 #include "virtio.h"
 #include "pc.h"
 #include "cpu.h"
-#include "balloon.h"
+#include "sysemu/balloon.h"
 #include "virtio-balloon.h"
-#include "kvm.h"
-#include "exec-memory.h"
+#include "sysemu/kvm.h"
+#include "exec/address-spaces.h"
+#include "qapi/visitor.h"
 
 #if defined(__linux__)
 #include <sys/mman.h>
@@ -36,6 +38,9 @@ typedef struct VirtIOBalloon
     uint64_t stats[VIRTIO_BALLOON_S_NR];
     VirtQueueElement stats_vq_elem;
     size_t stats_vq_offset;
+    QEMUTimer *stats_timer;
+    int64_t stats_last_update;
+    int64_t stats_poll_interval;
     DeviceState *qdev;
 } VirtIOBalloon;
 
@@ -53,6 +58,16 @@ static void balloon_page(void *addr, int deflate)
 #endif
 }
 
+static const char *balloon_stat_names[] = {
+   [VIRTIO_BALLOON_S_SWAP_IN] = "stat-swap-in",
+   [VIRTIO_BALLOON_S_SWAP_OUT] = "stat-swap-out",
+   [VIRTIO_BALLOON_S_MAJFLT] = "stat-major-faults",
+   [VIRTIO_BALLOON_S_MINFLT] = "stat-minor-faults",
+   [VIRTIO_BALLOON_S_MEMFREE] = "stat-free-memory",
+   [VIRTIO_BALLOON_S_MEMTOT] = "stat-total-memory",
+   [VIRTIO_BALLOON_S_NR] = NULL
+};
+
 /*
  * reset_stats - Mark all items in the stats array as unset
  *
@@ -67,6 +82,118 @@ static inline void reset_stats(VirtIOBalloon *dev)
     for (i = 0; i < VIRTIO_BALLOON_S_NR; dev->stats[i++] = -1);
 }
 
+static bool balloon_stats_supported(const VirtIOBalloon *s)
+{
+    return s->vdev.guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ);
+}
+
+static bool balloon_stats_enabled(const VirtIOBalloon *s)
+{
+    return s->stats_poll_interval > 0;
+}
+
+static void balloon_stats_destroy_timer(VirtIOBalloon *s)
+{
+    if (balloon_stats_enabled(s)) {
+        qemu_del_timer(s->stats_timer);
+        qemu_free_timer(s->stats_timer);
+        s->stats_timer = NULL;
+        s->stats_poll_interval = 0;
+    }
+}
+
+static void balloon_stats_change_timer(VirtIOBalloon *s, int secs)
+{
+    qemu_mod_timer(s->stats_timer, qemu_get_clock_ms(vm_clock) + secs * 1000);
+}
+
+static void balloon_stats_poll_cb(void *opaque)
+{
+    VirtIOBalloon *s = opaque;
+
+    if (!balloon_stats_supported(s)) {
+        /* re-schedule */
+        balloon_stats_change_timer(s, s->stats_poll_interval);
+        return;
+    }
+
+    virtqueue_push(s->svq, &s->stats_vq_elem, s->stats_vq_offset);
+    virtio_notify(&s->vdev, s->svq);
+}
+
+static void balloon_stats_get_all(Object *obj, struct Visitor *v,
+                                  void *opaque, const char *name, Error **errp)
+{
+    VirtIOBalloon *s = opaque;
+    int i;
+
+    if (!s->stats_last_update) {
+        error_setg(errp, "guest hasn't updated any stats yet");
+        return;
+    }
+
+    visit_start_struct(v, NULL, "guest-stats", name, 0, errp);
+    visit_type_int(v, &s->stats_last_update, "last-update", errp);
+
+    visit_start_struct(v, NULL, NULL, "stats", 0, errp);
+    for (i = 0; i < VIRTIO_BALLOON_S_NR; i++) {
+        visit_type_int64(v, (int64_t *) &s->stats[i], balloon_stat_names[i],
+                         errp);
+    }
+    visit_end_struct(v, errp);
+
+    visit_end_struct(v, errp);
+}
+
+static void balloon_stats_get_poll_interval(Object *obj, struct Visitor *v,
+                                            void *opaque, const char *name,
+                                            Error **errp)
+{
+    VirtIOBalloon *s = opaque;
+    visit_type_int(v, &s->stats_poll_interval, name, errp);
+}
+
+static void balloon_stats_set_poll_interval(Object *obj, struct Visitor *v,
+                                            void *opaque, const char *name,
+                                            Error **errp)
+{
+    VirtIOBalloon *s = opaque;
+    int64_t value;
+
+    visit_type_int(v, &value, name, errp);
+    if (error_is_set(errp)) {
+        return;
+    }
+
+    if (value < 0) {
+        error_setg(errp, "timer value must be greater than zero");
+        return;
+    }
+
+    if (value == s->stats_poll_interval) {
+        return;
+    }
+
+    if (value == 0) {
+        /* timer=0 disables the timer */
+        balloon_stats_destroy_timer(s);
+        return;
+    }
+
+    if (balloon_stats_enabled(s)) {
+        /* timer interval change */
+        s->stats_poll_interval = value;
+        balloon_stats_change_timer(s, value);
+        return;
+    }
+
+    /* create a new timer */
+    g_assert(s->stats_timer == NULL);
+    s->stats_timer = qemu_new_timer_ms(vm_clock, balloon_stats_poll_cb, s);
+    s->stats_poll_interval = value;
+    balloon_stats_change_timer(s, 0);
+}
+
 static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBalloon *s = to_virtio_balloon(vdev);
@@ -107,9 +234,10 @@ static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
     VirtQueueElement *elem = &s->stats_vq_elem;
     VirtIOBalloonStat stat;
     size_t offset = 0;
+    qemu_timeval tv;
 
     if (!virtqueue_pop(vq, elem)) {
-        return;
+        goto out;
     }
 
     /* Initialize the stats to get rid of any stale values.  This is only
@@ -128,6 +256,18 @@ static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
             s->stats[tag] = val;
     }
     s->stats_vq_offset = offset;
+
+    if (qemu_gettimeofday(&tv) < 0) {
+        fprintf(stderr, "warning: %s: failed to get time of day\n", __func__);
+        goto out;
+    }
+
+    s->stats_last_update = tv.tv_sec;
+
+out:
+    if (balloon_stats_enabled(s)) {
+        balloon_stats_change_timer(s, s->stats_poll_interval);
+    }
 }
 
 static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data)
@@ -164,28 +304,6 @@ static uint32_t virtio_balloon_get_features(VirtIODevice *vdev, uint32_t f)
 static void virtio_balloon_stat(void *opaque, BalloonInfo *info)
 {
     VirtIOBalloon *dev = opaque;
-
-#if 0
-    /* Disable guest-provided stats for now. For more details please check:
-     * https://bugzilla.redhat.com/show_bug.cgi?id=623903
-     *
-     * If you do enable it (which is probably not going to happen as we
-     * need a new command for it), remember that you also need to fill the
-     * appropriate members of the BalloonInfo structure so that the stats
-     * are returned to the client.
-     */
-    if (dev->vdev.guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ)) {
-        virtqueue_push(dev->svq, &dev->stats_vq_elem, dev->stats_vq_offset);
-        virtio_notify(&dev->vdev, dev->svq);
-        return;
-    }
-#endif
-
-    /* Stats are not supported.  Clear out any stale values that might
-     * have been set by a more featureful guest kernel.
-     */
-    reset_stats(dev);
-
     info->actual = ram_size - ((uint64_t) dev->actual <<
                                VIRTIO_BALLOON_PFN_SHIFT);
 }
@@ -255,12 +373,18 @@ VirtIODevice *virtio_balloon_init(DeviceState *dev)
     s->dvq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
     s->svq = virtio_add_queue(&s->vdev, 128, virtio_balloon_receive_stats);
 
-    reset_stats(s);
-
     s->qdev = dev;
     register_savevm(dev, "virtio-balloon", -1, 1,
                     virtio_balloon_save, virtio_balloon_load, s);
 
+    object_property_add(OBJECT(dev), "guest-stats", "guest statistics",
+                        balloon_stats_get_all, NULL, NULL, s, NULL);
+
+    object_property_add(OBJECT(dev), "guest-stats-polling-interval", "int",
+                        balloon_stats_get_poll_interval,
+                        balloon_stats_set_poll_interval,
+                        NULL, s, NULL);
+
     return &s->vdev;
 }
 
@@ -268,6 +392,7 @@ void virtio_balloon_exit(VirtIODevice *vdev)
 {
     VirtIOBalloon *s = DO_UPCAST(VirtIOBalloon, vdev, vdev);
 
+    balloon_stats_destroy_timer(s);
     qemu_remove_balloon_handler(s);
     unregister_savevm(s->qdev, "virtio-balloon", s);
     virtio_cleanup(vdev);
diff --git a/hw/virtio-balloon.h b/hw/virtio-balloon.h
index 73300dd..b1828f4 100644
--- a/hw/virtio-balloon.h
+++ b/hw/virtio-balloon.h
@@ -16,7 +16,7 @@
 #define _QEMU_VIRTIO_BALLOON_H
 
 #include "virtio.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 /* from Linux's linux/virtio_balloon.h */
 
diff --git a/hw/virtio-blk.c b/hw/virtio-blk.c
index e25cc96..34913ee 100644
--- a/hw/virtio-blk.c
+++ b/hw/virtio-blk.c
@@ -12,11 +12,14 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "trace.h"
 #include "hw/block-common.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "virtio-blk.h"
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+#include "hw/dataplane/virtio-blk.h"
+#endif
 #include "scsi-defs.h"
 #ifdef __linux__
 # include <scsi/sg.h>
@@ -33,6 +36,9 @@ typedef struct VirtIOBlock
     VirtIOBlkConf *blk;
     unsigned short sector_mask;
     DeviceState *qdev;
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    VirtIOBlockDataPlane *dataplane;
+#endif
 } VirtIOBlock;
 
 static VirtIOBlock *to_virtio_blk(VirtIODevice *vdev)
@@ -392,10 +398,14 @@ static void virtio_blk_handle_request(VirtIOBlockReq *req,
         qemu_iovec_init_external(&req->qiov, &req->elem.out_sg[1],
                                  req->elem.out_num - 1);
         virtio_blk_handle_write(req, mrb);
-    } else {
+    } else if (type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_BARRIER) {
+        /* VIRTIO_BLK_T_IN is 0, so we can't just & it. */
         qemu_iovec_init_external(&req->qiov, &req->elem.in_sg[0],
                                  req->elem.in_num - 1);
         virtio_blk_handle_read(req);
+    } else {
+        virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP);
+        g_free(req);
     }
 }
 
@@ -407,6 +417,16 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
         .num_writes = 0,
     };
 
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+     * dataplane here instead of waiting for .set_status().
+     */
+    if (s->dataplane) {
+        virtio_blk_data_plane_start(s->dataplane);
+        return;
+    }
+#endif
+
     while ((req = virtio_blk_get_request(s))) {
         virtio_blk_handle_request(req, &mrb);
     }
@@ -446,8 +466,9 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running,
 {
     VirtIOBlock *s = opaque;
 
-    if (!running)
+    if (!running) {
         return;
+    }
 
     if (!s->bh) {
         s->bh = qemu_bh_new(virtio_blk_dma_restart_bh, s);
@@ -457,6 +478,14 @@ static void virtio_blk_dma_restart_cb(void *opaque, int running,
 
 static void virtio_blk_reset(VirtIODevice *vdev)
 {
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    VirtIOBlock *s = to_virtio_blk(vdev);
+
+    if (s->dataplane) {
+        virtio_blk_data_plane_stop(s->dataplane);
+    }
+#endif
+
     /*
      * This should cancel pending requests, but can't do nicely until there
      * are per-device request lists.
@@ -524,6 +553,9 @@ static uint32_t virtio_blk_get_features(VirtIODevice *vdev, uint32_t features)
     features |= (1 << VIRTIO_BLK_F_BLK_SIZE);
     features |= (1 << VIRTIO_BLK_F_SCSI);
 
+    if (s->blk->config_wce) {
+        features |= (1 << VIRTIO_BLK_F_CONFIG_WCE);
+    }
     if (bdrv_enable_write_cache(s->bs))
         features |= (1 << VIRTIO_BLK_F_WCE);
 
@@ -538,6 +570,13 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
     VirtIOBlock *s = to_virtio_blk(vdev);
     uint32_t features;
 
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    if (s->dataplane && !(status & (VIRTIO_CONFIG_S_DRIVER |
+                                    VIRTIO_CONFIG_S_DRIVER_OK))) {
+        virtio_blk_data_plane_stop(s->dataplane);
+    }
+#endif
+
     if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return;
     }
@@ -635,6 +674,12 @@ VirtIODevice *virtio_blk_init(DeviceState *dev, VirtIOBlkConf *blk)
     s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
     s->vq = virtio_add_queue(&s->vdev, 128, virtio_blk_handle_output);
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    if (!virtio_blk_data_plane_create(&s->vdev, blk, &s->dataplane)) {
+        virtio_cleanup(&s->vdev);
+        return NULL;
+    }
+#endif
 
     qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
     s->qdev = dev;
@@ -652,6 +697,11 @@ VirtIODevice *virtio_blk_init(DeviceState *dev, VirtIOBlkConf *blk)
 void virtio_blk_exit(VirtIODevice *vdev)
 {
     VirtIOBlock *s = to_virtio_blk(vdev);
+
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    virtio_blk_data_plane_destroy(s->dataplane);
+    s->dataplane = NULL;
+#endif
     unregister_savevm(s->qdev, "virtio-blk", s);
     blockdev_mark_auto_del(s->bs);
     virtio_cleanup(vdev);
diff --git a/hw/virtio-blk.h b/hw/virtio-blk.h
index f0740d0..43ca492 100644
--- a/hw/virtio-blk.h
+++ b/hw/virtio-blk.h
@@ -105,10 +105,10 @@ struct VirtIOBlkConf
     char *serial;
     uint32_t scsi;
     uint32_t config_wce;
+    uint32_t data_plane;
 };
 
 #define DEFINE_VIRTIO_BLK_FEATURES(_state, _field) \
-        DEFINE_VIRTIO_COMMON_FEATURES(_state, _field), \
-        DEFINE_PROP_BIT("config-wce", _state, _field, VIRTIO_BLK_F_CONFIG_WCE, true)
+        DEFINE_VIRTIO_COMMON_FEATURES(_state, _field)
 
 #endif
diff --git a/hw/virtio-bus.c b/hw/virtio-bus.c
new file mode 100644
index 0000000..6045d8a
--- /dev/null
+++ b/hw/virtio-bus.c
@@ -0,0 +1,164 @@
+/*
+ * VirtioBus
+ *
+ *  Copyright (C) 2012 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "hw.h"
+#include "qemu/error-report.h"
+#include "qdev.h"
+#include "virtio-bus.h"
+#include "virtio.h"
+
+/* #define DEBUG_VIRTIO_BUS */
+
+#ifdef DEBUG_VIRTIO_BUS
+#define DPRINTF(fmt, ...) \
+do { printf("virtio_bus: " fmt , ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) do { } while (0)
+#endif
+
+/* Plug the VirtIODevice */
+int virtio_bus_plug_device(VirtIODevice *vdev)
+{
+    DeviceState *qdev = DEVICE(vdev);
+    BusState *qbus = BUS(qdev_get_parent_bus(qdev));
+    VirtioBusState *bus = VIRTIO_BUS(qbus);
+    VirtioBusClass *klass = VIRTIO_BUS_GET_CLASS(bus);
+    DPRINTF("%s: plug device.\n", qbus->name);
+
+    bus->vdev = vdev;
+
+    /*
+     * The lines below will disappear when we drop VirtIOBindings, at the end
+     * of the series.
+     */
+    bus->bindings.notify = klass->notify;
+    bus->bindings.save_config = klass->save_config;
+    bus->bindings.save_queue = klass->save_queue;
+    bus->bindings.load_config = klass->load_config;
+    bus->bindings.load_queue = klass->load_queue;
+    bus->bindings.load_done = klass->load_done;
+    bus->bindings.get_features = klass->get_features;
+    bus->bindings.query_guest_notifiers = klass->query_guest_notifiers;
+    bus->bindings.set_guest_notifiers = klass->set_guest_notifiers;
+    bus->bindings.set_host_notifier = klass->set_host_notifier;
+    bus->bindings.vmstate_change = klass->vmstate_change;
+    virtio_bind_device(bus->vdev, &bus->bindings, qbus->parent);
+
+    if (klass->device_plugged != NULL) {
+        klass->device_plugged(qbus->parent);
+    }
+
+    return 0;
+}
+
+/* Reset the virtio_bus */
+void virtio_bus_reset(VirtioBusState *bus)
+{
+    DPRINTF("%s: reset device.\n", qbus->name);
+    if (bus->vdev != NULL) {
+        virtio_reset(bus->vdev);
+    }
+}
+
+/* Destroy the VirtIODevice */
+void virtio_bus_destroy_device(VirtioBusState *bus)
+{
+    DeviceState *qdev;
+    BusState *qbus = BUS(bus);
+    VirtioBusClass *klass = VIRTIO_BUS_GET_CLASS(bus);
+    DPRINTF("%s: remove device.\n", qbus->name);
+
+    if (bus->vdev != NULL) {
+        if (klass->device_unplug != NULL) {
+            klass->device_unplug(qbus->parent);
+        }
+        qdev = DEVICE(bus->vdev);
+        qdev_free(qdev);
+        bus->vdev = NULL;
+    }
+}
+
+/* Get the device id of the plugged device. */
+uint16_t virtio_bus_get_vdev_id(VirtioBusState *bus)
+{
+    assert(bus->vdev != NULL);
+    return bus->vdev->device_id;
+}
+
+/* Get the config_len field of the plugged device. */
+size_t virtio_bus_get_vdev_config_len(VirtioBusState *bus)
+{
+    assert(bus->vdev != NULL);
+    return bus->vdev->config_len;
+}
+
+/* Get the features of the plugged device. */
+uint32_t virtio_bus_get_vdev_features(VirtioBusState *bus,
+                                    uint32_t requested_features)
+{
+    VirtioDeviceClass *k;
+    assert(bus->vdev != NULL);
+    k = VIRTIO_DEVICE_GET_CLASS(bus->vdev);
+    assert(k->get_features != NULL);
+    return k->get_features(bus->vdev, requested_features);
+}
+
+/* Get bad features of the plugged device. */
+uint32_t virtio_bus_get_vdev_bad_features(VirtioBusState *bus)
+{
+    VirtioDeviceClass *k;
+    assert(bus->vdev != NULL);
+    k = VIRTIO_DEVICE_GET_CLASS(bus->vdev);
+    if (k->bad_features != NULL) {
+        return k->bad_features(bus->vdev);
+    } else {
+        return 0;
+    }
+}
+
+/* Get config of the plugged device. */
+void virtio_bus_get_vdev_config(VirtioBusState *bus, uint8_t *config)
+{
+    VirtioDeviceClass *k;
+    assert(bus->vdev != NULL);
+    k = VIRTIO_DEVICE_GET_CLASS(bus->vdev);
+    if (k->get_config != NULL) {
+        k->get_config(bus->vdev, config);
+    }
+}
+
+static const TypeInfo virtio_bus_info = {
+    .name = TYPE_VIRTIO_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(VirtioBusState),
+    .abstract = true,
+    .class_size = sizeof(VirtioBusClass),
+};
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_bus_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/virtio-bus.h b/hw/virtio-bus.h
new file mode 100644
index 0000000..7584a0e
--- /dev/null
+++ b/hw/virtio-bus.h
@@ -0,0 +1,94 @@
+/*
+ * VirtioBus
+ *
+ *  Copyright (C) 2012 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef VIRTIO_BUS_H
+#define VIRTIO_BUS_H
+
+#include "qdev.h"
+#include "sysemu/sysemu.h"
+#include "virtio.h"
+
+#define TYPE_VIRTIO_BUS "virtio-bus"
+#define VIRTIO_BUS_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioBusClass, obj, TYPE_VIRTIO_BUS)
+#define VIRTIO_BUS_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioBusClass, klass, TYPE_VIRTIO_BUS)
+#define VIRTIO_BUS(obj) OBJECT_CHECK(VirtioBusState, (obj), TYPE_VIRTIO_BUS)
+
+typedef struct VirtioBusState VirtioBusState;
+
+typedef struct VirtioBusClass {
+    /* This is what a VirtioBus must implement */
+    BusClass parent;
+    void (*notify)(DeviceState *d, uint16_t vector);
+    void (*save_config)(DeviceState *d, QEMUFile *f);
+    void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_config)(DeviceState *d, QEMUFile *f);
+    int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_done)(DeviceState *d, QEMUFile *f);
+    unsigned (*get_features)(DeviceState *d);
+    bool (*query_guest_notifiers)(DeviceState *d);
+    int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assign);
+    int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
+    void (*vmstate_change)(DeviceState *d, bool running);
+    /*
+     * transport independent init function.
+     * This is called by virtio-bus just after the device is plugged.
+     */
+    void (*device_plugged)(DeviceState *d);
+    /*
+     * transport independent exit function.
+     * This is called by virtio-bus just before the device is unplugged.
+     */
+    void (*device_unplug)(DeviceState *d);
+} VirtioBusClass;
+
+struct VirtioBusState {
+    BusState parent_obj;
+    /*
+     * Only one VirtIODevice can be plugged on the bus.
+     */
+    VirtIODevice *vdev;
+    /*
+     * This will be removed at the end of the series.
+     */
+    VirtIOBindings bindings;
+};
+
+int virtio_bus_plug_device(VirtIODevice *vdev);
+void virtio_bus_reset(VirtioBusState *bus);
+void virtio_bus_destroy_device(VirtioBusState *bus);
+/* Get the device id of the plugged device. */
+uint16_t virtio_bus_get_vdev_id(VirtioBusState *bus);
+/* Get the config_len field of the plugged device. */
+size_t virtio_bus_get_vdev_config_len(VirtioBusState *bus);
+/* Get the features of the plugged device. */
+uint32_t virtio_bus_get_vdev_features(VirtioBusState *bus,
+                                    uint32_t requested_features);
+/* Get bad features of the plugged device. */
+uint32_t virtio_bus_get_vdev_bad_features(VirtioBusState *bus);
+/* Get config of the plugged device. */
+void virtio_bus_get_vdev_config(VirtioBusState *bus, uint8_t *config);
+
+#endif /* VIRTIO_BUS_H */
diff --git a/hw/virtio-console.c b/hw/virtio-console.c
index cffee3d..46072a0 100644
--- a/hw/virtio-console.c
+++ b/hw/virtio-console.c
@@ -10,8 +10,8 @@
  * the COPYING file in the top-level directory.
  */
 
-#include "qemu-char.h"
-#include "qemu-error.h"
+#include "char/char.h"
+#include "qemu/error-report.h"
 #include "trace.h"
 #include "virtio-serial.h"
 
@@ -142,7 +142,7 @@ static void virtconsole_class_init(ObjectClass *klass, void *data)
     dc->props = virtconsole_properties;
 }
 
-static TypeInfo virtconsole_info = {
+static const TypeInfo virtconsole_info = {
     .name          = "virtconsole",
     .parent        = TYPE_VIRTIO_SERIAL_PORT,
     .instance_size = sizeof(VirtConsole),
@@ -166,7 +166,7 @@ static void virtserialport_class_init(ObjectClass *klass, void *data)
     dc->props = virtserialport_properties;
 }
 
-static TypeInfo virtserialport_info = {
+static const TypeInfo virtserialport_info = {
     .name          = "virtserialport",
     .parent        = TYPE_VIRTIO_SERIAL_PORT,
     .instance_size = sizeof(VirtConsole),
diff --git a/hw/virtio-net.c b/hw/virtio-net.c
index 108ce07..573c669 100644
--- a/hw/virtio-net.c
+++ b/hw/virtio-net.c
@@ -11,13 +11,13 @@
  *
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
 #include "virtio.h"
-#include "net.h"
+#include "net/net.h"
 #include "net/checksum.h"
 #include "net/tap.h"
-#include "qemu-error.h"
-#include "qemu-timer.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
 #include "virtio-net.h"
 #include "vhost_net.h"
 
@@ -26,28 +26,33 @@
 #define MAC_TABLE_ENTRIES    64
 #define MAX_VLAN    (1 << 12)   /* Per 802.1Q definition */
 
+typedef struct VirtIONetQueue {
+    VirtQueue *rx_vq;
+    VirtQueue *tx_vq;
+    QEMUTimer *tx_timer;
+    QEMUBH *tx_bh;
+    int tx_waiting;
+    struct {
+        VirtQueueElement elem;
+        ssize_t len;
+    } async_tx;
+    struct VirtIONet *n;
+} VirtIONetQueue;
+
 typedef struct VirtIONet
 {
     VirtIODevice vdev;
     uint8_t mac[ETH_ALEN];
     uint16_t status;
-    VirtQueue *rx_vq;
-    VirtQueue *tx_vq;
+    VirtIONetQueue vqs[MAX_QUEUE_NUM];
     VirtQueue *ctrl_vq;
     NICState *nic;
-    QEMUTimer *tx_timer;
-    QEMUBH *tx_bh;
     uint32_t tx_timeout;
     int32_t tx_burst;
-    int tx_waiting;
     uint32_t has_vnet_hdr;
     size_t host_hdr_len;
     size_t guest_hdr_len;
     uint8_t has_ufo;
-    struct {
-        VirtQueueElement elem;
-        ssize_t len;
-    } async_tx;
     int mergeable_rx_bufs;
     uint8_t promisc;
     uint8_t allmulti;
@@ -65,8 +70,46 @@ typedef struct VirtIONet
     } mac_table;
     uint32_t *vlans;
     DeviceState *qdev;
+    int multiqueue;
+    uint16_t max_queues;
+    uint16_t curr_queues;
+    size_t config_size;
 } VirtIONet;
 
+/*
+ * Calculate the number of bytes up to and including the given 'field' of
+ * 'container'.
+ */
+#define endof(container, field) \
+    (offsetof(container, field) + sizeof(((container *)0)->field))
+
+typedef struct VirtIOFeature {
+    uint32_t flags;
+    size_t end;
+} VirtIOFeature;
+
+static VirtIOFeature feature_sizes[] = {
+    {.flags = 1 << VIRTIO_NET_F_MAC,
+     .end = endof(struct virtio_net_config, mac)},
+    {.flags = 1 << VIRTIO_NET_F_STATUS,
+     .end = endof(struct virtio_net_config, status)},
+    {.flags = 1 << VIRTIO_NET_F_MQ,
+     .end = endof(struct virtio_net_config, max_virtqueue_pairs)},
+    {}
+};
+
+static VirtIONetQueue *virtio_net_get_subqueue(NetClientState *nc)
+{
+    VirtIONet *n = qemu_get_nic_opaque(nc);
+
+    return &n->vqs[nc->queue_index];
+}
+
+static int vq2q(int queue_index)
+{
+    return queue_index / 2;
+}
+
 /* TODO
  * - we could suppress RX interrupt if we were so inclined.
  */
@@ -82,20 +125,22 @@ static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config)
     struct virtio_net_config netcfg;
 
     stw_p(&netcfg.status, n->status);
+    stw_p(&netcfg.max_virtqueue_pairs, n->max_queues);
     memcpy(netcfg.mac, n->mac, ETH_ALEN);
-    memcpy(config, &netcfg, sizeof(netcfg));
+    memcpy(config, &netcfg, n->config_size);
 }
 
 static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config)
 {
     VirtIONet *n = to_virtio_net(vdev);
-    struct virtio_net_config netcfg;
+    struct virtio_net_config netcfg = {};
 
-    memcpy(&netcfg, config, sizeof(netcfg));
+    memcpy(&netcfg, config, n->config_size);
 
-    if (memcmp(netcfg.mac, n->mac, ETH_ALEN)) {
+    if (!(n->vdev.guest_features >> VIRTIO_NET_F_CTRL_MAC_ADDR & 1) &&
+        memcmp(netcfg.mac, n->mac, ETH_ALEN)) {
         memcpy(n->mac, netcfg.mac, ETH_ALEN);
-        qemu_format_nic_info_str(&n->nic->nc, n->mac);
+        qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
     }
 }
 
@@ -107,34 +152,38 @@ static bool virtio_net_started(VirtIONet *n, uint8_t status)
 
 static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
 {
-    if (!n->nic->nc.peer) {
+    NetClientState *nc = qemu_get_queue(n->nic);
+    int queues = n->multiqueue ? n->max_queues : 1;
+
+    if (!nc->peer) {
         return;
     }
-    if (n->nic->nc.peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
         return;
     }
 
-    if (!tap_get_vhost_net(n->nic->nc.peer)) {
+    if (!tap_get_vhost_net(nc->peer)) {
         return;
     }
+
     if (!!n->vhost_started == virtio_net_started(n, status) &&
-                              !n->nic->nc.peer->link_down) {
+                              !nc->peer->link_down) {
         return;
     }
     if (!n->vhost_started) {
         int r;
-        if (!vhost_net_query(tap_get_vhost_net(n->nic->nc.peer), &n->vdev)) {
+        if (!vhost_net_query(tap_get_vhost_net(nc->peer), &n->vdev)) {
             return;
         }
-        r = vhost_net_start(tap_get_vhost_net(n->nic->nc.peer), &n->vdev);
+        n->vhost_started = 1;
+        r = vhost_net_start(&n->vdev, n->nic->ncs, queues);
         if (r < 0) {
             error_report("unable to start vhost net: %d: "
                          "falling back on userspace virtio", -r);
-        } else {
-            n->vhost_started = 1;
+            n->vhost_started = 0;
         }
     } else {
-        vhost_net_stop(tap_get_vhost_net(n->nic->nc.peer), &n->vdev);
+        vhost_net_stop(&n->vdev, n->nic->ncs, queues);
         n->vhost_started = 0;
     }
 }
@@ -142,32 +191,45 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status)
 {
     VirtIONet *n = to_virtio_net(vdev);
+    VirtIONetQueue *q;
+    int i;
+    uint8_t queue_status;
 
     virtio_net_vhost_status(n, status);
 
-    if (!n->tx_waiting) {
-        return;
-    }
+    for (i = 0; i < n->max_queues; i++) {
+        q = &n->vqs[i];
 
-    if (virtio_net_started(n, status) && !n->vhost_started) {
-        if (n->tx_timer) {
-            qemu_mod_timer(n->tx_timer,
-                           qemu_get_clock_ns(vm_clock) + n->tx_timeout);
+        if ((!n->multiqueue && i != 0) || i >= n->curr_queues) {
+            queue_status = 0;
         } else {
-            qemu_bh_schedule(n->tx_bh);
+            queue_status = status;
         }
-    } else {
-        if (n->tx_timer) {
-            qemu_del_timer(n->tx_timer);
+
+        if (!q->tx_waiting) {
+            continue;
+        }
+
+        if (virtio_net_started(n, queue_status) && !n->vhost_started) {
+            if (q->tx_timer) {
+                qemu_mod_timer(q->tx_timer,
+                               qemu_get_clock_ns(vm_clock) + n->tx_timeout);
+            } else {
+                qemu_bh_schedule(q->tx_bh);
+            }
         } else {
-            qemu_bh_cancel(n->tx_bh);
+            if (q->tx_timer) {
+                qemu_del_timer(q->tx_timer);
+            } else {
+                qemu_bh_cancel(q->tx_bh);
+            }
         }
     }
 }
 
 static void virtio_net_set_link_status(NetClientState *nc)
 {
-    VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
+    VirtIONet *n = qemu_get_nic_opaque(nc);
     uint16_t old_status = n->status;
 
     if (nc->link_down)
@@ -192,6 +254,8 @@ static void virtio_net_reset(VirtIODevice *vdev)
     n->nomulti = 0;
     n->nouni = 0;
     n->nobcast = 0;
+    /* multiqueue is disabled by default */
+    n->curr_queues = 1;
 
     /* Flush any MAC and VLAN filter table state */
     n->mac_table.in_use = 0;
@@ -199,18 +263,22 @@ static void virtio_net_reset(VirtIODevice *vdev)
     n->mac_table.multi_overflow = 0;
     n->mac_table.uni_overflow = 0;
     memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN);
+    memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac));
     memset(n->vlans, 0, MAX_VLAN >> 3);
 }
 
 static void peer_test_vnet_hdr(VirtIONet *n)
 {
-    if (!n->nic->nc.peer)
+    NetClientState *nc = qemu_get_queue(n->nic);
+    if (!nc->peer) {
         return;
+    }
 
-    if (n->nic->nc.peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP)
+    if (nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
         return;
+    }
 
-    n->has_vnet_hdr = tap_has_vnet_hdr(n->nic->nc.peer);
+    n->has_vnet_hdr = tap_has_vnet_hdr(nc->peer);
 }
 
 static int peer_has_vnet_hdr(VirtIONet *n)
@@ -223,28 +291,81 @@ static int peer_has_ufo(VirtIONet *n)
     if (!peer_has_vnet_hdr(n))
         return 0;
 
-    n->has_ufo = tap_has_ufo(n->nic->nc.peer);
+    n->has_ufo = tap_has_ufo(qemu_get_queue(n->nic)->peer);
 
     return n->has_ufo;
 }
 
 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs)
 {
+    int i;
+    NetClientState *nc;
+
     n->mergeable_rx_bufs = mergeable_rx_bufs;
 
     n->guest_hdr_len = n->mergeable_rx_bufs ?
         sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr);
 
-    if (peer_has_vnet_hdr(n) &&
-        tap_has_vnet_hdr_len(n->nic->nc.peer, n->guest_hdr_len)) {
-        tap_set_vnet_hdr_len(n->nic->nc.peer, n->guest_hdr_len);
-        n->host_hdr_len = n->guest_hdr_len;
+    for (i = 0; i < n->max_queues; i++) {
+        nc = qemu_get_subqueue(n->nic, i);
+
+        if (peer_has_vnet_hdr(n) &&
+            tap_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) {
+            tap_set_vnet_hdr_len(nc->peer, n->guest_hdr_len);
+            n->host_hdr_len = n->guest_hdr_len;
+        }
+    }
+}
+
+static int peer_attach(VirtIONet *n, int index)
+{
+    NetClientState *nc = qemu_get_subqueue(n->nic, index);
+
+    if (!nc->peer) {
+        return 0;
+    }
+
+    if (nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
+        return 0;
+    }
+
+    return tap_enable(nc->peer);
+}
+
+static int peer_detach(VirtIONet *n, int index)
+{
+    NetClientState *nc = qemu_get_subqueue(n->nic, index);
+
+    if (!nc->peer) {
+        return 0;
+    }
+
+    if (nc->peer->info->type !=  NET_CLIENT_OPTIONS_KIND_TAP) {
+        return 0;
+    }
+
+    return tap_disable(nc->peer);
+}
+
+static void virtio_net_set_queues(VirtIONet *n)
+{
+    int i;
+
+    for (i = 0; i < n->max_queues; i++) {
+        if (i < n->curr_queues) {
+            assert(!peer_attach(n, i));
+        } else {
+            assert(!peer_detach(n, i));
+        }
     }
 }
 
+static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue, int ctrl);
+
 static uint32_t virtio_net_get_features(VirtIODevice *vdev, uint32_t features)
 {
     VirtIONet *n = to_virtio_net(vdev);
+    NetClientState *nc = qemu_get_queue(n->nic);
 
     features |= (1 << VIRTIO_NET_F_MAC);
 
@@ -265,14 +386,13 @@ static uint32_t virtio_net_get_features(VirtIODevice *vdev, uint32_t features)
         features &= ~(0x1 << VIRTIO_NET_F_HOST_UFO);
     }
 
-    if (!n->nic->nc.peer ||
-        n->nic->nc.peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (!nc->peer || nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
         return features;
     }
-    if (!tap_get_vhost_net(n->nic->nc.peer)) {
+    if (!tap_get_vhost_net(nc->peer)) {
         return features;
     }
-    return vhost_net_get_features(tap_get_vhost_net(n->nic->nc.peer), features);
+    return vhost_net_get_features(tap_get_vhost_net(nc->peer), features);
 }
 
 static uint32_t virtio_net_bad_features(VirtIODevice *vdev)
@@ -293,66 +413,84 @@ static uint32_t virtio_net_bad_features(VirtIODevice *vdev)
 static void virtio_net_set_features(VirtIODevice *vdev, uint32_t features)
 {
     VirtIONet *n = to_virtio_net(vdev);
+    int i;
+
+    virtio_net_set_multiqueue(n, !!(features & (1 << VIRTIO_NET_F_MQ)),
+                              !!(features & (1 << VIRTIO_NET_F_CTRL_VQ)));
 
     virtio_net_set_mrg_rx_bufs(n, !!(features & (1 << VIRTIO_NET_F_MRG_RXBUF)));
 
     if (n->has_vnet_hdr) {
-        tap_set_offload(n->nic->nc.peer,
+        tap_set_offload(qemu_get_subqueue(n->nic, 0)->peer,
                         (features >> VIRTIO_NET_F_GUEST_CSUM) & 1,
                         (features >> VIRTIO_NET_F_GUEST_TSO4) & 1,
                         (features >> VIRTIO_NET_F_GUEST_TSO6) & 1,
                         (features >> VIRTIO_NET_F_GUEST_ECN)  & 1,
                         (features >> VIRTIO_NET_F_GUEST_UFO)  & 1);
     }
-    if (!n->nic->nc.peer ||
-        n->nic->nc.peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
-        return;
-    }
-    if (!tap_get_vhost_net(n->nic->nc.peer)) {
-        return;
+
+    for (i = 0;  i < n->max_queues; i++) {
+        NetClientState *nc = qemu_get_subqueue(n->nic, i);
+
+        if (!nc->peer || nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
+            continue;
+        }
+        if (!tap_get_vhost_net(nc->peer)) {
+            continue;
+        }
+        vhost_net_ack_features(tap_get_vhost_net(nc->peer), features);
     }
-    vhost_net_ack_features(tap_get_vhost_net(n->nic->nc.peer), features);
 }
 
 static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd,
-                                     VirtQueueElement *elem)
+                                     struct iovec *iov, unsigned int iov_cnt)
 {
     uint8_t on;
+    size_t s;
 
-    if (elem->out_num != 2 || elem->out_sg[1].iov_len != sizeof(on)) {
-        error_report("virtio-net ctrl invalid rx mode command");
-        exit(1);
+    s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on));
+    if (s != sizeof(on)) {
+        return VIRTIO_NET_ERR;
     }
 
-    on = ldub_p(elem->out_sg[1].iov_base);
-
-    if (cmd == VIRTIO_NET_CTRL_RX_MODE_PROMISC)
+    if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) {
         n->promisc = on;
-    else if (cmd == VIRTIO_NET_CTRL_RX_MODE_ALLMULTI)
+    } else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) {
         n->allmulti = on;
-    else if (cmd == VIRTIO_NET_CTRL_RX_MODE_ALLUNI)
+    } else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) {
         n->alluni = on;
-    else if (cmd == VIRTIO_NET_CTRL_RX_MODE_NOMULTI)
+    } else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) {
         n->nomulti = on;
-    else if (cmd == VIRTIO_NET_CTRL_RX_MODE_NOUNI)
+    } else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) {
         n->nouni = on;
-    else if (cmd == VIRTIO_NET_CTRL_RX_MODE_NOBCAST)
+    } else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) {
         n->nobcast = on;
-    else
+    } else {
         return VIRTIO_NET_ERR;
+    }
 
     return VIRTIO_NET_OK;
 }
 
 static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd,
-                                 VirtQueueElement *elem)
+                                 struct iovec *iov, unsigned int iov_cnt)
 {
     struct virtio_net_ctrl_mac mac_data;
+    size_t s;
 
-    if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET || elem->out_num != 3 ||
-        elem->out_sg[1].iov_len < sizeof(mac_data) ||
-        elem->out_sg[2].iov_len < sizeof(mac_data))
+    if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) {
+        if (iov_size(iov, iov_cnt) != sizeof(n->mac)) {
+            return VIRTIO_NET_ERR;
+        }
+        s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac));
+        assert(s == sizeof(n->mac));
+        qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
+        return VIRTIO_NET_OK;
+    }
+
+    if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) {
         return VIRTIO_NET_ERR;
+    }
 
     n->mac_table.in_use = 0;
     n->mac_table.first_multi = 0;
@@ -360,54 +498,72 @@ static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd,
     n->mac_table.multi_overflow = 0;
     memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN);
 
-    mac_data.entries = ldl_p(elem->out_sg[1].iov_base);
+    s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries,
+                   sizeof(mac_data.entries));
+    mac_data.entries = ldl_p(&mac_data.entries);
+    if (s != sizeof(mac_data.entries)) {
+        return VIRTIO_NET_ERR;
+    }
+    iov_discard_front(&iov, &iov_cnt, s);
 
-    if (sizeof(mac_data.entries) +
-        (mac_data.entries * ETH_ALEN) > elem->out_sg[1].iov_len)
+    if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) {
         return VIRTIO_NET_ERR;
+    }
 
     if (mac_data.entries <= MAC_TABLE_ENTRIES) {
-        memcpy(n->mac_table.macs, elem->out_sg[1].iov_base + sizeof(mac_data),
-               mac_data.entries * ETH_ALEN);
+        s = iov_to_buf(iov, iov_cnt, 0, n->mac_table.macs,
+                       mac_data.entries * ETH_ALEN);
+        if (s != mac_data.entries * ETH_ALEN) {
+            return VIRTIO_NET_ERR;
+        }
         n->mac_table.in_use += mac_data.entries;
     } else {
         n->mac_table.uni_overflow = 1;
     }
 
+    iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN);
+
     n->mac_table.first_multi = n->mac_table.in_use;
 
-    mac_data.entries = ldl_p(elem->out_sg[2].iov_base);
+    s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries,
+                   sizeof(mac_data.entries));
+    mac_data.entries = ldl_p(&mac_data.entries);
+    if (s != sizeof(mac_data.entries)) {
+        return VIRTIO_NET_ERR;
+    }
+
+    iov_discard_front(&iov, &iov_cnt, s);
 
-    if (sizeof(mac_data.entries) +
-        (mac_data.entries * ETH_ALEN) > elem->out_sg[2].iov_len)
+    if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) {
         return VIRTIO_NET_ERR;
+    }
 
-    if (mac_data.entries) {
-        if (n->mac_table.in_use + mac_data.entries <= MAC_TABLE_ENTRIES) {
-            memcpy(n->mac_table.macs + (n->mac_table.in_use * ETH_ALEN),
-                   elem->out_sg[2].iov_base + sizeof(mac_data),
-                   mac_data.entries * ETH_ALEN);
-            n->mac_table.in_use += mac_data.entries;
-        } else {
-            n->mac_table.multi_overflow = 1;
+    if (n->mac_table.in_use + mac_data.entries <= MAC_TABLE_ENTRIES) {
+        s = iov_to_buf(iov, iov_cnt, 0, n->mac_table.macs,
+                       mac_data.entries * ETH_ALEN);
+        if (s != mac_data.entries * ETH_ALEN) {
+            return VIRTIO_NET_ERR;
         }
+        n->mac_table.in_use += mac_data.entries;
+    } else {
+        n->mac_table.multi_overflow = 1;
     }
 
     return VIRTIO_NET_OK;
 }
 
 static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd,
-                                        VirtQueueElement *elem)
+                                        struct iovec *iov, unsigned int iov_cnt)
 {
     uint16_t vid;
+    size_t s;
 
-    if (elem->out_num != 2 || elem->out_sg[1].iov_len != sizeof(vid)) {
-        error_report("virtio-net ctrl invalid vlan command");
+    s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid));
+    vid = lduw_p(&vid);
+    if (s != sizeof(vid)) {
         return VIRTIO_NET_ERR;
     }
 
-    vid = lduw_p(elem->out_sg[1].iov_base);
-
     if (vid >= MAX_VLAN)
         return VIRTIO_NET_ERR;
 
@@ -421,36 +577,73 @@ static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd,
     return VIRTIO_NET_OK;
 }
 
+static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd,
+                                VirtQueueElement *elem)
+{
+    struct virtio_net_ctrl_mq s;
+
+    if (elem->out_num != 2 ||
+        elem->out_sg[1].iov_len != sizeof(struct virtio_net_ctrl_mq)) {
+        error_report("virtio-net ctrl invalid steering command");
+        return VIRTIO_NET_ERR;
+    }
+
+    if (cmd != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
+        return VIRTIO_NET_ERR;
+    }
+
+    memcpy(&s, elem->out_sg[1].iov_base, sizeof(struct virtio_net_ctrl_mq));
+
+    if (s.virtqueue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
+        s.virtqueue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
+        s.virtqueue_pairs > n->max_queues ||
+        !n->multiqueue) {
+        return VIRTIO_NET_ERR;
+    }
+
+    n->curr_queues = s.virtqueue_pairs;
+    /* stop the backend before changing the number of queues to avoid handling a
+     * disabled queue */
+    virtio_net_set_status(&n->vdev, n->vdev.status);
+    virtio_net_set_queues(n);
+
+    return VIRTIO_NET_OK;
+}
 static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIONet *n = to_virtio_net(vdev);
     struct virtio_net_ctrl_hdr ctrl;
     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
     VirtQueueElement elem;
+    size_t s;
+    struct iovec *iov;
+    unsigned int iov_cnt;
 
     while (virtqueue_pop(vq, &elem)) {
-        if ((elem.in_num < 1) || (elem.out_num < 1)) {
+        if (iov_size(elem.in_sg, elem.in_num) < sizeof(status) ||
+            iov_size(elem.out_sg, elem.out_num) < sizeof(ctrl)) {
             error_report("virtio-net ctrl missing headers");
             exit(1);
         }
 
-        if (elem.out_sg[0].iov_len < sizeof(ctrl) ||
-            elem.in_sg[elem.in_num - 1].iov_len < sizeof(status)) {
-            error_report("virtio-net ctrl header not in correct element");
-            exit(1);
+        iov = elem.out_sg;
+        iov_cnt = elem.out_num;
+        s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl));
+        iov_discard_front(&iov, &iov_cnt, sizeof(ctrl));
+        if (s != sizeof(ctrl)) {
+            status = VIRTIO_NET_ERR;
+        } else if (ctrl.class == VIRTIO_NET_CTRL_RX) {
+            status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, iov_cnt);
+        } else if (ctrl.class == VIRTIO_NET_CTRL_MAC) {
+            status = virtio_net_handle_mac(n, ctrl.cmd, iov, iov_cnt);
+        } else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) {
+            status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, iov_cnt);
+        } else if (ctrl.class == VIRTIO_NET_CTRL_MQ) {
+            status = virtio_net_handle_mq(n, ctrl.cmd, &elem);
         }
 
-        ctrl.class = ldub_p(elem.out_sg[0].iov_base);
-        ctrl.cmd = ldub_p(elem.out_sg[0].iov_base + sizeof(ctrl.class));
-
-        if (ctrl.class == VIRTIO_NET_CTRL_RX_MODE)
-            status = virtio_net_handle_rx_mode(n, ctrl.cmd, &elem);
-        else if (ctrl.class == VIRTIO_NET_CTRL_MAC)
-            status = virtio_net_handle_mac(n, ctrl.cmd, &elem);
-        else if (ctrl.class == VIRTIO_NET_CTRL_VLAN)
-            status = virtio_net_handle_vlan_table(n, ctrl.cmd, &elem);
-
-        stb_p(elem.in_sg[elem.in_num - 1].iov_base, status);
+        s = iov_from_buf(elem.in_sg, elem.in_num, 0, &status, sizeof(status));
+        assert(s == sizeof(status));
 
         virtqueue_push(vq, &elem, sizeof(status));
         virtio_notify(vdev, vq);
@@ -462,42 +655,52 @@ static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIONet *n = to_virtio_net(vdev);
+    int queue_index = vq2q(virtio_get_queue_index(vq));
 
-    qemu_flush_queued_packets(&n->nic->nc);
+    qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index));
 }
 
 static int virtio_net_can_receive(NetClientState *nc)
 {
-    VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
+    VirtIONet *n = qemu_get_nic_opaque(nc);
+    VirtIONetQueue *q = virtio_net_get_subqueue(nc);
+
     if (!n->vdev.vm_running) {
         return 0;
     }
 
-    if (!virtio_queue_ready(n->rx_vq) ||
-        !(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK))
+    if (nc->queue_index >= n->curr_queues) {
+        return 0;
+    }
+
+    if (!virtio_queue_ready(q->rx_vq) ||
+        !(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return 0;
+    }
 
     return 1;
 }
 
-static int virtio_net_has_buffers(VirtIONet *n, int bufsize)
+static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize)
 {
-    if (virtio_queue_empty(n->rx_vq) ||
+    VirtIONet *n = q->n;
+    if (virtio_queue_empty(q->rx_vq) ||
         (n->mergeable_rx_bufs &&
-         !virtqueue_avail_bytes(n->rx_vq, bufsize, 0))) {
-        virtio_queue_set_notification(n->rx_vq, 1);
+         !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) {
+        virtio_queue_set_notification(q->rx_vq, 1);
 
         /* To avoid a race condition where the guest has made some buffers
          * available after the above check but before notification was
          * enabled, check for available buffers again.
          */
-        if (virtio_queue_empty(n->rx_vq) ||
+        if (virtio_queue_empty(q->rx_vq) ||
             (n->mergeable_rx_bufs &&
-             !virtqueue_avail_bytes(n->rx_vq, bufsize, 0)))
+             !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) {
             return 0;
+        }
     }
 
-    virtio_queue_set_notification(n->rx_vq, 0);
+    virtio_queue_set_notification(q->rx_vq, 0);
     return 1;
 }
 
@@ -599,18 +802,21 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
 
 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
+    VirtIONet *n = qemu_get_nic_opaque(nc);
+    VirtIONetQueue *q = virtio_net_get_subqueue(nc);
     struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE];
     struct virtio_net_hdr_mrg_rxbuf mhdr;
     unsigned mhdr_cnt = 0;
     size_t offset, i, guest_offset;
 
-    if (!virtio_net_can_receive(&n->nic->nc))
+    if (!virtio_net_can_receive(nc)) {
         return -1;
+    }
 
     /* hdr_len refers to the header we supply to the guest */
-    if (!virtio_net_has_buffers(n, size + n->guest_hdr_len - n->host_hdr_len))
+    if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) {
         return 0;
+    }
 
     if (!receive_filter(n, buf, size))
         return size;
@@ -624,7 +830,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         total = 0;
 
-        if (virtqueue_pop(n->rx_vq, &elem) == 0) {
+        if (virtqueue_pop(q->rx_vq, &elem) == 0) {
             if (i == 0)
                 return -1;
             error_report("virtio-net unexpected empty queue: "
@@ -677,7 +883,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
         }
 
         /* signal other side */
-        virtqueue_fill(n->rx_vq, &elem, total, i++);
+        virtqueue_fill(q->rx_vq, &elem, total, i++);
     }
 
     if (mhdr_cnt) {
@@ -687,44 +893,47 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
                      &mhdr.num_buffers, sizeof mhdr.num_buffers);
     }
 
-    virtqueue_flush(n->rx_vq, i);
-    virtio_notify(&n->vdev, n->rx_vq);
+    virtqueue_flush(q->rx_vq, i);
+    virtio_notify(&n->vdev, q->rx_vq);
 
     return size;
 }
 
-static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq);
+static int32_t virtio_net_flush_tx(VirtIONetQueue *q);
 
 static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
 {
-    VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
+    VirtIONet *n = qemu_get_nic_opaque(nc);
+    VirtIONetQueue *q = virtio_net_get_subqueue(nc);
 
-    virtqueue_push(n->tx_vq, &n->async_tx.elem, 0);
-    virtio_notify(&n->vdev, n->tx_vq);
+    virtqueue_push(q->tx_vq, &q->async_tx.elem, 0);
+    virtio_notify(&n->vdev, q->tx_vq);
 
-    n->async_tx.elem.out_num = n->async_tx.len = 0;
+    q->async_tx.elem.out_num = q->async_tx.len = 0;
 
-    virtio_queue_set_notification(n->tx_vq, 1);
-    virtio_net_flush_tx(n, n->tx_vq);
+    virtio_queue_set_notification(q->tx_vq, 1);
+    virtio_net_flush_tx(q);
 }
 
 /* TX */
-static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
+static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
 {
+    VirtIONet *n = q->n;
     VirtQueueElement elem;
     int32_t num_packets = 0;
+    int queue_index = vq2q(virtio_get_queue_index(q->tx_vq));
     if (!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return num_packets;
     }
 
     assert(n->vdev.vm_running);
 
-    if (n->async_tx.elem.out_num) {
-        virtio_queue_set_notification(n->tx_vq, 0);
+    if (q->async_tx.elem.out_num) {
+        virtio_queue_set_notification(q->tx_vq, 0);
         return num_packets;
     }
 
-    while (virtqueue_pop(vq, &elem)) {
+    while (virtqueue_pop(q->tx_vq, &elem)) {
         ssize_t ret, len;
         unsigned int out_num = elem.out_num;
         struct iovec *out_sg = &elem.out_sg[0];
@@ -754,19 +963,19 @@ static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 
         len = n->guest_hdr_len;
 
-        ret = qemu_sendv_packet_async(&n->nic->nc, out_sg, out_num,
-                                      virtio_net_tx_complete);
+        ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index),
+                                      out_sg, out_num, virtio_net_tx_complete);
         if (ret == 0) {
-            virtio_queue_set_notification(n->tx_vq, 0);
-            n->async_tx.elem = elem;
-            n->async_tx.len  = len;
+            virtio_queue_set_notification(q->tx_vq, 0);
+            q->async_tx.elem = elem;
+            q->async_tx.len  = len;
             return -EBUSY;
         }
 
         len += ret;
 
-        virtqueue_push(vq, &elem, 0);
-        virtio_notify(&n->vdev, vq);
+        virtqueue_push(q->tx_vq, &elem, 0);
+        virtio_notify(&n->vdev, q->tx_vq);
 
         if (++num_packets >= n->tx_burst) {
             break;
@@ -778,22 +987,23 @@ static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIONet *n = to_virtio_net(vdev);
+    VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))];
 
     /* This happens when device was stopped but VCPU wasn't. */
     if (!n->vdev.vm_running) {
-        n->tx_waiting = 1;
+        q->tx_waiting = 1;
         return;
     }
 
-    if (n->tx_waiting) {
+    if (q->tx_waiting) {
         virtio_queue_set_notification(vq, 1);
-        qemu_del_timer(n->tx_timer);
-        n->tx_waiting = 0;
-        virtio_net_flush_tx(n, vq);
+        qemu_del_timer(q->tx_timer);
+        q->tx_waiting = 0;
+        virtio_net_flush_tx(q);
     } else {
-        qemu_mod_timer(n->tx_timer,
+        qemu_mod_timer(q->tx_timer,
                        qemu_get_clock_ns(vm_clock) + n->tx_timeout);
-        n->tx_waiting = 1;
+        q->tx_waiting = 1;
         virtio_queue_set_notification(vq, 0);
     }
 }
@@ -801,48 +1011,51 @@ static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq)
 static void virtio_net_handle_tx_bh(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIONet *n = to_virtio_net(vdev);
+    VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))];
 
-    if (unlikely(n->tx_waiting)) {
+    if (unlikely(q->tx_waiting)) {
         return;
     }
-    n->tx_waiting = 1;
+    q->tx_waiting = 1;
     /* This happens when device was stopped but VCPU wasn't. */
     if (!n->vdev.vm_running) {
         return;
     }
     virtio_queue_set_notification(vq, 0);
-    qemu_bh_schedule(n->tx_bh);
+    qemu_bh_schedule(q->tx_bh);
 }
 
 static void virtio_net_tx_timer(void *opaque)
 {
-    VirtIONet *n = opaque;
+    VirtIONetQueue *q = opaque;
+    VirtIONet *n = q->n;
     assert(n->vdev.vm_running);
 
-    n->tx_waiting = 0;
+    q->tx_waiting = 0;
 
     /* Just in case the driver is not ready on more */
     if (!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK))
         return;
 
-    virtio_queue_set_notification(n->tx_vq, 1);
-    virtio_net_flush_tx(n, n->tx_vq);
+    virtio_queue_set_notification(q->tx_vq, 1);
+    virtio_net_flush_tx(q);
 }
 
 static void virtio_net_tx_bh(void *opaque)
 {
-    VirtIONet *n = opaque;
+    VirtIONetQueue *q = opaque;
+    VirtIONet *n = q->n;
     int32_t ret;
 
     assert(n->vdev.vm_running);
 
-    n->tx_waiting = 0;
+    q->tx_waiting = 0;
 
     /* Just in case the driver is not ready on more */
     if (unlikely(!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)))
         return;
 
-    ret = virtio_net_flush_tx(n, n->tx_vq);
+    ret = virtio_net_flush_tx(q);
     if (ret == -EBUSY) {
         return; /* Notification re-enable handled by tx_complete */
     }
@@ -850,24 +1063,61 @@ static void virtio_net_tx_bh(void *opaque)
     /* If we flush a full burst of packets, assume there are
      * more coming and immediately reschedule */
     if (ret >= n->tx_burst) {
-        qemu_bh_schedule(n->tx_bh);
-        n->tx_waiting = 1;
+        qemu_bh_schedule(q->tx_bh);
+        q->tx_waiting = 1;
         return;
     }
 
     /* If less than a full burst, re-enable notification and flush
      * anything that may have come in while we weren't looking.  If
      * we find something, assume the guest is still active and reschedule */
-    virtio_queue_set_notification(n->tx_vq, 1);
-    if (virtio_net_flush_tx(n, n->tx_vq) > 0) {
-        virtio_queue_set_notification(n->tx_vq, 0);
-        qemu_bh_schedule(n->tx_bh);
-        n->tx_waiting = 1;
+    virtio_queue_set_notification(q->tx_vq, 1);
+    if (virtio_net_flush_tx(q) > 0) {
+        virtio_queue_set_notification(q->tx_vq, 0);
+        qemu_bh_schedule(q->tx_bh);
+        q->tx_waiting = 1;
     }
 }
 
+static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue, int ctrl)
+{
+    VirtIODevice *vdev = &n->vdev;
+    int i, max = multiqueue ? n->max_queues : 1;
+
+    n->multiqueue = multiqueue;
+
+    for (i = 2; i <= n->max_queues * 2 + 1; i++) {
+        virtio_del_queue(vdev, i);
+    }
+
+    for (i = 1; i < max; i++) {
+        n->vqs[i].rx_vq = virtio_add_queue(vdev, 256, virtio_net_handle_rx);
+        if (n->vqs[i].tx_timer) {
+            n->vqs[i].tx_vq =
+                virtio_add_queue(vdev, 256, virtio_net_handle_tx_timer);
+            n->vqs[i].tx_timer = qemu_new_timer_ns(vm_clock,
+                                                   virtio_net_tx_timer,
+                                                   &n->vqs[i]);
+        } else {
+            n->vqs[i].tx_vq =
+                virtio_add_queue(vdev, 256, virtio_net_handle_tx_bh);
+            n->vqs[i].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[i]);
+        }
+
+        n->vqs[i].tx_waiting = 0;
+        n->vqs[i].n = n;
+    }
+
+    if (ctrl) {
+        n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
+    }
+
+    virtio_net_set_queues(n);
+}
+
 static void virtio_net_save(QEMUFile *f, void *opaque)
 {
+    int i;
     VirtIONet *n = opaque;
 
     /* At this point, backend must be stopped, otherwise
@@ -876,7 +1126,7 @@ static void virtio_net_save(QEMUFile *f, void *opaque)
     virtio_save(&n->vdev, f);
 
     qemu_put_buffer(f, n->mac, ETH_ALEN);
-    qemu_put_be32(f, n->tx_waiting);
+    qemu_put_be32(f, n->vqs[0].tx_waiting);
     qemu_put_be32(f, n->mergeable_rx_bufs);
     qemu_put_be16(f, n->status);
     qemu_put_byte(f, n->promisc);
@@ -892,13 +1142,19 @@ static void virtio_net_save(QEMUFile *f, void *opaque)
     qemu_put_byte(f, n->nouni);
     qemu_put_byte(f, n->nobcast);
     qemu_put_byte(f, n->has_ufo);
+    if (n->max_queues > 1) {
+        qemu_put_be16(f, n->max_queues);
+        qemu_put_be16(f, n->curr_queues);
+        for (i = 1; i < n->curr_queues; i++) {
+            qemu_put_be32(f, n->vqs[i].tx_waiting);
+        }
+    }
 }
 
 static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
 {
     VirtIONet *n = opaque;
-    int i;
-    int ret;
+    int ret, i, link_down;
 
     if (version_id < 2 || version_id > VIRTIO_NET_VM_VERSION)
         return -EINVAL;
@@ -909,7 +1165,7 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
     }
 
     qemu_get_buffer(f, n->mac, ETH_ALEN);
-    n->tx_waiting = qemu_get_be32(f);
+    n->vqs[0].tx_waiting = qemu_get_be32(f);
 
     virtio_net_set_mrg_rx_bufs(n, qemu_get_be32(f));
 
@@ -951,7 +1207,7 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         }
 
         if (n->has_vnet_hdr) {
-            tap_set_offload(n->nic->nc.peer,
+            tap_set_offload(qemu_get_queue(n->nic)->peer,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_CSUM) & 1,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_TSO4) & 1,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_TSO6) & 1,
@@ -979,6 +1235,20 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         }
     }
 
+    if (n->max_queues > 1) {
+        if (n->max_queues != qemu_get_be16(f)) {
+            error_report("virtio-net: different max_queues ");
+            return -1;
+        }
+
+        n->curr_queues = qemu_get_be16(f);
+        for (i = 1; i < n->curr_queues; i++) {
+            n->vqs[i].tx_waiting = qemu_get_be32(f);
+        }
+    }
+
+    virtio_net_set_queues(n);
+
     /* Find the first multicast entry in the saved MAC filter */
     for (i = 0; i < n->mac_table.in_use; i++) {
         if (n->mac_table.macs[i * ETH_ALEN] & 1) {
@@ -989,14 +1259,17 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
 
     /* nc.link_down can't be migrated, so infer link_down according
      * to link status bit in n->status */
-    n->nic->nc.link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0;
+    link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0;
+    for (i = 0; i < n->max_queues; i++) {
+        qemu_get_subqueue(n->nic, i)->link_down = link_down;
+    }
 
     return 0;
 }
 
 static void virtio_net_cleanup(NetClientState *nc)
 {
-    VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
+    VirtIONet *n = qemu_get_nic_opaque(nc);
 
     n->nic = NULL;
 }
@@ -1010,15 +1283,40 @@ static NetClientInfo net_virtio_info = {
     .link_status_changed = virtio_net_set_link_status,
 };
 
+static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx)
+{
+    VirtIONet *n = to_virtio_net(vdev);
+    NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx));
+    assert(n->vhost_started);
+    return vhost_net_virtqueue_pending(tap_get_vhost_net(nc->peer), idx);
+}
+
+static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx,
+                                           bool mask)
+{
+    VirtIONet *n = to_virtio_net(vdev);
+    NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx));
+    assert(n->vhost_started);
+    vhost_net_virtqueue_mask(tap_get_vhost_net(nc->peer),
+                             vdev, idx, mask);
+}
+
 VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
-                              virtio_net_conf *net)
+                              virtio_net_conf *net, uint32_t host_features)
 {
     VirtIONet *n;
+    int i, config_size = 0;
+
+    for (i = 0; feature_sizes[i].flags != 0; i++) {
+        if (host_features & feature_sizes[i].flags) {
+            config_size = MAX(feature_sizes[i].end, config_size);
+        }
+    }
 
     n = (VirtIONet *)virtio_common_init("virtio-net", VIRTIO_ID_NET,
-                                        sizeof(struct virtio_net_config),
-                                        sizeof(VirtIONet));
+                                        config_size, sizeof(VirtIONet));
 
+    n->config_size = config_size;
     n->vdev.get_config = virtio_net_get_config;
     n->vdev.set_config = virtio_net_set_config;
     n->vdev.get_features = virtio_net_get_features;
@@ -1026,7 +1324,13 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
     n->vdev.bad_features = virtio_net_bad_features;
     n->vdev.reset = virtio_net_reset;
     n->vdev.set_status = virtio_net_set_status;
-    n->rx_vq = virtio_add_queue(&n->vdev, 256, virtio_net_handle_rx);
+    n->vdev.guest_notifier_mask = virtio_net_guest_notifier_mask;
+    n->vdev.guest_notifier_pending = virtio_net_guest_notifier_pending;
+    n->vqs[0].rx_vq = virtio_add_queue(&n->vdev, 256, virtio_net_handle_rx);
+    n->max_queues = conf->queues;
+    n->curr_queues = 1;
+    n->vqs[0].n = n;
+    n->tx_timeout = net->txtimer;
 
     if (net->tx && strcmp(net->tx, "timer") && strcmp(net->tx, "bh")) {
         error_report("virtio-net: "
@@ -1036,12 +1340,14 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
     }
 
     if (net->tx && !strcmp(net->tx, "timer")) {
-        n->tx_vq = virtio_add_queue(&n->vdev, 256, virtio_net_handle_tx_timer);
-        n->tx_timer = qemu_new_timer_ns(vm_clock, virtio_net_tx_timer, n);
-        n->tx_timeout = net->txtimer;
+        n->vqs[0].tx_vq = virtio_add_queue(&n->vdev, 256,
+                                           virtio_net_handle_tx_timer);
+        n->vqs[0].tx_timer = qemu_new_timer_ns(vm_clock, virtio_net_tx_timer,
+                                               &n->vqs[0]);
     } else {
-        n->tx_vq = virtio_add_queue(&n->vdev, 256, virtio_net_handle_tx_bh);
-        n->tx_bh = qemu_bh_new(virtio_net_tx_bh, n);
+        n->vqs[0].tx_vq = virtio_add_queue(&n->vdev, 256,
+                                           virtio_net_handle_tx_bh);
+        n->vqs[0].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[0]);
     }
     n->ctrl_vq = virtio_add_queue(&n->vdev, 64, virtio_net_handle_ctrl);
     qemu_macaddr_default_if_unset(&conf->macaddr);
@@ -1051,15 +1357,17 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
     n->nic = qemu_new_nic(&net_virtio_info, conf, object_get_typename(OBJECT(dev)), dev->id, n);
     peer_test_vnet_hdr(n);
     if (peer_has_vnet_hdr(n)) {
-        tap_using_vnet_hdr(n->nic->nc.peer, 1);
+        for (i = 0; i < n->max_queues; i++) {
+            tap_using_vnet_hdr(qemu_get_subqueue(n->nic, i)->peer, true);
+        }
         n->host_hdr_len = sizeof(struct virtio_net_hdr);
     } else {
         n->host_hdr_len = 0;
     }
 
-    qemu_format_nic_info_str(&n->nic->nc, conf->macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(n->nic), conf->macaddr.a);
 
-    n->tx_waiting = 0;
+    n->vqs[0].tx_waiting = 0;
     n->tx_burst = net->txburst;
     virtio_net_set_mrg_rx_bufs(n, 0);
     n->promisc = 1; /* for compatibility */
@@ -1080,24 +1388,30 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
 void virtio_net_exit(VirtIODevice *vdev)
 {
     VirtIONet *n = DO_UPCAST(VirtIONet, vdev, vdev);
+    int i;
 
     /* This will stop vhost backend if appropriate. */
     virtio_net_set_status(vdev, 0);
 
-    qemu_purge_queued_packets(&n->nic->nc);
-
     unregister_savevm(n->qdev, "virtio-net", n);
 
     g_free(n->mac_table.macs);
     g_free(n->vlans);
 
-    if (n->tx_timer) {
-        qemu_del_timer(n->tx_timer);
-        qemu_free_timer(n->tx_timer);
-    } else {
-        qemu_bh_delete(n->tx_bh);
+    for (i = 0; i < n->max_queues; i++) {
+        VirtIONetQueue *q = &n->vqs[i];
+        NetClientState *nc = qemu_get_subqueue(n->nic, i);
+
+        qemu_purge_queued_packets(nc);
+
+        if (q->tx_timer) {
+            qemu_del_timer(q->tx_timer);
+            qemu_free_timer(q->tx_timer);
+        } else {
+            qemu_bh_delete(q->tx_bh);
+        }
     }
 
-    qemu_del_net_client(&n->nic->nc);
+    qemu_del_nic(n->nic);
     virtio_cleanup(&n->vdev);
 }
diff --git a/hw/virtio-net.h b/hw/virtio-net.h
index 36aa463..e654c13 100644
--- a/hw/virtio-net.h
+++ b/hw/virtio-net.h
@@ -15,8 +15,7 @@
 #define _QEMU_VIRTIO_NET_H
 
 #include "virtio.h"
-#include "net.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 #define ETH_ALEN    6
 
@@ -44,6 +43,10 @@
 #define VIRTIO_NET_F_CTRL_RX    18      /* Control channel RX mode support */
 #define VIRTIO_NET_F_CTRL_VLAN  19      /* Control channel VLAN filtering */
 #define VIRTIO_NET_F_CTRL_RX_EXTRA 20   /* Extra RX mode control support */
+#define VIRTIO_NET_F_MQ         22      /* Device supports Receive Flow
+                                         * Steering */
+
+#define VIRTIO_NET_F_CTRL_MAC_ADDR   23 /* Set MAC address */
 
 #define VIRTIO_NET_S_LINK_UP    1       /* Link is up */
 
@@ -72,35 +75,10 @@ struct virtio_net_config
     uint8_t mac[ETH_ALEN];
     /* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */
     uint16_t status;
+    /* Max virtqueue pairs supported by the device */
+    uint16_t max_virtqueue_pairs;
 } QEMU_PACKED;
 
-/* This is the first element of the scatter-gather list.  If you don't
- * specify GSO or CSUM features, you can simply ignore the header. */
-struct virtio_net_hdr
-{
-#define VIRTIO_NET_HDR_F_NEEDS_CSUM     1       // Use csum_start, csum_offset
-#define VIRTIO_NET_HDR_F_DATA_VALID	2	// Csum is valid
-    uint8_t flags;
-#define VIRTIO_NET_HDR_GSO_NONE         0       // Not a GSO frame
-#define VIRTIO_NET_HDR_GSO_TCPV4        1       // GSO frame, IPv4 TCP (TSO)
-#define VIRTIO_NET_HDR_GSO_UDP          3       // GSO frame, IPv4 UDP (UFO)
-#define VIRTIO_NET_HDR_GSO_TCPV6        4       // GSO frame, IPv6 TCP
-#define VIRTIO_NET_HDR_GSO_ECN          0x80    // TCP has ECN set
-    uint8_t gso_type;
-    uint16_t hdr_len;
-    uint16_t gso_size;
-    uint16_t csum_start;
-    uint16_t csum_offset;
-};
-
-/* This is the version of the header to use when the MRG_RXBUF
- * feature has been negotiated. */
-struct virtio_net_hdr_mrg_rxbuf
-{
-    struct virtio_net_hdr hdr;
-    uint16_t num_buffers;   /* Number of merged rx buffers */
-};
-
 /*
  * Control virtqueue data structures
  *
@@ -125,16 +103,16 @@ typedef uint8_t virtio_net_ctrl_ack;
  * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
  * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
  */
-#define VIRTIO_NET_CTRL_RX_MODE    0
- #define VIRTIO_NET_CTRL_RX_MODE_PROMISC      0
- #define VIRTIO_NET_CTRL_RX_MODE_ALLMULTI     1
- #define VIRTIO_NET_CTRL_RX_MODE_ALLUNI       2
- #define VIRTIO_NET_CTRL_RX_MODE_NOMULTI      3
- #define VIRTIO_NET_CTRL_RX_MODE_NOUNI        4
- #define VIRTIO_NET_CTRL_RX_MODE_NOBCAST      5
+#define VIRTIO_NET_CTRL_RX    0
+ #define VIRTIO_NET_CTRL_RX_PROMISC      0
+ #define VIRTIO_NET_CTRL_RX_ALLMULTI     1
+ #define VIRTIO_NET_CTRL_RX_ALLUNI       2
+ #define VIRTIO_NET_CTRL_RX_NOMULTI      3
+ #define VIRTIO_NET_CTRL_RX_NOUNI        4
+ #define VIRTIO_NET_CTRL_RX_NOBCAST      5
 
 /*
- * Control the MAC filter table.
+ * Control the MAC
  *
  * The MAC filter table is managed by the hypervisor, the guest should
  * assume the size is infinite.  Filtering should be considered
@@ -147,6 +125,10 @@ typedef uint8_t virtio_net_ctrl_ack;
  * first sg list contains unicast addresses, the second is for multicast.
  * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
  * is available.
+ *
+ * The ADDR_SET command requests one out scatterlist, it contains a
+ * 6 bytes MAC address. This functionality is present if the
+ * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
  */
 struct virtio_net_ctrl_mac {
     uint32_t entries;
@@ -154,6 +136,7 @@ struct virtio_net_ctrl_mac {
 };
 #define VIRTIO_NET_CTRL_MAC    1
  #define VIRTIO_NET_CTRL_MAC_TABLE_SET        0
+ #define VIRTIO_NET_CTRL_MAC_ADDR_SET         1
 
 /*
  * Control VLAN filtering
@@ -168,6 +151,26 @@ struct virtio_net_ctrl_mac {
  #define VIRTIO_NET_CTRL_VLAN_ADD             0
  #define VIRTIO_NET_CTRL_VLAN_DEL             1
 
+/*
+ * Control Multiqueue
+ *
+ * The command VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
+ * enables multiqueue, specifying the number of the transmit and
+ * receive queues that will be used. After the command is consumed and acked by
+ * the device, the device will not steer new packets on receive virtqueues
+ * other than specified nor read from transmit virtqueues other than specified.
+ * Accordingly, driver should not transmit new packets  on virtqueues other than
+ * specified.
+ */
+struct virtio_net_ctrl_mq {
+    uint16_t virtqueue_pairs;
+};
+
+#define VIRTIO_NET_CTRL_MQ   4
+ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET        0
+ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN        1
+ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX        0x8000
+
 #define DEFINE_VIRTIO_NET_FEATURES(_state, _field) \
         DEFINE_VIRTIO_COMMON_FEATURES(_state, _field), \
         DEFINE_PROP_BIT("csum", _state, _field, VIRTIO_NET_F_CSUM, true), \
@@ -186,5 +189,8 @@ struct virtio_net_ctrl_mac {
         DEFINE_PROP_BIT("ctrl_vq", _state, _field, VIRTIO_NET_F_CTRL_VQ, true), \
         DEFINE_PROP_BIT("ctrl_rx", _state, _field, VIRTIO_NET_F_CTRL_RX, true), \
         DEFINE_PROP_BIT("ctrl_vlan", _state, _field, VIRTIO_NET_F_CTRL_VLAN, true), \
-        DEFINE_PROP_BIT("ctrl_rx_extra", _state, _field, VIRTIO_NET_F_CTRL_RX_EXTRA, true)
+        DEFINE_PROP_BIT("ctrl_rx_extra", _state, _field, VIRTIO_NET_F_CTRL_RX_EXTRA, true), \
+        DEFINE_PROP_BIT("ctrl_mac_addr", _state, _field, VIRTIO_NET_F_CTRL_MAC_ADDR, true), \
+        DEFINE_PROP_BIT("mq", _state, _field, VIRTIO_NET_F_MQ, false)
+
 #endif
diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
index 71f4fb5..a869f53 100644
--- a/hw/virtio-pci.c
+++ b/hw/virtio-pci.c
@@ -22,16 +22,16 @@
 #include "virtio-net.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
-#include "pci.h"
-#include "qemu-error.h"
-#include "msi.h"
-#include "msix.h"
-#include "net.h"
+#include "pci/pci.h"
+#include "qemu/error-report.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
 #include "loader.h"
-#include "kvm.h"
-#include "blockdev.h"
+#include "sysemu/kvm.h"
+#include "sysemu/blockdev.h"
 #include "virtio-pci.h"
-#include "range.h"
+#include "qemu/range.h"
+#include "virtio-bus.h"
 
 /* from Linux's linux/virtio_pci.h */
 
@@ -97,35 +97,48 @@
 bool virtio_is_big_endian(void);
 
 /* virtio device */
+/* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
+static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d)
+{
+    return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
+}
 
-static void virtio_pci_notify(void *opaque, uint16_t vector)
+/* DeviceState to VirtIOPCIProxy. Note: used on datapath,
+ * be careful and test performance if you change this.
+ */
+static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
+}
+
+static void virtio_pci_notify(DeviceState *d, uint16_t vector)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d);
     if (msix_enabled(&proxy->pci_dev))
         msix_notify(&proxy->pci_dev, vector);
     else
         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 }
 
-static void virtio_pci_save_config(void * opaque, QEMUFile *f)
+static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     pci_device_save(&proxy->pci_dev, f);
     msix_save(&proxy->pci_dev, f);
     if (msix_present(&proxy->pci_dev))
         qemu_put_be16(f, proxy->vdev->config_vector);
 }
 
-static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
+static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     if (msix_present(&proxy->pci_dev))
         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 }
 
-static int virtio_pci_load_config(void * opaque, QEMUFile *f)
+static int virtio_pci_load_config(DeviceState *d, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     int ret;
     ret = pci_device_load(&proxy->pci_dev, f);
     if (ret) {
@@ -144,9 +157,9 @@ static int virtio_pci_load_config(void * opaque, QEMUFile *f)
     return 0;
 }
 
-static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
+static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     uint16_t vector;
     if (msix_present(&proxy->pci_dev)) {
         qemu_get_be16s(f, &vector);
@@ -244,7 +257,7 @@ static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 
 void virtio_pci_reset(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     virtio_pci_stop_ioeventfd(proxy);
     virtio_reset(proxy->vdev);
     msix_unuse_all_vectors(&proxy->pci_dev);
@@ -464,9 +477,9 @@ static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
     }
 }
 
-static unsigned virtio_pci_get_features(void *opaque)
+static unsigned virtio_pci_get_features(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     return proxy->host_features;
 }
 
@@ -475,8 +488,6 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
                                         unsigned int vector,
                                         MSIMessage msg)
 {
-    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
-    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
     int ret;
 
@@ -488,23 +499,34 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
         irqfd->virq = ret;
     }
     irqfd->users++;
-
-    ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, irqfd->virq);
-    if (ret < 0) {
-        if (--irqfd->users == 0) {
-            kvm_irqchip_release_virq(kvm_state, irqfd->virq);
-        }
-        return ret;
-    }
-
-    virtio_queue_set_guest_notifier_fd_handler(vq, true, true);
     return 0;
 }
 
 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
-                                             unsigned int queue_no,
                                              unsigned int vector)
 {
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    if (--irqfd->users == 0) {
+        kvm_irqchip_release_virq(kvm_state, irqfd->virq);
+    }
+}
+
+static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy,
+                                 unsigned int queue_no,
+                                 unsigned int vector)
+{
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
+    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
+    int ret;
+    ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, irqfd->virq);
+    return ret;
+}
+
+static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
+                                      unsigned int queue_no,
+                                      unsigned int vector)
+{
     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
@@ -512,29 +534,143 @@ static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
 
     ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, n, irqfd->virq);
     assert(ret == 0);
+}
 
-    if (--irqfd->users == 0) {
-        kvm_irqchip_release_virq(kvm_state, irqfd->virq);
+static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs)
+{
+    PCIDevice *dev = &proxy->pci_dev;
+    VirtIODevice *vdev = proxy->vdev;
+    unsigned int vector;
+    int ret, queue_no;
+    MSIMessage msg;
+
+    for (queue_no = 0; queue_no < nvqs; queue_no++) {
+        if (!virtio_queue_get_num(vdev, queue_no)) {
+            break;
+        }
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector >= msix_nr_vectors_allocated(dev)) {
+            continue;
+        }
+        msg = msix_get_message(dev, vector);
+        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
+        if (ret < 0) {
+            goto undo;
+        }
+        /* If guest supports masking, set up irqfd now.
+         * Otherwise, delay until unmasked in the frontend.
+         */
+        if (proxy->vdev->guest_notifier_mask) {
+            ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
+            if (ret < 0) {
+                kvm_virtio_pci_vq_vector_release(proxy, vector);
+                goto undo;
+            }
+        }
     }
+    return 0;
 
-    virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
+undo:
+    while (--queue_no >= 0) {
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector >= msix_nr_vectors_allocated(dev)) {
+            continue;
+        }
+        if (proxy->vdev->guest_notifier_mask) {
+            kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
+        }
+        kvm_virtio_pci_vq_vector_release(proxy, vector);
+    }
+    return ret;
 }
 
-static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector,
+static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
+{
+    PCIDevice *dev = &proxy->pci_dev;
+    VirtIODevice *vdev = proxy->vdev;
+    unsigned int vector;
+    int queue_no;
+
+    for (queue_no = 0; queue_no < nvqs; queue_no++) {
+        if (!virtio_queue_get_num(vdev, queue_no)) {
+            break;
+        }
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector >= msix_nr_vectors_allocated(dev)) {
+            continue;
+        }
+        /* If guest supports masking, clean up irqfd now.
+         * Otherwise, it was cleaned when masked in the frontend.
+         */
+        if (proxy->vdev->guest_notifier_mask) {
+            kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
+        }
+        kvm_virtio_pci_vq_vector_release(proxy, vector);
+    }
+}
+
+static int kvm_virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
+                                        unsigned int queue_no,
+                                        unsigned int vector,
+                                        MSIMessage msg)
+{
+    VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
+    EventNotifier *n = virtio_queue_get_guest_notifier(vq);
+    VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
+    int ret = 0;
+
+    if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
+        ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    /* If guest supports masking, irqfd is already setup, unmask it.
+     * Otherwise, set it up now.
+     */
+    if (proxy->vdev->guest_notifier_mask) {
+        proxy->vdev->guest_notifier_mask(proxy->vdev, queue_no, false);
+        /* Test after unmasking to avoid losing events. */
+        if (proxy->vdev->guest_notifier_pending &&
+            proxy->vdev->guest_notifier_pending(proxy->vdev, queue_no)) {
+            event_notifier_set(n);
+        }
+    } else {
+        ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
+    }
+    return ret;
+}
+
+static void kvm_virtio_pci_vq_vector_mask(VirtIOPCIProxy *proxy,
+                                             unsigned int queue_no,
+                                             unsigned int vector)
+{
+    /* If guest supports masking, keep irqfd but mask it.
+     * Otherwise, clean it up now.
+     */ 
+    if (proxy->vdev->guest_notifier_mask) {
+        proxy->vdev->guest_notifier_mask(proxy->vdev, queue_no, true);
+    } else {
+        kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
+    }
+}
+
+static int kvm_virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector,
                                      MSIMessage msg)
 {
     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
     VirtIODevice *vdev = proxy->vdev;
     int ret, queue_no;
 
-    for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
+    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
         if (!virtio_queue_get_num(vdev, queue_no)) {
             break;
         }
         if (virtio_queue_vector(vdev, queue_no) != vector) {
             continue;
         }
-        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
+        ret = kvm_virtio_pci_vq_vector_unmask(proxy, queue_no, vector, msg);
         if (ret < 0) {
             goto undo;
         }
@@ -546,31 +682,64 @@ undo:
         if (virtio_queue_vector(vdev, queue_no) != vector) {
             continue;
         }
-        kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
+        kvm_virtio_pci_vq_vector_mask(proxy, queue_no, vector);
     }
     return ret;
 }
 
-static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector)
+static void kvm_virtio_pci_vector_mask(PCIDevice *dev, unsigned vector)
 {
     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
     VirtIODevice *vdev = proxy->vdev;
     int queue_no;
 
-    for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
+    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
         if (!virtio_queue_get_num(vdev, queue_no)) {
             break;
         }
         if (virtio_queue_vector(vdev, queue_no) != vector) {
             continue;
         }
-        kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
+        kvm_virtio_pci_vq_vector_mask(proxy, queue_no, vector);
     }
 }
 
-static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
+static void kvm_virtio_pci_vector_poll(PCIDevice *dev,
+                                       unsigned int vector_start,
+                                       unsigned int vector_end)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
+    VirtIODevice *vdev = proxy->vdev;
+    int queue_no;
+    unsigned int vector;
+    EventNotifier *notifier;
+    VirtQueue *vq;
+
+    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
+        if (!virtio_queue_get_num(vdev, queue_no)) {
+            break;
+        }
+        vector = virtio_queue_vector(vdev, queue_no);
+        if (vector < vector_start || vector >= vector_end ||
+            !msix_is_masked(dev, vector)) {
+            continue;
+        }
+        vq = virtio_get_queue(vdev, queue_no);
+        notifier = virtio_queue_get_guest_notifier(vq);
+        if (vdev->guest_notifier_pending) {
+            if (vdev->guest_notifier_pending(vdev, queue_no)) {
+                msix_set_pending(dev, vector);
+            }
+        } else if (event_notifier_test_and_clear(notifier)) {
+            msix_set_pending(dev, vector);
+        }
+    }
+}
+
+static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
+                                         bool with_irqfd)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 
@@ -579,72 +748,94 @@ static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
         if (r < 0) {
             return r;
         }
-        virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
+        virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
     } else {
-        virtio_queue_set_guest_notifier_fd_handler(vq, false, false);
+        virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
         event_notifier_cleanup(notifier);
     }
 
     return 0;
 }
 
-static bool virtio_pci_query_guest_notifiers(void *opaque)
+static bool virtio_pci_query_guest_notifiers(DeviceState *d)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     return msix_enabled(&proxy->pci_dev);
 }
 
-static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
+static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     VirtIODevice *vdev = proxy->vdev;
     int r, n;
+    bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
+        kvm_msi_via_irqfd_enabled();
+
+    nvqs = MIN(nvqs, VIRTIO_PCI_QUEUE_MAX);
+
+    /* When deassigning, pass a consistent nvqs value
+     * to avoid leaking notifiers.
+     */
+    assert(assign || nvqs == proxy->nvqs_with_notifiers);
+
+    proxy->nvqs_with_notifiers = nvqs;
 
     /* Must unset vector notifier while guest notifier is still assigned */
-    if (kvm_msi_via_irqfd_enabled() && !assign) {
+    if (proxy->vector_irqfd && !assign) {
         msix_unset_vector_notifiers(&proxy->pci_dev);
+        kvm_virtio_pci_vector_release(proxy, nvqs);
         g_free(proxy->vector_irqfd);
         proxy->vector_irqfd = NULL;
     }
 
-    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
+    for (n = 0; n < nvqs; n++) {
         if (!virtio_queue_get_num(vdev, n)) {
             break;
         }
 
-        r = virtio_pci_set_guest_notifier(opaque, n, assign);
+        r = virtio_pci_set_guest_notifier(d, n, assign,
+                                          kvm_msi_via_irqfd_enabled());
         if (r < 0) {
             goto assign_error;
         }
     }
 
     /* Must set vector notifier after guest notifier has been assigned */
-    if (kvm_msi_via_irqfd_enabled() && assign) {
+    if (with_irqfd && assign) {
         proxy->vector_irqfd =
             g_malloc0(sizeof(*proxy->vector_irqfd) *
                       msix_nr_vectors_allocated(&proxy->pci_dev));
-        r = msix_set_vector_notifiers(&proxy->pci_dev,
-                                      kvm_virtio_pci_vector_use,
-                                      kvm_virtio_pci_vector_release);
+        r = kvm_virtio_pci_vector_use(proxy, nvqs);
         if (r < 0) {
             goto assign_error;
         }
+        r = msix_set_vector_notifiers(&proxy->pci_dev,
+                                      kvm_virtio_pci_vector_unmask,
+                                      kvm_virtio_pci_vector_mask,
+                                      kvm_virtio_pci_vector_poll);
+        if (r < 0) {
+            goto notifiers_error;
+        }
     }
 
     return 0;
 
+notifiers_error:
+    assert(assign);
+    kvm_virtio_pci_vector_release(proxy, nvqs);
+
 assign_error:
     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
     assert(assign);
     while (--n >= 0) {
-        virtio_pci_set_guest_notifier(opaque, n, !assign);
+        virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd);
     }
     return r;
 }
 
-static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
+static int virtio_pci_set_host_notifier(DeviceState *d, int n, bool assign)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 
     /* Stop using ioeventfd for virtqueue kick if the device starts using host
      * notifiers.  This makes it easy to avoid stepping on each others' toes.
@@ -660,9 +851,9 @@ static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
     return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
 }
 
-static void virtio_pci_vmstate_change(void *opaque, bool running)
+static void virtio_pci_vmstate_change(DeviceState *d, bool running)
 {
-    VirtIOPCIProxy *proxy = opaque;
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 
     if (running) {
         /* Try to find out if the guest has bus master disabled, but is
@@ -727,7 +918,7 @@ void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev)
         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
     }
 
-    virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
+    virtio_bind_device(vdev, &virtio_pci_bindings, DEVICE(proxy));
     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
@@ -806,7 +997,8 @@ static int virtio_net_init_pci(PCIDevice *pci_dev)
     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
     VirtIODevice *vdev;
 
-    vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
+    vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net,
+                           proxy->host_features);
 
     vdev->nvectors = proxy->nvectors;
     virtio_init_pci(proxy, vdev);
@@ -897,6 +1089,9 @@ static Property virtio_blk_properties[] = {
 #endif
     DEFINE_PROP_BIT("config-wce", VirtIOPCIProxy, blk.config_wce, 0, true),
     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
+#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
+    DEFINE_PROP_BIT("x-data-plane", VirtIOPCIProxy, blk.data_plane, 0, false),
+#endif
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
     DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
     DEFINE_PROP_END_OF_LIST(),
@@ -917,7 +1112,7 @@ static void virtio_blk_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_blk_properties;
 }
 
-static TypeInfo virtio_blk_info = {
+static const TypeInfo virtio_blk_info = {
     .name          = "virtio-blk-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -951,7 +1146,7 @@ static void virtio_net_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_net_properties;
 }
 
-static TypeInfo virtio_net_info = {
+static const TypeInfo virtio_net_info = {
     .name          = "virtio-net-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -982,7 +1177,7 @@ static void virtio_serial_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_serial_properties;
 }
 
-static TypeInfo virtio_serial_info = {
+static const TypeInfo virtio_serial_info = {
     .name          = "virtio-serial-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -1010,7 +1205,7 @@ static void virtio_balloon_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_balloon_properties;
 }
 
-static TypeInfo virtio_balloon_info = {
+static const TypeInfo virtio_balloon_info = {
     .name          = "virtio-balloon-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -1053,7 +1248,7 @@ static void virtio_rng_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_rng_properties;
 }
 
-static TypeInfo virtio_rng_info = {
+static const TypeInfo virtio_rng_info = {
     .name          = "virtio-rng-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
@@ -1111,13 +1306,165 @@ static void virtio_scsi_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_scsi_properties;
 }
 
-static TypeInfo virtio_scsi_info = {
+static const TypeInfo virtio_scsi_info = {
     .name          = "virtio-scsi-pci",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VirtIOPCIProxy),
     .class_init    = virtio_scsi_class_init,
 };
 
+/*
+ * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
+ */
+
+/* This is called by virtio-bus just after the device is plugged. */
+static void virtio_pci_device_plugged(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtioBusState *bus = &proxy->bus;
+    uint8_t *config;
+    uint32_t size;
+
+    proxy->vdev = bus->vdev;
+
+    config = proxy->pci_dev.config;
+    if (proxy->class_code) {
+        pci_config_set_class(config, proxy->class_code);
+    }
+    pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
+                 pci_get_word(config + PCI_VENDOR_ID));
+    pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
+    config[PCI_INTERRUPT_PIN] = 1;
+
+    if (proxy->nvectors &&
+        msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors, 1)) {
+        proxy->nvectors = 0;
+    }
+
+    proxy->pci_dev.config_write = virtio_write_config;
+
+    size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
+         + virtio_bus_get_vdev_config_len(bus);
+    if (size & (size - 1)) {
+        size = 1 << qemu_fls(size);
+    }
+
+    memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy,
+                          "virtio-pci", size);
+    pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
+                     &proxy->bar);
+
+    if (!kvm_has_many_ioeventfds()) {
+        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
+    }
+
+    proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
+    proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
+    proxy->host_features = virtio_bus_get_vdev_features(bus,
+                                                      proxy->host_features);
+}
+
+/* This is called by virtio-bus just before the device is unplugged. */
+static void virtio_pci_device_unplug(DeviceState *d)
+{
+    VirtIOPCIProxy *dev = VIRTIO_PCI(d);
+    virtio_pci_stop_ioeventfd(dev);
+}
+
+static int virtio_pci_init(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *dev = VIRTIO_PCI(pci_dev);
+    VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
+    virtio_pci_bus_new(&dev->bus, dev);
+    if (k->init != NULL) {
+        return k->init(dev);
+    }
+    return 0;
+}
+
+static void virtio_pci_exit(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
+    VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    BusState *qbus = BUS(&proxy->bus);
+    virtio_bus_destroy_device(bus);
+    qbus_free(qbus);
+    virtio_exit_pci(pci_dev);
+}
+
+/*
+ * This will be renamed virtio_pci_reset at the end of the series.
+ * virtio_pci_reset is still in use at this moment.
+ */
+static void virtio_pci_rst(DeviceState *qdev)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
+    VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    virtio_pci_stop_ioeventfd(proxy);
+    virtio_bus_reset(bus);
+    msix_unuse_all_vectors(&proxy->pci_dev);
+    proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
+}
+
+static void virtio_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = virtio_pci_init;
+    k->exit = virtio_pci_exit;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
+    k->revision = VIRTIO_PCI_ABI_VERSION;
+    k->class_id = PCI_CLASS_OTHERS;
+    dc->reset = virtio_pci_rst;
+}
+
+static const TypeInfo virtio_pci_info = {
+    .name          = TYPE_VIRTIO_PCI,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VirtIOPCIProxy),
+    .class_init    = virtio_pci_class_init,
+    .class_size    = sizeof(VirtioPCIClass),
+    .abstract      = true,
+};
+
+/* virtio-pci-bus */
+
+void virtio_pci_bus_new(VirtioBusState *bus, VirtIOPCIProxy *dev)
+{
+    DeviceState *qdev = DEVICE(dev);
+    BusState *qbus;
+    qbus_create_inplace((BusState *)bus, TYPE_VIRTIO_PCI_BUS, qdev, NULL);
+    qbus = BUS(bus);
+    qbus->allow_hotplug = 0;
+}
+
+static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
+{
+    BusClass *bus_class = BUS_CLASS(klass);
+    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
+    bus_class->max_dev = 1;
+    k->notify = virtio_pci_notify;
+    k->save_config = virtio_pci_save_config;
+    k->load_config = virtio_pci_load_config;
+    k->save_queue = virtio_pci_save_queue;
+    k->load_queue = virtio_pci_load_queue;
+    k->get_features = virtio_pci_get_features;
+    k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
+    k->set_host_notifier = virtio_pci_set_host_notifier;
+    k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
+    k->vmstate_change = virtio_pci_vmstate_change;
+    k->device_plugged = virtio_pci_device_plugged;
+    k->device_unplug = virtio_pci_device_unplug;
+}
+
+static const TypeInfo virtio_pci_bus_info = {
+    .name          = TYPE_VIRTIO_PCI_BUS,
+    .parent        = TYPE_VIRTIO_BUS,
+    .instance_size = sizeof(VirtioPCIBusState),
+    .class_init    = virtio_pci_bus_class_init,
+};
+
 static void virtio_pci_register_types(void)
 {
     type_register_static(&virtio_blk_info);
@@ -1126,6 +1473,8 @@ static void virtio_pci_register_types(void)
     type_register_static(&virtio_balloon_info);
     type_register_static(&virtio_scsi_info);
     type_register_static(&virtio_rng_info);
+    type_register_static(&virtio_pci_bus_info);
+    type_register_static(&virtio_pci_info);
 }
 
 type_init(virtio_pci_register_types)
diff --git a/hw/virtio-pci.h b/hw/virtio-pci.h
index b58d9a2..d24957c 100644
--- a/hw/virtio-pci.h
+++ b/hw/virtio-pci.h
@@ -15,11 +15,28 @@
 #ifndef QEMU_VIRTIO_PCI_H
 #define QEMU_VIRTIO_PCI_H
 
+#include "hw/pci/msi.h"
 #include "virtio-blk.h"
 #include "virtio-net.h"
 #include "virtio-rng.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
+#include "virtio-bus.h"
+
+typedef struct VirtIOPCIProxy VirtIOPCIProxy;
+
+/* virtio-pci-bus */
+
+typedef struct VirtioBusState VirtioPCIBusState;
+typedef struct VirtioBusClass VirtioPCIBusClass;
+
+#define TYPE_VIRTIO_PCI_BUS "virtio-pci-bus"
+#define VIRTIO_PCI_BUS(obj) \
+        OBJECT_CHECK(VirtioPCIBusState, (obj), TYPE_VIRTIO_PCI_BUS)
+#define VIRTIO_PCI_BUS_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioPCIBusClass, obj, TYPE_VIRTIO_PCI_BUS)
+#define VIRTIO_PCI_BUS_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
 
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
@@ -27,11 +44,28 @@
 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
 
 typedef struct {
+    MSIMessage msg;
     int virq;
     unsigned int users;
 } VirtIOIRQFD;
 
-typedef struct {
+/*
+ * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
+ */
+#define TYPE_VIRTIO_PCI "virtio-pci"
+#define VIRTIO_PCI_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioPCIClass, obj, TYPE_VIRTIO_PCI)
+#define VIRTIO_PCI_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioPCIClass, klass, TYPE_VIRTIO_PCI)
+#define VIRTIO_PCI(obj) \
+        OBJECT_CHECK(VirtIOPCIProxy, (obj), TYPE_VIRTIO_PCI)
+
+typedef struct VirtioPCIClass {
+    PCIDeviceClass parent_class;
+    int (*init)(VirtIOPCIProxy *vpci_dev);
+} VirtioPCIClass;
+
+struct VirtIOPCIProxy {
     PCIDevice pci_dev;
     VirtIODevice *vdev;
     MemoryRegion bar;
@@ -51,10 +85,13 @@ typedef struct {
     bool ioeventfd_disabled;
     bool ioeventfd_started;
     VirtIOIRQFD *vector_irqfd;
-} VirtIOPCIProxy;
+    int nvqs_with_notifiers;
+    VirtioBusState bus;
+};
 
 void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev);
 void virtio_pci_reset(DeviceState *d);
+void virtio_pci_bus_new(VirtioBusState *bus, VirtIOPCIProxy *dev);
 
 /* Virtio ABI version, if we increment this, we break the guest driver. */
 #define VIRTIO_PCI_ABI_VERSION          0
diff --git a/hw/virtio-rng.c b/hw/virtio-rng.c
index a73ef8e..e063127 100644
--- a/hw/virtio-rng.c
+++ b/hw/virtio-rng.c
@@ -9,7 +9,7 @@
  * top-level directory.
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
 #include "qdev.h"
 #include "virtio.h"
 #include "virtio-rng.h"
diff --git a/hw/virtio-scsi.c b/hw/virtio-scsi.c
index bfe1860..0715865 100644
--- a/hw/virtio-scsi.c
+++ b/hw/virtio-scsi.c
@@ -565,6 +565,10 @@ static void virtio_scsi_reset(VirtIODevice *vdev)
 {
     VirtIOSCSI *s = (VirtIOSCSI *)vdev;
 
+    s->resetting++;
+    qbus_reset_all(&s->bus.qbus);
+    s->resetting--;
+
     s->sense_size = VIRTIO_SCSI_SENSE_SIZE;
     s->cdb_size = VIRTIO_SCSI_CDB_SIZE;
     s->events_dropped = false;
diff --git a/hw/virtio-scsi.h b/hw/virtio-scsi.h
index 91924f6..8d9d15f 100644
--- a/hw/virtio-scsi.h
+++ b/hw/virtio-scsi.h
@@ -15,8 +15,7 @@
 #define _QEMU_VIRTIO_SCSI_H
 
 #include "virtio.h"
-#include "net.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 /* The ID for virtio_scsi */
 #define VIRTIO_ID_SCSI  8
diff --git a/hw/virtio-serial-bus.c b/hw/virtio-serial-bus.c
index 155da58..aa7d0d7 100644
--- a/hw/virtio-serial-bus.c
+++ b/hw/virtio-serial-bus.c
@@ -18,9 +18,9 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "iov.h"
-#include "monitor.h"
-#include "qemu-queue.h"
+#include "qemu/iov.h"
+#include "monitor/monitor.h"
+#include "qemu/queue.h"
 #include "sysbus.h"
 #include "trace.h"
 #include "virtio-serial.h"
@@ -36,6 +36,15 @@ struct VirtIOSerialBus {
     uint32_t max_nr_ports;
 };
 
+typedef struct VirtIOSerialPostLoad {
+    QEMUTimer *timer;
+    uint32_t nr_active_ports;
+    struct {
+        VirtIOSerialPort *port;
+        uint8_t host_connected;
+    } *connected;
+} VirtIOSerialPostLoad;
+
 struct VirtIOSerial {
     VirtIODevice vdev;
 
@@ -54,14 +63,7 @@ struct VirtIOSerial {
 
     struct virtio_console_config config;
 
-    struct {
-        QEMUTimer *timer;
-        int nr_active_ports;
-        struct {
-            VirtIOSerialPort *port;
-            uint8_t host_connected;
-        } *connected;
-    } post_load;
+    struct VirtIOSerialPostLoad *post_load;
 };
 
 static VirtIOSerialPort *find_port_by_id(VirtIOSerial *vser, uint32_t id)
@@ -215,13 +217,12 @@ static void flush_queued_data(VirtIOSerialPort *port)
     do_flush_queued_data(port, port->ovq, &port->vser->vdev);
 }
 
-static size_t send_control_msg(VirtIOSerialPort *port, void *buf, size_t len)
+static size_t send_control_msg(VirtIOSerial *vser, void *buf, size_t len)
 {
     VirtQueueElement elem;
     VirtQueue *vq;
-    struct virtio_console_control *cpkt;
 
-    vq = port->vser->c_ivq;
+    vq = vser->c_ivq;
     if (!virtio_queue_ready(vq)) {
         return 0;
     }
@@ -229,25 +230,24 @@ static size_t send_control_msg(VirtIOSerialPort *port, void *buf, size_t len)
         return 0;
     }
 
-    cpkt = (struct virtio_console_control *)buf;
-    stl_p(&cpkt->id, port->id);
     memcpy(elem.in_sg[0].iov_base, buf, len);
 
     virtqueue_push(vq, &elem, len);
-    virtio_notify(&port->vser->vdev, vq);
+    virtio_notify(&vser->vdev, vq);
     return len;
 }
 
-static size_t send_control_event(VirtIOSerialPort *port, uint16_t event,
-                                 uint16_t value)
+static size_t send_control_event(VirtIOSerial *vser, uint32_t port_id,
+                                 uint16_t event, uint16_t value)
 {
     struct virtio_console_control cpkt;
 
+    stl_p(&cpkt.id, port_id);
     stw_p(&cpkt.event, event);
     stw_p(&cpkt.value, value);
 
-    trace_virtio_serial_send_control_event(port->id, event, value);
-    return send_control_msg(port, &cpkt, sizeof(cpkt));
+    trace_virtio_serial_send_control_event(port_id, event, value);
+    return send_control_msg(vser, &cpkt, sizeof(cpkt));
 }
 
 /* Functions for use inside qemu to open and read from/write to ports */
@@ -259,7 +259,7 @@ int virtio_serial_open(VirtIOSerialPort *port)
     }
     /* Send port open notification to the guest */
     port->host_connected = true;
-    send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
+    send_control_event(port->vser, port->id, VIRTIO_CONSOLE_PORT_OPEN, 1);
 
     return 0;
 }
@@ -274,7 +274,7 @@ int virtio_serial_close(VirtIOSerialPort *port)
     port->throttled = false;
     discard_vq_data(port->ovq, &port->vser->vdev);
 
-    send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
+    send_control_event(port->vser, port->id, VIRTIO_CONSOLE_PORT_OPEN, 0);
 
     return 0;
 }
@@ -363,7 +363,7 @@ static void handle_control_message(VirtIOSerial *vser, void *buf, size_t len)
          * ports we have here.
          */
         QTAILQ_FOREACH(port, &vser->ports, next) {
-            send_control_event(port, VIRTIO_CONSOLE_PORT_ADD, 1);
+            send_control_event(vser, port->id, VIRTIO_CONSOLE_PORT_ADD, 1);
         }
         return;
     }
@@ -394,10 +394,11 @@ static void handle_control_message(VirtIOSerial *vser, void *buf, size_t len)
          * up to hvc.
          */
         if (vsc->is_console) {
-            send_control_event(port, VIRTIO_CONSOLE_CONSOLE_PORT, 1);
+            send_control_event(vser, port->id, VIRTIO_CONSOLE_CONSOLE_PORT, 1);
         }
 
         if (port->name) {
+            stl_p(&cpkt.id, port->id);
             stw_p(&cpkt.event, VIRTIO_CONSOLE_PORT_NAME);
             stw_p(&cpkt.value, 1);
 
@@ -408,12 +409,12 @@ static void handle_control_message(VirtIOSerial *vser, void *buf, size_t len)
             memcpy(buffer + sizeof(cpkt), port->name, strlen(port->name));
             buffer[buffer_len - 1] = 0;
 
-            send_control_msg(port, buffer, buffer_len);
+            send_control_msg(vser, buffer, buffer_len);
             g_free(buffer);
         }
 
         if (port->host_connected) {
-            send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
+            send_control_event(vser, port->id, VIRTIO_CONSOLE_PORT_OPEN, 1);
         }
 
         /*
@@ -637,31 +638,91 @@ static void virtio_serial_save(QEMUFile *f, void *opaque)
 
 static void virtio_serial_post_load_timer_cb(void *opaque)
 {
-    int i;
+    uint32_t i;
     VirtIOSerial *s = opaque;
     VirtIOSerialPort *port;
     uint8_t host_connected;
 
-    for (i = 0 ; i < s->post_load.nr_active_ports; ++i) {
-        port = s->post_load.connected[i].port;
-        host_connected = s->post_load.connected[i].host_connected;
+    if (!s->post_load) {
+        return;
+    }
+    for (i = 0 ; i < s->post_load->nr_active_ports; ++i) {
+        port = s->post_load->connected[i].port;
+        host_connected = s->post_load->connected[i].host_connected;
         if (host_connected != port->host_connected) {
             /*
              * We have to let the guest know of the host connection
              * status change
              */
-            send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN,
+            send_control_event(s, port->id, VIRTIO_CONSOLE_PORT_OPEN,
                                port->host_connected);
         }
     }
-    g_free(s->post_load.connected);
-    s->post_load.connected = NULL;
+    g_free(s->post_load->connected);
+    qemu_free_timer(s->post_load->timer);
+    g_free(s->post_load);
+    s->post_load = NULL;
+}
+
+static int fetch_active_ports_list(QEMUFile *f, int version_id,
+                                   VirtIOSerial *s, uint32_t nr_active_ports)
+{
+    uint32_t i;
+
+    s->post_load = g_malloc0(sizeof(*s->post_load));
+    s->post_load->nr_active_ports = nr_active_ports;
+    s->post_load->connected =
+        g_malloc0(sizeof(*s->post_load->connected) * nr_active_ports);
+
+    s->post_load->timer = qemu_new_timer_ns(vm_clock,
+                                            virtio_serial_post_load_timer_cb,
+                                            s);
+
+    /* Items in struct VirtIOSerialPort */
+    for (i = 0; i < nr_active_ports; i++) {
+        VirtIOSerialPort *port;
+        uint32_t id;
+
+        id = qemu_get_be32(f);
+        port = find_port_by_id(s, id);
+        if (!port) {
+            return -EINVAL;
+        }
+
+        port->guest_connected = qemu_get_byte(f);
+        s->post_load->connected[i].port = port;
+        s->post_load->connected[i].host_connected = qemu_get_byte(f);
+
+        if (version_id > 2) {
+            uint32_t elem_popped;
+
+            qemu_get_be32s(f, &elem_popped);
+            if (elem_popped) {
+                qemu_get_be32s(f, &port->iov_idx);
+                qemu_get_be64s(f, &port->iov_offset);
+
+                qemu_get_buffer(f, (unsigned char *)&port->elem,
+                                sizeof(port->elem));
+                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
+                                 port->elem.in_num, 1);
+                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
+                                 port->elem.out_num, 1);
+
+                /*
+                 *  Port was throttled on source machine.  Let's
+                 *  unthrottle it here so data starts flowing again.
+                 */
+                virtio_serial_throttle_port(port, false);
+            }
+        }
+    }
+    qemu_mod_timer(s->post_load->timer, 1);
+    return 0;
 }
 
 static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
 {
     VirtIOSerial *s = opaque;
-    VirtIOSerialPort *port;
     uint32_t max_nr_ports, nr_active_ports, ports_map;
     unsigned int i;
     int ret;
@@ -705,48 +766,12 @@ static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
 
     qemu_get_be32s(f, &nr_active_ports);
 
-    s->post_load.nr_active_ports = nr_active_ports;
-    s->post_load.connected =
-        g_malloc0(sizeof(*s->post_load.connected) * nr_active_ports);
-
-    /* Items in struct VirtIOSerialPort */
-    for (i = 0; i < nr_active_ports; i++) {
-        uint32_t id;
-
-        id = qemu_get_be32(f);
-        port = find_port_by_id(s, id);
-        if (!port) {
-            return -EINVAL;
-        }
-
-        port->guest_connected = qemu_get_byte(f);
-        s->post_load.connected[i].port = port;
-        s->post_load.connected[i].host_connected = qemu_get_byte(f);
-
-        if (version_id > 2) {
-            uint32_t elem_popped;
-
-            qemu_get_be32s(f, &elem_popped);
-            if (elem_popped) {
-                qemu_get_be32s(f, &port->iov_idx);
-                qemu_get_be64s(f, &port->iov_offset);
-
-                qemu_get_buffer(f, (unsigned char *)&port->elem,
-                                sizeof(port->elem));
-                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
-                                 port->elem.in_num, 1);
-                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
-                                 port->elem.out_num, 1);
-
-                /*
-                 *  Port was throttled on source machine.  Let's
-                 *  unthrottle it here so data starts flowing again.
-                 */
-                virtio_serial_throttle_port(port, false);
-            }
+    if (nr_active_ports) {
+        ret = fetch_active_ports_list(f, version_id, s, nr_active_ports);
+        if (ret) {
+            return ret;
         }
     }
-    qemu_mod_timer(s->post_load.timer, 1);
     return 0;
 }
 
@@ -815,9 +840,7 @@ static void mark_port_added(VirtIOSerial *vser, uint32_t port_id)
 static void add_port(VirtIOSerial *vser, uint32_t port_id)
 {
     mark_port_added(vser, port_id);
-
-    send_control_event(find_port_by_id(vser, port_id),
-                       VIRTIO_CONSOLE_PORT_ADD, 1);
+    send_control_event(vser, port_id, VIRTIO_CONSOLE_PORT_ADD, 1);
 }
 
 static void remove_port(VirtIOSerial *vser, uint32_t port_id)
@@ -829,10 +852,16 @@ static void remove_port(VirtIOSerial *vser, uint32_t port_id)
     vser->ports_map[i] &= ~(1U << (port_id % 32));
 
     port = find_port_by_id(vser, port_id);
+    /*
+     * This function is only called from qdev's unplug callback; if we
+     * get a NULL port here, we're in trouble.
+     */
+    assert(port);
+
     /* Flush out any unconsumed buffers first */
     discard_vq_data(port->ovq, &port->vser->vdev);
 
-    send_control_event(port, VIRTIO_CONSOLE_PORT_REMOVE, 1);
+    send_control_event(vser, port->id, VIRTIO_CONSOLE_PORT_REMOVE, 1);
 }
 
 static int virtser_port_qdev_init(DeviceState *qdev)
@@ -989,6 +1018,8 @@ VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *conf)
 
     vser->qdev = dev;
 
+    vser->post_load = NULL;
+
     /*
      * Register for the savevm section with the virtio-console name
      * to preserve backward compat
@@ -996,9 +1027,6 @@ VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *conf)
     register_savevm(dev, "virtio-console", -1, 3, virtio_serial_save,
                     virtio_serial_load, vser);
 
-    vser->post_load.timer = qemu_new_timer_ns(vm_clock,
-            virtio_serial_post_load_timer_cb, vser);
-
     return vdev;
 }
 
@@ -1011,9 +1039,12 @@ void virtio_serial_exit(VirtIODevice *vdev)
     g_free(vser->ivqs);
     g_free(vser->ovqs);
     g_free(vser->ports_map);
-    g_free(vser->post_load.connected);
-    qemu_free_timer(vser->post_load.timer);
-
+    if (vser->post_load) {
+        g_free(vser->post_load->connected);
+        qemu_del_timer(vser->post_load->timer);
+        qemu_free_timer(vser->post_load->timer);
+        g_free(vser->post_load);
+    }
     virtio_cleanup(vdev);
 }
 
@@ -1027,7 +1058,7 @@ static void virtio_serial_port_class_init(ObjectClass *klass, void *data)
     k->props = virtser_props;
 }
 
-static TypeInfo virtio_serial_port_type_info = {
+static const TypeInfo virtio_serial_port_type_info = {
     .name = TYPE_VIRTIO_SERIAL_PORT,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(VirtIOSerialPort),
diff --git a/hw/virtio.c b/hw/virtio.c
index f40a8c5..e259348 100644
--- a/hw/virtio.c
+++ b/hw/virtio.c
@@ -14,9 +14,10 @@
 #include <inttypes.h>
 
 #include "trace.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 #include "virtio.h"
-#include "qemu-barrier.h"
+#include "qemu/atomic.h"
+#include "virtio-bus.h"
 
 /* The alignment to use between consumer and producer parts of vring.
  * x86 pagesize again. */
@@ -72,6 +73,8 @@ struct VirtQueue
     /* Notification enabled? */
     bool notification;
 
+    uint16_t queue_index;
+
     int inuse;
 
     uint16_t vector;
@@ -700,6 +703,15 @@ VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
     return &vdev->vq[i];
 }
 
+void virtio_del_queue(VirtIODevice *vdev, int n)
+{
+    if (n < 0 || n >= VIRTIO_PCI_QUEUE_MAX) {
+        abort();
+    }
+
+    vdev->vq[n].vring.num = 0;
+}
+
 void virtio_irq(VirtQueue *vq)
 {
     trace_virtio_irq(vq);
@@ -875,11 +887,16 @@ int virtio_load(VirtIODevice *vdev, QEMUFile *f)
     return 0;
 }
 
-void virtio_cleanup(VirtIODevice *vdev)
+void virtio_common_cleanup(VirtIODevice *vdev)
 {
     qemu_del_vm_change_state_handler(vdev->vmstate);
     g_free(vdev->config);
     g_free(vdev->vq);
+}
+
+void virtio_cleanup(VirtIODevice *vdev)
+{
+    virtio_common_cleanup(vdev);
     g_free(vdev);
 }
 
@@ -902,14 +919,10 @@ static void virtio_vmstate_change(void *opaque, int running, RunState state)
     }
 }
 
-VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
-                                 size_t config_size, size_t struct_size)
+void virtio_init(VirtIODevice *vdev, const char *name,
+                 uint16_t device_id, size_t config_size)
 {
-    VirtIODevice *vdev;
     int i;
-
-    vdev = g_malloc0(struct_size);
-
     vdev->device_id = device_id;
     vdev->status = 0;
     vdev->isr = 0;
@@ -917,25 +930,34 @@ VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
     vdev->config_vector = VIRTIO_NO_VECTOR;
     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_PCI_QUEUE_MAX);
     vdev->vm_running = runstate_is_running();
-    for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
         vdev->vq[i].vdev = vdev;
+        vdev->vq[i].queue_index = i;
     }
 
     vdev->name = name;
     vdev->config_len = config_size;
-    if (vdev->config_len)
+    if (vdev->config_len) {
         vdev->config = g_malloc0(config_size);
-    else
+    } else {
         vdev->config = NULL;
+    }
+    vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
+                                                     vdev);
+}
 
-    vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change, vdev);
-
+VirtIODevice *virtio_common_init(const char *name, uint16_t device_id,
+                                 size_t config_size, size_t struct_size)
+{
+    VirtIODevice *vdev;
+    vdev = g_malloc0(struct_size);
+    virtio_init(vdev, name, device_id, config_size);
     return vdev;
 }
 
 void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding,
-                        void *opaque)
+                        DeviceState *opaque)
 {
     vdev->binding = binding;
     vdev->binding_opaque = opaque;
@@ -999,6 +1021,11 @@ VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
     return vdev->vq + n;
 }
 
+uint16_t virtio_get_queue_index(VirtQueue *vq)
+{
+    return vq->queue_index;
+}
+
 static void virtio_queue_guest_notifier_read(EventNotifier *n)
 {
     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
@@ -1056,3 +1083,39 @@ EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
 {
     return &vq->host_notifier;
 }
+
+static int virtio_device_init(DeviceState *qdev)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(qdev);
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(qdev);
+    assert(k->init != NULL);
+    if (k->init(vdev) < 0) {
+        return -1;
+    }
+    virtio_bus_plug_device(vdev);
+    return 0;
+}
+
+static void virtio_device_class_init(ObjectClass *klass, void *data)
+{
+    /* Set the default value here. */
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->init = virtio_device_init;
+    dc->bus_type = TYPE_VIRTIO_BUS;
+}
+
+static const TypeInfo virtio_device_info = {
+    .name = TYPE_VIRTIO_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(VirtIODevice),
+    .class_init = virtio_device_class_init,
+    .abstract = true,
+    .class_size = sizeof(VirtioDeviceClass),
+};
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_device_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/virtio.h b/hw/virtio.h
index 7c17f7b..1e206b8 100644
--- a/hw/virtio.h
+++ b/hw/virtio.h
@@ -15,10 +15,10 @@
 #define _QEMU_VIRTIO_H
 
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "qdev.h"
-#include "sysemu.h"
-#include "event_notifier.h"
+#include "sysemu/sysemu.h"
+#include "qemu/event_notifier.h"
 #ifdef CONFIG_LINUX
 #include "9p.h"
 #endif
@@ -91,25 +91,34 @@ typedef struct VirtQueueElement
 } VirtQueueElement;
 
 typedef struct {
-    void (*notify)(void * opaque, uint16_t vector);
-    void (*save_config)(void * opaque, QEMUFile *f);
-    void (*save_queue)(void * opaque, int n, QEMUFile *f);
-    int (*load_config)(void * opaque, QEMUFile *f);
-    int (*load_queue)(void * opaque, int n, QEMUFile *f);
-    int (*load_done)(void * opaque, QEMUFile *f);
-    unsigned (*get_features)(void * opaque);
-    bool (*query_guest_notifiers)(void * opaque);
-    int (*set_guest_notifiers)(void * opaque, bool assigned);
-    int (*set_host_notifier)(void * opaque, int n, bool assigned);
-    void (*vmstate_change)(void * opaque, bool running);
+    void (*notify)(DeviceState *d, uint16_t vector);
+    void (*save_config)(DeviceState *d, QEMUFile *f);
+    void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_config)(DeviceState *d, QEMUFile *f);
+    int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
+    int (*load_done)(DeviceState *d, QEMUFile *f);
+    unsigned (*get_features)(DeviceState *d);
+    bool (*query_guest_notifiers)(DeviceState *d);
+    int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assigned);
+    int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
+    void (*vmstate_change)(DeviceState *d, bool running);
 } VirtIOBindings;
 
 #define VIRTIO_PCI_QUEUE_MAX 64
 
 #define VIRTIO_NO_VECTOR 0xffff
 
+#define TYPE_VIRTIO_DEVICE "virtio-device"
+#define VIRTIO_DEVICE_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtioDeviceClass, obj, TYPE_VIRTIO_DEVICE)
+#define VIRTIO_DEVICE_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtioDeviceClass, klass, TYPE_VIRTIO_DEVICE)
+#define VIRTIO_DEVICE(obj) \
+        OBJECT_CHECK(VirtIODevice, (obj), TYPE_VIRTIO_DEVICE)
+
 struct VirtIODevice
 {
+    DeviceState parent_obj;
     const char *name;
     uint8_t status;
     uint8_t isr;
@@ -119,6 +128,10 @@ struct VirtIODevice
     void *config;
     uint16_t config_vector;
     int nvectors;
+    /*
+     * Function pointers will be removed at the end of the series as they are in
+     * VirtioDeviceClass.
+     */
     uint32_t (*get_features)(VirtIODevice *vdev, uint32_t requested_features);
     uint32_t (*bad_features)(VirtIODevice *vdev);
     void (*set_features)(VirtIODevice *vdev, uint32_t val);
@@ -126,18 +139,50 @@ struct VirtIODevice
     void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
     void (*reset)(VirtIODevice *vdev);
     void (*set_status)(VirtIODevice *vdev, uint8_t val);
+    /* Test and clear event pending status.
+     * Should be called after unmask to avoid losing events.
+     * If backend does not support masking,
+     * must check in frontend instead.
+     */
+    bool (*guest_notifier_pending)(VirtIODevice *vdev, int n);
+    /* Mask/unmask events from this vq. Any events reported
+     * while masked will become pending.
+     * If backend does not support masking,
+     * must mask in frontend instead.
+     */
+    void (*guest_notifier_mask)(VirtIODevice *vdev, int n, bool mask);
+
     VirtQueue *vq;
     const VirtIOBindings *binding;
-    void *binding_opaque;
+    DeviceState *binding_opaque;
     uint16_t device_id;
     bool vm_running;
     VMChangeStateEntry *vmstate;
 };
 
+typedef struct VirtioDeviceClass {
+    /* This is what a VirtioDevice must implement */
+    DeviceClass parent;
+    int (*init)(VirtIODevice *vdev);
+    uint32_t (*get_features)(VirtIODevice *vdev, uint32_t requested_features);
+    uint32_t (*bad_features)(VirtIODevice *vdev);
+    void (*set_features)(VirtIODevice *vdev, uint32_t val);
+    void (*get_config)(VirtIODevice *vdev, uint8_t *config);
+    void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
+    void (*reset)(VirtIODevice *vdev);
+    void (*set_status)(VirtIODevice *vdev, uint8_t val);
+} VirtioDeviceClass;
+
+void virtio_init(VirtIODevice *vdev, const char *name,
+                         uint16_t device_id, size_t config_size);
+void virtio_common_cleanup(VirtIODevice *vdev);
+
 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
                             void (*handle_output)(VirtIODevice *,
                                                   VirtQueue *));
 
+void virtio_del_queue(VirtIODevice *vdev, int n);
+
 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len);
 void virtqueue_flush(VirtQueue *vq, unsigned int count);
@@ -191,14 +236,15 @@ void virtio_update_irq(VirtIODevice *vdev);
 int virtio_set_features(VirtIODevice *vdev, uint32_t val);
 
 void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding,
-                        void *opaque);
+                        DeviceState *opaque);
 
 /* Base devices.  */
 typedef struct VirtIOBlkConf VirtIOBlkConf;
 VirtIODevice *virtio_blk_init(DeviceState *dev, VirtIOBlkConf *blk);
 struct virtio_net_conf;
 VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
-                              struct virtio_net_conf *net);
+                              struct virtio_net_conf *net,
+                              uint32_t host_features);
 typedef struct virtio_serial_conf virtio_serial_conf;
 VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *serial);
 VirtIODevice *virtio_balloon_init(DeviceState *dev);
@@ -235,6 +281,7 @@ hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n);
 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n);
 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx);
 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n);
+uint16_t virtio_get_queue_index(VirtQueue *vq);
 int virtio_queue_get_id(VirtQueue *vq);
 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq);
 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
diff --git a/hw/vmmouse.c b/hw/vmmouse.c
index 6338efa..b9afc2c 100644
--- a/hw/vmmouse.c
+++ b/hw/vmmouse.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 #include "hw.h"
-#include "console.h"
+#include "ui/console.h"
 #include "ps2.h"
 #include "pc.h"
 #include "qdev.h"
@@ -252,7 +252,6 @@ static void vmmouse_reset(DeviceState *d)
 {
     VMMouseState *s = container_of(d, VMMouseState, dev.qdev);
 
-    s->status = 0xffff;
     s->queue_size = VMMOUSE_QUEUE_SIZE;
 
     vmmouse_disable(s);
@@ -287,7 +286,7 @@ static void vmmouse_class_initfn(ObjectClass *klass, void *data)
     dc->props = vmmouse_properties;
 }
 
-static TypeInfo vmmouse_info = {
+static const TypeInfo vmmouse_info = {
     .name          = "vmmouse",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(VMMouseState),
diff --git a/hw/vmport.c b/hw/vmport.c
index 3ab3a14..faead3a 100644
--- a/hw/vmport.c
+++ b/hw/vmport.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "isa.h"
 #include "pc.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "qdev.h"
 
 //#define VMPORT_DEBUG
@@ -155,7 +155,7 @@ static void vmport_class_initfn(ObjectClass *klass, void *data)
     dc->no_user = 1;
 }
 
-static TypeInfo vmport_info = {
+static const TypeInfo vmport_info = {
     .name          = "vmport",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(VMPortState),
diff --git a/hw/vmware_vga.c b/hw/vmware_vga.c
index 834588d..cd15ee4 100644
--- a/hw/vmware_vga.c
+++ b/hw/vmware_vga.c
@@ -23,8 +23,8 @@
  */
 #include "hw.h"
 #include "loader.h"
-#include "console.h"
-#include "pci.h"
+#include "ui/console.h"
+#include "pci/pci.h"
 
 #undef VERBOSE
 #define HW_RECT_ACCEL
@@ -296,6 +296,15 @@ static inline void vmsvga_update_rect(struct vmsvga_state_s *s,
     uint8_t *src;
     uint8_t *dst;
 
+    if (x < 0) {
+        fprintf(stderr, "%s: update x was < 0 (%d)\n", __func__, x);
+        w += x;
+        x = 0;
+    }
+    if (w < 0) {
+        fprintf(stderr, "%s: update w was < 0 (%d)\n", __func__, w);
+        w = 0;
+    }
     if (x + w > ds_get_width(s->vga.ds)) {
         fprintf(stderr, "%s: update width too large x: %d, w: %d\n",
                 __func__, x, w);
@@ -303,6 +312,15 @@ static inline void vmsvga_update_rect(struct vmsvga_state_s *s,
         w = ds_get_width(s->vga.ds) - x;
     }
 
+    if (y < 0) {
+        fprintf(stderr, "%s: update y was < 0 (%d)\n",  __func__, y);
+        h += y;
+        y = 0;
+    }
+    if (h < 0) {
+        fprintf(stderr, "%s: update h was < 0 (%d)\n",  __func__, h);
+        h = 0;
+    }
     if (y + h > ds_get_height(s->vga.ds)) {
         fprintf(stderr, "%s: update height too large y: %d, h: %d\n",
                 __func__, y, h);
@@ -1244,7 +1262,7 @@ static void vmsvga_class_init(ObjectClass *klass, void *data)
     dc->props = vga_vmware_properties;
 }
 
-static TypeInfo vmsvga_info = {
+static const TypeInfo vmsvga_info = {
     .name          = "vmware-svga",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(struct pci_vmsvga_state_s),
diff --git a/hw/vt82c686.c b/hw/vt82c686.c
index 5d7c00c..2d8e398 100644
--- a/hw/vt82c686.c
+++ b/hw/vt82c686.c
@@ -15,18 +15,19 @@
 #include "vt82c686.h"
 #include "i2c.h"
 #include "smbus.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "isa.h"
 #include "sysbus.h"
 #include "mips.h"
 #include "apm.h"
 #include "acpi.h"
 #include "pm_smbus.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "exec/address-spaces.h"
 
 typedef uint32_t pci_addr_t;
-#include "pci_host.h"
+#include "pci/pci_host.h"
 //#define DEBUG_VT82C686B
 
 #ifdef DEBUG_VT82C686B
@@ -159,6 +160,7 @@ static void vt82c686b_write_config(PCIDevice * d, uint32_t address,
 
 typedef struct VT686PMState {
     PCIDevice dev;
+    MemoryRegion io;
     ACPIREGS ar;
     APMState apm;
     PMSMBus smb;
@@ -195,92 +197,17 @@ static void pm_tmr_timer(ACPIREGS *ar)
     pm_update_sci(s);
 }
 
-static void pm_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
-{
-    VT686PMState *s = opaque;
-
-    addr &= 0x0f;
-    switch (addr) {
-    case 0x00:
-        acpi_pm1_evt_write_sts(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x02:
-        acpi_pm1_evt_write_en(&s->ar, val);
-        pm_update_sci(s);
-        break;
-    case 0x04:
-        acpi_pm1_cnt_write(&s->ar, val, 0);
-        break;
-    default:
-        break;
-    }
-    DPRINTF("PM writew port=0x%04x val=0x%02x\n", addr, val);
-}
-
-static uint32_t pm_ioport_readw(void *opaque, uint32_t addr)
-{
-    VT686PMState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x0f;
-    switch (addr) {
-    case 0x00:
-        val = acpi_pm1_evt_get_sts(&s->ar);
-        break;
-    case 0x02:
-        val = s->ar.pm1.evt.en;
-        break;
-    case 0x04:
-        val = s->ar.pm1.cnt.cnt;
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    DPRINTF("PM readw port=0x%04x val=0x%02x\n", addr, val);
-    return val;
-}
-
-static void pm_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
-{
-    addr &= 0x0f;
-    DPRINTF("PM writel port=0x%04x val=0x%08x\n", addr, val);
-}
-
-static uint32_t pm_ioport_readl(void *opaque, uint32_t addr)
-{
-    VT686PMState *s = opaque;
-    uint32_t val;
-
-    addr &= 0x0f;
-    switch (addr) {
-    case 0x08:
-        val = acpi_pm_tmr_get(&s->ar);
-        break;
-    default:
-        val = 0;
-        break;
-    }
-    DPRINTF("PM readl port=0x%04x val=0x%08x\n", addr, val);
-    return val;
-}
-
 static void pm_io_space_update(VT686PMState *s)
 {
     uint32_t pm_io_base;
 
-    if (s->dev.config[0x80] & 1) {
-        pm_io_base = pci_get_long(s->dev.config + 0x40);
-        pm_io_base &= 0xffc0;
+    pm_io_base = pci_get_long(s->dev.config + 0x40);
+    pm_io_base &= 0xffc0;
 
-        /* XXX: need to improve memory and ioport allocation */
-        DPRINTF("PM: mapping to 0x%x\n", pm_io_base);
-        register_ioport_write(pm_io_base, 64, 2, pm_ioport_writew, s);
-        register_ioport_read(pm_io_base, 64, 2, pm_ioport_readw, s);
-        register_ioport_write(pm_io_base, 64, 4, pm_ioport_writel, s);
-        register_ioport_read(pm_io_base, 64, 4, pm_ioport_readl, s);
-    }
+    memory_region_transaction_begin();
+    memory_region_set_enabled(&s->io, s->dev.config[0x80] & 1);
+    memory_region_set_address(&s->io, pm_io_base);
+    memory_region_transaction_commit();
 }
 
 static void pm_write_config(PCIDevice *d,
@@ -357,7 +284,7 @@ static void via_ac97_class_init(ObjectClass *klass, void *data)
     dc->desc = "AC97";
 }
 
-static TypeInfo via_ac97_info = {
+static const TypeInfo via_ac97_info = {
     .name          = "VT82C686B_AC97",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686AC97State),
@@ -398,7 +325,7 @@ static void via_mc97_class_init(ObjectClass *klass, void *data)
     dc->desc = "MC97";
 }
 
-static TypeInfo via_mc97_info = {
+static const TypeInfo via_mc97_info = {
     .name          = "VT82C686B_MC97",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686MC97State),
@@ -424,15 +351,18 @@ static int vt82c686b_pm_initfn(PCIDevice *dev)
     pci_conf[0x90] = s->smb_io_base | 1;
     pci_conf[0x91] = s->smb_io_base >> 8;
     pci_conf[0xd2] = 0x90;
-    register_ioport_write(s->smb_io_base, 0xf, 1, smb_ioport_writeb, &s->smb);
-    register_ioport_read(s->smb_io_base, 0xf, 1, smb_ioport_readb, &s->smb);
+    pm_smbus_init(&s->dev.qdev, &s->smb);
+    memory_region_add_subregion(get_system_io(), s->smb_io_base, &s->smb.io);
 
-    apm_init(&s->apm, NULL, s);
+    apm_init(dev, &s->apm, NULL, s);
 
-    acpi_pm_tmr_init(&s->ar, pm_tmr_timer);
-    acpi_pm1_cnt_init(&s->ar);
+    memory_region_init(&s->io, "vt82c686-pm", 64);
+    memory_region_set_enabled(&s->io, false);
+    memory_region_add_subregion(get_system_io(), 0, &s->io);
 
-    pm_smbus_init(&s->dev.qdev, &s->smb);
+    acpi_pm_tmr_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_evt_init(&s->ar, pm_tmr_timer, &s->io);
+    acpi_pm1_cnt_init(&s->ar, &s->io);
 
     return 0;
 }
@@ -474,7 +404,7 @@ static void via_pm_class_init(ObjectClass *klass, void *data)
     dc->props = via_pm_properties;
 }
 
-static TypeInfo via_pm_info = {
+static const TypeInfo via_pm_info = {
     .name          = "VT82C686B_PM",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686PMState),
@@ -541,7 +471,7 @@ static void via_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_via;
 }
 
-static TypeInfo via_info = {
+static const TypeInfo via_info = {
     .name          = "VT82C686B",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT82C686BState),
diff --git a/hw/watchdog.c b/hw/watchdog.c
index b52aced..072d256 100644
--- a/hw/watchdog.c
+++ b/hw/watchdog.c
@@ -20,12 +20,12 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-queue.h"
-#include "qemu-objects.h"
-#include "monitor.h"
-#include "sysemu.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "qemu/queue.h"
+#include "qapi/qmp/types.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "hw/watchdog.h"
 
 /* Possible values for action parameter. */
@@ -66,7 +66,7 @@ int select_watchdog(const char *p)
     QLIST_FOREACH(model, &watchdog_list, entry) {
         if (strcasecmp(model->wdt_name, p) == 0) {
             /* add the device */
-            opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0, NULL);
+            opts = qemu_opts_create_nofail(qemu_find_opts("device"));
             qemu_opt_set(opts, "driver", p);
             return 0;
         }
diff --git a/hw/watchdog.h b/hw/watchdog.h
index c12a293..3e9a970 100644
--- a/hw/watchdog.h
+++ b/hw/watchdog.h
@@ -22,7 +22,7 @@
 #ifndef QEMU_WATCHDOG_H
 #define QEMU_WATCHDOG_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 struct WatchdogTimerModel {
     QLIST_ENTRY(WatchdogTimerModel) entry;
diff --git a/hw/wdt_i6300esb.c b/hw/wdt_i6300esb.c
index da15c73..37ce362 100644
--- a/hw/wdt_i6300esb.c
+++ b/hw/wdt_i6300esb.c
@@ -22,10 +22,10 @@
 #include <inttypes.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "watchdog.h"
 #include "hw.h"
-#include "pci.h"
+#include "pci/pci.h"
 
 /*#define I6300ESB_DEBUG 1*/
 
@@ -439,7 +439,7 @@ static void i6300esb_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_i6300esb;
 }
 
-static TypeInfo i6300esb_info = {
+static const TypeInfo i6300esb_info = {
     .name          = "i6300esb",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(I6300State),
diff --git a/hw/wdt_ib700.c b/hw/wdt_ib700.c
index 7f6c21d..599a86f 100644
--- a/hw/wdt_ib700.c
+++ b/hw/wdt_ib700.c
@@ -20,7 +20,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "watchdog.h"
 #include "hw.h"
 #include "isa.h"
@@ -129,7 +129,7 @@ static void wdt_ib700_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_ib700;
 }
 
-static TypeInfo wdt_ib700_info = {
+static const TypeInfo wdt_ib700_info = {
     .name          = "ib700",
     .parent        = TYPE_ISA_DEVICE,
     .instance_size = sizeof(IB700State),
diff --git a/hw/wm8750.c b/hw/wm8750.c
index 44f138f..d3ea5ba 100644
--- a/hw/wm8750.c
+++ b/hw/wm8750.c
@@ -632,7 +632,7 @@ static void wm8750_fini(I2CSlave *i2c)
 void wm8750_data_req_set(DeviceState *dev,
                 void (*data_req)(void *, int, int), void *opaque)
 {
-    WM8750State *s = FROM_I2C_SLAVE(WM8750State, I2C_SLAVE_FROM_QDEV(dev));
+    WM8750State *s = FROM_I2C_SLAVE(WM8750State, I2C_SLAVE(dev));
     s->data_req = data_req;
     s->opaque = opaque;
 }
@@ -701,7 +701,7 @@ static void wm8750_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_wm8750;
 }
 
-static TypeInfo wm8750_info = {
+static const TypeInfo wm8750_info = {
     .name          = "wm8750",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(WM8750State),
diff --git a/hw/xen-host-pci-device.h b/hw/xen-host-pci-device.h
index 0079dac..942b24d 100644
--- a/hw/xen-host-pci-device.h
+++ b/hw/xen-host-pci-device.h
@@ -1,7 +1,7 @@
 #ifndef XEN_HOST_PCI_DEVICE_H
 #define XEN_HOST_PCI_DEVICE_H
 
-#include "pci.h"
+#include "pci/pci.h"
 
 enum {
     XEN_HOST_PCI_REGION_TYPE_IO = 1 << 1,
diff --git a/hw/xen.h b/hw/xen.h
index e3cca7f..6235f91 100644
--- a/hw/xen.h
+++ b/hw/xen.h
@@ -21,9 +21,9 @@ enum xen_mode {
 extern uint32_t xen_domid;
 extern enum xen_mode xen_mode;
 
-extern int xen_allowed;
+extern bool xen_allowed;
 
-static inline int xen_enabled(void)
+static inline bool xen_enabled(void)
 {
 #if defined(CONFIG_XEN_BACKEND) && !defined(CONFIG_NO_XEN)
     return xen_allowed;
diff --git a/hw/xen_apic.c b/hw/xen_apic.c
index fc45366..1d1d15c 100644
--- a/hw/xen_apic.c
+++ b/hw/xen_apic.c
@@ -10,7 +10,7 @@
  * later. See the COPYING file in the top-level directory.
  */
 #include "hw/apic_internal.h"
-#include "hw/msi.h"
+#include "hw/pci/msi.h"
 #include "xen.h"
 
 static uint64_t xen_apic_mem_read(void *opaque, hwaddr addr,
@@ -80,7 +80,7 @@ static void xen_apic_class_init(ObjectClass *klass, void *data)
     k->external_nmi = xen_apic_external_nmi;
 }
 
-static TypeInfo xen_apic_info = {
+static const TypeInfo xen_apic_info = {
     .name = "xen-apic",
     .parent = TYPE_APIC_COMMON,
     .instance_size = sizeof(APICCommonState),
diff --git a/hw/xen_backend.c b/hw/xen_backend.c
index f83a1e1..3fa3009 100644
--- a/hw/xen_backend.c
+++ b/hw/xen_backend.c
@@ -35,8 +35,8 @@
 #include <sys/signal.h>
 
 #include "hw.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
+#include "char/char.h"
+#include "qemu/log.h"
 #include "xen_backend.h"
 
 #include <xen/grant_table.h>
diff --git a/hw/xen_backend.h b/hw/xen_backend.h
index fea86dd..f37afb1 100644
--- a/hw/xen_backend.h
+++ b/hw/xen_backend.h
@@ -2,9 +2,8 @@
 #define QEMU_HW_XEN_BACKEND_H 1
 
 #include "xen_common.h"
-#include "sysemu.h"
-#include "net.h"
-#include "net/hub.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
 
 /* ------------------------------------------------------------- */
 
diff --git a/hw/xen_common.h b/hw/xen_common.h
index 727757a..95bc9a7 100644
--- a/hw/xen_common.h
+++ b/hw/xen_common.h
@@ -16,7 +16,7 @@
 
 #include "hw.h"
 #include "xen.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 /*
  * We don't support Xen prior to 3.3.0.
diff --git a/hw/xen_console.c b/hw/xen_console.c
index 9426d73..44141f8 100644
--- a/hw/xen_console.c
+++ b/hw/xen_console.c
@@ -30,7 +30,7 @@
 #include <sys/mman.h>
 
 #include "hw.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "xen_backend.h"
 
 #include <xen/io/console.h>
@@ -184,7 +184,11 @@ static int con_init(struct XenDevice *xendev)
 
     /* setup */
     dom = xs_get_domain_path(xenstore, con->xendev.dom);
-    snprintf(con->console, sizeof(con->console), "%s/console", dom);
+    if (!xendev->dev) {
+        snprintf(con->console, sizeof(con->console), "%s/console", dom);
+    } else {
+        snprintf(con->console, sizeof(con->console), "%s/device/console/%d", dom, xendev->dev);
+    }
     free(dom);
 
     type = xenstore_read_str(con->console, "type");
@@ -223,10 +227,16 @@ static int con_initialise(struct XenDevice *xendev)
     if (xenstore_read_int(con->console, "limit", &limit) == 0)
 	con->buffer.max_capacity = limit;
 
-    con->sring = xc_map_foreign_range(xen_xc, con->xendev.dom,
-				      XC_PAGE_SIZE,
-				      PROT_READ|PROT_WRITE,
-				      con->ring_ref);
+    if (!xendev->dev) {
+        con->sring = xc_map_foreign_range(xen_xc, con->xendev.dom,
+                                          XC_PAGE_SIZE,
+                                          PROT_READ|PROT_WRITE,
+                                          con->ring_ref);
+    } else {
+        con->sring = xc_gnttab_map_grant_ref(xendev->gnttabdev, con->xendev.dom,
+                                             con->ring_ref,
+                                             PROT_READ|PROT_WRITE);
+    }
     if (!con->sring)
 	return -1;
 
@@ -255,7 +265,11 @@ static void con_disconnect(struct XenDevice *xendev)
     xen_be_unbind_evtchn(&con->xendev);
 
     if (con->sring) {
-	munmap(con->sring, XC_PAGE_SIZE);
+        if (!xendev->gnttabdev) {
+            munmap(con->sring, XC_PAGE_SIZE);
+        } else {
+            xc_gnttab_munmap(xendev->gnttabdev, con->sring, 1);
+        }
 	con->sring = NULL;
     }
 }
@@ -273,7 +287,7 @@ static void con_event(struct XenDevice *xendev)
 
 struct XenDevOps xen_console_ops = {
     .size       = sizeof(struct XenConsole),
-    .flags      = DEVOPS_FLAG_IGNORE_STATE,
+    .flags      = DEVOPS_FLAG_IGNORE_STATE|DEVOPS_FLAG_NEED_GNTDEV,
     .init       = con_init,
     .initialise = con_initialise,
     .event      = con_event,
diff --git a/hw/xen_devconfig.c b/hw/xen_devconfig.c
index d83e8d0..e2ba741 100644
--- a/hw/xen_devconfig.c
+++ b/hw/xen_devconfig.c
@@ -1,5 +1,5 @@
 #include "xen_backend.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 /* ------------------------------------------------------------- */
 
diff --git a/hw/xen_disk.c b/hw/xen_disk.c
index e6bb2f2..7fea871 100644
--- a/hw/xen_disk.c
+++ b/hw/xen_disk.c
@@ -36,10 +36,9 @@
 #include <sys/uio.h>
 
 #include "hw.h"
-#include "qemu-char.h"
 #include "xen_backend.h"
 #include "xen_blkif.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 /* ------------------------------------------------------------- */
 
@@ -52,6 +51,13 @@ static int max_requests = 32;
 #define BLOCK_SIZE  512
 #define IOCB_COUNT  (BLKIF_MAX_SEGMENTS_PER_REQUEST + 2)
 
+struct PersistentGrant {
+    void *page;
+    struct XenBlkDev *blkdev;
+};
+
+typedef struct PersistentGrant PersistentGrant;
+
 struct ioreq {
     blkif_request_t     req;
     int16_t             status;
@@ -69,6 +75,7 @@ struct ioreq {
     int                 prot;
     void                *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     void                *pages;
+    int                 num_unmap;
 
     /* aio status */
     int                 aio_inflight;
@@ -105,6 +112,12 @@ struct XenBlkDev {
     int                 requests_inflight;
     int                 requests_finished;
 
+    /* Persistent grants extension */
+    gboolean            feature_persistent;
+    GTree               *persistent_gnts;
+    unsigned int        persistent_gnt_count;
+    unsigned int        max_grants;
+
     /* qemu block driver */
     DriveInfo           *dinfo;
     BlockDriverState    *bs;
@@ -113,6 +126,54 @@ struct XenBlkDev {
 
 /* ------------------------------------------------------------- */
 
+static void ioreq_reset(struct ioreq *ioreq)
+{
+    memset(&ioreq->req, 0, sizeof(ioreq->req));
+    ioreq->status = 0;
+    ioreq->start = 0;
+    ioreq->presync = 0;
+    ioreq->postsync = 0;
+    ioreq->mapped = 0;
+
+    memset(ioreq->domids, 0, sizeof(ioreq->domids));
+    memset(ioreq->refs, 0, sizeof(ioreq->refs));
+    ioreq->prot = 0;
+    memset(ioreq->page, 0, sizeof(ioreq->page));
+    ioreq->pages = NULL;
+
+    ioreq->aio_inflight = 0;
+    ioreq->aio_errors = 0;
+
+    ioreq->blkdev = NULL;
+    memset(&ioreq->list, 0, sizeof(ioreq->list));
+    memset(&ioreq->acct, 0, sizeof(ioreq->acct));
+
+    qemu_iovec_reset(&ioreq->v);
+}
+
+static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
+{
+    uint ua = GPOINTER_TO_UINT(a);
+    uint ub = GPOINTER_TO_UINT(b);
+    return (ua > ub) - (ua < ub);
+}
+
+static void destroy_grant(gpointer pgnt)
+{
+    PersistentGrant *grant = pgnt;
+    XenGnttab gnt = grant->blkdev->xendev.gnttabdev;
+
+    if (xc_gnttab_munmap(gnt, grant->page, 1) != 0) {
+        xen_be_printf(&grant->blkdev->xendev, 0,
+                      "xc_gnttab_munmap failed: %s\n",
+                      strerror(errno));
+    }
+    grant->blkdev->persistent_gnt_count--;
+    xen_be_printf(&grant->blkdev->xendev, 3,
+                  "unmapped grant %p\n", grant->page);
+    g_free(grant);
+}
+
 static struct ioreq *ioreq_start(struct XenBlkDev *blkdev)
 {
     struct ioreq *ioreq = NULL;
@@ -130,7 +191,6 @@ static struct ioreq *ioreq_start(struct XenBlkDev *blkdev)
         /* get one from freelist */
         ioreq = QLIST_FIRST(&blkdev->freelist);
         QLIST_REMOVE(ioreq, list);
-        qemu_iovec_reset(&ioreq->v);
     }
     QLIST_INSERT_HEAD(&blkdev->inflight, ioreq, list);
     blkdev->requests_inflight++;
@@ -154,7 +214,7 @@ static void ioreq_release(struct ioreq *ioreq, bool finish)
     struct XenBlkDev *blkdev = ioreq->blkdev;
 
     QLIST_REMOVE(ioreq, list);
-    memset(ioreq, 0, sizeof(*ioreq));
+    ioreq_reset(ioreq);
     ioreq->blkdev = blkdev;
     QLIST_INSERT_HEAD(&blkdev->freelist, ioreq, list);
     if (finish) {
@@ -183,12 +243,11 @@ static int ioreq_parse(struct ioreq *ioreq)
     case BLKIF_OP_READ:
         ioreq->prot = PROT_WRITE; /* to memory */
         break;
-    case BLKIF_OP_WRITE_BARRIER:
+    case BLKIF_OP_FLUSH_DISKCACHE:
+        ioreq->presync = 1;
         if (!ioreq->req.nr_segments) {
-            ioreq->presync = 1;
             return 0;
         }
-        ioreq->presync = ioreq->postsync = 1;
         /* fall through */
     case BLKIF_OP_WRITE:
         ioreq->prot = PROT_READ; /* from memory */
@@ -242,21 +301,21 @@ static void ioreq_unmap(struct ioreq *ioreq)
     XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
     int i;
 
-    if (ioreq->v.niov == 0 || ioreq->mapped == 0) {
+    if (ioreq->num_unmap == 0 || ioreq->mapped == 0) {
         return;
     }
     if (batch_maps) {
         if (!ioreq->pages) {
             return;
         }
-        if (xc_gnttab_munmap(gnt, ioreq->pages, ioreq->v.niov) != 0) {
+        if (xc_gnttab_munmap(gnt, ioreq->pages, ioreq->num_unmap) != 0) {
             xen_be_printf(&ioreq->blkdev->xendev, 0, "xc_gnttab_munmap failed: %s\n",
                           strerror(errno));
         }
-        ioreq->blkdev->cnt_map -= ioreq->v.niov;
+        ioreq->blkdev->cnt_map -= ioreq->num_unmap;
         ioreq->pages = NULL;
     } else {
-        for (i = 0; i < ioreq->v.niov; i++) {
+        for (i = 0; i < ioreq->num_unmap; i++) {
             if (!ioreq->page[i]) {
                 continue;
             }
@@ -274,41 +333,120 @@ static void ioreq_unmap(struct ioreq *ioreq)
 static int ioreq_map(struct ioreq *ioreq)
 {
     XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
-    int i;
+    uint32_t domids[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    uint32_t refs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    void *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    int i, j, new_maps = 0;
+    PersistentGrant *grant;
+    /* domids and refs variables will contain the information necessary
+     * to map the grants that are needed to fulfill this request.
+     *
+     * After mapping the needed grants, the page array will contain the
+     * memory address of each granted page in the order specified in ioreq
+     * (disregarding if it's a persistent grant or not).
+     */
 
     if (ioreq->v.niov == 0 || ioreq->mapped == 1) {
         return 0;
     }
-    if (batch_maps) {
+    if (ioreq->blkdev->feature_persistent) {
+        for (i = 0; i < ioreq->v.niov; i++) {
+            grant = g_tree_lookup(ioreq->blkdev->persistent_gnts,
+                                    GUINT_TO_POINTER(ioreq->refs[i]));
+
+            if (grant != NULL) {
+                page[i] = grant->page;
+                xen_be_printf(&ioreq->blkdev->xendev, 3,
+                              "using persistent-grant %" PRIu32 "\n",
+                              ioreq->refs[i]);
+            } else {
+                    /* Add the grant to the list of grants that
+                     * should be mapped
+                     */
+                    domids[new_maps] = ioreq->domids[i];
+                    refs[new_maps] = ioreq->refs[i];
+                    page[i] = NULL;
+                    new_maps++;
+            }
+        }
+        /* Set the protection to RW, since grants may be reused later
+         * with a different protection than the one needed for this request
+         */
+        ioreq->prot = PROT_WRITE | PROT_READ;
+    } else {
+        /* All grants in the request should be mapped */
+        memcpy(refs, ioreq->refs, sizeof(refs));
+        memcpy(domids, ioreq->domids, sizeof(domids));
+        memset(page, 0, sizeof(page));
+        new_maps = ioreq->v.niov;
+    }
+
+    if (batch_maps && new_maps) {
         ioreq->pages = xc_gnttab_map_grant_refs
-            (gnt, ioreq->v.niov, ioreq->domids, ioreq->refs, ioreq->prot);
+            (gnt, new_maps, domids, refs, ioreq->prot);
         if (ioreq->pages == NULL) {
             xen_be_printf(&ioreq->blkdev->xendev, 0,
                           "can't map %d grant refs (%s, %d maps)\n",
-                          ioreq->v.niov, strerror(errno), ioreq->blkdev->cnt_map);
+                          new_maps, strerror(errno), ioreq->blkdev->cnt_map);
             return -1;
         }
-        for (i = 0; i < ioreq->v.niov; i++) {
-            ioreq->v.iov[i].iov_base = ioreq->pages + i * XC_PAGE_SIZE +
-                (uintptr_t)ioreq->v.iov[i].iov_base;
+        for (i = 0, j = 0; i < ioreq->v.niov; i++) {
+            if (page[i] == NULL) {
+                page[i] = ioreq->pages + (j++) * XC_PAGE_SIZE;
+            }
         }
-        ioreq->blkdev->cnt_map += ioreq->v.niov;
-    } else  {
-        for (i = 0; i < ioreq->v.niov; i++) {
+        ioreq->blkdev->cnt_map += new_maps;
+    } else if (new_maps)  {
+        for (i = 0; i < new_maps; i++) {
             ioreq->page[i] = xc_gnttab_map_grant_ref
-                (gnt, ioreq->domids[i], ioreq->refs[i], ioreq->prot);
+                (gnt, domids[i], refs[i], ioreq->prot);
             if (ioreq->page[i] == NULL) {
                 xen_be_printf(&ioreq->blkdev->xendev, 0,
                               "can't map grant ref %d (%s, %d maps)\n",
-                              ioreq->refs[i], strerror(errno), ioreq->blkdev->cnt_map);
+                              refs[i], strerror(errno), ioreq->blkdev->cnt_map);
                 ioreq_unmap(ioreq);
                 return -1;
             }
-            ioreq->v.iov[i].iov_base = ioreq->page[i] + (uintptr_t)ioreq->v.iov[i].iov_base;
             ioreq->blkdev->cnt_map++;
         }
+        for (i = 0, j = 0; i < ioreq->v.niov; i++) {
+            if (page[i] == NULL) {
+                page[i] = ioreq->page[j++];
+            }
+        }
+    }
+    if (ioreq->blkdev->feature_persistent) {
+        while ((ioreq->blkdev->persistent_gnt_count < ioreq->blkdev->max_grants)
+              && new_maps) {
+            /* Go through the list of newly mapped grants and add as many
+             * as possible to the list of persistently mapped grants.
+             *
+             * Since we start at the end of ioreq->page(s), we only need
+             * to decrease new_maps to prevent this granted pages from
+             * being unmapped in ioreq_unmap.
+             */
+            grant = g_malloc0(sizeof(*grant));
+            new_maps--;
+            if (batch_maps) {
+                grant->page = ioreq->pages + (new_maps) * XC_PAGE_SIZE;
+            } else {
+                grant->page = ioreq->page[new_maps];
+            }
+            grant->blkdev = ioreq->blkdev;
+            xen_be_printf(&ioreq->blkdev->xendev, 3,
+                          "adding grant %" PRIu32 " page: %p\n",
+                          refs[new_maps], grant->page);
+            g_tree_insert(ioreq->blkdev->persistent_gnts,
+                          GUINT_TO_POINTER(refs[new_maps]),
+                          grant);
+            ioreq->blkdev->persistent_gnt_count++;
+        }
+    }
+    for (i = 0; i < ioreq->v.niov; i++) {
+        ioreq->v.iov[i].iov_base += (uintptr_t)page[i];
     }
     ioreq->mapped = 1;
+    ioreq->num_unmap = new_maps;
     return 0;
 }
 
@@ -370,7 +508,7 @@ static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
                        qemu_aio_complete, ioreq);
         break;
     case BLKIF_OP_WRITE:
-    case BLKIF_OP_WRITE_BARRIER:
+    case BLKIF_OP_FLUSH_DISKCACHE:
         if (!ioreq->req.nr_segments) {
             break;
         }
@@ -655,7 +793,8 @@ static int blk_init(struct XenDevice *xendev)
                   blkdev->file_size, blkdev->file_size >> 20);
 
     /* fill info */
-    xenstore_write_be_int(&blkdev->xendev, "feature-barrier", 1);
+    xenstore_write_be_int(&blkdev->xendev, "feature-flush-cache", 1);
+    xenstore_write_be_int(&blkdev->xendev, "feature-persistent", 1);
     xenstore_write_be_int(&blkdev->xendev, "info",            info);
     xenstore_write_be_int(&blkdev->xendev, "sector-size",     blkdev->file_blk);
     xenstore_write_be_int(&blkdev->xendev, "sectors",
@@ -679,6 +818,7 @@ out_error:
 static int blk_connect(struct XenDevice *xendev)
 {
     struct XenBlkDev *blkdev = container_of(xendev, struct XenBlkDev, xendev);
+    int pers;
 
     if (xenstore_read_fe_int(&blkdev->xendev, "ring-ref", &blkdev->ring_ref) == -1) {
         return -1;
@@ -687,6 +827,11 @@ static int blk_connect(struct XenDevice *xendev)
                              &blkdev->xendev.remote_port) == -1) {
         return -1;
     }
+    if (xenstore_read_fe_int(&blkdev->xendev, "feature-persistent", &pers)) {
+        blkdev->feature_persistent = FALSE;
+    } else {
+        blkdev->feature_persistent = !!pers;
+    }
 
     blkdev->protocol = BLKIF_PROTOCOL_NATIVE;
     if (blkdev->xendev.protocol) {
@@ -730,6 +875,15 @@ static int blk_connect(struct XenDevice *xendev)
     }
     }
 
+    if (blkdev->feature_persistent) {
+        /* Init persistent grants */
+        blkdev->max_grants = max_requests * BLKIF_MAX_SEGMENTS_PER_REQUEST;
+        blkdev->persistent_gnts = g_tree_new_full((GCompareDataFunc)int_cmp,
+                                             NULL, NULL,
+                                             (GDestroyNotify)destroy_grant);
+        blkdev->persistent_gnt_count = 0;
+    }
+
     xen_be_bind_evtchn(&blkdev->xendev);
 
     xen_be_printf(&blkdev->xendev, 1, "ok: proto %s, ring-ref %d, "
@@ -770,6 +924,11 @@ static int blk_free(struct XenDevice *xendev)
         blk_disconnect(xendev);
     }
 
+    /* Free persistent grants */
+    if (blkdev->feature_persistent) {
+        g_tree_destroy(blkdev->persistent_gnts);
+    }
+
     while (!QLIST_EMPTY(&blkdev->freelist)) {
         ioreq = QLIST_FIRST(&blkdev->freelist);
         QLIST_REMOVE(ioreq, list);
diff --git a/hw/xen_domainbuild.c b/hw/xen_domainbuild.c
index db14974..a4272f0 100644
--- a/hw/xen_domainbuild.c
+++ b/hw/xen_domainbuild.c
@@ -1,8 +1,8 @@
 #include <signal.h>
 #include "xen_backend.h"
 #include "xen_domainbuild.h"
-#include "qemu-timer.h"
-#include "qemu-log.h"
+#include "qemu/timer.h"
+#include "qemu/log.h"
 
 #include <xenguest.h>
 
diff --git a/hw/xen_machine_pv.c b/hw/xen_machine_pv.c
index 4264703..66e8981 100644
--- a/hw/xen_machine_pv.c
+++ b/hw/xen_machine_pv.c
@@ -27,7 +27,7 @@
 #include "boards.h"
 #include "xen_backend.h"
 #include "xen_domainbuild.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 
 static void xen_init_pv(QEMUMachineInitArgs *args)
 {
@@ -115,6 +115,7 @@ static QEMUMachine xenpv_machine = {
     .init = xen_init_pv,
     .max_cpus = 1,
     .default_machine_opts = "accel=xen",
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void xenpv_machine_init(void)
diff --git a/hw/xen_nic.c b/hw/xen_nic.c
index cf7d559..34961c2 100644
--- a/hw/xen_nic.c
+++ b/hw/xen_nic.c
@@ -36,10 +36,9 @@
 #include <sys/wait.h>
 
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 #include "net/checksum.h"
 #include "net/util.h"
-#include "qemu-char.h"
 #include "xen_backend.h"
 
 #include <xen/io/netif.h>
@@ -186,9 +185,11 @@ static void net_tx_packets(struct XenNetDev *netdev)
                 }
                 memcpy(tmpbuf, page + txreq.offset, txreq.size);
                 net_checksum_calculate(tmpbuf, txreq.size);
-                qemu_send_packet(&netdev->nic->nc, tmpbuf, txreq.size);
+                qemu_send_packet(qemu_get_queue(netdev->nic), tmpbuf,
+                                 txreq.size);
             } else {
-                qemu_send_packet(&netdev->nic->nc, page + txreq.offset, txreq.size);
+                qemu_send_packet(qemu_get_queue(netdev->nic),
+                                 page + txreq.offset, txreq.size);
             }
             xc_gnttab_munmap(netdev->xendev.gnttabdev, page, 1);
             net_tx_response(netdev, &txreq, NETIF_RSP_OKAY);
@@ -235,7 +236,7 @@ static void net_rx_response(struct XenNetDev *netdev,
 
 static int net_rx_ok(NetClientState *nc)
 {
-    struct XenNetDev *netdev = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XenNetDev *netdev = qemu_get_nic_opaque(nc);
     RING_IDX rc, rp;
 
     if (netdev->xendev.be_state != XenbusStateConnected) {
@@ -256,7 +257,7 @@ static int net_rx_ok(NetClientState *nc)
 
 static ssize_t net_rx_packet(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    struct XenNetDev *netdev = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XenNetDev *netdev = qemu_get_nic_opaque(nc);
     netif_rx_request_t rxreq;
     RING_IDX rc, rp;
     void *page;
@@ -325,12 +326,11 @@ static int net_init(struct XenDevice *xendev)
         return -1;
     }
 
-    netdev->conf.peer = NULL;
-
     netdev->nic = qemu_new_nic(&net_xen_info, &netdev->conf,
                                "xen", NULL, netdev);
 
-    snprintf(netdev->nic->nc.info_str, sizeof(netdev->nic->nc.info_str),
+    snprintf(qemu_get_queue(netdev->nic)->info_str,
+             sizeof(qemu_get_queue(netdev->nic)->info_str),
              "nic: xenbus vif macaddr=%s", netdev->mac);
 
     /* fill info */
@@ -406,7 +406,7 @@ static void net_disconnect(struct XenDevice *xendev)
         netdev->rxs = NULL;
     }
     if (netdev->nic) {
-        qemu_del_net_client(&netdev->nic->nc);
+        qemu_del_nic(netdev->nic);
         netdev->nic = NULL;
     }
 }
@@ -415,7 +415,7 @@ static void net_event(struct XenDevice *xendev)
 {
     struct XenNetDev *netdev = container_of(xendev, struct XenNetDev, xendev);
     net_tx_packets(netdev);
-    qemu_flush_queued_packets(&netdev->nic->nc);
+    qemu_flush_queued_packets(qemu_get_queue(netdev->nic));
 }
 
 static int net_free(struct XenDevice *xendev)
diff --git a/hw/xen_platform.c b/hw/xen_platform.c
index a54e7a2..8866468 100644
--- a/hw/xen_platform.c
+++ b/hw/xen_platform.c
@@ -27,13 +27,12 @@
 
 #include "hw.h"
 #include "pc.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "irq.h"
 #include "xen_common.h"
-#include "net.h"
 #include "xen_backend.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #include <xenguest.h>
 
@@ -280,7 +279,8 @@ static void platform_fixed_ioport_init(PCIXenPlatformState* s)
 
 /* Xen Platform PCI Device */
 
-static uint32_t xen_platform_ioport_readb(void *opaque, uint32_t addr)
+static uint64_t xen_platform_ioport_readb(void *opaque, hwaddr addr,
+                                          unsigned int size)
 {
     if (addr == 0) {
         return platform_fixed_ioport_readb(opaque, 0);
@@ -289,30 +289,28 @@ static uint32_t xen_platform_ioport_readb(void *opaque, uint32_t addr)
     }
 }
 
-static void xen_platform_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void xen_platform_ioport_writeb(void *opaque, hwaddr addr,
+                                       uint64_t val, unsigned int size)
 {
     PCIXenPlatformState *s = opaque;
 
     switch (addr) {
     case 0: /* Platform flags */
-        platform_fixed_ioport_writeb(opaque, 0, val);
+        platform_fixed_ioport_writeb(opaque, 0, (uint32_t)val);
         break;
     case 8:
-        log_writeb(s, val);
+        log_writeb(s, (uint32_t)val);
         break;
     default:
         break;
     }
 }
 
-static MemoryRegionPortio xen_pci_portio[] = {
-    { 0, 0x100, 1, .read = xen_platform_ioport_readb, },
-    { 0, 0x100, 1, .write = xen_platform_ioport_writeb, },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps xen_pci_io_ops = {
-    .old_portio = xen_pci_portio,
+    .read  = xen_platform_ioport_readb,
+    .write = xen_platform_ioport_writeb,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
 };
 
 static void platform_ioport_bar_setup(PCIXenPlatformState *d)
@@ -421,7 +419,7 @@ static void xen_platform_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_xen_platform;
 }
 
-static TypeInfo xen_platform_info = {
+static const TypeInfo xen_platform_info = {
     .name          = "xen-platform",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIXenPlatformState),
diff --git a/hw/xen_pt.c b/hw/xen_pt.c
index 7a3846e..9db5f6e 100644
--- a/hw/xen_pt.c
+++ b/hw/xen_pt.c
@@ -54,12 +54,12 @@
 
 #include <sys/ioctl.h>
 
-#include "pci.h"
+#include "pci/pci.h"
 #include "xen.h"
 #include "xen_backend.h"
 #include "xen_pt.h"
-#include "range.h"
-#include "exec-memory.h"
+#include "qemu/range.h"
+#include "exec/address-spaces.h"
 
 #define XEN_PT_NR_IRQS (256)
 static uint8_t xen_pt_mapped_machine_irq[XEN_PT_NR_IRQS] = {0};
@@ -671,7 +671,8 @@ static int xen_pt_initfn(PCIDevice *d)
     s->is_virtfn = s->real_device.is_virtfn;
     if (s->is_virtfn) {
         XEN_PT_LOG(d, "%04x:%02x:%02x.%d is a SR-IOV Virtual Function\n",
-                   s->real_device.domain, bus, slot, func);
+                   s->real_device.domain, s->real_device.bus,
+                   s->real_device.dev, s->real_device.func);
     }
 
     /* Initialize virtualized PCI configuration (Extended 256 Bytes) */
@@ -752,7 +753,7 @@ out:
     memory_listener_register(&s->memory_listener, &address_space_memory);
     memory_listener_register(&s->io_listener, &address_space_io);
     XEN_PT_LOG(d, "Real physical device %02x:%02x.%d registered successfuly!\n",
-               bus, slot, func);
+               s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function);
 
     return 0;
 }
@@ -828,7 +829,7 @@ static void xen_pci_passthrough_class_init(ObjectClass *klass, void *data)
     dc->props = xen_pci_passthrough_properties;
 };
 
-static TypeInfo xen_pci_passthrough_info = {
+static const TypeInfo xen_pci_passthrough_info = {
     .name = "xen-pci-passthrough",
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XenPCIPassthroughState),
diff --git a/hw/xen_pt.h b/hw/xen_pt.h
index f15e69a..e349730 100644
--- a/hw/xen_pt.h
+++ b/hw/xen_pt.h
@@ -3,7 +3,7 @@
 
 #include "qemu-common.h"
 #include "xen_common.h"
-#include "pci.h"
+#include "pci/pci.h"
 #include "xen-host-pci-device.h"
 
 void xen_pt_log(const PCIDevice *d, const char *f, ...) GCC_FMT_ATTR(2, 3);
diff --git a/hw/xen_pt_config_init.c b/hw/xen_pt_config_init.c
index 0a5f82c..54a179a 100644
--- a/hw/xen_pt_config_init.c
+++ b/hw/xen_pt_config_init.c
@@ -12,7 +12,7 @@
  * This file implements direct PCI assignment to a HVM guest
  */
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "xen_backend.h"
 #include "xen_pt.h"
 
diff --git a/hw/xen_pt_msi.c b/hw/xen_pt_msi.c
index 6807672..db757cd 100644
--- a/hw/xen_pt_msi.c
+++ b/hw/xen_pt_msi.c
@@ -321,7 +321,7 @@ static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr)
 
     pirq = entry->pirq;
 
-    rc = msi_msix_setup(s, entry->data, entry->data, &pirq, true, entry_nr,
+    rc = msi_msix_setup(s, entry->addr, entry->data, &pirq, true, entry_nr,
                         entry->pirq == XEN_PT_UNASSIGNED_PIRQ);
     if (rc) {
         return rc;
diff --git a/hw/xenfb.c b/hw/xenfb.c
index 442a63a..903efd3 100644
--- a/hw/xenfb.c
+++ b/hw/xenfb.c
@@ -36,8 +36,8 @@
 #include <time.h>
 
 #include "hw.h"
-#include "console.h"
-#include "qemu-char.h"
+#include "ui/console.h"
+#include "char/char.h"
 #include "xen_backend.h"
 
 #include <xen/event_channel.h>
diff --git a/hw/xgmac.c b/hw/xgmac.c
index ec50c74..5072298 100644
--- a/hw/xgmac.c
+++ b/hw/xgmac.c
@@ -25,9 +25,9 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
-#include "net.h"
+#include "char/char.h"
+#include "qemu/log.h"
+#include "net/net.h"
 #include "net/checksum.h"
 
 #ifdef DEBUG_XGMAC
@@ -235,7 +235,7 @@ static void xgmac_enet_send(struct XgmacState *s)
         frame_size += len;
         if (bd.ctl_stat & 0x20000000) {
             /* Last buffer in frame.  */
-            qemu_send_packet(&s->nic->nc, frame, len);
+            qemu_send_packet(qemu_get_queue(s->nic), frame, len);
             ptr = frame;
             frame_size = 0;
             s->regs[DMA_STATUS] |= DMA_STATUS_TI | DMA_STATUS_NIS;
@@ -310,7 +310,7 @@ static const MemoryRegionOps enet_mem_ops = {
 
 static int eth_can_rx(NetClientState *nc)
 {
-    struct XgmacState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XgmacState *s = qemu_get_nic_opaque(nc);
 
     /* RX enabled?  */
     return s->regs[DMA_CONTROL] & DMA_CONTROL_SR;
@@ -318,7 +318,7 @@ static int eth_can_rx(NetClientState *nc)
 
 static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    struct XgmacState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XgmacState *s = qemu_get_nic_opaque(nc);
     static const unsigned char sa_bcast[6] = {0xff, 0xff, 0xff,
                                               0xff, 0xff, 0xff};
     int unicast, broadcast, multicast;
@@ -366,7 +366,7 @@ out:
 
 static void eth_cleanup(NetClientState *nc)
 {
-    struct XgmacState *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XgmacState *s = qemu_get_nic_opaque(nc);
     s->nic = NULL;
 }
 
@@ -391,7 +391,7 @@ static int xgmac_enet_init(SysBusDevice *dev)
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_xgmac_enet_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     s->regs[XGMAC_ADDR_HIGH(0)] = (s->conf.macaddr.a[5] << 8) |
                                    s->conf.macaddr.a[4];
@@ -418,7 +418,7 @@ static void xgmac_enet_class_init(ObjectClass *klass, void *data)
     dc->props = xgmac_properties;
 }
 
-static TypeInfo xgmac_enet_info = {
+static const TypeInfo xgmac_enet_info = {
     .name          = "xgmac",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct XgmacState),
diff --git a/hw/xics.c b/hw/xics.c
index 1da3106..9ef0d61 100644
--- a/hw/xics.c
+++ b/hw/xics.c
@@ -26,6 +26,7 @@
  */
 
 #include "hw.h"
+#include "trace.h"
 #include "hw/spapr.h"
 #include "hw/xics.h"
 
@@ -66,6 +67,8 @@ static void icp_check_ipi(struct icp_state *icp, int server)
         return;
     }
 
+    trace_xics_icp_check_ipi(server, ss->mfrr);
+
     if (XISR(ss)) {
         ics_reject(icp->ics, XISR(ss));
     }
@@ -120,11 +123,13 @@ static void icp_set_mfrr(struct icp_state *icp, int server, uint8_t mfrr)
 
 static uint32_t icp_accept(struct icp_server_state *ss)
 {
-    uint32_t xirr;
+    uint32_t xirr = ss->xirr;
 
     qemu_irq_lower(ss->output);
-    xirr = ss->xirr;
     ss->xirr = ss->pending_priority << 24;
+
+    trace_xics_icp_accept(xirr, ss->xirr);
+
     return xirr;
 }
 
@@ -134,6 +139,7 @@ static void icp_eoi(struct icp_state *icp, int server, uint32_t xirr)
 
     /* Send EOI -> ICS */
     ss->xirr = (ss->xirr & ~CPPR_MASK) | (xirr & CPPR_MASK);
+    trace_xics_icp_eoi(server, xirr, ss->xirr);
     ics_eoi(icp->ics, xirr & XISR_MASK);
     if (!XISR(ss)) {
         icp_resend(icp, server);
@@ -144,6 +150,8 @@ static void icp_irq(struct icp_state *icp, int server, int nr, uint8_t priority)
 {
     struct icp_server_state *ss = icp->ss + server;
 
+    trace_xics_icp_irq(server, nr, priority);
+
     if ((priority >= CPPR(ss))
         || (XISR(ss) && (ss->pending_priority <= priority))) {
         ics_reject(icp->ics, nr);
@@ -153,6 +161,7 @@ static void icp_irq(struct icp_state *icp, int server, int nr, uint8_t priority)
         }
         ss->xirr = (ss->xirr & ~XISR_MASK) | (nr & XISR_MASK);
         ss->pending_priority = priority;
+        trace_xics_icp_raise(ss->xirr, ss->pending_priority);
         qemu_irq_raise(ss->output);
     }
 }
@@ -170,13 +179,13 @@ struct ics_irq_state {
 #define XICS_STATUS_REJECTED           0x4
 #define XICS_STATUS_MASKED_PENDING     0x8
     uint8_t status;
-    bool lsi;
 };
 
 struct ics_state {
     int nr_irqs;
     int offset;
     qemu_irq *qirqs;
+    bool *islsi;
     struct ics_irq_state *irqs;
     struct icp_state *icp;
 };
@@ -217,10 +226,12 @@ static void set_irq_msi(struct ics_state *ics, int srcno, int val)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
+    trace_xics_set_irq_msi(srcno, srcno + ics->offset);
+
     if (val) {
         if (irq->priority == 0xff) {
             irq->status |= XICS_STATUS_MASKED_PENDING;
-            /* masked pending */ ;
+            trace_xics_masked_pending();
         } else  {
             icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
         }
@@ -231,6 +242,7 @@ static void set_irq_lsi(struct ics_state *ics, int srcno, int val)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
+    trace_xics_set_irq_lsi(srcno, srcno + ics->offset);
     if (val) {
         irq->status |= XICS_STATUS_ASSERTED;
     } else {
@@ -242,9 +254,8 @@ static void set_irq_lsi(struct ics_state *ics, int srcno, int val)
 static void ics_set_irq(void *opaque, int srcno, int val)
 {
     struct ics_state *ics = (struct ics_state *)opaque;
-    struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (irq->lsi) {
+    if (ics->islsi[srcno]) {
         set_irq_lsi(ics, srcno, val);
     } else {
         set_irq_msi(ics, srcno, val);
@@ -279,7 +290,9 @@ static void ics_write_xive(struct ics_state *ics, int nr, int server,
     irq->priority = priority;
     irq->saved_priority = saved_priority;
 
-    if (irq->lsi) {
+    trace_xics_ics_write_xive(nr, srcno, server, priority);
+
+    if (ics->islsi[srcno]) {
         write_xive_lsi(ics, srcno);
     } else {
         write_xive_msi(ics, srcno);
@@ -290,6 +303,7 @@ static void ics_reject(struct ics_state *ics, int nr)
 {
     struct ics_irq_state *irq = ics->irqs + nr - ics->offset;
 
+    trace_xics_ics_reject(nr, nr - ics->offset);
     irq->status |= XICS_STATUS_REJECTED; /* Irrelevant but harmless for LSI */
     irq->status &= ~XICS_STATUS_SENT; /* Irrelevant but harmless for MSI */
 }
@@ -299,10 +313,8 @@ static void ics_resend(struct ics_state *ics)
     int i;
 
     for (i = 0; i < ics->nr_irqs; i++) {
-        struct ics_irq_state *irq = ics->irqs + i;
-
         /* FIXME: filter by server#? */
-        if (irq->lsi) {
+        if (ics->islsi[i]) {
             resend_lsi(ics, i);
         } else {
             resend_msi(ics, i);
@@ -315,7 +327,9 @@ static void ics_eoi(struct ics_state *ics, int nr)
     int srcno = nr - ics->offset;
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (irq->lsi) {
+    trace_xics_ics_eoi(nr);
+
+    if (ics->islsi[srcno]) {
         irq->status &= ~XICS_STATUS_SENT;
     }
 }
@@ -337,16 +351,16 @@ void xics_set_irq_type(struct icp_state *icp, int irq, bool lsi)
 {
     assert(ics_valid_irq(icp->ics, irq));
 
-    icp->ics->irqs[irq - icp->ics->offset].lsi = lsi;
+    icp->ics->islsi[irq - icp->ics->offset] = lsi;
 }
 
 static target_ulong h_cppr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
-    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     target_ulong cppr = args[0];
 
-    icp_set_cppr(spapr->icp, env->cpu_index, cppr);
+    icp_set_cppr(spapr->icp, cs->cpu_index, cppr);
     return H_SUCCESS;
 }
 
@@ -362,14 +376,13 @@ static target_ulong h_ipi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
 
     icp_set_mfrr(spapr->icp, server, mfrr);
     return H_SUCCESS;
-
 }
 
 static target_ulong h_xirr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
-    CPUPPCState *env = &cpu->env;
-    uint32_t xirr = icp_accept(spapr->icp->ss + env->cpu_index);
+    CPUState *cs = CPU(cpu);
+    uint32_t xirr = icp_accept(spapr->icp->ss + cs->cpu_index);
 
     args[0] = xirr;
     return H_SUCCESS;
@@ -378,10 +391,10 @@ static target_ulong h_xirr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
 static target_ulong h_eoi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                           target_ulong opcode, target_ulong *args)
 {
-    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     target_ulong xirr = args[0];
 
-    icp_eoi(spapr->icp, env->cpu_index, xirr);
+    icp_eoi(spapr->icp, cs->cpu_index, xirr);
     return H_SUCCESS;
 }
 
@@ -495,16 +508,14 @@ static void xics_reset(void *opaque)
 
     for (i = 0; i < icp->nr_servers; i++) {
         icp->ss[i].xirr = 0;
-        icp->ss[i].pending_priority = 0;
+        icp->ss[i].pending_priority = 0xff;
         icp->ss[i].mfrr = 0xff;
         /* Make all outputs are deasserted */
         qemu_set_irq(icp->ss[i].output, 0);
     }
 
+    memset(ics->irqs, 0, sizeof(struct ics_irq_state) * ics->nr_irqs);
     for (i = 0; i < ics->nr_irqs; i++) {
-        /* Reset everything *except* the type */
-        ics->irqs[i].server = 0;
-        ics->irqs[i].status = 0;
         ics->irqs[i].priority = 0xff;
         ics->irqs[i].saved_priority = 0xff;
     }
@@ -513,14 +524,16 @@ static void xics_reset(void *opaque)
 struct icp_state *xics_system_init(int nr_irqs)
 {
     CPUPPCState *env;
+    CPUState *cpu;
     int max_server_num;
     struct icp_state *icp;
     struct ics_state *ics;
 
     max_server_num = -1;
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->cpu_index > max_server_num) {
-            max_server_num = env->cpu_index;
+        cpu = CPU(ppc_env_get_cpu(env));
+        if (cpu->cpu_index > max_server_num) {
+            max_server_num = cpu->cpu_index;
         }
     }
 
@@ -529,7 +542,8 @@ struct icp_state *xics_system_init(int nr_irqs)
     icp->ss = g_malloc0(icp->nr_servers*sizeof(struct icp_server_state));
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        struct icp_server_state *ss = &icp->ss[env->cpu_index];
+        cpu = CPU(ppc_env_get_cpu(env));
+        struct icp_server_state *ss = &icp->ss[cpu->cpu_index];
 
         switch (PPC_INPUT(env)) {
         case PPC_FLAGS_INPUT_POWER7:
@@ -549,8 +563,9 @@ struct icp_state *xics_system_init(int nr_irqs)
 
     ics = g_malloc0(sizeof(*ics));
     ics->nr_irqs = nr_irqs;
-    ics->offset = 16;
+    ics->offset = XICS_IRQ_BASE;
     ics->irqs = g_malloc0(nr_irqs * sizeof(struct ics_irq_state));
+    ics->islsi = g_malloc0(nr_irqs * sizeof(bool));
 
     icp->ics = ics;
     ics->icp = icp;
diff --git a/hw/xics.h b/hw/xics.h
index 6817268..c3bf008 100644
--- a/hw/xics.h
+++ b/hw/xics.h
@@ -28,6 +28,7 @@
 #define __XICS_H__
 
 #define XICS_IPI        0x2
+#define XICS_IRQ_BASE   0x10
 
 struct icp_state;
 
diff --git a/hw/xilinx.h b/hw/xilinx.h
index 9323fd0..09bc2e4 100644
--- a/hw/xilinx.h
+++ b/hw/xilinx.h
@@ -1,6 +1,10 @@
+#ifndef HW_XILINX_H
+#define HW_XILINX_H 1
+
+
 #include "stream.h"
 #include "qemu-common.h"
-#include "net.h"
+#include "net/net.h"
 
 static inline DeviceState *
 xilinx_intc_create(hwaddr base, qemu_irq irq, int kind_of_intr)
@@ -10,8 +14,8 @@ xilinx_intc_create(hwaddr base, qemu_irq irq, int kind_of_intr)
     dev = qdev_create(NULL, "xlnx.xps-intc");
     qdev_prop_set_uint32(dev, "kind-of-intr", kind_of_intr);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
@@ -25,8 +29,8 @@ xilinx_timer_create(hwaddr base, qemu_irq irq, int oto, int freq)
     qdev_prop_set_uint32(dev, "one-timer-only", oto);
     qdev_prop_set_uint32(dev, "clock-frequency", freq);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
@@ -44,22 +48,17 @@ xilinx_ethlite_create(NICInfo *nd, hwaddr base, qemu_irq irq,
     qdev_prop_set_uint32(dev, "tx-ping-pong", txpingpong);
     qdev_prop_set_uint32(dev, "rx-ping-pong", rxpingpong);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
     return dev;
 }
 
-static inline DeviceState *
-xilinx_axiethernet_create(NICInfo *nd, StreamSlave *peer,
-                          hwaddr base, qemu_irq irq,
-                          int txmem, int rxmem)
+static inline void
+xilinx_axiethernet_init(DeviceState *dev, NICInfo *nd, StreamSlave *peer,
+                        hwaddr base, qemu_irq irq, int txmem, int rxmem)
 {
-    DeviceState *dev;
     Error *errp = NULL;
 
-    qemu_check_nic_model(nd, "xlnx.axi-ethernet");
-
-    dev = qdev_create(NULL, "xlnx.axi-ethernet");
     qdev_set_nic_properties(dev, nd);
     qdev_prop_set_uint32(dev, "rxmem", rxmem);
     qdev_prop_set_uint32(dev, "txmem", txmem);
@@ -67,16 +66,13 @@ xilinx_axiethernet_create(NICInfo *nd, StreamSlave *peer,
                              &errp);
     assert_no_error(errp);
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
-
-    return dev;
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
 }
 
 static inline void
-xilinx_axiethernetdma_init(DeviceState *dev, StreamSlave *peer,
-                           hwaddr base, qemu_irq irq,
-                           qemu_irq irq2, int freqhz)
+xilinx_axidma_init(DeviceState *dev, StreamSlave *peer, hwaddr base,
+                   qemu_irq irq, qemu_irq irq2, int freqhz)
 {
     Error *errp = NULL;
 
@@ -86,7 +82,9 @@ xilinx_axiethernetdma_init(DeviceState *dev, StreamSlave *peer,
     assert_no_error(errp);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
-    sysbus_connect_irq(sysbus_from_qdev(dev), 1, irq2);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, irq2);
 }
+
+#endif
diff --git a/hw/xilinx_axidma.c b/hw/xilinx_axidma.c
index 4575da1..cc51584 100644
--- a/hw/xilinx_axidma.c
+++ b/hw/xilinx_axidma.c
@@ -23,10 +23,9 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "ptimer.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "qdev-addr.h"
 
 #include "stream.h"
@@ -445,7 +444,7 @@ static void axidma_write(void *opaque, hwaddr addr,
             break;
         default:
             D(qemu_log("%s: ch=%d addr=" TARGET_FMT_plx " v=%x\n",
-                  __func__, sid, addr * 4, value));
+                  __func__, sid, addr * 4, (unsigned)value));
             s->regs[addr] = value;
             break;
     }
@@ -504,7 +503,7 @@ static void axidma_class_init(ObjectClass *klass, void *data)
     ssc->push = axidma_push;
 }
 
-static TypeInfo axidma_info = {
+static const TypeInfo axidma_info = {
     .name          = "xlnx.axi-dma",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct XilinxAXIDMA),
diff --git a/hw/xilinx_axienet.c b/hw/xilinx_axienet.c
index baae02b..34e344c 100644
--- a/hw/xilinx_axienet.c
+++ b/hw/xilinx_axienet.c
@@ -23,9 +23,8 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
-#include "qemu-log.h"
-#include "net.h"
+#include "qemu/log.h"
+#include "net/net.h"
 #include "net/checksum.h"
 
 #include "stream.h"
@@ -591,6 +590,10 @@ static void enet_write(void *opaque, hwaddr addr,
             s->maddr[s->fmi & 3][addr & 1] = value;
             break;
 
+        case R_IS:
+            s->regs[addr] &= ~value;
+            break;
+
         case 0x8000 ... 0x83ff:
             s->ext_mtable[addr - 0x8000] = value;
             break;
@@ -614,7 +617,7 @@ static const MemoryRegionOps enet_ops = {
 
 static int eth_can_rx(NetClientState *nc)
 {
-    struct XilinxAXIEnet *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XilinxAXIEnet *s = qemu_get_nic_opaque(nc);
 
     /* RX enabled?  */
     return !axienet_rx_resetting(s) && axienet_rx_enabled(s);
@@ -637,7 +640,7 @@ static int enet_match_addr(const uint8_t *buf, uint32_t f0, uint32_t f1)
 
 static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    struct XilinxAXIEnet *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XilinxAXIEnet *s = qemu_get_nic_opaque(nc);
     static const unsigned char sa_bcast[6] = {0xff, 0xff, 0xff,
                                               0xff, 0xff, 0xff};
     static const unsigned char sa_ipmcast[3] = {0x01, 0x00, 0x52};
@@ -782,7 +785,7 @@ static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 static void eth_cleanup(NetClientState *nc)
 {
     /* FIXME.  */
-    struct XilinxAXIEnet *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct XilinxAXIEnet *s = qemu_get_nic_opaque(nc);
     g_free(s->rxmem);
     g_free(s);
 }
@@ -823,7 +826,7 @@ axienet_stream_push(StreamSlave *obj, uint8_t *buf, size_t size, uint32_t *hdr)
         buf[write_off + 1] = csum & 0xff;
     }
 
-    qemu_send_packet(&s->nic->nc, buf, size);
+    qemu_send_packet(qemu_get_queue(s->nic), buf, size);
 
     s->stats.tx_bytes += size;
     s->regs[R_IS] |= IS_TX_COMPLETE;
@@ -850,7 +853,7 @@ static int xilinx_enet_init(SysBusDevice *dev)
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_xilinx_enet_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 
     tdk_init(&s->TEMAC.phy);
     mdio_attach(&s->TEMAC.mdio_bus, &s->TEMAC.phy, s->c_phyaddr);
@@ -890,7 +893,7 @@ static void xilinx_enet_class_init(ObjectClass *klass, void *data)
     ssc->push = axienet_stream_push;
 }
 
-static TypeInfo xilinx_enet_info = {
+static const TypeInfo xilinx_enet_info = {
     .name          = "xlnx.axi-ethernet",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct XilinxAXIEnet),
diff --git a/hw/xilinx_ethlite.c b/hw/xilinx_ethlite.c
index 13bd456..21c6f8c 100644
--- a/hw/xilinx_ethlite.c
+++ b/hw/xilinx_ethlite.c
@@ -24,7 +24,7 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "net.h"
+#include "net/net.h"
 
 #define D(x)
 #define R_TX_BUF0     0
@@ -89,7 +89,7 @@ eth_read(void *opaque, hwaddr addr, unsigned int size)
         case R_RX_CTRL1:
         case R_RX_CTRL0:
             r = s->regs[addr];
-            D(qemu_log("%s %x=%x\n", __func__, addr * 4, r));
+            D(qemu_log("%s " TARGET_FMT_plx "=%x\n", __func__, addr * 4, r));
             break;
 
         default:
@@ -115,9 +115,10 @@ eth_write(void *opaque, hwaddr addr,
             if (addr == R_TX_CTRL1)
                 base = 0x800 / 4;
 
-            D(qemu_log("%s addr=%x val=%x\n", __func__, addr * 4, value));
+            D(qemu_log("%s addr=" TARGET_FMT_plx " val=%x\n",
+                       __func__, addr * 4, value));
             if ((value & (CTRL_P | CTRL_S)) == CTRL_S) {
-                qemu_send_packet(&s->nic->nc,
+                qemu_send_packet(qemu_get_queue(s->nic),
                                  (void *) &s->regs[base],
                                  s->regs[base + R_TX_LEN0]);
                 D(qemu_log("eth_tx %d\n", s->regs[base + R_TX_LEN0]));
@@ -135,12 +136,16 @@ eth_write(void *opaque, hwaddr addr,
             break;
 
         /* Keep these native.  */
+        case R_RX_CTRL0:
+        case R_RX_CTRL1:
+            if (!(value & CTRL_S)) {
+                qemu_flush_queued_packets(qemu_get_queue(s->nic));
+            }
         case R_TX_LEN0:
         case R_TX_LEN1:
         case R_TX_GIE0:
-        case R_RX_CTRL0:
-        case R_RX_CTRL1:
-            D(qemu_log("%s addr=%x val=%x\n", __func__, addr * 4, value));
+            D(qemu_log("%s addr=" TARGET_FMT_plx " val=%x\n",
+                       __func__, addr * 4, value));
             s->regs[addr] = value;
             break;
 
@@ -162,15 +167,15 @@ static const MemoryRegionOps eth_ops = {
 
 static int eth_can_rx(NetClientState *nc)
 {
-    struct xlx_ethlite *s = DO_UPCAST(NICState, nc, nc)->opaque;
-    int r;
-    r = !(s->regs[R_RX_CTRL0] & CTRL_S);
-    return r;
+    struct xlx_ethlite *s = qemu_get_nic_opaque(nc);
+    unsigned int rxbase = s->rxbuf * (0x800 / 4);
+
+    return !(s->regs[rxbase + R_RX_CTRL0] & CTRL_S);
 }
 
 static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    struct xlx_ethlite *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct xlx_ethlite *s = qemu_get_nic_opaque(nc);
     unsigned int rxbase = s->rxbuf * (0x800 / 4);
 
     /* DA filter.  */
@@ -182,7 +187,7 @@ static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
         return -1;
     }
 
-    D(qemu_log("%s %d rxbase=%x\n", __func__, size, rxbase));
+    D(qemu_log("%s %zd rxbase=%x\n", __func__, size, rxbase));
     memcpy(&s->regs[rxbase + R_RX_BUF0], buf, size);
 
     s->regs[rxbase + R_RX_CTRL0] |= CTRL_S;
@@ -196,7 +201,7 @@ static ssize_t eth_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 
 static void eth_cleanup(NetClientState *nc)
 {
-    struct xlx_ethlite *s = DO_UPCAST(NICState, nc, nc)->opaque;
+    struct xlx_ethlite *s = qemu_get_nic_opaque(nc);
 
     s->nic = NULL;
 }
@@ -223,7 +228,7 @@ static int xilinx_ethlite_init(SysBusDevice *dev)
     qemu_macaddr_default_if_unset(&s->conf.macaddr);
     s->nic = qemu_new_nic(&net_xilinx_ethlite_info, &s->conf,
                           object_get_typename(OBJECT(dev)), dev->qdev.id, s);
-    qemu_format_nic_info_str(&s->nic->nc, s->conf.macaddr.a);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
     return 0;
 }
 
@@ -243,7 +248,7 @@ static void xilinx_ethlite_class_init(ObjectClass *klass, void *data)
     dc->props = xilinx_ethlite_properties;
 }
 
-static TypeInfo xilinx_ethlite_info = {
+static const TypeInfo xilinx_ethlite_info = {
     .name          = "xlnx.xps-ethernetlite",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct xlx_ethlite),
diff --git a/hw/xilinx_intc.c b/hw/xilinx_intc.c
index 7765079..0c34149 100644
--- a/hw/xilinx_intc.c
+++ b/hw/xilinx_intc.c
@@ -175,7 +175,7 @@ static void xilinx_intc_class_init(ObjectClass *klass, void *data)
     dc->props = xilinx_intc_properties;
 }
 
-static TypeInfo xilinx_intc_info = {
+static const TypeInfo xilinx_intc_info = {
     .name          = "xlnx.xps-intc",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct xlx_pic),
diff --git a/hw/xilinx_spi.c b/hw/xilinx_spi.c
index 0390274..be581c2 100644
--- a/hw/xilinx_spi.c
+++ b/hw/xilinx_spi.c
@@ -25,8 +25,8 @@
  */
 
 #include "sysbus.h"
-#include "sysemu.h"
-#include "qemu-log.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
 #include "fifo.h"
 
 #include "ssi.h"
@@ -370,7 +370,7 @@ static void xilinx_spi_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_xilinx_spi;
 }
 
-static TypeInfo xilinx_spi_info = {
+static const TypeInfo xilinx_spi_info = {
     .name           = "xlnx.xps-spi",
     .parent         = TYPE_SYS_BUS_DEVICE,
     .instance_size  = sizeof(XilinxSPI),
diff --git a/hw/xilinx_spips.c b/hw/xilinx_spips.c
index ee7656d..42e019d 100644
--- a/hw/xilinx_spips.c
+++ b/hw/xilinx_spips.c
@@ -23,12 +23,12 @@
  */
 
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "ptimer.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "fifo.h"
 #include "ssi.h"
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 #ifdef XILINX_SPIPS_ERR_DEBUG
 #define DB_PRINT(...) do { \
diff --git a/hw/xilinx_timer.c b/hw/xilinx_timer.c
index 2b01dc2..aa162ef 100644
--- a/hw/xilinx_timer.c
+++ b/hw/xilinx_timer.c
@@ -24,7 +24,7 @@
 
 #include "sysbus.h"
 #include "ptimer.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #define D(x)
 
@@ -240,7 +240,7 @@ static void xilinx_timer_class_init(ObjectClass *klass, void *data)
     dc->props = xilinx_timer_properties;
 }
 
-static TypeInfo xilinx_timer_info = {
+static const TypeInfo xilinx_timer_info = {
     .name          = "xlnx.xps-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct timerblock),
diff --git a/hw/xilinx_uartlite.c b/hw/xilinx_uartlite.c
index d20fc41..9963982 100644
--- a/hw/xilinx_uartlite.c
+++ b/hw/xilinx_uartlite.c
@@ -23,7 +23,7 @@
  */
 
 #include "sysbus.h"
-#include "qemu-char.h"
+#include "char/char.h"
 
 #define DUART(x)
 
@@ -97,6 +97,7 @@ uart_read(void *opaque, hwaddr addr, unsigned int size)
                 s->rx_fifo_len--;
             uart_update_status(s);
             uart_update_irq(s);
+            qemu_chr_accept_input(s->chr);
             break;
 
         default:
@@ -182,12 +183,8 @@ static void uart_rx(void *opaque, const uint8_t *buf, int size)
 static int uart_can_rx(void *opaque)
 {
     struct xlx_uartlite *s = opaque;
-    int r;
 
-    r = s->rx_fifo_len < sizeof(s->rx_fifo);
-    if (!r)
-        printf("cannot receive!\n");
-    return r;
+    return s->rx_fifo_len < sizeof(s->rx_fifo);
 }
 
 static void uart_event(void *opaque, int event)
@@ -219,7 +216,7 @@ static void xilinx_uartlite_class_init(ObjectClass *klass, void *data)
     sdc->init = xilinx_uartlite_init;
 }
 
-static TypeInfo xilinx_uartlite_info = {
+static const TypeInfo xilinx_uartlite_info = {
     .name          = "xlnx.xps-uartlite",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof (struct xlx_uartlite),
diff --git a/hw/xilinx_zynq.c b/hw/xilinx_zynq.c
index 1f12a3d..311f791 100644
--- a/hw/xilinx_zynq.c
+++ b/hw/xilinx_zynq.c
@@ -17,12 +17,12 @@
 
 #include "sysbus.h"
 #include "arm-misc.h"
-#include "net.h"
-#include "exec-memory.h"
-#include "sysemu.h"
+#include "net/net.h"
+#include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "flash.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
 #include "loader.h"
 #include "ssi.h"
 
@@ -46,7 +46,7 @@ static void gem_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     dev = qdev_create(NULL, "cadence_gem");
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(s, 0, base);
     sysbus_connect_irq(s, 0, irq);
 }
@@ -57,6 +57,7 @@ static inline void zynq_init_spi_flashes(uint32_t base_addr, qemu_irq irq,
     DeviceState *dev;
     SysBusDevice *busdev;
     SSIBus *spi;
+    DeviceState *flash_dev;
     int i, j;
     int num_busses =  is_qspi ? NUM_QSPI_BUSSES : 1;
     int num_ss = is_qspi ? NUM_QSPI_FLASHES : NUM_SPI_FLASHES;
@@ -66,7 +67,7 @@ static inline void zynq_init_spi_flashes(uint32_t base_addr, qemu_irq irq,
     qdev_prop_set_uint8(dev, "num-ss-bits", num_ss);
     qdev_prop_set_uint8(dev, "num-busses", num_busses);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, base_addr);
     if (is_qspi) {
         sysbus_mmio_map(busdev, 1, 0xFC000000);
@@ -81,11 +82,11 @@ static inline void zynq_init_spi_flashes(uint32_t base_addr, qemu_irq irq,
         spi = (SSIBus *)qdev_get_child_bus(dev, bus_name);
 
         for (j = 0; j < num_ss; ++j) {
-            dev = ssi_create_slave_no_init(spi, "m25p80");
-            qdev_prop_set_string(dev, "partname", "n25q128");
-            qdev_init_nofail(dev);
+            flash_dev = ssi_create_slave_no_init(spi, "m25p80");
+            qdev_prop_set_string(flash_dev, "partname", "n25q128");
+            qdev_init_nofail(flash_dev);
 
-            cs_line = qdev_get_gpio_in(dev, 0);
+            cs_line = qdev_get_gpio_in(flash_dev, 0);
             sysbus_connect_irq(busdev, i * num_ss + j + 1, cs_line);
         }
     }
@@ -149,12 +150,12 @@ static void zynq_init(QEMUMachineInitArgs *args)
 
     dev = qdev_create(NULL, "xilinx,zynq_slcr");
     qdev_init_nofail(dev);
-    sysbus_mmio_map(sysbus_from_qdev(dev), 0, 0xF8000000);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xF8000000);
 
     dev = qdev_create(NULL, "a9mpcore_priv");
     qdev_prop_set_uint32(dev, "num-cpu", 1);
     qdev_init_nofail(dev);
-    busdev = sysbus_from_qdev(dev);
+    busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xF8F00000);
     sysbus_connect_irq(busdev, 0, cpu_irq);
 
@@ -167,7 +168,7 @@ static void zynq_init(QEMUMachineInitArgs *args)
     zynq_init_spi_flashes(0xE000D000, pic[51-IRQ_OFFSET], true);
 
     sysbus_create_simple("xlnx,ps7-usb", 0xE0002000, pic[53-IRQ_OFFSET]);
-    sysbus_create_simple("xlnx,ps7-usb", 0xE0003000, pic[75-IRQ_OFFSET]);
+    sysbus_create_simple("xlnx,ps7-usb", 0xE0003000, pic[76-IRQ_OFFSET]);
 
     sysbus_create_simple("cadence_uart", 0xE0000000, pic[59-IRQ_OFFSET]);
     sysbus_create_simple("cadence_uart", 0xE0001000, pic[82-IRQ_OFFSET]);
@@ -200,9 +201,10 @@ static QEMUMachine zynq_machine = {
     .name = "xilinx-zynq-a9",
     .desc = "Xilinx Zynq Platform Baseboard for Cortex-A9",
     .init = zynq_init,
-    .use_scsi = 1,
+    .block_default_type = IF_SCSI,
     .max_cpus = 1,
-    .no_sdcard = 1
+    .no_sdcard = 1,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void zynq_machine_init(void)
diff --git a/hw/xio3130_downstream.c b/hw/xio3130_downstream.c
index 0d8a5e7..7f00bc8 100644
--- a/hw/xio3130_downstream.c
+++ b/hw/xio3130_downstream.c
@@ -19,9 +19,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_ids.h"
-#include "msi.h"
-#include "pcie.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/pcie.h"
 #include "xio3130_downstream.h"
 
 #define PCI_DEVICE_ID_TI_XIO3130D       0x8233  /* downstream port */
@@ -193,7 +193,7 @@ static void xio3130_downstream_class_init(ObjectClass *klass, void *data)
     dc->props = xio3130_downstream_properties;
 }
 
-static TypeInfo xio3130_downstream_info = {
+static const TypeInfo xio3130_downstream_info = {
     .name          = "xio3130-downstream",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIESlot),
diff --git a/hw/xio3130_downstream.h b/hw/xio3130_downstream.h
index 010487f..559dff6 100644
--- a/hw/xio3130_downstream.h
+++ b/hw/xio3130_downstream.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_XIO3130_DOWNSTREAM_H
 #define QEMU_XIO3130_DOWNSTREAM_H
 
-#include "pcie_port.h"
+#include "pci/pcie_port.h"
 
 PCIESlot *xio3130_downstream_init(PCIBus *bus, int devfn, bool multifunction,
                                   const char *bus_name, pci_map_irq_fn map_irq,
diff --git a/hw/xio3130_upstream.c b/hw/xio3130_upstream.c
index d46b86c..70b15d3 100644
--- a/hw/xio3130_upstream.c
+++ b/hw/xio3130_upstream.c
@@ -19,9 +19,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "pci_ids.h"
-#include "msi.h"
-#include "pcie.h"
+#include "pci/pci_ids.h"
+#include "pci/msi.h"
+#include "pci/pcie.h"
 #include "xio3130_upstream.h"
 
 #define PCI_DEVICE_ID_TI_XIO3130U       0x8232  /* upstream port */
@@ -167,7 +167,7 @@ static void xio3130_upstream_class_init(ObjectClass *klass, void *data)
     dc->props = xio3130_upstream_properties;
 }
 
-static TypeInfo xio3130_upstream_info = {
+static const TypeInfo xio3130_upstream_info = {
     .name          = "x3130-upstream",
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIEPort),
diff --git a/hw/xio3130_upstream.h b/hw/xio3130_upstream.h
index e996997..fa09656 100644
--- a/hw/xio3130_upstream.h
+++ b/hw/xio3130_upstream.h
@@ -1,7 +1,7 @@
 #ifndef QEMU_XIO3130_UPSTREAM_H
 #define QEMU_XIO3130_UPSTREAM_H
 
-#include "pcie_port.h"
+#include "pci/pcie_port.h"
 
 PCIEPort *xio3130_upstream_init(PCIBus *bus, int devfn, bool multifunction,
                                 const char *bus_name, pci_map_irq_fn map_irq,
diff --git a/hw/xtensa_lx60.c b/hw/xtensa_lx60.c
index 4c42edc..a810b9e 100644
--- a/hw/xtensa_lx60.c
+++ b/hw/xtensa_lx60.c
@@ -25,17 +25,18 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 #include "serial.h"
-#include "net.h"
+#include "net/net.h"
 #include "sysbus.h"
 #include "flash.h"
-#include "blockdev.h"
+#include "sysemu/blockdev.h"
+#include "char/char.h"
 #include "xtensa_bootparam.h"
 
 typedef struct LxBoardDesc {
@@ -130,7 +131,7 @@ static void lx60_net_init(MemoryRegion *address_space,
     qdev_set_nic_properties(dev, nd);
     qdev_init_nofail(dev);
 
-    s = sysbus_from_qdev(dev);
+    s = SYS_BUS_DEVICE(dev);
     sysbus_connect_irq(s, 0, irq);
     memory_region_add_subregion(address_space, base,
             sysbus_mmio_get_region(s, 0));
@@ -294,6 +295,7 @@ static QEMUMachine xtensa_lx60_machine = {
     .desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
     .init = xtensa_lx60_init,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static QEMUMachine xtensa_lx200_machine = {
@@ -301,6 +303,7 @@ static QEMUMachine xtensa_lx200_machine = {
     .desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
     .init = xtensa_lx200_init,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void xtensa_lx_machines_init(void)
diff --git a/hw/xtensa_pic.c b/hw/xtensa_pic.c
index 1ec70cd..97d36be 100644
--- a/hw/xtensa_pic.c
+++ b/hw/xtensa_pic.c
@@ -26,8 +26,8 @@
  */
 
 #include "hw.h"
-#include "qemu-log.h"
-#include "qemu-timer.h"
+#include "qemu/log.h"
+#include "qemu/timer.h"
 
 void xtensa_advance_ccount(CPUXtensaState *env, uint32_t d)
 {
diff --git a/hw/xtensa_sim.c b/hw/xtensa_sim.c
index 0d633e4..864e57c 100644
--- a/hw/xtensa_sim.c
+++ b/hw/xtensa_sim.c
@@ -25,12 +25,12 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "boards.h"
 #include "loader.h"
 #include "elf.h"
-#include "memory.h"
-#include "exec-memory.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
 static uint64_t translate_phys_addr(void *env, uint64_t addr)
 {
@@ -106,6 +106,7 @@ static QEMUMachine xtensa_sim_machine = {
     .is_default = true,
     .init = xtensa_sim_init,
     .max_cpus = 4,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void xtensa_sim_machine_init(void)
diff --git a/hw/z2.c b/hw/z2.c
index f62b806..731550f 100644
--- a/hw/z2.c
+++ b/hw/z2.c
@@ -18,12 +18,12 @@
 #include "i2c.h"
 #include "ssi.h"
 #include "boards.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "flash.h"
-#include "blockdev.h"
-#include "console.h"
+#include "sysemu/blockdev.h"
+#include "ui/console.h"
 #include "audio/audio.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #ifdef DEBUG_Z2
 #define DPRINTF(fmt, ...) \
@@ -185,7 +185,7 @@ static void zipit_lcd_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_zipit_lcd_state;
 }
 
-static TypeInfo zipit_lcd_info = {
+static const TypeInfo zipit_lcd_info = {
     .name          = "zipit-lcd",
     .parent        = TYPE_SSI_SLAVE,
     .instance_size = sizeof(ZipitLCD),
@@ -288,7 +288,7 @@ static void aer915_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_aer915_state;
 }
 
-static TypeInfo aer915_info = {
+static const TypeInfo aer915_info = {
     .name          = "aer915",
     .parent        = TYPE_I2C_SLAVE,
     .instance_size = sizeof(AER915State),
@@ -373,6 +373,7 @@ static QEMUMachine z2_machine = {
     .name = "z2",
     .desc = "Zipit Z2 (PXA27x)",
     .init = z2_init,
+    DEFAULT_MACHINE_OPTIONS,
 };
 
 static void z2_machine_init(void)
diff --git a/hw/zaurus.c b/hw/zaurus.c
index d77b34e..2defe3b 100644
--- a/hw/zaurus.c
+++ b/hw/zaurus.c
@@ -236,7 +236,7 @@ static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
     dc->props = scoop_sysbus_properties;
 }
 
-static TypeInfo scoop_sysbus_info = {
+static const TypeInfo scoop_sysbus_info = {
     .name          = "scoop",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ScoopInfo),
diff --git a/hw/zynq_slcr.c b/hw/zynq_slcr.c
index dde4306..27b00f0 100644
--- a/hw/zynq_slcr.c
+++ b/hw/zynq_slcr.c
@@ -15,9 +15,9 @@
  */
 
 #include "hw.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "sysbus.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #ifdef ZYNQ_ARM_SLCR_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -160,7 +160,7 @@ static void zynq_slcr_reset(DeviceState *d)
 {
     int i;
     ZynqSLCRState *s =
-            FROM_SYSBUS(ZynqSLCRState, sysbus_from_qdev(d));
+            FROM_SYSBUS(ZynqSLCRState, SYS_BUS_DEVICE(d));
 
     DB_PRINT("RESET\n");
 
@@ -334,7 +334,7 @@ static uint64_t zynq_slcr_read(void *opaque, hwaddr offset,
 {
     uint32_t ret = zynq_slcr_read_imp(opaque, offset);
 
-    DB_PRINT("addr: %08x data: %08x\n", offset, ret);
+    DB_PRINT("addr: %08x data: %08x\n", (unsigned)offset, (unsigned)ret);
     return ret;
 }
 
@@ -343,7 +343,7 @@ static void zynq_slcr_write(void *opaque, hwaddr offset,
 {
     ZynqSLCRState *s = (ZynqSLCRState *)opaque;
 
-    DB_PRINT("offset: %08x data: %08x\n", offset, (unsigned)val);
+    DB_PRINT("offset: %08x data: %08x\n", (unsigned)offset, (unsigned)val);
 
     switch (offset) {
     case 0x00: /* SCL */
@@ -476,7 +476,8 @@ static void zynq_slcr_write(void *opaque, hwaddr offset,
             break;
         default:
         bad_reg:
-            DB_PRINT("Bad register write %x <= %08x\n", (int)offset, val);
+            DB_PRINT("Bad register write %x <= %08x\n", (int)offset,
+                     (unsigned)val);
         }
     } else {
         DB_PRINT("SCLR registers are locked. Unlock them first\n");
@@ -520,7 +521,7 @@ static void zynq_slcr_class_init(ObjectClass *klass, void *data)
     dc->reset = zynq_slcr_reset;
 }
 
-static TypeInfo zynq_slcr_info = {
+static const TypeInfo zynq_slcr_info = {
     .class_init = zynq_slcr_class_init,
     .name  = "xilinx,zynq_slcr",
     .parent = TYPE_SYS_BUS_DEVICE,
diff --git a/aes.h b/include/block/aes.h
index a0167eb..a0167eb 100644
--- a/aes.h
+++ b/include/block/aes.h
diff --git a/qemu-aio.h b/include/block/aio.h
index 3889fe9..8eda924 100644
--- a/qemu-aio.h
+++ b/include/block/aio.h
@@ -15,8 +15,8 @@
 #define QEMU_AIO_H
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "event_notifier.h"
+#include "qemu/queue.h"
+#include "qemu/event_notifier.h"
 
 typedef struct BlockDriverAIOCB BlockDriverAIOCB;
 typedef void BlockDriverCompletionFunc(void *opaque, int ret);
@@ -162,10 +162,6 @@ void qemu_bh_cancel(QEMUBH *bh);
  */
 void qemu_bh_delete(QEMUBH *bh);
 
-/* Flush any pending AIO operation. This function will block until all
- * outstanding AIO operations have been completed or cancelled. */
-void aio_flush(AioContext *ctx);
-
 /* Return whether there are any pending callbacks from the GSource
  * attached to the AioContext.
  *
@@ -177,16 +173,14 @@ bool aio_pending(AioContext *ctx);
  * aio as a result of executing I/O completion or bh callbacks.
  *
  * If there is no pending AIO operation or completion (bottom half),
- * return false.  If there are pending bottom halves, return true.
+ * return false.  If there are pending AIO operations of bottom halves,
+ * return true.
  *
  * If there are no pending bottom halves, but there are pending AIO
  * operations, it may not be possible to make any progress without
  * blocking.  If @blocking is true, this function will wait until one
  * or more AIO events have completed, to ensure something has moved
  * before returning.
- *
- * If @blocking is false, this function will also return false if the
- * function cannot make any progress without blocking.
  */
 bool aio_poll(AioContext *ctx, bool blocking);
 
@@ -196,7 +190,7 @@ typedef int (AioFlushHandler)(void *opaque);
 
 /* Register a file descriptor and associated callbacks.  Behaves very similarly
  * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
- * be invoked when using either qemu_aio_wait() or qemu_aio_flush().
+ * be invoked when using qemu_aio_wait().
  *
  * Code that invokes AIO completion functions should rely on this function
  * instead of qemu_set_fd_handler[2].
@@ -211,7 +205,7 @@ void aio_set_fd_handler(AioContext *ctx,
 
 /* Register an event notifier and associated callbacks.  Behaves very similarly
  * to event_notifier_set_handler.  Unlike event_notifier_set_handler, these callbacks
- * will be invoked when using either qemu_aio_wait() or qemu_aio_flush().
+ * will be invoked when using qemu_aio_wait().
  *
  * Code that invokes AIO completion functions should rely on this function
  * instead of event_notifier_set_handler.
@@ -228,7 +222,6 @@ GSource *aio_get_g_source(AioContext *ctx);
 
 /* Functions to operate on the main QEMU AioContext.  */
 
-void qemu_aio_flush(void);
 bool qemu_aio_wait(void);
 void qemu_aio_set_event_notifier(EventNotifier *notifier,
                                  EventNotifierHandler *io_read,
diff --git a/block.h b/include/block/block.h
index 722c620..5c3b911 100644
--- a/block.h
+++ b/include/block/block.h
@@ -1,11 +1,11 @@
 #ifndef BLOCK_H
 #define BLOCK_H
 
-#include "qemu-aio.h"
+#include "block/aio.h"
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-coroutine.h"
-#include "qobject.h"
+#include "qemu/option.h"
+#include "block/coroutine.h"
+#include "qapi/qmp/qobject.h"
 #include "qapi-types.h"
 
 /* block.c */
@@ -309,6 +309,10 @@ int bdrv_get_flags(BlockDriverState *bs);
 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
                           const uint8_t *buf, int nb_sectors);
 int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi);
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t sector_num, int nb_sectors,
+                            int64_t *cluster_sector_num,
+                            int *cluster_nb_sectors);
 
 const char *bdrv_get_encrypted_filename(BlockDriverState *bs);
 void bdrv_get_backing_filename(BlockDriverState *bs,
@@ -343,20 +347,20 @@ int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
                       int64_t pos, int size);
 
-int bdrv_img_create(const char *filename, const char *fmt,
-                    const char *base_filename, const char *base_fmt,
-                    char *options, uint64_t img_size, int flags);
+void bdrv_img_create(const char *filename, const char *fmt,
+                     const char *base_filename, const char *base_fmt,
+                     char *options, uint64_t img_size, int flags, Error **errp);
 
 void bdrv_set_buffer_alignment(BlockDriverState *bs, int align);
 void *qemu_blockalign(BlockDriverState *bs, size_t size);
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
 
-#define BDRV_SECTORS_PER_DIRTY_CHUNK 2048
-
-void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable);
+struct HBitmapIter;
+void bdrv_set_dirty_tracking(BlockDriverState *bs, int granularity);
 int bdrv_get_dirty(BlockDriverState *bs, int64_t sector);
 void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
 void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
-int64_t bdrv_get_next_dirty(BlockDriverState *bs, int64_t sector);
+void bdrv_dirty_iter_init(BlockDriverState *bs, struct HBitmapIter *hbi);
 int64_t bdrv_get_dirty_count(BlockDriverState *bs);
 
 void bdrv_enable_copy_on_read(BlockDriverState *bs);
@@ -365,6 +369,15 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs);
 void bdrv_set_in_use(BlockDriverState *bs, int in_use);
 int bdrv_in_use(BlockDriverState *bs);
 
+#ifdef CONFIG_LINUX_AIO
+int raw_get_aio_fd(BlockDriverState *bs);
+#else
+static inline int raw_get_aio_fd(BlockDriverState *bs)
+{
+    return -ENOTSUP;
+}
+#endif
+
 enum BlockAcctType {
     BDRV_ACCT_READ,
     BDRV_ACCT_WRITE,
@@ -431,4 +444,9 @@ typedef enum {
 #define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
 void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
 
+int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
+                           const char *tag);
+int bdrv_debug_resume(BlockDriverState *bs, const char *tag);
+bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag);
+
 #endif
diff --git a/block_int.h b/include/block/block_int.h
index 9deedb8..eaad53e 100644
--- a/block_int.h
+++ b/include/block/block_int.h
@@ -24,14 +24,15 @@
 #ifndef BLOCK_INT_H
 #define BLOCK_INT_H
 
-#include "block.h"
-#include "qemu-option.h"
-#include "qemu-queue.h"
-#include "qemu-coroutine.h"
-#include "qemu-timer.h"
+#include "block/block.h"
+#include "qemu/option.h"
+#include "qemu/queue.h"
+#include "block/coroutine.h"
+#include "qemu/timer.h"
 #include "qapi-types.h"
-#include "qerror.h"
-#include "monitor.h"
+#include "qapi/qmp/qerror.h"
+#include "monitor/monitor.h"
+#include "qemu/hbitmap.h"
 
 #define BLOCK_FLAG_ENCRYPT          1
 #define BLOCK_FLAG_COMPAT6          4
@@ -55,6 +56,7 @@
 #define BLOCK_OPT_SUBFMT            "subformat"
 #define BLOCK_OPT_COMPAT_LEVEL      "compat"
 #define BLOCK_OPT_LAZY_REFCOUNTS    "lazy_refcounts"
+#define BLOCK_OPT_ADAPTER_TYPE      "adapter_type"
 
 typedef struct BdrvTrackedRequest BdrvTrackedRequest;
 
@@ -190,6 +192,12 @@ struct BlockDriver {
 
     void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
 
+    /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
+    int (*bdrv_debug_breakpoint)(BlockDriverState *bs, const char *event,
+        const char *tag);
+    int (*bdrv_debug_resume)(BlockDriverState *bs, const char *tag);
+    bool (*bdrv_debug_is_suspended)(BlockDriverState *bs, const char *tag);
+
     /*
      * Returns 1 if newly created images are guaranteed to contain only
      * zeros, 0 otherwise.
@@ -269,8 +277,7 @@ struct BlockDriverState {
     bool iostatus_enabled;
     BlockDeviceIoStatus iostatus;
     char device_name[32];
-    unsigned long *dirty_bitmap;
-    int64_t dirty_count;
+    HBitmap *dirty_bitmap;
     int in_use; /* users other than guest access, eg. block migration */
     QTAILQ_ENTRY(BlockDriverState) list;
 
@@ -338,6 +345,8 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
  * @bs: Block device to operate on.
  * @target: Block device to write to.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @granularity: The chosen granularity for the dirty bitmap.
+ * @buf_size: The amount of data that can be in flight at one time.
  * @mode: Whether to collapse all images in the chain to the target.
  * @on_source_error: The action to take upon error reading from the source.
  * @on_target_error: The action to take upon error writing to the target.
@@ -351,8 +360,8 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
  * @bs will be switched to read from @target.
  */
 void mirror_start(BlockDriverState *bs, BlockDriverState *target,
-                  int64_t speed, MirrorSyncMode mode,
-                  BlockdevOnError on_source_error,
+                  int64_t speed, int64_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockDriverCompletionFunc *cb,
                   void *opaque, Error **errp);
diff --git a/blockjob.h b/include/block/blockjob.h
index 3792b73..c290d07 100644
--- a/blockjob.h
+++ b/include/block/blockjob.h
@@ -25,7 +25,7 @@
 #ifndef BLOCKJOB_H
 #define BLOCKJOB_H 1
 
-#include "block.h"
+#include "block/block.h"
 
 /**
  * BlockJobType:
diff --git a/qemu-coroutine.h b/include/block/coroutine.h
index 34c15d4..c31fae3 100644
--- a/qemu-coroutine.h
+++ b/include/block/coroutine.h
@@ -16,8 +16,8 @@
 #define QEMU_COROUTINE_H
 
 #include <stdbool.h>
-#include "qemu-queue.h"
-#include "qemu-timer.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
 
 /**
  * Coroutines are a mechanism for stack switching and can be used for
diff --git a/qemu-coroutine-int.h b/include/block/coroutine_int.h
index 0f1bd80..17eb71e 100644
--- a/qemu-coroutine-int.h
+++ b/include/block/coroutine_int.h
@@ -25,8 +25,8 @@
 #ifndef QEMU_COROUTINE_INT_H
 #define QEMU_COROUTINE_INT_H
 
-#include "qemu-queue.h"
-#include "qemu-coroutine.h"
+#include "qemu/queue.h"
+#include "block/coroutine.h"
 
 typedef enum {
     COROUTINE_YIELD = 1,
diff --git a/nbd.h b/include/block/nbd.h
index 344f05b..344f05b 100644
--- a/nbd.h
+++ b/include/block/nbd.h
diff --git a/thread-pool.h b/include/block/thread-pool.h
index 378a4ac..200703e 100644
--- a/thread-pool.h
+++ b/include/block/thread-pool.h
@@ -19,10 +19,10 @@
 #define QEMU_THREAD_POOL_H 1
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "qemu-thread.h"
-#include "qemu-coroutine.h"
-#include "block_int.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "block/coroutine.h"
+#include "block/block_int.h"
 
 typedef int ThreadPoolFunc(void *opaque);
 
diff --git a/include/bt/bt.h b/include/bt/bt.h
new file mode 100644
index 0000000..2bc6d53
--- /dev/null
+++ b/include/bt/bt.h
@@ -0,0 +1,20 @@
+#ifndef BT_HOST_H
+#define BT_HOST_H
+
+/* BT HCI info */
+
+struct HCIInfo {
+    int (*bdaddr_set)(struct HCIInfo *hci, const uint8_t *bd_addr);
+    void (*cmd_send)(struct HCIInfo *hci, const uint8_t *data, int len);
+    void (*sco_send)(struct HCIInfo *hci, const uint8_t *data, int len);
+    void (*acl_send)(struct HCIInfo *hci, const uint8_t *data, int len);
+    void *opaque;
+    void (*evt_recv)(void *opaque, const uint8_t *data, int len);
+    void (*acl_recv)(void *opaque, const uint8_t *data, int len);
+};
+
+/* bt-host.c */
+struct HCIInfo *bt_host_hci(const char *id);
+struct HCIInfo *qemu_next_hci(void);
+
+#endif
diff --git a/qemu-char.h b/include/char/char.h
index a121e04..c91ce3c 100644
--- a/qemu-char.h
+++ b/include/char/char.h
@@ -2,13 +2,13 @@
 #define QEMU_CHAR_H
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-aio.h"
-#include "qobject.h"
-#include "qstring.h"
-#include "main-loop.h"
+#include "qemu/queue.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "block/aio.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/main-loop.h"
 
 /* character device */
 
@@ -75,6 +75,7 @@ struct CharDriverState {
     char *filename;
     int opened;
     int avail_connections;
+    QemuOpts *opts;
     QTAILQ_ENTRY(CharDriverState) next;
 };
 
@@ -89,7 +90,8 @@ struct CharDriverState {
  * Returns: a new character backend
  */
 CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
-                                    void (*init)(struct CharDriverState *s));
+                                    void (*init)(struct CharDriverState *s),
+                                    Error **errp);
 
 /**
  * @qemu_chr_new:
diff --git a/config.h b/include/config.h
index e20f786..e20f786 100644
--- a/config.h
+++ b/include/config.h
diff --git a/dis-asm.h b/include/disas/bfd.h
index 3944b3c..3944b3c 100644
--- a/dis-asm.h
+++ b/include/disas/bfd.h
diff --git a/disas.h b/include/disas/disas.h
index c13ca9a..c13ca9a 100644
--- a/disas.h
+++ b/include/disas/disas.h
diff --git a/elf.h b/include/elf.h
index a21ea53..a21ea53 100644
--- a/elf.h
+++ b/include/elf.h
diff --git a/exec-memory.h b/include/exec/address-spaces.h
index ac1d07d..3d12cdd 100644
--- a/exec-memory.h
+++ b/include/exec/address-spaces.h
@@ -19,7 +19,7 @@
  * you're one of them.
  */
 
-#include "memory.h"
+#include "exec/memory.h"
 
 #ifndef CONFIG_USER_ONLY
 
diff --git a/cpu-all.h b/include/exec/cpu-all.h
index c9c51b8..249e046 100644
--- a/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -20,8 +20,9 @@
 #define CPU_ALL_H
 
 #include "qemu-common.h"
-#include "qemu-tls.h"
-#include "cpu-common.h"
+#include "qemu/tls.h"
+#include "exec/cpu-common.h"
+#include "qemu/thread.h"
 
 /* some important defines:
  *
@@ -180,7 +181,7 @@ static inline void tswap64s(uint64_t *s)
 
 #if defined(CONFIG_USER_ONLY)
 #include <assert.h>
-#include "qemu-types.h"
+#include "exec/user/abitypes.h"
 
 /* On some host systems the guest address space is reserved on the host.
  * This allows the guest address space to be offset to a convenient location.
@@ -353,7 +354,6 @@ int page_check_range(target_ulong start, target_ulong len, int flags);
 #endif
 
 CPUArchState *cpu_copy(CPUArchState *env);
-CPUArchState *qemu_get_cpu(int cpu);
 
 #define CPU_DUMP_CODE 0x00010000
 #define CPU_DUMP_FPU 0x00020000 /* dump FPU register state, not just integer */
@@ -487,15 +487,23 @@ typedef struct RAMBlock {
     ram_addr_t length;
     uint32_t flags;
     char idstr[256];
-    QLIST_ENTRY(RAMBlock) next;
+    /* Reads can take either the iothread or the ramlist lock.
+     * Writes must take both locks.
+     */
+    QTAILQ_ENTRY(RAMBlock) next;
 #if defined(__linux__) && !defined(TARGET_S390X)
     int fd;
 #endif
 } RAMBlock;
 
 typedef struct RAMList {
+    QemuMutex mutex;
+    /* Protected by the iothread lock.  */
     uint8_t *phys_dirty;
-    QLIST_HEAD(, RAMBlock) blocks;
+    RAMBlock *mru_block;
+    /* Protected by the ramlist lock.  */
+    QTAILQ_HEAD(, RAMBlock) blocks;
+    uint32_t version;
 } RAMList;
 extern RAMList ram_list;
 
@@ -514,6 +522,8 @@ extern int mem_prealloc;
 
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
 ram_addr_t last_ram_offset(void);
+void qemu_mutex_lock_ramlist(void);
+void qemu_mutex_unlock_ramlist(void);
 #endif /* !CONFIG_USER_ONLY */
 
 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
diff --git a/cpu-common.h b/include/exec/cpu-common.h
index d2fbafa..2e5f11f 100644
--- a/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -3,14 +3,26 @@
 
 /* CPU interfaces that are target independent.  */
 
-#include "hwaddr.h"
+#include "exec/hwaddr.h"
 
 #ifndef NEED_CPU_H
-#include "poison.h"
+#include "exec/poison.h"
 #endif
 
-#include "bswap.h"
-#include "qemu-queue.h"
+#include "qemu/bswap.h"
+#include "qemu/queue.h"
+
+/**
+ * CPUListState:
+ * @cpu_fprintf: Print function.
+ * @file: File to print to using @cpu_fprint.
+ *
+ * State commonly used for iterating over CPU models.
+ */
+typedef struct CPUListState {
+    fprintf_function cpu_fprintf;
+    FILE *file;
+} CPUListState;
 
 #if !defined(CONFIG_USER_ONLY)
 
diff --git a/cpu-defs.h b/include/exec/cpu-defs.h
index 3669241..2911b9f 100644
--- a/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -27,9 +27,9 @@
 #include <setjmp.h>
 #include <inttypes.h>
 #include <signal.h>
-#include "osdep.h"
-#include "qemu-queue.h"
-#include "hwaddr.h"
+#include "qemu/osdep.h"
+#include "qemu/queue.h"
+#include "exec/hwaddr.h"
 
 #ifndef TARGET_LONG_BITS
 #error TARGET_LONG_BITS must be defined before including this header
@@ -134,10 +134,6 @@ typedef struct icount_decr_u16 {
 } icount_decr_u16;
 #endif
 
-struct kvm_run;
-struct KVMState;
-struct qemu_work_item;
-
 typedef struct CPUBreakpoint {
     target_ulong pc;
     int flags; /* BP_* */
@@ -195,19 +191,11 @@ typedef struct CPUWatchpoint {
     int exception_index;                                                \
                                                                         \
     CPUArchState *next_cpu; /* next CPU sharing TB cache */                 \
-    int cpu_index; /* CPU index (informative) */                        \
     uint32_t host_tid; /* host thread ID */                             \
-    int numa_node; /* NUMA node this cpu is belonging to  */            \
-    int nr_cores;  /* number of cores within this CPU package */        \
-    int nr_threads;/* number of threads within this CPU */              \
     int running; /* Nonzero if cpu is currently running(usermode).  */  \
     /* user data */                                                     \
     void *opaque;                                                       \
                                                                         \
-    const char *cpu_model_str;                                          \
-    struct KVMState *kvm_state;                                         \
-    struct kvm_run *kvm_run;                                            \
-    int kvm_fd;                                                         \
-    int kvm_vcpu_dirty;
+    const char *cpu_model_str;
 
 #endif
diff --git a/cputlb.h b/include/exec/cputlb.h
index 733c885..733c885 100644
--- a/cputlb.h
+++ b/include/exec/cputlb.h
diff --git a/def-helper.h b/include/exec/def-helper.h
index 022a9ce..022a9ce 100644
--- a/def-helper.h
+++ b/include/exec/def-helper.h
diff --git a/exec-all.h b/include/exec/exec-all.h
index 21aacda..d235ef8 100644
--- a/exec-all.h
+++ b/include/exec/exec-all.h
@@ -70,11 +70,7 @@ typedef struct TranslationBlock TranslationBlock;
 
 #define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
 
-extern target_ulong gen_opc_pc[OPC_BUF_SIZE];
-extern uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
-extern uint16_t gen_opc_icount[OPC_BUF_SIZE];
-
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 void gen_intermediate_code(CPUArchState *env, struct TranslationBlock *tb);
 void gen_intermediate_code_pc(CPUArchState *env, struct TranslationBlock *tb);
@@ -84,8 +80,8 @@ void restore_state_to_opc(CPUArchState *env, struct TranslationBlock *tb,
 void cpu_gen_init(void);
 int cpu_gen_code(CPUArchState *env, struct TranslationBlock *tb,
                  int *gen_code_size_ptr);
-int cpu_restore_state(struct TranslationBlock *tb,
-                      CPUArchState *env, uintptr_t searched_pc);
+bool cpu_restore_state(CPUArchState *env, uintptr_t searched_pc);
+
 void QEMU_NORETURN cpu_resume_from_signal(CPUArchState *env1, void *puc);
 void QEMU_NORETURN cpu_io_recompile(CPUArchState *env, uintptr_t retaddr);
 TranslationBlock *tb_gen_code(CPUArchState *env, 
@@ -279,9 +275,7 @@ static inline void tb_add_jump(TranslationBlock *tb, int n,
     }
 }
 
-TranslationBlock *tb_find_pc(uintptr_t pc_ptr);
-
-#include "qemu-lock.h"
+#include "exec/spinlock.h"
 
 extern spinlock_t tb_lock;
 
@@ -290,14 +284,8 @@ extern int tb_invalidated_flag;
 /* The return address may point to the start of the next instruction.
    Subtracting one gets us the call instruction itself.  */
 #if defined(CONFIG_TCG_INTERPRETER)
-/* Softmmu, Alpha, MIPS, SH4 and SPARC user mode emulations call GETPC().
-   For all others, GETPC remains undefined (which makes TCI a little faster. */
-# if defined(CONFIG_SOFTMMU) || \
-    defined(TARGET_ALPHA) || defined(TARGET_MIPS) || \
-    defined(TARGET_SH4) || defined(TARGET_SPARC)
 extern uintptr_t tci_tb_ptr;
-#  define GETPC() tci_tb_ptr
-# endif
+# define GETPC() tci_tb_ptr
 #elif defined(__s390__) && !defined(__s390x__)
 # define GETPC() \
     (((uintptr_t)__builtin_return_address(0) & 0x7fffffffUL) - 1)
@@ -359,22 +347,22 @@ void io_mem_write(struct MemoryRegion *mr, hwaddr addr,
 void tlb_fill(CPUArchState *env1, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr);
 
-#include "softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 #define ACCESS_TYPE (NB_MMU_MODES + 1)
 #define MEMSUFFIX _code
 
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
diff --git a/gdbstub.h b/include/exec/gdbstub.h
index 668de66..49231fe 100644
--- a/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -35,7 +35,8 @@ static inline int cpu_index(CPUArchState *env)
 #if defined(CONFIG_USER_ONLY) && defined(CONFIG_USE_NPTL)
     return env->host_tid;
 #else
-    return env->cpu_index + 1;
+    CPUState *cpu = ENV_GET_CPU(env);
+    return cpu->cpu_index + 1;
 #endif
 }
 
diff --git a/gen-icount.h b/include/exec/gen-icount.h
index 248cf5b..8043b3b 100644
--- a/gen-icount.h
+++ b/include/exec/gen-icount.h
@@ -1,4 +1,7 @@
-#include "qemu-timer.h"
+#ifndef GEN_ICOUNT_H
+#define GEN_ICOUNT_H 1
+
+#include "qemu/timer.h"
 
 /* Helpers for instruction counting code generation.  */
 
@@ -46,3 +49,5 @@ static inline void gen_io_end(void)
     tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUArchState, can_do_io));
     tcg_temp_free_i32(tmp);
 }
+
+#endif
diff --git a/hwaddr.h b/include/exec/hwaddr.h
index 251cf92..251cf92 100644
--- a/hwaddr.h
+++ b/include/exec/hwaddr.h
diff --git a/ioport.h b/include/exec/ioport.h
index 23441cb..fc28350 100644
--- a/ioport.h
+++ b/include/exec/ioport.h
@@ -25,7 +25,7 @@
 #define IOPORT_H
 
 #include "qemu-common.h"
-#include "iorange.h"
+#include "exec/iorange.h"
 
 typedef uint32_t pio_addr_t;
 #define FMT_pioaddr     PRIx32
diff --git a/iorange.h b/include/exec/iorange.h
index cd980a8..cd980a8 100644
--- a/iorange.h
+++ b/include/exec/iorange.h
diff --git a/memory-internal.h b/include/exec/memory-internal.h
index 1da2400..1da2400 100644
--- a/memory-internal.h
+++ b/include/exec/memory-internal.h
diff --git a/memory.h b/include/exec/memory.h
index 9462bfd..2322732 100644
--- a/memory.h
+++ b/include/exec/memory.h
@@ -19,12 +19,12 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include "qemu-common.h"
-#include "cpu-common.h"
-#include "hwaddr.h"
-#include "qemu-queue.h"
-#include "iorange.h"
-#include "ioport.h"
-#include "int128.h"
+#include "exec/cpu-common.h"
+#include "exec/hwaddr.h"
+#include "qemu/queue.h"
+#include "exec/iorange.h"
+#include "exec/ioport.h"
+#include "qemu/int128.h"
 
 typedef struct MemoryRegionOps MemoryRegionOps;
 typedef struct MemoryRegion MemoryRegion;
@@ -454,6 +454,22 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size);
 
 /**
+ * memory_region_test_and_clear_dirty: Check whether a range of bytes is dirty
+ *                                     for a specified client. It clears them.
+ *
+ * Checks whether a range of bytes has been written to since the last
+ * call to memory_region_reset_dirty() with the same @client.  Dirty logging
+ * must be enabled.
+ *
+ * @mr: the memory region being queried.
+ * @addr: the address (relative to the start of the region) being queried.
+ * @size: the size of the range being queried.
+ * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
+ *          %DIRTY_MEMORY_VGA.
+ */
+bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
+                                        hwaddr size, unsigned client);
+/**
  * memory_region_sync_dirty_bitmap: Synchronize a region's dirty bitmap with
  *                                  any external TLBs (e.g. kvm)
  *
diff --git a/poison.h b/include/exec/poison.h
index 7d7b23b..7d7b23b 100644
--- a/poison.h
+++ b/include/exec/poison.h
diff --git a/softmmu-semi.h b/include/exec/softmmu-semi.h
index bcb979a..93798b9 100644
--- a/softmmu-semi.h
+++ b/include/exec/softmmu-semi.h
@@ -6,6 +6,8 @@
  *
  * This code is licensed under the GPL
  */
+#ifndef SOFTMMU_SEMI_H
+#define SOFTMMU_SEMI_H 1
 
 static inline uint32_t softmmu_tget32(CPUArchState *env, uint32_t addr)
 {
@@ -71,3 +73,5 @@ static void softmmu_unlock_user(CPUArchState *env, void *p, target_ulong addr,
     free(p);
 }
 #define unlock_user(s, args, len) softmmu_unlock_user(env, s, args, len)
+
+#endif
diff --git a/softmmu_defs.h b/include/exec/softmmu_defs.h
index 1f25e33..1f25e33 100644
--- a/softmmu_defs.h
+++ b/include/exec/softmmu_defs.h
diff --git a/softmmu_exec.h b/include/exec/softmmu_exec.h
index 8c73985..3e4e886 100644
--- a/softmmu_exec.h
+++ b/include/exec/softmmu_exec.h
@@ -19,37 +19,37 @@
 #define ldul_executive  ldl_executive
 #define ldul_supervisor ldl_supervisor
 
-#include "softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 #define ACCESS_TYPE 0
 #define MEMSUFFIX MMU_MODE0_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 
 #define ACCESS_TYPE 1
 #define MEMSUFFIX MMU_MODE1_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 
@@ -58,16 +58,16 @@
 #define ACCESS_TYPE 2
 #define MEMSUFFIX MMU_MODE2_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 3) */
@@ -77,16 +77,16 @@
 #define ACCESS_TYPE 3
 #define MEMSUFFIX MMU_MODE3_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 4) */
@@ -96,16 +96,16 @@
 #define ACCESS_TYPE 4
 #define MEMSUFFIX MMU_MODE4_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 5) */
@@ -115,16 +115,16 @@
 #define ACCESS_TYPE 5
 #define MEMSUFFIX MMU_MODE5_SUFFIX
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 6) */
@@ -137,16 +137,16 @@
 #define ACCESS_TYPE (NB_MMU_MODES)
 #define MEMSUFFIX _data
 #define DATA_SIZE 1
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 2
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 4
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 
 #define DATA_SIZE 8
-#include "softmmu_header.h"
+#include "exec/softmmu_header.h"
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
 
diff --git a/softmmu_header.h b/include/exec/softmmu_header.h
index d8d9c81..d8d9c81 100644
--- a/softmmu_header.h
+++ b/include/exec/softmmu_header.h
diff --git a/softmmu_template.h b/include/exec/softmmu_template.h
index ce30d8b..b219191 100644
--- a/softmmu_template.h
+++ b/include/exec/softmmu_template.h
@@ -21,8 +21,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "qemu-timer.h"
-#include "memory.h"
+#include "qemu/timer.h"
+#include "exec/memory.h"
 
 #define DATA_SIZE (1 << SHIFT)
 
diff --git a/qemu-lock.h b/include/exec/spinlock.h
index a72edda..a72edda 100644
--- a/qemu-lock.h
+++ b/include/exec/spinlock.h
diff --git a/linux-user/qemu-types.h b/include/exec/user/abitypes.h
index fe7f662..fe7f662 100644
--- a/linux-user/qemu-types.h
+++ b/include/exec/user/abitypes.h
diff --git a/thunk.h b/include/exec/user/thunk.h
index 87025c3..87025c3 100644
--- a/thunk.h
+++ b/include/exec/user/thunk.h
diff --git a/fpu/softfloat.h b/include/fpu/softfloat.h
index d8999b3..f3927e2 100644
--- a/fpu/softfloat.h
+++ b/include/fpu/softfloat.h
@@ -44,7 +44,7 @@ these four paragraphs for those parts of this code that are retained.
 
 #include <inttypes.h>
 #include "config-host.h"
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 /*----------------------------------------------------------------------------
 | Each of the following `typedef's defines the most convenient type that holds
@@ -237,6 +237,7 @@ float64 int64_to_float64( int64 STATUS_PARAM );
 float64 uint64_to_float64( uint64 STATUS_PARAM );
 floatx80 int64_to_floatx80( int64 STATUS_PARAM );
 float128 int64_to_float128( int64 STATUS_PARAM );
+float128 uint64_to_float128( uint64 STATUS_PARAM );
 
 /*----------------------------------------------------------------------------
 | Software half-precision conversion routines.
@@ -630,6 +631,8 @@ INLINE int float128_is_any_nan(float128 a)
         ((a.low != 0) || ((a.high & 0xffffffffffffLL) != 0));
 }
 
+#define float128_zero make_float128(0, 0)
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated quadruple-precision NaN.
 *----------------------------------------------------------------------------*/
diff --git a/libfdt_env.h b/include/libfdt_env.h
index 90d7f3b..3667d4c 100644
--- a/libfdt_env.h
+++ b/include/libfdt_env.h
@@ -19,18 +19,18 @@
 #ifndef _LIBFDT_ENV_H
 #define _LIBFDT_ENV_H
 
-#include "bswap.h"
+#include "qemu/bswap.h"
 
 #ifdef HOST_WORDS_BIGENDIAN
-#define fdt32_to_cpu(x)		(x)
-#define cpu_to_fdt32(x)		(x)
-#define fdt64_to_cpu(x)		(x)
-#define cpu_to_fdt64(x)		(x)
+#define fdt32_to_cpu(x)  (x)
+#define cpu_to_fdt32(x)  (x)
+#define fdt64_to_cpu(x)  (x)
+#define cpu_to_fdt64(x)  (x)
 #else
-#define fdt32_to_cpu(x)		(bswap_32((x)))
-#define cpu_to_fdt32(x)		(bswap_32((x)))
-#define fdt64_to_cpu(x)		(bswap_64((x)))
-#define cpu_to_fdt64(x)		(bswap_64((x)))
+#define fdt32_to_cpu(x)  bswap32(x)
+#define cpu_to_fdt32(x)  bswap32(x)
+#define fdt64_to_cpu(x)  bswap64(x)
+#define cpu_to_fdt64(x)  bswap64(x)
 #endif
 
 #endif /* _LIBFDT_ENV_H */
diff --git a/block-migration.h b/include/migration/block.h
index ffa8ac0..ffa8ac0 100644
--- a/block-migration.h
+++ b/include/migration/block.h
diff --git a/migration.h b/include/migration/migration.h
index c3a23cc..a8c9639 100644
--- a/migration.h
+++ b/include/migration/migration.h
@@ -14,11 +14,12 @@
 #ifndef QEMU_MIGRATION_H
 #define QEMU_MIGRATION_H
 
-#include "qdict.h"
+#include "qapi/qmp/qdict.h"
 #include "qemu-common.h"
-#include "notify.h"
-#include "error.h"
-#include "vmstate.h"
+#include "qemu/thread.h"
+#include "qemu/notify.h"
+#include "qapi/error.h"
+#include "migration/vmstate.h"
 #include "qapi-types.h"
 
 struct MigrationParams {
@@ -31,6 +32,13 @@ typedef struct MigrationState MigrationState;
 struct MigrationState
 {
     int64_t bandwidth_limit;
+    size_t bytes_xfer;
+    size_t xfer_limit;
+    uint8_t *buffer;
+    size_t buffer_size;
+    size_t buffer_capacity;
+    QemuThread thread;
+
     QEMUFile *file;
     int fd;
     int state;
@@ -45,6 +53,7 @@ struct MigrationState
     int64_t dirty_pages_rate;
     bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
     int64_t xbzrle_cache_size;
+    bool complete;
 };
 
 void process_incoming_migration(QEMUFile *f);
@@ -77,10 +86,6 @@ void migrate_fd_error(MigrationState *s);
 
 void migrate_fd_connect(MigrationState *s);
 
-ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
-                              size_t size);
-void migrate_fd_put_ready(MigrationState *s);
-int migrate_fd_wait_for_unfreeze(MigrationState *s);
 int migrate_fd_close(MigrationState *s);
 
 void add_migration_state_change_notifier(Notifier *notify);
@@ -127,5 +132,4 @@ int migrate_use_xbzrle(void);
 int64_t migrate_xbzrle_cache_size(void);
 
 int64_t xbzrle_cache_resize(int64_t new_size);
-
 #endif
diff --git a/include/qemu/page_cache.h b/include/migration/page_cache.h
index 3839ac7..3839ac7 100644
--- a/include/qemu/page_cache.h
+++ b/include/migration/page_cache.h
diff --git a/qemu-file.h b/include/migration/qemu-file.h
index d64bdbb..46fc11d 100644
--- a/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -81,6 +81,7 @@ QEMUFile *qemu_popen(FILE *popen_file, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
 int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
+int64_t qemu_ftell(QEMUFile *f);
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
 void qemu_put_byte(QEMUFile *f, int v);
 
@@ -113,11 +114,6 @@ int64_t qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
 int64_t qemu_file_get_rate_limit(QEMUFile *f);
 int qemu_file_get_error(QEMUFile *f);
 
-/* Try to send any outstanding data.  This function is useful when output is
- * halted due to rate limiting or EAGAIN errors occur as it can be used to
- * resume output. */
-int qemu_file_put_notify(QEMUFile *f);
-
 static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
 {
     qemu_put_be64(f, *pv);
diff --git a/vmstate.h b/include/migration/vmstate.h
index 623af0a..f27276c 100644
--- a/vmstate.h
+++ b/include/migration/vmstate.h
@@ -35,6 +35,7 @@ typedef struct SaveVMHandlers {
     int (*save_live_setup)(QEMUFile *f, void *opaque);
     int (*save_live_iterate)(QEMUFile *f, void *opaque);
     int (*save_live_complete)(QEMUFile *f, void *opaque);
+    uint64_t (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size);
     void (*cancel)(void *opaque);
     LoadStateHandler *load_state;
     bool (*is_active)(void *opaque);
diff --git a/monitor.h b/include/monitor/monitor.h
index b4ef955..87fb49c 100644
--- a/monitor.h
+++ b/include/monitor/monitor.h
@@ -2,11 +2,10 @@
 #define MONITOR_H
 
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qerror.h"
-#include "qdict.h"
-#include "block.h"
-#include "readline.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qdict.h"
+#include "block/block.h"
+#include "monitor/readline.h"
 
 extern Monitor *cur_mon;
 extern Monitor *default_mon;
diff --git a/readline.h b/include/monitor/readline.h
index fc9806e..fc9806e 100644
--- a/readline.h
+++ b/include/monitor/readline.h
diff --git a/net/checksum.h b/include/net/checksum.h
index 1f05298..1f05298 100644
--- a/net/checksum.h
+++ b/include/net/checksum.h
diff --git a/net.h b/include/net/net.h
index 04fda1d..43a045e 100644
--- a/net.h
+++ b/include/net/net.h
@@ -1,32 +1,40 @@
 #ifndef QEMU_NET_H
 #define QEMU_NET_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qdict.h"
-#include "qemu-option.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu/option.h"
 #include "net/queue.h"
-#include "vmstate.h"
+#include "migration/vmstate.h"
 #include "qapi-types.h"
 
+#define MAX_QUEUE_NUM 1024
+
 struct MACAddr {
     uint8_t a[6];
 };
 
 /* qdev nic properties */
 
+typedef struct NICPeers {
+    NetClientState *ncs[MAX_QUEUE_NUM];
+} NICPeers;
+
 typedef struct NICConf {
     MACAddr macaddr;
-    NetClientState *peer;
+    NICPeers peers;
     int32_t bootindex;
+    int32_t queues;
 } NICConf;
 
 #define DEFINE_NIC_PROPERTIES(_state, _conf)                            \
     DEFINE_PROP_MACADDR("mac",   _state, _conf.macaddr),                \
-    DEFINE_PROP_VLAN("vlan",     _state, _conf.peer),                   \
-    DEFINE_PROP_NETDEV("netdev", _state, _conf.peer),                   \
+    DEFINE_PROP_VLAN("vlan",     _state, _conf.peers),                   \
+    DEFINE_PROP_NETDEV("netdev", _state, _conf.peers),                   \
     DEFINE_PROP_INT32("bootindex", _state, _conf.bootindex, -1)
 
+
 /* Net clients */
 
 typedef void (NetPoll)(NetClientState *, bool enable);
@@ -35,6 +43,7 @@ typedef ssize_t (NetReceive)(NetClientState *, const uint8_t *, size_t);
 typedef ssize_t (NetReceiveIOV)(NetClientState *, const struct iovec *, int);
 typedef void (NetCleanup) (NetClientState *);
 typedef void (LinkStatusChanged)(NetClientState *);
+typedef void (NetClientDestructor)(NetClientState *);
 
 typedef struct NetClientInfo {
     NetClientOptionsKind type;
@@ -58,16 +67,20 @@ struct NetClientState {
     char *name;
     char info_str[256];
     unsigned receive_disabled : 1;
+    NetClientDestructor *destructor;
+    unsigned int queue_index;
 };
 
 typedef struct NICState {
-    NetClientState nc;
+    NetClientState ncs[MAX_QUEUE_NUM];
     NICConf *conf;
     void *opaque;
     bool peer_deleted;
 } NICState;
 
 NetClientState *qemu_find_netdev(const char *id);
+int qemu_find_net_clients_except(const char *id, NetClientState **ncs,
+                                 NetClientOptionsKind type, int max);
 NetClientState *qemu_new_net_client(NetClientInfo *info,
                                     NetClientState *peer,
                                     const char *model,
@@ -77,6 +90,11 @@ NICState *qemu_new_nic(NetClientInfo *info,
                        const char *model,
                        const char *name,
                        void *opaque);
+void qemu_del_nic(NICState *nic);
+NetClientState *qemu_get_subqueue(NICState *nic, int queue_index);
+NetClientState *qemu_get_queue(NICState *nic);
+NICState *qemu_get_nic(NetClientState *nc);
+void *qemu_get_nic_opaque(NetClientState *nc);
 void qemu_del_net_client(NetClientState *nc);
 NetClientState *qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                                               const char *client_str);
@@ -112,7 +130,7 @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                             void *opaque);
 
 void print_net_client(Monitor *mon, NetClientState *nc);
-void do_info_network(Monitor *mon);
+void do_info_network(Monitor *mon, const QDict *qdict);
 
 /* NIC info */
 
@@ -133,20 +151,6 @@ extern int nb_nics;
 extern NICInfo nd_table[MAX_NICS];
 extern int default_net;
 
-/* BT HCI info */
-
-struct HCIInfo {
-    int (*bdaddr_set)(struct HCIInfo *hci, const uint8_t *bd_addr);
-    void (*cmd_send)(struct HCIInfo *hci, const uint8_t *data, int len);
-    void (*sco_send)(struct HCIInfo *hci, const uint8_t *data, int len);
-    void (*acl_send)(struct HCIInfo *hci, const uint8_t *data, int len);
-    void *opaque;
-    void (*evt_recv)(void *opaque, const uint8_t *data, int len);
-    void (*acl_recv)(void *opaque, const uint8_t *data, int len);
-};
-
-struct HCIInfo *qemu_next_hci(void);
-
 /* from net.c */
 extern const char *legacy_tftp_prefix;
 extern const char *legacy_bootp_filename;
@@ -161,6 +165,9 @@ void net_host_device_remove(Monitor *mon, const QDict *qdict);
 void netdev_add(QemuOpts *opts, Error **errp);
 int qmp_netdev_add(Monitor *mon, const QDict *qdict, QObject **ret);
 
+int net_hub_id_for_client(NetClientState *nc, int *id);
+NetClientState *net_hub_port_find(int hub_id);
+
 #define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
 #define DEFAULT_NETWORK_DOWN_SCRIPT "/etc/qemu-ifdown"
 #define DEFAULT_BRIDGE_HELPER CONFIG_QEMU_HELPERDIR "/qemu-bridge-helper"
diff --git a/net/queue.h b/include/net/queue.h
index fc02b33..fc02b33 100644
--- a/net/queue.h
+++ b/include/net/queue.h
diff --git a/net/slirp.h b/include/net/slirp.h
index 2ca09b6..0502389 100644
--- a/net/slirp.h
+++ b/include/net/slirp.h
@@ -25,8 +25,8 @@
 #define QEMU_NET_SLIRP_H
 
 #include "qemu-common.h"
-#include "qdict.h"
-#include "qemu-option.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu/option.h"
 #include "qapi-types.h"
 
 #ifdef CONFIG_SLIRP
@@ -40,7 +40,7 @@ int net_slirp_parse_legacy(QemuOptsList *opts_list, const char *optarg, int *ret
 
 int net_slirp_smb(const char *exported_dir);
 
-void do_info_usernet(Monitor *mon);
+void do_info_usernet(Monitor *mon, const QDict *qdict);
 
 #endif
 
diff --git a/include/net/tap.h b/include/net/tap.h
new file mode 100644
index 0000000..a994f20
--- /dev/null
+++ b/include/net/tap.h
@@ -0,0 +1,69 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_NET_TAP_H
+#define QEMU_NET_TAP_H
+
+#include "qemu-common.h"
+#include "qapi-types.h"
+
+bool tap_has_ufo(NetClientState *nc);
+int tap_has_vnet_hdr(NetClientState *nc);
+int tap_has_vnet_hdr_len(NetClientState *nc, int len);
+void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr);
+void tap_set_offload(NetClientState *nc, int csum, int tso4, int tso6, int ecn, int ufo);
+void tap_set_vnet_hdr_len(NetClientState *nc, int len);
+int tap_enable(NetClientState *nc);
+int tap_disable(NetClientState *nc);
+
+int tap_get_fd(NetClientState *nc);
+
+struct vhost_net;
+struct vhost_net *tap_get_vhost_net(NetClientState *nc);
+
+struct virtio_net_hdr
+{
+#define VIRTIO_NET_HDR_F_NEEDS_CSUM     1       // Use csum_start, csum_offset
+#define VIRTIO_NET_HDR_F_DATA_VALID    2       // Csum is valid
+    uint8_t flags;
+#define VIRTIO_NET_HDR_GSO_NONE         0       // Not a GSO frame
+#define VIRTIO_NET_HDR_GSO_TCPV4        1       // GSO frame, IPv4 TCP (TSO)
+#define VIRTIO_NET_HDR_GSO_UDP          3       // GSO frame, IPv4 UDP (UFO)
+#define VIRTIO_NET_HDR_GSO_TCPV6        4       // GSO frame, IPv6 TCP
+#define VIRTIO_NET_HDR_GSO_ECN          0x80    // TCP has ECN set
+    uint8_t gso_type;
+    uint16_t hdr_len;
+    uint16_t gso_size;
+    uint16_t csum_start;
+    uint16_t csum_offset;
+};
+
+struct virtio_net_hdr_mrg_rxbuf
+{
+    struct virtio_net_hdr hdr;
+    uint16_t num_buffers;   /* Number of merged rx buffers */
+};
+
+#endif /* QEMU_NET_TAP_H */
diff --git a/qapi/qapi-dealloc-visitor.h b/include/qapi/dealloc-visitor.h
index 5842bc7..cf4c36d 100644
--- a/qapi/qapi-dealloc-visitor.h
+++ b/include/qapi/dealloc-visitor.h
@@ -14,7 +14,7 @@
 #ifndef QAPI_DEALLOC_VISITOR_H
 #define QAPI_DEALLOC_VISITOR_H
 
-#include "qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 typedef struct QapiDeallocVisitor QapiDeallocVisitor;
 
diff --git a/error.h b/include/qapi/error.h
index 4d52e73..5cd2f0c 100644
--- a/error.h
+++ b/include/qapi/error.h
@@ -12,7 +12,7 @@
 #ifndef ERROR_H
 #define ERROR_H
 
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include "qapi-types.h"
 #include <stdbool.h>
 
diff --git a/qapi/opts-visitor.h b/include/qapi/opts-visitor.h
index ea1a395..5939eee 100644
--- a/qapi/opts-visitor.h
+++ b/include/qapi/opts-visitor.h
@@ -13,8 +13,8 @@
 #ifndef OPTS_VISITOR_H
 #define OPTS_VISITOR_H
 
-#include "qapi-visit-core.h"
-#include "qemu-option.h"
+#include "qapi/visitor.h"
+#include "qemu/option.h"
 
 typedef struct OptsVisitor OptsVisitor;
 
diff --git a/qapi/qmp-input-visitor.h b/include/qapi/qmp-input-visitor.h
index e0a48a5..3ed499c 100644
--- a/qapi/qmp-input-visitor.h
+++ b/include/qapi/qmp-input-visitor.h
@@ -14,8 +14,8 @@
 #ifndef QMP_INPUT_VISITOR_H
 #define QMP_INPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
-#include "qobject.h"
+#include "qapi/visitor.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QmpInputVisitor QmpInputVisitor;
 
diff --git a/qapi/qmp-output-visitor.h b/include/qapi/qmp-output-visitor.h
index 4a649c2..2266770 100644
--- a/qapi/qmp-output-visitor.h
+++ b/include/qapi/qmp-output-visitor.h
@@ -14,8 +14,8 @@
 #ifndef QMP_OUTPUT_VISITOR_H
 #define QMP_OUTPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
-#include "qobject.h"
+#include "qapi/visitor.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QmpOutputVisitor QmpOutputVisitor;
 
diff --git a/qapi/qmp-core.h b/include/qapi/qmp/dispatch.h
index 00446cf..1ce11f5 100644
--- a/qapi/qmp-core.h
+++ b/include/qapi/qmp/dispatch.h
@@ -14,9 +14,9 @@
 #ifndef QMP_CORE_H
 #define QMP_CORE_H
 
-#include "qobject.h"
-#include "qdict.h"
-#include "error.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/error.h"
 
 typedef void (QmpCommandFunc)(QDict *, QObject **, Error **);
 
diff --git a/json-lexer.h b/include/qapi/qmp/json-lexer.h
index 10bc0a7..cdff046 100644
--- a/json-lexer.h
+++ b/include/qapi/qmp/json-lexer.h
@@ -14,8 +14,8 @@
 #ifndef QEMU_JSON_LEXER_H
 #define QEMU_JSON_LEXER_H
 
-#include "qstring.h"
-#include "qlist.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qlist.h"
 
 typedef enum json_token_type {
     JSON_OPERATOR = 100,
diff --git a/json-parser.h b/include/qapi/qmp/json-parser.h
index 8f2b5ec..44d88f3 100644
--- a/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -15,8 +15,8 @@
 #define QEMU_JSON_PARSER_H
 
 #include "qemu-common.h"
-#include "qlist.h"
-#include "error.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/error.h"
 
 QObject *json_parser_parse(QList *tokens, va_list *ap);
 QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp);
diff --git a/json-streamer.h b/include/qapi/qmp/json-streamer.h
index f09bc4d..823f7d7 100644
--- a/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -14,8 +14,8 @@
 #ifndef QEMU_JSON_STREAMER_H
 #define QEMU_JSON_STREAMER_H
 
-#include "qlist.h"
-#include "json-lexer.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/json-lexer.h"
 
 typedef struct JSONMessageParser
 {
diff --git a/qbool.h b/include/qapi/qmp/qbool.h
index fe66fcd..c4eaab9 100644
--- a/qbool.h
+++ b/include/qapi/qmp/qbool.h
@@ -15,7 +15,7 @@
 #define QBOOL_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QBool {
     QObject_HEAD;
diff --git a/qdict.h b/include/qapi/qmp/qdict.h
index 929d8d2..6d9a4be 100644
--- a/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -13,9 +13,9 @@
 #ifndef QDICT_H
 #define QDICT_H
 
-#include "qobject.h"
-#include "qlist.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qlist.h"
+#include "qemu/queue.h"
 #include <stdint.h>
 
 #define QDICT_BUCKET_MAX 512
diff --git a/qerror.h b/include/qapi/qmp/qerror.h
index 8db4309..6c0a18d 100644
--- a/qerror.h
+++ b/include/qapi/qmp/qerror.h
@@ -12,10 +12,10 @@
 #ifndef QERROR_H
 #define QERROR_H
 
-#include "qdict.h"
-#include "qstring.h"
-#include "qemu-error.h"
-#include "error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "qapi-types.h"
 #include <stdarg.h>
 
diff --git a/qfloat.h b/include/qapi/qmp/qfloat.h
index 9d67876..a865844 100644
--- a/qfloat.h
+++ b/include/qapi/qmp/qfloat.h
@@ -15,7 +15,7 @@
 #define QFLOAT_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QFloat {
     QObject_HEAD;
diff --git a/qint.h b/include/qapi/qmp/qint.h
index 6b1a15c..48a41b0 100644
--- a/qint.h
+++ b/include/qapi/qmp/qint.h
@@ -14,7 +14,7 @@
 #define QINT_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QInt {
     QObject_HEAD;
diff --git a/qjson.h b/include/qapi/qmp/qjson.h
index 1190d8a..73351ed 100644
--- a/qjson.h
+++ b/include/qapi/qmp/qjson.h
@@ -15,9 +15,9 @@
 #define QJSON_H
 
 #include <stdarg.h>
-#include "compiler.h"
-#include "qobject.h"
-#include "qstring.h"
+#include "qemu/compiler.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
 
 QObject *qobject_from_json(const char *string) GCC_FMT_ATTR(1, 0);
 QObject *qobject_from_jsonf(const char *string, ...) GCC_FMT_ATTR(1, 2);
diff --git a/qlist.h b/include/qapi/qmp/qlist.h
index ae776f9..382f04c 100644
--- a/qlist.h
+++ b/include/qapi/qmp/qlist.h
@@ -13,10 +13,9 @@
 #ifndef QLIST_H
 #define QLIST_H
 
-#include "qobject.h"
-#include "qemu-queue.h"
-#include "qemu-common.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
+#include "qemu/queue.h"
 
 typedef struct QListEntry {
     QObject *value;
diff --git a/qobject.h b/include/qapi/qmp/qobject.h
index 9124649..9124649 100644
--- a/qobject.h
+++ b/include/qapi/qmp/qobject.h
diff --git a/qstring.h b/include/qapi/qmp/qstring.h
index 84ccd96..0e690f4 100644
--- a/qstring.h
+++ b/include/qapi/qmp/qstring.h
@@ -14,7 +14,7 @@
 #define QSTRING_H
 
 #include <stdint.h>
-#include "qobject.h"
+#include "qapi/qmp/qobject.h"
 
 typedef struct QString {
     QObject_HEAD;
diff --git a/qemu-objects.h b/include/qapi/qmp/types.h
index c53fbaa..7782ec5 100644
--- a/qemu-objects.h
+++ b/include/qapi/qmp/types.h
@@ -13,13 +13,13 @@
 #ifndef QEMU_OBJECTS_H
 #define QEMU_OBJECTS_H
 
-#include "qobject.h"
-#include "qint.h"
-#include "qfloat.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qdict.h"
-#include "qlist.h"
-#include "qjson.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qjson.h"
 
 #endif /* QEMU_OBJECTS_H */
diff --git a/qapi/string-input-visitor.h b/include/qapi/string-input-visitor.h
index d269d42..089243c 100644
--- a/qapi/string-input-visitor.h
+++ b/include/qapi/string-input-visitor.h
@@ -13,7 +13,7 @@
 #ifndef STRING_INPUT_VISITOR_H
 #define STRING_INPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 typedef struct StringInputVisitor StringInputVisitor;
 
diff --git a/qapi/string-output-visitor.h b/include/qapi/string-output-visitor.h
index 8868454..ec81e42 100644
--- a/qapi/string-output-visitor.h
+++ b/include/qapi/string-output-visitor.h
@@ -13,7 +13,7 @@
 #ifndef STRING_OUTPUT_VISITOR_H
 #define STRING_OUTPUT_VISITOR_H
 
-#include "qapi-visit-core.h"
+#include "qapi/visitor.h"
 
 typedef struct StringOutputVisitor StringOutputVisitor;
 
diff --git a/include/qapi/visitor-impl.h b/include/qapi/visitor-impl.h
new file mode 100644
index 0000000..5159964
--- /dev/null
+++ b/include/qapi/visitor-impl.h
@@ -0,0 +1,63 @@
+/*
+ * Core Definitions for QAPI Visitor implementations
+ *
+ * Copyright (C) 2012 Red Hat, Inc.
+ *
+ * Author: Paolo Bonizni <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+#ifndef QAPI_VISITOR_IMPL_H
+#define QAPI_VISITOR_IMPL_H
+
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+
+struct Visitor
+{
+    /* Must be set */
+    void (*start_struct)(Visitor *v, void **obj, const char *kind,
+                         const char *name, size_t size, Error **errp);
+    void (*end_struct)(Visitor *v, Error **errp);
+
+    void (*start_list)(Visitor *v, const char *name, Error **errp);
+    GenericList *(*next_list)(Visitor *v, GenericList **list, Error **errp);
+    void (*end_list)(Visitor *v, Error **errp);
+
+    void (*type_enum)(Visitor *v, int *obj, const char *strings[],
+                      const char *kind, const char *name, Error **errp);
+
+    void (*type_int)(Visitor *v, int64_t *obj, const char *name, Error **errp);
+    void (*type_bool)(Visitor *v, bool *obj, const char *name, Error **errp);
+    void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
+    void (*type_number)(Visitor *v, double *obj, const char *name,
+                        Error **errp);
+
+    /* May be NULL */
+    void (*start_optional)(Visitor *v, bool *present, const char *name,
+                           Error **errp);
+    void (*end_optional)(Visitor *v, Error **errp);
+
+    void (*start_handle)(Visitor *v, void **obj, const char *kind,
+                         const char *name, Error **errp);
+    void (*end_handle)(Visitor *v, Error **errp);
+    void (*type_uint8)(Visitor *v, uint8_t *obj, const char *name, Error **errp);
+    void (*type_uint16)(Visitor *v, uint16_t *obj, const char *name, Error **errp);
+    void (*type_uint32)(Visitor *v, uint32_t *obj, const char *name, Error **errp);
+    void (*type_uint64)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
+    void (*type_int8)(Visitor *v, int8_t *obj, const char *name, Error **errp);
+    void (*type_int16)(Visitor *v, int16_t *obj, const char *name, Error **errp);
+    void (*type_int32)(Visitor *v, int32_t *obj, const char *name, Error **errp);
+    void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
+    /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
+    void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
+};
+
+void input_type_enum(Visitor *v, int *obj, const char *strings[],
+                     const char *kind, const char *name, Error **errp);
+void output_type_enum(Visitor *v, int *obj, const char *strings[],
+                      const char *kind, const char *name, Error **errp);
+
+#endif
diff --git a/qapi/qapi-visit-core.h b/include/qapi/visitor.h
index 60aceda..1fef18c 100644
--- a/qapi/qapi-visit-core.h
+++ b/include/qapi/visitor.h
@@ -13,7 +13,7 @@
 #ifndef QAPI_VISITOR_CORE_H
 #define QAPI_VISITOR_CORE_H
 
-#include "qapi/qapi-types-core.h"
+#include "qapi/error.h"
 #include <stdlib.h>
 
 typedef struct GenericList
@@ -24,46 +24,6 @@ typedef struct GenericList
 
 typedef struct Visitor Visitor;
 
-struct Visitor
-{
-    /* Must be set */
-    void (*start_struct)(Visitor *v, void **obj, const char *kind,
-                         const char *name, size_t size, Error **errp);
-    void (*end_struct)(Visitor *v, Error **errp);
-
-    void (*start_list)(Visitor *v, const char *name, Error **errp);
-    GenericList *(*next_list)(Visitor *v, GenericList **list, Error **errp);
-    void (*end_list)(Visitor *v, Error **errp);
-
-    void (*type_enum)(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name, Error **errp);
-
-    void (*type_int)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    void (*type_bool)(Visitor *v, bool *obj, const char *name, Error **errp);
-    void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
-    void (*type_number)(Visitor *v, double *obj, const char *name,
-                        Error **errp);
-
-    /* May be NULL */
-    void (*start_optional)(Visitor *v, bool *present, const char *name,
-                           Error **errp);
-    void (*end_optional)(Visitor *v, Error **errp);
-
-    void (*start_handle)(Visitor *v, void **obj, const char *kind,
-                         const char *name, Error **errp);
-    void (*end_handle)(Visitor *v, Error **errp);
-    void (*type_uint8)(Visitor *v, uint8_t *obj, const char *name, Error **errp);
-    void (*type_uint16)(Visitor *v, uint16_t *obj, const char *name, Error **errp);
-    void (*type_uint32)(Visitor *v, uint32_t *obj, const char *name, Error **errp);
-    void (*type_uint64)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-    void (*type_int8)(Visitor *v, int8_t *obj, const char *name, Error **errp);
-    void (*type_int16)(Visitor *v, int16_t *obj, const char *name, Error **errp);
-    void (*type_int32)(Visitor *v, int32_t *obj, const char *name, Error **errp);
-    void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
-    void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-};
-
 void visit_start_handle(Visitor *v, void **obj, const char *kind,
                         const char *name, Error **errp);
 void visit_end_handle(Visitor *v, Error **errp);
diff --git a/qemu-common.h b/include/qemu-common.h
index cef264c..80016ad 100644
--- a/qemu-common.h
+++ b/include/qemu-common.h
@@ -12,8 +12,9 @@
 #ifndef QEMU_COMMON_H
 #define QEMU_COMMON_H
 
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include "config-host.h"
+#include "qemu/typedefs.h"
 
 #if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
 #define WORDS_ALIGNED
@@ -21,15 +22,6 @@
 
 #define TFR(expr) do { if ((expr) != -1) break; } while (errno == EINTR)
 
-typedef struct QEMUTimer QEMUTimer;
-typedef struct QEMUFile QEMUFile;
-typedef struct QEMUBH QEMUBH;
-typedef struct DeviceState DeviceState;
-
-struct Monitor;
-typedef struct Monitor Monitor;
-typedef struct MigrationParams MigrationParams;
-
 /* we put basic includes here to avoid repeating them in device drivers */
 #include <stdlib.h>
 #include <stdio.h>
@@ -51,11 +43,11 @@ typedef struct MigrationParams MigrationParams;
 #include <glib.h>
 
 #ifdef _WIN32
-#include "qemu-os-win32.h"
+#include "sysemu/os-win32.h"
 #endif
 
 #ifdef CONFIG_POSIX
-#include "qemu-os-posix.h"
+#include "sysemu/os-posix.h"
 #endif
 
 #ifndef O_LARGEFILE
@@ -76,6 +68,9 @@ typedef struct MigrationParams MigrationParams;
 #if !defined(ECANCELED)
 #define ECANCELED 4097
 #endif
+#if !defined(EMEDIUMTYPE)
+#define EMEDIUMTYPE 4098
+#endif
 #ifndef TIME_MAX
 #define TIME_MAX LONG_MAX
 #endif
@@ -130,8 +125,8 @@ extern int use_icount;
 /* FIXME: Remove NEED_CPU_H.  */
 #ifndef NEED_CPU_H
 
-#include "osdep.h"
-#include "bswap.h"
+#include "qemu/osdep.h"
+#include "qemu/bswap.h"
 
 #else
 
@@ -178,6 +173,10 @@ int qemu_fdatasync(int fd);
 int fcntl_setfl(int fd, int flag);
 int qemu_parse_fd(const char *param);
 
+int parse_uint(const char *s, unsigned long long *value, char **endptr,
+               int base);
+int parse_uint_full(const char *s, unsigned long long *value, int base);
+
 /*
  * strtosz() suffixes used to specify the default treatment of an
  * argument passed to strtosz() without an explicit suffix.
@@ -258,48 +257,6 @@ struct ParallelIOArg {
 
 typedef int (*DMA_transfer_handler) (void *opaque, int nchan, int pos, int size);
 
-/* A load of opaque types so that device init declarations don't have to
-   pull in all the real definitions.  */
-typedef struct NICInfo NICInfo;
-typedef struct HCIInfo HCIInfo;
-typedef struct AudioState AudioState;
-typedef struct BlockDriverState BlockDriverState;
-typedef struct DriveInfo DriveInfo;
-typedef struct DisplayState DisplayState;
-typedef struct DisplayChangeListener DisplayChangeListener;
-typedef struct DisplaySurface DisplaySurface;
-typedef struct PixelFormat PixelFormat;
-typedef struct QemuConsole QemuConsole;
-typedef struct CharDriverState CharDriverState;
-typedef struct MACAddr MACAddr;
-typedef struct NetClientState NetClientState;
-typedef struct i2c_bus i2c_bus;
-typedef struct ISABus ISABus;
-typedef struct ISADevice ISADevice;
-typedef struct SMBusDevice SMBusDevice;
-typedef struct PCIHostState PCIHostState;
-typedef struct PCIExpressHost PCIExpressHost;
-typedef struct PCIBus PCIBus;
-typedef struct PCIDevice PCIDevice;
-typedef struct PCIExpressDevice PCIExpressDevice;
-typedef struct PCIBridge PCIBridge;
-typedef struct PCIEAERMsg PCIEAERMsg;
-typedef struct PCIEAERLog PCIEAERLog;
-typedef struct PCIEAERErr PCIEAERErr;
-typedef struct PCIEPort PCIEPort;
-typedef struct PCIESlot PCIESlot;
-typedef struct MSIMessage MSIMessage;
-typedef struct SerialState SerialState;
-typedef struct PCMCIACardState PCMCIACardState;
-typedef struct MouseTransformInfo MouseTransformInfo;
-typedef struct uWireSlave uWireSlave;
-typedef struct I2SCodec I2SCodec;
-typedef struct SSIBus SSIBus;
-typedef struct EventNotifier EventNotifier;
-typedef struct VirtIODevice VirtIODevice;
-typedef struct QEMUSGList QEMUSGList;
-typedef struct SHPCDevice SHPCDevice;
-
 typedef uint64_t pcibus_t;
 
 typedef enum LostTickPolicy {
@@ -338,7 +295,9 @@ struct qemu_work_item {
 };
 
 #ifdef CONFIG_USER_ONLY
-#define qemu_init_vcpu(env) do { } while (0)
+static inline void qemu_init_vcpu(void *env)
+{
+}
 #else
 void qemu_init_vcpu(void *env);
 #endif
@@ -379,6 +338,9 @@ void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
 void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
 void qemu_iovec_concat(QEMUIOVector *dst,
                        QEMUIOVector *src, size_t soffset, size_t sbytes);
+void qemu_iovec_concat_iov(QEMUIOVector *dst,
+                           struct iovec *src_iov, unsigned int src_cnt,
+                           size_t soffset, size_t sbytes);
 void qemu_iovec_destroy(QEMUIOVector *qiov);
 void qemu_iovec_reset(QEMUIOVector *qiov);
 size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
@@ -458,7 +420,7 @@ static inline bool is_power_of_2(uint64_t value)
 /* round down to the nearest power of 2*/
 int64_t pow2floor(int64_t value);
 
-#include "module.h"
+#include "qemu/module.h"
 
 /*
  * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
diff --git a/acl.h b/include/qemu/acl.h
index 0ef7804..116487e 100644
--- a/acl.h
+++ b/include/qemu/acl.h
@@ -25,7 +25,7 @@
 #ifndef __QEMU_ACL_H__
 #define __QEMU_ACL_H__
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 typedef struct qemu_acl_entry qemu_acl_entry;
 typedef struct qemu_acl qemu_acl;
diff --git a/qemu-barrier.h b/include/qemu/atomic.h
index faa83d2..96a194b 100644
--- a/qemu-barrier.h
+++ b/include/qemu/atomic.h
@@ -6,7 +6,7 @@
 
 #if defined(__i386__)
 
-#include "compiler.h"        /* QEMU_GNUC_PREREQ */
+#include "qemu/compiler.h"        /* QEMU_GNUC_PREREQ */
 
 /*
  * Because of the strongly ordered x86 storage model, wmb() and rmb() are nops
diff --git a/bitmap.h b/include/qemu/bitmap.h
index 08755eb..308bbb7 100644
--- a/bitmap.h
+++ b/include/qemu/bitmap.h
@@ -13,7 +13,7 @@
 #define BITMAP_H
 
 #include "qemu-common.h"
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 /*
  * The available bitmap operations and their rough meaning in the
diff --git a/bitops.h b/include/qemu/bitops.h
index 74e14e5..8b88791 100644
--- a/bitops.h
+++ b/include/qemu/bitops.h
@@ -13,6 +13,7 @@
 #define BITOPS_H
 
 #include "qemu-common.h"
+#include "host-utils.h"
 
 #define BITS_PER_BYTE           CHAR_BIT
 #define BITS_PER_LONG           (sizeof (unsigned long) * BITS_PER_BYTE)
@@ -23,41 +24,29 @@
 #define BITS_TO_LONGS(nr)	DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
 
 /**
- * bitops_ffs - find first bit in word.
+ * bitops_ctzl - count trailing zeroes in word.
  * @word: The word to search
  *
- * Undefined if no bit exists, so code should check against 0 first.
+ * Returns -1 if no bit exists.  Note that compared to the C library
+ * routine ffsl, this one returns one less.
  */
-static unsigned long bitops_ffsl(unsigned long word)
+static unsigned long bitops_ctzl(unsigned long word)
 {
-	int num = 0;
+#if QEMU_GNUC_PREREQ(3, 4)
+    return __builtin_ffsl(word) - 1;
+#else
+    if (!word) {
+        return -1;
+    }
 
-#if LONG_MAX > 0x7FFFFFFF
-	if ((word & 0xffffffff) == 0) {
-		num += 32;
-		word >>= 32;
-	}
+    if (sizeof(long) == 4) {
+        return ctz32(word);
+    } else if (sizeof(long) == 8) {
+        return ctz64(word);
+    } else {
+        abort();
+    }
 #endif
-	if ((word & 0xffff) == 0) {
-		num += 16;
-		word >>= 16;
-	}
-	if ((word & 0xff) == 0) {
-		num += 8;
-		word >>= 8;
-	}
-	if ((word & 0xf) == 0) {
-		num += 4;
-		word >>= 4;
-	}
-	if ((word & 0x3) == 0) {
-		num += 2;
-		word >>= 2;
-	}
-	if ((word & 0x1) == 0) {
-		num += 1;
-        }
-	return num;
 }
 
 /**
@@ -99,14 +88,14 @@ static inline unsigned long bitops_flsl(unsigned long word)
 }
 
 /**
- * ffz - find first zero in word.
+ * cto - count trailing ones in word.
  * @word: The word to search
  *
- * Undefined if no zero exists, so code should check against ~0UL first.
+ * Returns -1 if all bit are set.
  */
-static inline unsigned long ffz(unsigned long word)
+static inline unsigned long bitops_ctol(unsigned long word)
 {
-    return bitops_ffsl(~word);
+    return bitops_ctzl(~word);
 }
 
 /**
diff --git a/include/qemu/bswap.h b/include/qemu/bswap.h
new file mode 100644
index 0000000..d3af35d
--- /dev/null
+++ b/include/qemu/bswap.h
@@ -0,0 +1,477 @@
+#ifndef BSWAP_H
+#define BSWAP_H
+
+#include "config-host.h"
+#include <inttypes.h>
+#include <limits.h>
+#include "fpu/softfloat.h"
+
+#ifdef CONFIG_MACHINE_BSWAP_H
+# include <sys/endian.h>
+# include <sys/types.h>
+# include <machine/bswap.h>
+#elif defined(CONFIG_BYTESWAP_H)
+# include <byteswap.h>
+
+static inline uint16_t bswap16(uint16_t x)
+{
+    return bswap_16(x);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return bswap_32(x);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return bswap_64(x);
+}
+# else
+static inline uint16_t bswap16(uint16_t x)
+{
+    return (((x & 0x00ff) << 8) |
+            ((x & 0xff00) >> 8));
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return (((x & 0x000000ffU) << 24) |
+            ((x & 0x0000ff00U) <<  8) |
+            ((x & 0x00ff0000U) >>  8) |
+            ((x & 0xff000000U) >> 24));
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return (((x & 0x00000000000000ffULL) << 56) |
+            ((x & 0x000000000000ff00ULL) << 40) |
+            ((x & 0x0000000000ff0000ULL) << 24) |
+            ((x & 0x00000000ff000000ULL) <<  8) |
+            ((x & 0x000000ff00000000ULL) >>  8) |
+            ((x & 0x0000ff0000000000ULL) >> 24) |
+            ((x & 0x00ff000000000000ULL) >> 40) |
+            ((x & 0xff00000000000000ULL) >> 56));
+}
+#endif /* ! CONFIG_MACHINE_BSWAP_H */
+
+static inline void bswap16s(uint16_t *s)
+{
+    *s = bswap16(*s);
+}
+
+static inline void bswap32s(uint32_t *s)
+{
+    *s = bswap32(*s);
+}
+
+static inline void bswap64s(uint64_t *s)
+{
+    *s = bswap64(*s);
+}
+
+#if defined(HOST_WORDS_BIGENDIAN)
+#define be_bswap(v, size) (v)
+#define le_bswap(v, size) glue(bswap, size)(v)
+#define be_bswaps(v, size)
+#define le_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
+#else
+#define le_bswap(v, size) (v)
+#define be_bswap(v, size) glue(bswap, size)(v)
+#define le_bswaps(v, size)
+#define be_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
+#endif
+
+#define CPU_CONVERT(endian, size, type)\
+static inline type endian ## size ## _to_cpu(type v)\
+{\
+    return glue(endian, _bswap)(v, size);\
+}\
+\
+static inline type cpu_to_ ## endian ## size(type v)\
+{\
+    return glue(endian, _bswap)(v, size);\
+}\
+\
+static inline void endian ## size ## _to_cpus(type *p)\
+{\
+    glue(endian, _bswaps)(p, size);\
+}\
+\
+static inline void cpu_to_ ## endian ## size ## s(type *p)\
+{\
+    glue(endian, _bswaps)(p, size);\
+}\
+\
+static inline type endian ## size ## _to_cpup(const type *p)\
+{\
+    return glue(glue(endian, size), _to_cpu)(*p);\
+}\
+\
+static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\
+{\
+    *p = glue(glue(cpu_to_, endian), size)(v);\
+}
+
+CPU_CONVERT(be, 16, uint16_t)
+CPU_CONVERT(be, 32, uint32_t)
+CPU_CONVERT(be, 64, uint64_t)
+
+CPU_CONVERT(le, 16, uint16_t)
+CPU_CONVERT(le, 32, uint32_t)
+CPU_CONVERT(le, 64, uint64_t)
+
+/* len must be one of 1, 2, 4 */
+static inline uint32_t qemu_bswap_len(uint32_t value, int len)
+{
+    return bswap32(value) >> (32 - 8 * len);
+}
+
+/* Unions for reinterpreting between floats and integers.  */
+
+typedef union {
+    float32 f;
+    uint32_t l;
+} CPU_FloatU;
+
+typedef union {
+    float64 d;
+#if defined(HOST_WORDS_BIGENDIAN)
+    struct {
+        uint32_t upper;
+        uint32_t lower;
+    } l;
+#else
+    struct {
+        uint32_t lower;
+        uint32_t upper;
+    } l;
+#endif
+    uint64_t ll;
+} CPU_DoubleU;
+
+typedef union {
+     floatx80 d;
+     struct {
+         uint64_t lower;
+         uint16_t upper;
+     } l;
+} CPU_LDoubleU;
+
+typedef union {
+    float128 q;
+#if defined(HOST_WORDS_BIGENDIAN)
+    struct {
+        uint32_t upmost;
+        uint32_t upper;
+        uint32_t lower;
+        uint32_t lowest;
+    } l;
+    struct {
+        uint64_t upper;
+        uint64_t lower;
+    } ll;
+#else
+    struct {
+        uint32_t lowest;
+        uint32_t lower;
+        uint32_t upper;
+        uint32_t upmost;
+    } l;
+    struct {
+        uint64_t lower;
+        uint64_t upper;
+    } ll;
+#endif
+} CPU_QuadU;
+
+/* unaligned/endian-independent pointer access */
+
+/*
+ * the generic syntax is:
+ *
+ * load: ld{type}{sign}{size}{endian}_p(ptr)
+ *
+ * store: st{type}{size}{endian}_p(ptr, val)
+ *
+ * Note there are small differences with the softmmu access API!
+ *
+ * type is:
+ * (empty): integer access
+ *   f    : float access
+ *
+ * sign is:
+ * (empty): for floats or 32 bit size
+ *   u    : unsigned
+ *   s    : signed
+ *
+ * size is:
+ *   b: 8 bits
+ *   w: 16 bits
+ *   l: 32 bits
+ *   q: 64 bits
+ *
+ * endian is:
+ * (empty): host endian
+ *   be   : big endian
+ *   le   : little endian
+ */
+
+static inline int ldub_p(const void *ptr)
+{
+    return *(uint8_t *)ptr;
+}
+
+static inline int ldsb_p(const void *ptr)
+{
+    return *(int8_t *)ptr;
+}
+
+static inline void stb_p(void *ptr, int v)
+{
+    *(uint8_t *)ptr = v;
+}
+
+/* Any compiler worth its salt will turn these memcpy into native unaligned
+   operations.  Thus we don't need to play games with packed attributes, or
+   inline byte-by-byte stores.  */
+
+static inline int lduw_p(const void *ptr)
+{
+    uint16_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline int ldsw_p(const void *ptr)
+{
+    int16_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stw_p(void *ptr, uint16_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline int ldl_p(const void *ptr)
+{
+    int32_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stl_p(void *ptr, uint32_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline uint64_t ldq_p(const void *ptr)
+{
+    uint64_t r;
+    memcpy(&r, ptr, sizeof(r));
+    return r;
+}
+
+static inline void stq_p(void *ptr, uint64_t v)
+{
+    memcpy(ptr, &v, sizeof(v));
+}
+
+static inline int lduw_le_p(const void *ptr)
+{
+    return (uint16_t)le_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldsw_le_p(const void *ptr)
+{
+    return (int16_t)le_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldl_le_p(const void *ptr)
+{
+    return le_bswap(ldl_p(ptr), 32);
+}
+
+static inline uint64_t ldq_le_p(const void *ptr)
+{
+    return le_bswap(ldq_p(ptr), 64);
+}
+
+static inline void stw_le_p(void *ptr, int v)
+{
+    stw_p(ptr, le_bswap(v, 16));
+}
+
+static inline void stl_le_p(void *ptr, int v)
+{
+    stl_p(ptr, le_bswap(v, 32));
+}
+
+static inline void stq_le_p(void *ptr, uint64_t v)
+{
+    stq_p(ptr, le_bswap(v, 64));
+}
+
+/* float access */
+
+static inline float32 ldfl_le_p(const void *ptr)
+{
+    CPU_FloatU u;
+    u.l = ldl_le_p(ptr);
+    return u.f;
+}
+
+static inline void stfl_le_p(void *ptr, float32 v)
+{
+    CPU_FloatU u;
+    u.f = v;
+    stl_le_p(ptr, u.l);
+}
+
+static inline float64 ldfq_le_p(const void *ptr)
+{
+    CPU_DoubleU u;
+    u.ll = ldq_le_p(ptr);
+    return u.d;
+}
+
+static inline void stfq_le_p(void *ptr, float64 v)
+{
+    CPU_DoubleU u;
+    u.d = v;
+    stq_le_p(ptr, u.ll);
+}
+
+static inline int lduw_be_p(const void *ptr)
+{
+    return (uint16_t)be_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldsw_be_p(const void *ptr)
+{
+    return (int16_t)be_bswap(lduw_p(ptr), 16);
+}
+
+static inline int ldl_be_p(const void *ptr)
+{
+    return be_bswap(ldl_p(ptr), 32);
+}
+
+static inline uint64_t ldq_be_p(const void *ptr)
+{
+    return be_bswap(ldq_p(ptr), 64);
+}
+
+static inline void stw_be_p(void *ptr, int v)
+{
+    stw_p(ptr, be_bswap(v, 16));
+}
+
+static inline void stl_be_p(void *ptr, int v)
+{
+    stl_p(ptr, be_bswap(v, 32));
+}
+
+static inline void stq_be_p(void *ptr, uint64_t v)
+{
+    stq_p(ptr, be_bswap(v, 64));
+}
+
+/* float access */
+
+static inline float32 ldfl_be_p(const void *ptr)
+{
+    CPU_FloatU u;
+    u.l = ldl_be_p(ptr);
+    return u.f;
+}
+
+static inline void stfl_be_p(void *ptr, float32 v)
+{
+    CPU_FloatU u;
+    u.f = v;
+    stl_be_p(ptr, u.l);
+}
+
+static inline float64 ldfq_be_p(const void *ptr)
+{
+    CPU_DoubleU u;
+    u.ll = ldq_be_p(ptr);
+    return u.d;
+}
+
+static inline void stfq_be_p(void *ptr, float64 v)
+{
+    CPU_DoubleU u;
+    u.d = v;
+    stq_be_p(ptr, u.ll);
+}
+
+/* Legacy unaligned versions.  Note that we never had a complete set.  */
+
+static inline void cpu_to_le16wu(uint16_t *p, uint16_t v)
+{
+    stw_le_p(p, v);
+}
+
+static inline void cpu_to_le32wu(uint32_t *p, uint32_t v)
+{
+    stl_le_p(p, v);
+}
+
+static inline uint16_t le16_to_cpupu(const uint16_t *p)
+{
+    return lduw_le_p(p);
+}
+
+static inline uint32_t le32_to_cpupu(const uint32_t *p)
+{
+    return ldl_le_p(p);
+}
+
+static inline uint32_t be32_to_cpupu(const uint32_t *p)
+{
+    return ldl_be_p(p);
+}
+
+static inline void cpu_to_be16wu(uint16_t *p, uint16_t v)
+{
+    stw_be_p(p, v);
+}
+
+static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
+{
+    stl_be_p(p, v);
+}
+
+static inline void cpu_to_be64wu(uint64_t *p, uint64_t v)
+{
+    stq_be_p(p, v);
+}
+
+static inline void cpu_to_32wu(uint32_t *p, uint32_t v)
+{
+    stl_p(p, v);
+}
+
+static inline unsigned long leul_to_cpu(unsigned long v)
+{
+    /* In order to break an include loop between here and
+       qemu-common.h, don't rely on HOST_LONG_BITS.  */
+#if ULONG_MAX == UINT32_MAX
+    return le_bswap(v, 32);
+#elif ULONG_MAX == UINT64_MAX
+    return le_bswap(v, 64);
+#else
+# error Unknown sizeof long
+#endif
+}
+
+#undef le_bswap
+#undef be_bswap
+#undef le_bswaps
+#undef be_bswaps
+
+#endif /* BSWAP_H */
diff --git a/cache-utils.h b/include/qemu/cache-utils.h
index 2c57f78..2c57f78 100644
--- a/cache-utils.h
+++ b/include/qemu/cache-utils.h
diff --git a/compatfd.h b/include/qemu/compatfd.h
index 6b04877..6b04877 100644
--- a/compatfd.h
+++ b/include/qemu/compatfd.h
diff --git a/compiler.h b/include/qemu/compiler.h
index 2f7998b..2f7998b 100644
--- a/compiler.h
+++ b/include/qemu/compiler.h
diff --git a/qemu-config.h b/include/qemu/config-file.h
index 812c4c5..ccfccae 100644
--- a/qemu-config.h
+++ b/include/qemu/config-file.h
@@ -2,13 +2,9 @@
 #define QEMU_CONFIG_H
 
 #include <stdio.h>
-#include "qemu-option.h"
-#include "error.h"
-
-extern QemuOptsList qemu_fsdev_opts;
-extern QemuOptsList qemu_virtfs_opts;
-extern QemuOptsList qemu_spice_opts;
-extern QemuOptsList qemu_sandbox_opts;
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "qemu/option.h"
 
 QemuOptsList *qemu_find_opts(const char *group);
 QemuOptsList *qemu_find_opts_err(const char *group, Error **errp);
diff --git a/envlist.h b/include/qemu/envlist.h
index b9addcc..b9addcc 100644
--- a/envlist.h
+++ b/include/qemu/envlist.h
diff --git a/qemu-error.h b/include/qemu/error-report.h
index 93d74b4..c902cc1 100644
--- a/qemu-error.h
+++ b/include/qemu/error-report.h
@@ -13,6 +13,8 @@
 #ifndef QEMU_ERROR_H
 #define QEMU_ERROR_H
 
+#include <stdarg.h>
+
 typedef struct Location {
     /* all members are private to qemu-error.c */
     enum { LOC_NONE, LOC_CMDLINE, LOC_FILE } kind;
diff --git a/event_notifier.h b/include/qemu/event_notifier.h
index 88b57af..88b57af 100644
--- a/event_notifier.h
+++ b/include/qemu/event_notifier.h
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
new file mode 100644
index 0000000..250de03
--- /dev/null
+++ b/include/qemu/hbitmap.h
@@ -0,0 +1,208 @@
+/*
+ * Hierarchical Bitmap Data Type
+ *
+ * Copyright Red Hat, Inc., 2012
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef HBITMAP_H
+#define HBITMAP_H 1
+
+#include <limits.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include "bitops.h"
+
+typedef struct HBitmap HBitmap;
+typedef struct HBitmapIter HBitmapIter;
+
+#define BITS_PER_LEVEL         (BITS_PER_LONG == 32 ? 5 : 6)
+
+/* For 32-bit, the largest that fits in a 4 GiB address space.
+ * For 64-bit, the number of sectors in 1 PiB.  Good luck, in
+ * either case... :)
+ */
+#define HBITMAP_LOG_MAX_SIZE   (BITS_PER_LONG == 32 ? 34 : 41)
+
+/* We need to place a sentinel in level 0 to speed up iteration.  Thus,
+ * we do this instead of HBITMAP_LOG_MAX_SIZE / BITS_PER_LEVEL.  The
+ * difference is that it allocates an extra level when HBITMAP_LOG_MAX_SIZE
+ * is an exact multiple of BITS_PER_LEVEL.
+ */
+#define HBITMAP_LEVELS         ((HBITMAP_LOG_MAX_SIZE / BITS_PER_LEVEL) + 1)
+
+struct HBitmapIter {
+    const HBitmap *hb;
+
+    /* Copied from hb for access in the inline functions (hb is opaque).  */
+    int granularity;
+
+    /* Entry offset into the last-level array of longs.  */
+    size_t pos;
+
+    /* The currently-active path in the tree.  Each item of cur[i] stores
+     * the bits (i.e. the subtrees) yet to be processed under that node.
+     */
+    unsigned long cur[HBITMAP_LEVELS];
+};
+
+/**
+ * hbitmap_alloc:
+ * @size: Number of bits in the bitmap.
+ * @granularity: Granularity of the bitmap.  Aligned groups of 2^@granularity
+ * bits will be represented by a single bit.  Each operation on a
+ * range of bits first rounds the bits to determine which group they land
+ * in, and then affect the entire set; iteration will only visit the first
+ * bit of each group.
+ *
+ * Allocate a new HBitmap.
+ */
+HBitmap *hbitmap_alloc(uint64_t size, int granularity);
+
+/**
+ * hbitmap_empty:
+ * @hb: HBitmap to operate on.
+ *
+ * Return whether the bitmap is empty.
+ */
+bool hbitmap_empty(const HBitmap *hb);
+
+/**
+ * hbitmap_granularity:
+ * @hb: HBitmap to operate on.
+ *
+ * Return the granularity of the HBitmap.
+ */
+int hbitmap_granularity(const HBitmap *hb);
+
+/**
+ * hbitmap_count:
+ * @hb: HBitmap to operate on.
+ *
+ * Return the number of bits set in the HBitmap.
+ */
+uint64_t hbitmap_count(const HBitmap *hb);
+
+/**
+ * hbitmap_set:
+ * @hb: HBitmap to operate on.
+ * @start: First bit to set (0-based).
+ * @count: Number of bits to set.
+ *
+ * Set a consecutive range of bits in an HBitmap.
+ */
+void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count);
+
+/**
+ * hbitmap_reset:
+ * @hb: HBitmap to operate on.
+ * @start: First bit to reset (0-based).
+ * @count: Number of bits to reset.
+ *
+ * Reset a consecutive range of bits in an HBitmap.
+ */
+void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count);
+
+/**
+ * hbitmap_get:
+ * @hb: HBitmap to operate on.
+ * @item: Bit to query (0-based).
+ *
+ * Return whether the @item-th bit in an HBitmap is set.
+ */
+bool hbitmap_get(const HBitmap *hb, uint64_t item);
+
+/**
+ * hbitmap_free:
+ * @hb: HBitmap to operate on.
+ *
+ * Free an HBitmap and all of its associated memory.
+ */
+void hbitmap_free(HBitmap *hb);
+
+/**
+ * hbitmap_iter_init:
+ * @hbi: HBitmapIter to initialize.
+ * @hb: HBitmap to iterate on.
+ * @first: First bit to visit (0-based, must be strictly less than the
+ * size of the bitmap).
+ *
+ * Set up @hbi to iterate on the HBitmap @hb.  hbitmap_iter_next will return
+ * the lowest-numbered bit that is set in @hb, starting at @first.
+ *
+ * Concurrent setting of bits is acceptable, and will at worst cause the
+ * iteration to miss some of those bits.  Resetting bits before the current
+ * position of the iterator is also okay.  However, concurrent resetting of
+ * bits can lead to unexpected behavior if the iterator has not yet reached
+ * those bits.
+ */
+void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first);
+
+/* hbitmap_iter_skip_words:
+ * @hbi: HBitmapIter to operate on.
+ *
+ * Internal function used by hbitmap_iter_next and hbitmap_iter_next_word.
+ */
+unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi);
+
+/**
+ * hbitmap_iter_next:
+ * @hbi: HBitmapIter to operate on.
+ *
+ * Return the next bit that is set in @hbi's associated HBitmap,
+ * or -1 if all remaining bits are zero.
+ */
+static inline int64_t hbitmap_iter_next(HBitmapIter *hbi)
+{
+    unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+    int64_t item;
+
+    if (cur == 0) {
+        cur = hbitmap_iter_skip_words(hbi);
+        if (cur == 0) {
+            return -1;
+        }
+    }
+
+    /* The next call will resume work from the next bit.  */
+    hbi->cur[HBITMAP_LEVELS - 1] = cur & (cur - 1);
+    item = ((uint64_t)hbi->pos << BITS_PER_LEVEL) + bitops_ctzl(cur);
+
+    return item << hbi->granularity;
+}
+
+/**
+ * hbitmap_iter_next_word:
+ * @hbi: HBitmapIter to operate on.
+ * @p_cur: Location where to store the next non-zero word.
+ *
+ * Return the index of the next nonzero word that is set in @hbi's
+ * associated HBitmap, and set *p_cur to the content of that word
+ * (bits before the index that was passed to hbitmap_iter_init are
+ * trimmed on the first call).  Return -1, and set *p_cur to zero,
+ * if all remaining words are zero.
+ */
+static inline size_t hbitmap_iter_next_word(HBitmapIter *hbi, unsigned long *p_cur)
+{
+    unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1];
+
+    if (cur == 0) {
+        cur = hbitmap_iter_skip_words(hbi);
+        if (cur == 0) {
+            *p_cur = 0;
+            return -1;
+        }
+    }
+
+    /* The next call will resume work from the next word.  */
+    hbi->cur[HBITMAP_LEVELS - 1] = 0;
+    *p_cur = cur;
+    return hbi->pos;
+}
+
+
+#endif
diff --git a/host-utils.h b/include/qemu/host-utils.h
index 821db93..81c9a75 100644
--- a/host-utils.h
+++ b/include/qemu/host-utils.h
@@ -22,8 +22,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef HOST_UTILS_H
+#define HOST_UTILS_H 1
 
-#include "compiler.h"   /* QEMU_GNUC_PREREQ */
+#include "qemu/compiler.h"   /* QEMU_GNUC_PREREQ */
 
 #if defined(__x86_64__)
 #define __HAVE_FAST_MULU64__
@@ -234,3 +236,5 @@ static inline int ctpop64(uint64_t val)
     return val;
 #endif
 }
+
+#endif
diff --git a/int128.h b/include/qemu/int128.h
index b3864b6..b3864b6 100644
--- a/int128.h
+++ b/include/qemu/int128.h
diff --git a/iov.h b/include/qemu/iov.h
index 34c8ec9..68d25f2 100644
--- a/iov.h
+++ b/include/qemu/iov.h
@@ -11,6 +11,9 @@
  * the COPYING file in the top-level directory.
  */
 
+#ifndef IOV_H
+#define IOV_H
+
 #include "qemu-common.h"
 
 /**
@@ -95,3 +98,18 @@ void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
 unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
                  const struct iovec *iov, unsigned int iov_cnt,
                  size_t offset, size_t bytes);
+
+/*
+ * Remove a given number of bytes from the front or back of a vector.
+ * This may update iov and/or iov_cnt to exclude iovec elements that are
+ * no longer required.
+ *
+ * The number of bytes actually discarded is returned.  This number may be
+ * smaller than requested if the vector is too small.
+ */
+size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
+                         size_t bytes);
+size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
+                        size_t bytes);
+
+#endif
diff --git a/qemu-log.h b/include/qemu/log.h
index 344eca3..58f69cb 100644
--- a/qemu-log.h
+++ b/include/qemu/log.h
@@ -3,7 +3,7 @@
 
 #include <stdarg.h>
 #ifdef NEED_CPU_H
-#include "disas.h"
+#include "disas/disas.h"
 #endif
 
 /* Private global variables, don't use */
diff --git a/main-loop.h b/include/qemu/main-loop.h
index 326c742..e8059c3 100644
--- a/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -25,7 +25,7 @@
 #ifndef QEMU_MAIN_LOOP_H
 #define QEMU_MAIN_LOOP_H 1
 
-#include "qemu-aio.h"
+#include "block/aio.h"
 
 #define SIG_IPI SIGUSR1
 
diff --git a/module.h b/include/qemu/module.h
index c4ccd57..c4ccd57 100644
--- a/module.h
+++ b/include/qemu/module.h
diff --git a/notify.h b/include/qemu/notify.h
index 03cf26c..4e2e7f0 100644
--- a/notify.h
+++ b/include/qemu/notify.h
@@ -14,7 +14,7 @@
 #ifndef QEMU_NOTIFY_H
 #define QEMU_NOTIFY_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 typedef struct Notifier Notifier;
 
diff --git a/qemu-option.h b/include/qemu/option.h
index ca72986..ba197cd 100644
--- a/qemu-option.h
+++ b/include/qemu/option.h
@@ -27,9 +27,9 @@
 #define QEMU_OPTIONS_H
 
 #include <stdint.h>
-#include "qemu-queue.h"
-#include "error.h"
-#include "qdict.h"
+#include "qemu/queue.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qdict.h"
 
 enum QEMUOptionParType {
     OPT_FLAG,
@@ -126,6 +126,7 @@ int qemu_opt_set(QemuOpts *opts, const char *name, const char *value);
 void qemu_opt_set_err(QemuOpts *opts, const char *name, const char *value,
                       Error **errp);
 int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val);
+int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val);
 typedef int (*qemu_opt_loopfunc)(const char *name, const char *value, void *opaque);
 int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
                      int abort_on_failure);
@@ -133,6 +134,7 @@ int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
 QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id);
 QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
                            int fail_if_exists, Error **errp);
+QemuOpts *qemu_opts_create_nofail(QemuOptsList *list);
 void qemu_opts_reset(QemuOptsList *list);
 void qemu_opts_loc_restore(QemuOpts *opts);
 int qemu_opts_set(QemuOptsList *list, const char *id,
diff --git a/qemu-option-internal.h b/include/qemu/option_int.h
index 19fdc1c..8212fa4 100644
--- a/qemu-option-internal.h
+++ b/include/qemu/option_int.h
@@ -26,7 +26,8 @@
 #ifndef QEMU_OPTIONS_INTERNAL_H
 #define QEMU_OPTIONS_INTERNAL_H
 
-#include "qemu-option.h"
+#include "qemu/option.h"
+#include "qemu/error-report.h"
 
 struct QemuOpt {
     const char   *name;
diff --git a/osdep.h b/include/qemu/osdep.h
index 87d3b9c..87d3b9c 100644
--- a/osdep.h
+++ b/include/qemu/osdep.h
diff --git a/qemu-queue.h b/include/qemu/queue.h
index 9288cd8..d433b90 100644
--- a/qemu-queue.h
+++ b/include/qemu/queue.h
@@ -78,7 +78,7 @@
  * For details on the use of these macros, see the queue(3) manual page.
  */
 
-#include "qemu-barrier.h" /* for smp_wmb() */
+#include "qemu/atomic.h" /* for smp_wmb() */
 
 /*
  * List definitions.
diff --git a/range.h b/include/qemu/range.h
index 3502372..3502372 100644
--- a/range.h
+++ b/include/qemu/range.h
diff --git a/include/qemu/rng-random.h b/include/qemu/rng-random.h
index 6249290..4332772 100644
--- a/include/qemu/rng-random.h
+++ b/include/qemu/rng-random.h
@@ -12,7 +12,7 @@
 #ifndef QEMU_RNG_RANDOM_H
 #define QEMU_RNG_RANDOM_H
 
-#include "qemu/object.h"
+#include "qom/object.h"
 
 #define TYPE_RNG_RANDOM "rng-random"
 #define RNG_RANDOM(obj) OBJECT_CHECK(RndRandom, (obj), TYPE_RNG_RANDOM)
diff --git a/include/qemu/rng.h b/include/qemu/rng.h
index d094bf8..509abd0 100644
--- a/include/qemu/rng.h
+++ b/include/qemu/rng.h
@@ -13,9 +13,9 @@
 #ifndef QEMU_RNG_H
 #define QEMU_RNG_H
 
-#include "qemu/object.h"
+#include "qom/object.h"
 #include "qemu-common.h"
-#include "error.h"
+#include "qapi/error.h"
 
 #define TYPE_RNG_BACKEND "rng-backend"
 #define RNG_BACKEND(obj) \
diff --git a/qemu_socket.h b/include/qemu/sockets.h
index 02490ad..803ae17 100644
--- a/qemu_socket.h
+++ b/include/qemu/sockets.h
@@ -26,9 +26,9 @@ int inet_aton(const char *cp, struct in_addr *ia);
 
 #endif /* !_WIN32 */
 
-#include "qemu-option.h"
-#include "error.h"
-#include "qerror.h"
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 
 /* misc helpers */
 int qemu_socket(int domain, int type, int protocol);
diff --git a/qemu-thread-posix.h b/include/qemu/thread-posix.h
index 380bae2..0f30dcc 100644
--- a/qemu-thread-posix.h
+++ b/include/qemu/thread-posix.h
@@ -12,7 +12,7 @@ struct QemuCond {
 };
 
 struct QemuSemaphore {
-#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+#if defined(__APPLE__) || defined(__NetBSD__)
     pthread_mutex_t lock;
     pthread_cond_t cond;
     int count;
diff --git a/qemu-thread-win32.h b/include/qemu/thread-win32.h
index 13adb95..13adb95 100644
--- a/qemu-thread-win32.h
+++ b/include/qemu/thread-win32.h
diff --git a/qemu-thread.h b/include/qemu/thread.h
index 3ee2f6b..c02404b 100644
--- a/qemu-thread.h
+++ b/include/qemu/thread.h
@@ -10,9 +10,9 @@ typedef struct QemuSemaphore QemuSemaphore;
 typedef struct QemuThread QemuThread;
 
 #ifdef _WIN32
-#include "qemu-thread-win32.h"
+#include "qemu/thread-win32.h"
 #else
-#include "qemu-thread-posix.h"
+#include "qemu/thread-posix.h"
 #endif
 
 #define QEMU_THREAD_JOINABLE 0
diff --git a/qemu-timer.h b/include/qemu/timer.h
index da7e97c..1766b2d 100644
--- a/qemu-timer.h
+++ b/include/qemu/timer.h
@@ -2,8 +2,8 @@
 #define QEMU_TIMER_H
 
 #include "qemu-common.h"
-#include "main-loop.h"
-#include "notify.h"
+#include "qemu/main-loop.h"
+#include "qemu/notify.h"
 
 #ifdef __FreeBSD__
 #include <sys/param.h>
diff --git a/qemu-tls.h b/include/qemu/tls.h
index b92ea9d..b92ea9d 100644
--- a/qemu-tls.h
+++ b/include/qemu/tls.h
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
new file mode 100644
index 0000000..fd532a2
--- /dev/null
+++ b/include/qemu/typedefs.h
@@ -0,0 +1,61 @@
+#ifndef QEMU_TYPEDEFS_H
+#define QEMU_TYPEDEFS_H
+
+/* A load of opaque types so that device init declarations don't have to
+   pull in all the real definitions.  */
+typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUFile QEMUFile;
+typedef struct QEMUBH QEMUBH;
+
+struct Monitor;
+typedef struct Monitor Monitor;
+typedef struct MigrationParams MigrationParams;
+
+typedef struct Property Property;
+typedef struct PropertyInfo PropertyInfo;
+typedef struct CompatProperty CompatProperty;
+typedef struct DeviceState DeviceState;
+typedef struct BusState BusState;
+typedef struct BusClass BusClass;
+
+typedef struct NICInfo NICInfo;
+typedef struct HCIInfo HCIInfo;
+typedef struct AudioState AudioState;
+typedef struct BlockDriverState BlockDriverState;
+typedef struct DriveInfo DriveInfo;
+typedef struct DisplayState DisplayState;
+typedef struct DisplayChangeListener DisplayChangeListener;
+typedef struct DisplaySurface DisplaySurface;
+typedef struct PixelFormat PixelFormat;
+typedef struct QemuConsole QemuConsole;
+typedef struct CharDriverState CharDriverState;
+typedef struct MACAddr MACAddr;
+typedef struct NetClientState NetClientState;
+typedef struct i2c_bus i2c_bus;
+typedef struct ISABus ISABus;
+typedef struct ISADevice ISADevice;
+typedef struct SMBusDevice SMBusDevice;
+typedef struct PCIHostState PCIHostState;
+typedef struct PCIExpressHost PCIExpressHost;
+typedef struct PCIBus PCIBus;
+typedef struct PCIDevice PCIDevice;
+typedef struct PCIExpressDevice PCIExpressDevice;
+typedef struct PCIBridge PCIBridge;
+typedef struct PCIEAERMsg PCIEAERMsg;
+typedef struct PCIEAERLog PCIEAERLog;
+typedef struct PCIEAERErr PCIEAERErr;
+typedef struct PCIEPort PCIEPort;
+typedef struct PCIESlot PCIESlot;
+typedef struct MSIMessage MSIMessage;
+typedef struct SerialState SerialState;
+typedef struct PCMCIACardState PCMCIACardState;
+typedef struct MouseTransformInfo MouseTransformInfo;
+typedef struct uWireSlave uWireSlave;
+typedef struct I2SCodec I2SCodec;
+typedef struct SSIBus SSIBus;
+typedef struct EventNotifier EventNotifier;
+typedef struct VirtIODevice VirtIODevice;
+typedef struct QEMUSGList QEMUSGList;
+typedef struct SHPCDevice SHPCDevice;
+
+#endif /* QEMU_TYPEDEFS_H */
diff --git a/uri.h b/include/qemu/uri.h
index de99b3b..de99b3b 100644
--- a/uri.h
+++ b/include/qemu/uri.h
diff --git a/qemu-xattr.h b/include/qemu/xattr.h
index f910d96..f910d96 100644
--- a/qemu-xattr.h
+++ b/include/qemu/xattr.h
diff --git a/include/qemu/cpu.h b/include/qom/cpu.h
index 61b7698..46f2247 100644
--- a/include/qemu/cpu.h
+++ b/include/qom/cpu.h
@@ -20,8 +20,8 @@
 #ifndef QEMU_CPU_H
 #define QEMU_CPU_H
 
-#include "qemu/object.h"
-#include "qemu-thread.h"
+#include "hw/qdev-core.h"
+#include "qemu/thread.h"
 
 /**
  * SECTION:cpu
@@ -40,31 +40,47 @@ typedef struct CPUState CPUState;
 
 /**
  * CPUClass:
+ * @class_by_name: Callback to map -cpu command line model name to an
+ * instantiatable CPU type.
  * @reset: Callback to reset the #CPUState to its initial state.
  *
  * Represents a CPU family or model.
  */
 typedef struct CPUClass {
     /*< private >*/
-    ObjectClass parent_class;
+    DeviceClass parent_class;
     /*< public >*/
 
+    ObjectClass *(*class_by_name)(const char *cpu_model);
+
     void (*reset)(CPUState *cpu);
 } CPUClass;
 
+struct KVMState;
+struct kvm_run;
+
 /**
  * CPUState:
+ * @cpu_index: CPU index (informative).
+ * @nr_cores: Number of cores within this CPU package.
+ * @nr_threads: Number of threads within this CPU.
+ * @numa_node: NUMA node this CPU is belonging to.
  * @created: Indicates whether the CPU thread has been successfully created.
  * @stop: Indicates a pending stop request.
  * @stopped: Indicates the CPU has been artificially stopped.
+ * @kvm_fd: vCPU file descriptor for KVM.
  *
  * State of one CPU core or thread.
  */
 struct CPUState {
     /*< private >*/
-    Object parent_obj;
+    DeviceState parent_obj;
     /*< public >*/
 
+    int nr_cores;
+    int nr_threads;
+    int numa_node;
+
     struct QemuThread *thread;
 #ifdef _WIN32
     HANDLE hThread;
@@ -77,7 +93,13 @@ struct CPUState {
     bool stop;
     bool stopped;
 
+    int kvm_fd;
+    bool kvm_vcpu_dirty;
+    struct KVMState *kvm_state;
+    struct kvm_run *kvm_run;
+
     /* TODO Move common fields from CPUArchState here. */
+    int cpu_index; /* used by alpha TCG */
 };
 
 
@@ -88,6 +110,17 @@ struct CPUState {
 void cpu_reset(CPUState *cpu);
 
 /**
+ * cpu_class_by_name:
+ * @typename: The CPU base type.
+ * @cpu_model: The model string without any parameters.
+ *
+ * Looks up a CPU #ObjectClass matching name @cpu_model.
+ *
+ * Returns: A #CPUClass or %NULL if not matching class is found.
+ */
+ObjectClass *cpu_class_by_name(const char *typename, const char *cpu_model);
+
+/**
  * qemu_cpu_has_work:
  * @cpu: The vCPU to check.
  *
@@ -136,5 +169,15 @@ bool cpu_is_stopped(CPUState *cpu);
  */
 void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data);
 
+/**
+ * qemu_get_cpu:
+ * @index: The CPUState@cpu_index value of the CPU to obtain.
+ *
+ * Gets a CPU matching @index.
+ *
+ * Returns: The CPU or %NULL if there is no matching CPU.
+ */
+CPUState *qemu_get_cpu(int index);
+
 
 #endif
diff --git a/include/qemu/object.h b/include/qom/object.h
index ed1f47f..cf094e7 100644
--- a/include/qemu/object.h
+++ b/include/qom/object.h
@@ -17,7 +17,7 @@
 #include <glib.h>
 #include <stdint.h>
 #include <stdbool.h>
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 struct Visitor;
 struct Error;
@@ -65,7 +65,7 @@ typedef struct InterfaceInfo InterfaceInfo;
  *     int reg0, reg1, reg2;
  * } MyDevice;
  *
- * static TypeInfo my_device_info = {
+ * static const TypeInfo my_device_info = {
  *     .name = TYPE_MY_DEVICE,
  *     .parent = TYPE_DEVICE,
  *     .instance_size = sizeof(MyDevice),
@@ -138,7 +138,7 @@ typedef struct InterfaceInfo InterfaceInfo;
  *     dc->reset = my_device_reset;
  * }
  *
- * static TypeInfo my_device_info = {
+ * static const TypeInfo my_device_info = {
  *     .name = TYPE_MY_DEVICE,
  *     .parent = TYPE_DEVICE,
  *     .instance_size = sizeof(MyDevice),
@@ -147,9 +147,9 @@ typedef struct InterfaceInfo InterfaceInfo;
  *   </programlisting>
  * </example>
  *
- * Introducing new virtual functions requires a class to define its own
- * struct and to add a .class_size member to the TypeInfo.  Each function
- * will also have a wrapper to call it easily:
+ * Introducing new virtual methods requires a class to define its own
+ * struct and to add a .class_size member to the #TypeInfo.  Each method
+ * will also have a wrapper function to call it easily:
  *
  * <example>
  *   <title>Defining an abstract class</title>
@@ -163,7 +163,7 @@ typedef struct InterfaceInfo InterfaceInfo;
  *     void (*frobnicate) (MyDevice *obj);
  * } MyDeviceClass;
  *
- * static TypeInfo my_device_info = {
+ * static const TypeInfo my_device_info = {
  *     .name = TYPE_MY_DEVICE,
  *     .parent = TYPE_DEVICE,
  *     .instance_size = sizeof(MyDevice),
@@ -186,6 +186,104 @@ typedef struct InterfaceInfo InterfaceInfo;
  * similar to normal types except for the fact that are only defined by
  * their classes and never carry any state.  You can dynamically cast an object
  * to one of its #Interface types and vice versa.
+ *
+ * # Methods #
+ *
+ * A <emphasis>method</emphasis> is a function within the namespace scope of
+ * a class. It usually operates on the object instance by passing it as a
+ * strongly-typed first argument.
+ * If it does not operate on an object instance, it is dubbed
+ * <emphasis>class method</emphasis>.
+ *
+ * Methods cannot be overloaded. That is, the #ObjectClass and method name
+ * uniquely identity the function to be called; the signature does not vary
+ * except for trailing varargs.
+ *
+ * Methods are always <emphasis>virtual</emphasis>. Overriding a method in
+ * #TypeInfo.class_init of a subclass leads to any user of the class obtained
+ * via OBJECT_GET_CLASS() accessing the overridden function.
+ * The original function is not automatically invoked. It is the responsability
+ * of the overriding class to determine whether and when to invoke the method
+ * being overridden.
+ *
+ * To invoke the method being overridden, the preferred solution is to store
+ * the original value in the overriding class before overriding the method.
+ * This corresponds to |[ {super,base}.method(...) ]| in Java and C#
+ * respectively; this frees the overriding class from hardcoding its parent
+ * class, which someone might choose to change at some point.
+ *
+ * <example>
+ *   <title>Overriding a virtual method</title>
+ *   <programlisting>
+ * typedef struct MyState MyState;
+ *
+ * typedef void (*MyDoSomething)(MyState *obj);
+ *
+ * typedef struct MyClass {
+ *     ObjectClass parent_class;
+ *
+ *     MyDoSomething do_something;
+ * } MyClass;
+ *
+ * static void my_do_something(MyState *obj)
+ * {
+ *     // do something
+ * }
+ *
+ * static void my_class_init(ObjectClass *oc, void *data)
+ * {
+ *     MyClass *mc = MY_CLASS(oc);
+ *
+ *     mc->do_something = my_do_something;
+ * }
+ *
+ * static const TypeInfo my_type_info = {
+ *     .name = TYPE_MY,
+ *     .parent = TYPE_OBJECT,
+ *     .instance_size = sizeof(MyState),
+ *     .class_size = sizeof(MyClass),
+ *     .class_init = my_class_init,
+ * };
+ *
+ * typedef struct DerivedClass {
+ *     MyClass parent_class;
+ *
+ *     MyDoSomething parent_do_something;
+ * } MyClass;
+ *
+ * static void derived_do_something(MyState *obj)
+ * {
+ *     DerivedClass *dc = DERIVED_GET_CLASS(obj);
+ *
+ *     // do something here
+ *     dc->parent_do_something(obj);
+ *     // do something else here
+ * }
+ *
+ * static void derived_class_init(ObjectClass *oc, void *data)
+ * {
+ *     MyClass *mc = MY_CLASS(oc);
+ *     DerivedClass *dc = DERIVED_CLASS(oc);
+ *
+ *     dc->parent_do_something = mc->do_something;
+ *     mc->do_something = derived_do_something;
+ * }
+ *
+ * static const TypeInfo derived_type_info = {
+ *     .name = TYPE_DERIVED,
+ *     .parent = TYPE_MY,
+ *     .class_size = sizeof(DerivedClass),
+ *     .class_init = my_class_init,
+ * };
+ *   </programlisting>
+ * </example>
+ *
+ * Alternatively, object_class_by_name() can be used to obtain the class and
+ * its non-overridden methods for a specific type. This would correspond to
+ * |[ MyClass::method(...) ]| in C++.
+ *
+ * The first example of such a QOM method was #CPUClass.reset,
+ * another example is #DeviceClass.realize.
  */
 
 
@@ -455,9 +553,9 @@ struct InterfaceClass
  * object_new:
  * @typename: The name of the type of the object to instantiate.
  *
- * This function will initialize a new object using heap allocated memory.  This
- * function should be paired with object_delete() to free the resources
- * associated with the object.
+ * This function will initialize a new object using heap allocated memory.
+ * The returned object has a reference count of 1, and will be freed when
+ * the last reference is dropped.
  *
  * Returns: The newly allocated and instantiated object.
  */
@@ -467,30 +565,22 @@ Object *object_new(const char *typename);
  * object_new_with_type:
  * @type: The type of the object to instantiate.
  *
- * This function will initialize a new object using heap allocated memory.  This
- * function should be paired with object_delete() to free the resources
- * associated with the object.
+ * This function will initialize a new object using heap allocated memory.
+ * The returned object has a reference count of 1, and will be freed when
+ * the last reference is dropped.
  *
  * Returns: The newly allocated and instantiated object.
  */
 Object *object_new_with_type(Type type);
 
 /**
- * object_delete:
- * @obj: The object to free.
- *
- * Finalize an object and then free the memory associated with it.  This should
- * be paired with object_new() to free the resources associated with an object.
- */
-void object_delete(Object *obj);
-
-/**
  * object_initialize_with_type:
  * @obj: A pointer to the memory to be used for the object.
  * @type: The type of the object to instantiate.
  *
  * This function will initialize an object.  The memory for the object should
- * have already been allocated.
+ * have already been allocated.  The returned object has a reference count of 1,
+ * and will be finalized when the last reference is dropped.
  */
 void object_initialize_with_type(void *data, Type type);
 
@@ -500,7 +590,8 @@ void object_initialize_with_type(void *data, Type type);
  * @typename: The name of the type of the object to instantiate.
  *
  * This function will initialize an object.  The memory for the object should
- * have already been allocated.
+ * have already been allocated.  The returned object has a reference count of 1,
+ * and will be finalized when the last reference is dropped.
  */
 void object_initialize(void *obj, const char *typename);
 
@@ -593,6 +684,14 @@ ObjectClass *object_class_get_parent(ObjectClass *klass);
 const char *object_class_get_name(ObjectClass *klass);
 
 /**
+ * object_class_is_abstract:
+ * @klass: The class to obtain the abstractness for.
+ *
+ * Returns: %true if @klass is abstract, %false otherwise.
+ */
+bool object_class_is_abstract(ObjectClass *klass);
+
+/**
  * object_class_by_name:
  * @typename: The QOM typename to obtain the class for.
  *
@@ -900,7 +999,7 @@ Object *object_resolve_path_type(const char *path, const char *typename,
  *
  * Returns: The resolved object or NULL on path lookup failure.
  */
-Object *object_resolve_path_component(Object *parent, gchar *part);
+Object *object_resolve_path_component(Object *parent, const gchar *part);
 
 /**
  * object_property_add_child:
@@ -935,6 +1034,11 @@ void object_property_add_child(Object *obj, const char *name,
  * between objects.
  *
  * Links form the graph in the object model.
+ *
+ * Ownership of the pointer that @child points to is transferred to the
+ * link property.  The reference count for <code>*@child</code> is
+ * managed by the property from after the function returns till the
+ * property is deleted with object_property_del().
  */
 void object_property_add_link(Object *obj, const char *name,
                               const char *type, Object **child,
diff --git a/include/qemu/qom-qobject.h b/include/qom/qom-qobject.h
index f9dff12..77cd717 100644
--- a/include/qemu/qom-qobject.h
+++ b/include/qom/qom-qobject.h
@@ -13,7 +13,7 @@
 #ifndef QEMU_QOM_QOBJECT_H
 #define QEMU_QOM_QOBJECT_H
 
-#include "qemu/object.h"
+#include "qom/object.h"
 
 /*
  * object_property_get_qobject:
diff --git a/arch_init.h b/include/sysemu/arch_init.h
index 5fc780c..5fc780c 100644
--- a/arch_init.h
+++ b/include/sysemu/arch_init.h
diff --git a/balloon.h b/include/sysemu/balloon.h
index b803a00..bd9d395 100644
--- a/balloon.h
+++ b/include/sysemu/balloon.h
@@ -14,7 +14,7 @@
 #ifndef _QEMU_BALLOON_H
 #define _QEMU_BALLOON_H
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "qapi-types.h"
 
 typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
diff --git a/blockdev.h b/include/sysemu/blockdev.h
index 5f27b64..1fe5332 100644
--- a/blockdev.h
+++ b/include/sysemu/blockdev.h
@@ -10,17 +10,22 @@
 #ifndef BLOCKDEV_H
 #define BLOCKDEV_H
 
-#include "block.h"
-#include "error.h"
-#include "qemu-queue.h"
+#include "block/block.h"
+#include "qapi/error.h"
+#include "qemu/queue.h"
 
 void blockdev_mark_auto_del(BlockDriverState *bs);
 void blockdev_auto_del(BlockDriverState *bs);
 
 typedef enum {
     IF_DEFAULT = -1,            /* for use with drive_add() only */
+    /*
+     * IF_IDE must be zero, because we want QEMUMachine member
+     * block_default_type to default-initialize to IF_IDE
+     */
+    IF_IDE = 0,
     IF_NONE,
-    IF_IDE, IF_SCSI, IF_FLOPPY, IF_PFLASH, IF_MTD, IF_SD, IF_VIRTIO, IF_XEN,
+    IF_SCSI, IF_FLOPPY, IF_PFLASH, IF_MTD, IF_SD, IF_VIRTIO, IF_XEN,
     IF_COUNT
 } BlockInterfaceType;
 
@@ -51,7 +56,7 @@ DriveInfo *drive_get_by_blockdev(BlockDriverState *bs);
 QemuOpts *drive_def(const char *optstr);
 QemuOpts *drive_add(BlockInterfaceType type, int index, const char *file,
                     const char *optstr);
-DriveInfo *drive_init(QemuOpts *arg, int default_to_scsi);
+DriveInfo *drive_init(QemuOpts *arg, BlockInterfaceType block_default_type);
 
 /* device-hotplug */
 
diff --git a/cpus.h b/include/sysemu/cpus.h
index 81bd817..f7f6854 100644
--- a/cpus.h
+++ b/include/sysemu/cpus.h
@@ -13,9 +13,16 @@ void cpu_synchronize_all_post_init(void);
 
 void qtest_clock_warp(int64_t dest);
 
+#ifndef CONFIG_USER_ONLY
 /* vl.c */
 extern int smp_cores;
 extern int smp_threads;
+#else
+/* *-user doesn't have configurable SMP topology */
+#define smp_cores   1
+#define smp_threads 1
+#endif
+
 void set_numa_modes(void);
 void set_cpu_log(const char *optarg);
 void set_cpu_log_filename(const char *optarg);
diff --git a/device_tree.h b/include/sysemu/device_tree.h
index f0b3f35..f0b3f35 100644
--- a/device_tree.h
+++ b/include/sysemu/device_tree.h
diff --git a/dma.h b/include/sysemu/dma.h
index eedf878..a52c93a 100644
--- a/dma.h
+++ b/include/sysemu/dma.h
@@ -11,10 +11,10 @@
 #define DMA_H
 
 #include <stdio.h>
-#include "memory.h"
+#include "exec/memory.h"
 #include "hw/hw.h"
-#include "block.h"
-#include "kvm.h"
+#include "block/block.h"
+#include "sysemu/kvm.h"
 
 typedef struct DMAContext DMAContext;
 typedef struct ScatterGatherEntry ScatterGatherEntry;
diff --git a/dump.h b/include/sysemu/dump.h
index e25b7cf..e25b7cf 100644
--- a/dump.h
+++ b/include/sysemu/dump.h
diff --git a/kvm.h b/include/sysemu/kvm.h
index 72d866a..f2d97b5 100644
--- a/kvm.h
+++ b/include/sysemu/kvm.h
@@ -16,14 +16,30 @@
 
 #include <errno.h>
 #include "config-host.h"
-#include "qemu-queue.h"
+#include "qemu/queue.h"
+#include "qom/cpu.h"
 
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
+#else
+/* These constants must never be used at runtime if kvm_enabled() is false.
+ * They exist so we don't need #ifdefs around KVM-specific code that already
+ * checks kvm_enabled() properly.
+ */
+#define KVM_CPUID_SIGNATURE      0
+#define KVM_CPUID_FEATURES       0
+#define KVM_FEATURE_CLOCKSOURCE  0
+#define KVM_FEATURE_NOP_IO_DELAY 0
+#define KVM_FEATURE_MMU_OP       0
+#define KVM_FEATURE_CLOCKSOURCE2 0
+#define KVM_FEATURE_ASYNC_PF     0
+#define KVM_FEATURE_STEAL_TIME   0
+#define KVM_FEATURE_PV_EOI       0
+#define KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 0
 #endif
 
-extern int kvm_allowed;
+extern bool kvm_allowed;
 extern bool kvm_kernel_irqchip;
 extern bool kvm_async_interrupts_allowed;
 extern bool kvm_irqfds_allowed;
@@ -120,9 +136,9 @@ int kvm_has_many_ioeventfds(void);
 int kvm_has_gsi_routing(void);
 int kvm_has_intx_set_mask(void);
 
-#ifdef NEED_CPU_H
-int kvm_init_vcpu(CPUArchState *env);
+int kvm_init_vcpu(CPUState *cpu);
 
+#ifdef NEED_CPU_H
 int kvm_cpu_exec(CPUArchState *env);
 
 #if !defined(CONFIG_USER_ONLY)
@@ -143,7 +159,7 @@ int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap);
 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset);
 #endif
 
-int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr);
+int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_on_sigbus(int code, void *addr);
 
 /* internal API */
@@ -152,20 +168,20 @@ int kvm_ioctl(KVMState *s, int type, ...);
 
 int kvm_vm_ioctl(KVMState *s, int type, ...);
 
-int kvm_vcpu_ioctl(CPUArchState *env, int type, ...);
+int kvm_vcpu_ioctl(CPUState *cpu, int type, ...);
 
 /* Arch specific hooks */
 
 extern const KVMCapabilityInfo kvm_arch_required_capabilities[];
 
-void kvm_arch_pre_run(CPUArchState *env, struct kvm_run *run);
-void kvm_arch_post_run(CPUArchState *env, struct kvm_run *run);
+void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run);
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run);
 
-int kvm_arch_handle_exit(CPUArchState *env, struct kvm_run *run);
+int kvm_arch_handle_exit(CPUState *cpu, struct kvm_run *run);
 
-int kvm_arch_process_async_events(CPUArchState *env);
+int kvm_arch_process_async_events(CPUState *cpu);
 
-int kvm_arch_get_registers(CPUArchState *env);
+int kvm_arch_get_registers(CPUState *cpu);
 
 /* state subset only touched by the VCPU itself during runtime */
 #define KVM_PUT_RUNTIME_STATE   1
@@ -174,15 +190,18 @@ int kvm_arch_get_registers(CPUArchState *env);
 /* full state set, modified during initialization or on vmload */
 #define KVM_PUT_FULL_STATE      3
 
-int kvm_arch_put_registers(CPUArchState *env, int level);
+int kvm_arch_put_registers(CPUState *cpu, int level);
 
 int kvm_arch_init(KVMState *s);
 
-int kvm_arch_init_vcpu(CPUArchState *env);
+int kvm_arch_init_vcpu(CPUState *cpu);
+
+/* Returns VCPU ID to be used on KVM_CREATE_VCPU ioctl() */
+unsigned long kvm_arch_vcpu_id(CPUState *cpu);
 
-void kvm_arch_reset_vcpu(CPUArchState *env);
+void kvm_arch_reset_vcpu(CPUState *cpu);
 
-int kvm_arch_on_sigbus_vcpu(CPUArchState *env, int code, void *addr);
+int kvm_arch_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_arch_on_sigbus(int code, void *addr);
 
 void kvm_arch_init_irq_routing(KVMState *s);
@@ -207,14 +226,14 @@ struct kvm_sw_breakpoint {
 
 QTAILQ_HEAD(kvm_sw_breakpoint_head, kvm_sw_breakpoint);
 
-struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
+struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
                                                  target_ulong pc);
 
-int kvm_sw_breakpoints_active(CPUArchState *env);
+int kvm_sw_breakpoints_active(CPUState *cpu);
 
-int kvm_arch_insert_sw_breakpoint(CPUArchState *current_env,
+int kvm_arch_insert_sw_breakpoint(CPUState *current_cpu,
                                   struct kvm_sw_breakpoint *bp);
-int kvm_arch_remove_sw_breakpoint(CPUArchState *current_env,
+int kvm_arch_remove_sw_breakpoint(CPUState *current_cpu,
                                   struct kvm_sw_breakpoint *bp);
 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
                                   target_ulong len, int type);
@@ -222,9 +241,9 @@ int kvm_arch_remove_hw_breakpoint(target_ulong addr,
                                   target_ulong len, int type);
 void kvm_arch_remove_all_hw_breakpoints(void);
 
-void kvm_arch_update_guest_debug(CPUArchState *env, struct kvm_guest_debug *dbg);
+void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg);
 
-bool kvm_arch_stop_on_emulation_error(CPUArchState *env);
+bool kvm_arch_stop_on_emulation_error(CPUState *cpu);
 
 int kvm_check_extension(KVMState *s, unsigned int extension);
 
diff --git a/memory_mapping.h b/include/sysemu/memory_mapping.h
index d5ba46c..1256125 100644
--- a/memory_mapping.h
+++ b/include/sysemu/memory_mapping.h
@@ -14,7 +14,7 @@
 #ifndef MEMORY_MAPPING_H
 #define MEMORY_MAPPING_H
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 /* The physical and virtual address in the memory mapping are contiguous. */
 typedef struct MemoryMapping {
diff --git a/qemu-os-posix.h b/include/sysemu/os-posix.h
index 7f198e4..7f198e4 100644
--- a/qemu-os-posix.h
+++ b/include/sysemu/os-posix.h
diff --git a/qemu-os-win32.h b/include/sysemu/os-win32.h
index d0e9234..bf9edeb 100644
--- a/qemu-os-win32.h
+++ b/include/sysemu/os-win32.h
@@ -73,6 +73,8 @@ struct tm *gmtime_r(const time_t *timep, struct tm *result);
 #undef localtime_r
 struct tm *localtime_r(const time_t *timep, struct tm *result);
 
+char *strtok_r(char *str, const char *delim, char **saveptr);
+
 static inline void os_setup_signal_handling(void) {}
 static inline void os_daemonize(void) {}
 static inline void os_setup_post(void) {}
diff --git a/qtest.h b/include/sysemu/qtest.h
index 723a4f9..9a0c6b3 100644
--- a/qtest.h
+++ b/include/sysemu/qtest.h
@@ -17,7 +17,7 @@
 #include "qemu-common.h"
 
 #if !defined(CONFIG_USER_ONLY)
-extern int qtest_allowed;
+extern bool qtest_allowed;
 extern const char *qtest_chrdev;
 extern const char *qtest_log;
 
diff --git a/qemu-seccomp.h b/include/sysemu/seccomp.h
index b2fc3f8..1189fa2 100644
--- a/qemu-seccomp.h
+++ b/include/sysemu/seccomp.h
@@ -16,7 +16,7 @@
 #define QEMU_SECCOMP_H
 
 #include <seccomp.h>
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 int seccomp_start(void);
 #endif
diff --git a/sysemu.h b/include/sysemu/sysemu.h
index f5ac664..1d9599e 100644
--- a/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -2,13 +2,13 @@
 #define SYSEMU_H
 /* Misc. things related to the system emulator.  */
 
-#include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-queue.h"
-#include "qemu-timer.h"
+#include "qemu/typedefs.h"
+#include "qemu/option.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
 #include "qapi-types.h"
-#include "notify.h"
-#include "main-loop.h"
+#include "qemu/notify.h"
+#include "qemu/main-loop.h"
 
 /* vl.c */
 
@@ -68,7 +68,7 @@ void qemu_add_machine_init_done_notifier(Notifier *notify);
 void do_savevm(Monitor *mon, const QDict *qdict);
 int load_vmstate(const char *name);
 void do_delvm(Monitor *mon, const QDict *qdict);
-void do_info_snapshots(Monitor *mon);
+void do_info_snapshots(Monitor *mon, const QDict *qdict);
 
 void qemu_announce_self(void);
 
@@ -77,7 +77,8 @@ int qemu_savevm_state_begin(QEMUFile *f,
                             const MigrationParams *params);
 int qemu_savevm_state_iterate(QEMUFile *f);
 int qemu_savevm_state_complete(QEMUFile *f);
-void qemu_savevm_state_cancel(QEMUFile *f);
+void qemu_savevm_state_cancel(void);
+uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
 int qemu_loadvm_state(QEMUFile *f);
 
 /* SLIRP */
@@ -121,7 +122,7 @@ extern int semihosting_enabled;
 extern int old_param;
 extern int boot_menu;
 extern uint8_t *boot_splash_filedata;
-extern int boot_splash_filedata_size;
+extern size_t boot_splash_filedata_size;
 extern uint8_t qemu_extra_params_fw[2];
 extern QEMUClock *rtc_clock;
 
@@ -145,8 +146,7 @@ extern unsigned int nb_prom_envs;
 
 /* pci-hotplug */
 void pci_device_hot_add(Monitor *mon, const QDict *qdict);
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict,
-                      DriveInfo *dinfo, int type);
+int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo);
 void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict);
 
 /* generic hotplug */
@@ -171,7 +171,7 @@ extern CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 
 void do_usb_add(Monitor *mon, const QDict *qdict);
 void do_usb_del(Monitor *mon, const QDict *qdict);
-void usb_info(Monitor *mon);
+void usb_info(Monitor *mon, const QDict *qdict);
 
 void rtc_change_mon_event(struct tm *tm);
 
@@ -179,8 +179,16 @@ void register_devices(void);
 
 void add_boot_device_path(int32_t bootindex, DeviceState *dev,
                           const char *suffix);
-char *get_boot_devices_list(uint32_t *size);
+char *get_boot_devices_list(size_t *size);
 
 bool usb_enabled(bool default_usb);
 
+extern QemuOptsList qemu_drive_opts;
+extern QemuOptsList qemu_chardev_opts;
+extern QemuOptsList qemu_device_opts;
+extern QemuOptsList qemu_netdev_opts;
+extern QemuOptsList qemu_net_opts;
+extern QemuOptsList qemu_global_opts;
+extern QemuOptsList qemu_mon_opts;
+
 #endif
diff --git a/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index c598040..c598040 100644
--- a/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
diff --git a/include/trace.h b/include/trace.h
new file mode 100644
index 0000000..c15f498
--- /dev/null
+++ b/include/trace.h
@@ -0,0 +1,6 @@
+#ifndef TRACE_H
+#define TRACE_H
+
+#include "trace/generated-tracers.h"
+
+#endif  /* TRACE_H */
diff --git a/console.h b/include/ui/console.h
index 638ad4d..753af2b 100644
--- a/console.h
+++ b/include/ui/console.h
@@ -1,14 +1,13 @@
 #ifndef CONSOLE_H
 #define CONSOLE_H
 
-#include "qemu-char.h"
-#include "qemu-pixman.h"
-#include "qdict.h"
-#include "notify.h"
-#include "monitor.h"
+#include "ui/qemu-pixman.h"
+#include "qapi/qmp/qdict.h"
+#include "qemu/notify.h"
+#include "monitor/monitor.h"
 #include "trace.h"
 #include "qapi-types.h"
-#include "error.h"
+#include "qapi/error.h"
 
 /* keyboard/mouse support */
 
@@ -235,6 +234,16 @@ static inline void unregister_displaychangelistener(DisplayState *ds,
 static inline void dpy_gfx_update(DisplayState *s, int x, int y, int w, int h)
 {
     struct DisplayChangeListener *dcl;
+    int width = pixman_image_get_width(s->surface->image);
+    int height = pixman_image_get_height(s->surface->image);
+
+    x = MAX(x, 0);
+    y = MAX(y, 0);
+    x = MIN(x, width);
+    y = MIN(y, height);
+    w = MIN(w, width - x);
+    h = MIN(h, height - y);
+
     QLIST_FOREACH(dcl, &s->listeners, next) {
         if (dcl->dpy_gfx_update) {
             dcl->dpy_gfx_update(s, x, y, w, h);
diff --git a/hw/pixel_ops.h b/include/ui/pixel_ops.h
index d390adf..d390adf 100644
--- a/hw/pixel_ops.h
+++ b/include/ui/pixel_ops.h
diff --git a/qemu-pixman.h b/include/ui/qemu-pixman.h
index bee55eb..b032f52 100644
--- a/qemu-pixman.h
+++ b/include/ui/qemu-pixman.h
@@ -6,9 +6,16 @@
 #ifndef QEMU_PIXMAN_H
 #define QEMU_PIXMAN_H
 
+/* pixman-0.16.0 headers have a redundant declaration */
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#endif
 #include <pixman.h>
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic error "-Wredundant-decls"
+#endif
 
-#include "console.h"
+#include "qemu/typedefs.h"
 
 /*
  * pixman image formats are defined to be native endian,
@@ -31,7 +38,7 @@ pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf);
 pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
                                            int width);
 void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
-                              int width, int y);
+                              int width, int x, int y);
 pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
                                           pixman_image_t *image);
 void qemu_pixman_image_unref(pixman_image_t *image);
diff --git a/ui/qemu-spice.h b/include/ui/qemu-spice.h
index 3299da8..5a78fd7 100644
--- a/ui/qemu-spice.h
+++ b/include/ui/qemu-spice.h
@@ -22,10 +22,9 @@
 
 #include <spice.h>
 
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "qemu-char.h"
-#include "monitor.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "monitor/monitor.h"
 
 extern int using_spice;
 
@@ -46,9 +45,13 @@ void do_info_spice_print(Monitor *mon, const QObject *data);
 void do_info_spice(Monitor *mon, QObject **ret_data);
 
 CharDriverState *qemu_chr_open_spice(QemuOpts *opts);
+#if SPICE_SERVER_VERSION >= 0x000c02
+CharDriverState *qemu_chr_open_spice_port(QemuOpts *opts);
+void qemu_spice_register_ports(void);
+#endif
 
 #else  /* CONFIG_SPICE */
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 #define using_spice 0
 static inline int qemu_spice_set_passwd(const char *passwd,
diff --git a/ui/spice-display.h b/include/ui/spice-display.h
index 38b6ea9..46f9530 100644
--- a/ui/spice-display.h
+++ b/include/ui/spice-display.h
@@ -19,9 +19,10 @@
 #include <spice/enums.h>
 #include <spice/qxl_dev.h>
 
-#include "qemu-thread.h"
-#include "qemu-pixman.h"
-#include "sysemu.h"
+#include "qemu/thread.h"
+#include "ui/qemu-pixman.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 #define NUM_MEMSLOTS 8
 #define MEMSLOT_GENERATION_BITS 8
diff --git a/iohandler.c b/iohandler.c
index 60460a6..2523adc 100644
--- a/iohandler.c
+++ b/iohandler.c
@@ -24,10 +24,9 @@
 
 #include "config-host.h"
 #include "qemu-common.h"
-#include "qemu-char.h"
-#include "qemu-queue.h"
-#include "qemu-aio.h"
-#include "main-loop.h"
+#include "qemu/queue.h"
+#include "block/aio.h"
+#include "qemu/main-loop.h"
 
 #ifndef _WIN32
 #include <sys/wait.h>
diff --git a/ioport.c b/ioport.c
index 6e4ca0d..a0ac2a0 100644
--- a/ioport.c
+++ b/ioport.c
@@ -25,9 +25,9 @@
  * splitted out ioport related stuffs from vl.c.
  */
 
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "trace.h"
-#include "memory.h"
+#include "exec/memory.h"
 
 /***********************************************************/
 /* IO Port */
diff --git a/kvm-all.c b/kvm-all.c
index 8e9a8d8..04ec2d5 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -21,18 +21,18 @@
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
-#include "qemu-barrier.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "sysemu.h"
+#include "qemu/atomic.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "sysemu/sysemu.h"
 #include "hw/hw.h"
-#include "hw/msi.h"
-#include "gdbstub.h"
-#include "kvm.h"
-#include "bswap.h"
-#include "memory.h"
-#include "exec-memory.h"
-#include "event_notifier.h"
+#include "hw/pci/msi.h"
+#include "exec/gdbstub.h"
+#include "sysemu/kvm.h"
+#include "qemu/bswap.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "qemu/event_notifier.h"
 
 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
@@ -209,12 +209,12 @@ static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 
 static void kvm_reset_vcpu(void *opaque)
 {
-    CPUArchState *env = opaque;
+    CPUState *cpu = opaque;
 
-    kvm_arch_reset_vcpu(env);
+    kvm_arch_reset_vcpu(cpu);
 }
 
-int kvm_init_vcpu(CPUArchState *env)
+int kvm_init_vcpu(CPUState *cpu)
 {
     KVMState *s = kvm_state;
     long mmap_size;
@@ -222,15 +222,15 @@ int kvm_init_vcpu(CPUArchState *env)
 
     DPRINTF("kvm_init_vcpu\n");
 
-    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
+    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
     if (ret < 0) {
         DPRINTF("kvm_create_vcpu failed\n");
         goto err;
     }
 
-    env->kvm_fd = ret;
-    env->kvm_state = s;
-    env->kvm_vcpu_dirty = 1;
+    cpu->kvm_fd = ret;
+    cpu->kvm_state = s;
+    cpu->kvm_vcpu_dirty = true;
 
     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
     if (mmap_size < 0) {
@@ -239,9 +239,9 @@ int kvm_init_vcpu(CPUArchState *env)
         goto err;
     }
 
-    env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
-                        env->kvm_fd, 0);
-    if (env->kvm_run == MAP_FAILED) {
+    cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                        cpu->kvm_fd, 0);
+    if (cpu->kvm_run == MAP_FAILED) {
         ret = -errno;
         DPRINTF("mmap'ing vcpu state failed\n");
         goto err;
@@ -249,13 +249,13 @@ int kvm_init_vcpu(CPUArchState *env)
 
     if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
         s->coalesced_mmio_ring =
-            (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
+            (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
     }
 
-    ret = kvm_arch_init_vcpu(env);
+    ret = kvm_arch_init_vcpu(cpu);
     if (ret == 0) {
-        qemu_register_reset(kvm_reset_vcpu, env);
-        kvm_arch_reset_vcpu(env);
+        qemu_register_reset(kvm_reset_vcpu, cpu);
+        kvm_arch_reset_vcpu(cpu);
     }
 err:
     return ret;
@@ -991,8 +991,6 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
         }
     }
     clear_gsi(s, virq);
-
-    kvm_irqchip_commit_routes(s);
 }
 
 static unsigned int kvm_hash_msi(uint32_t data)
@@ -1183,6 +1181,11 @@ static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
     abort();
 }
+
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+{
+    return -ENOSYS;
+}
 #endif /* !KVM_CAP_IRQ_ROUTING */
 
 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
@@ -1437,6 +1440,8 @@ static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
 
 static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     fprintf(stderr, "KVM internal error.");
     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
         int i;
@@ -1451,7 +1456,7 @@ static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
     }
     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
         fprintf(stderr, "emulation failure\n");
-        if (!kvm_arch_stop_on_emulation_error(env)) {
+        if (!kvm_arch_stop_on_emulation_error(cpu)) {
             cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
             return EXCP_INTERRUPT;
         }
@@ -1488,13 +1493,13 @@ void kvm_flush_coalesced_mmio_buffer(void)
     s->coalesced_flush_in_progress = false;
 }
 
-static void do_kvm_cpu_synchronize_state(void *_env)
+static void do_kvm_cpu_synchronize_state(void *arg)
 {
-    CPUArchState *env = _env;
+    CPUState *cpu = arg;
 
-    if (!env->kvm_vcpu_dirty) {
-        kvm_arch_get_registers(env);
-        env->kvm_vcpu_dirty = 1;
+    if (!cpu->kvm_vcpu_dirty) {
+        kvm_arch_get_registers(cpu);
+        cpu->kvm_vcpu_dirty = true;
     }
 }
 
@@ -1502,42 +1507,47 @@ void kvm_cpu_synchronize_state(CPUArchState *env)
 {
     CPUState *cpu = ENV_GET_CPU(env);
 
-    if (!env->kvm_vcpu_dirty) {
-        run_on_cpu(cpu, do_kvm_cpu_synchronize_state, env);
+    if (!cpu->kvm_vcpu_dirty) {
+        run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
     }
 }
 
 void kvm_cpu_synchronize_post_reset(CPUArchState *env)
 {
-    kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
-    env->kvm_vcpu_dirty = 0;
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
+    cpu->kvm_vcpu_dirty = false;
 }
 
 void kvm_cpu_synchronize_post_init(CPUArchState *env)
 {
-    kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
-    env->kvm_vcpu_dirty = 0;
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
+    cpu->kvm_vcpu_dirty = false;
 }
 
 int kvm_cpu_exec(CPUArchState *env)
 {
-    struct kvm_run *run = env->kvm_run;
+    CPUState *cpu = ENV_GET_CPU(env);
+    struct kvm_run *run = cpu->kvm_run;
     int ret, run_ret;
 
     DPRINTF("kvm_cpu_exec()\n");
 
-    if (kvm_arch_process_async_events(env)) {
+    if (kvm_arch_process_async_events(cpu)) {
         env->exit_request = 0;
         return EXCP_HLT;
     }
 
     do {
-        if (env->kvm_vcpu_dirty) {
-            kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
-            env->kvm_vcpu_dirty = 0;
+        if (cpu->kvm_vcpu_dirty) {
+            kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
+            cpu->kvm_vcpu_dirty = false;
         }
 
-        kvm_arch_pre_run(env, run);
+        kvm_arch_pre_run(cpu, run);
         if (env->exit_request) {
             DPRINTF("interrupt exit requested\n");
             /*
@@ -1549,10 +1559,10 @@ int kvm_cpu_exec(CPUArchState *env)
         }
         qemu_mutex_unlock_iothread();
 
-        run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
+        run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
 
         qemu_mutex_lock_iothread();
-        kvm_arch_post_run(env, run);
+        kvm_arch_post_run(cpu, run);
 
         if (run_ret < 0) {
             if (run_ret == -EINTR || run_ret == -EAGAIN) {
@@ -1602,7 +1612,7 @@ int kvm_cpu_exec(CPUArchState *env)
             break;
         default:
             DPRINTF("kvm_arch_handle_exit\n");
-            ret = kvm_arch_handle_exit(env, run);
+            ret = kvm_arch_handle_exit(cpu, run);
             break;
         }
     } while (ret == 0);
@@ -1650,7 +1660,7 @@ int kvm_vm_ioctl(KVMState *s, int type, ...)
     return ret;
 }
 
-int kvm_vcpu_ioctl(CPUArchState *env, int type, ...)
+int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
 {
     int ret;
     void *arg;
@@ -1660,7 +1670,7 @@ int kvm_vcpu_ioctl(CPUArchState *env, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
-    ret = ioctl(env->kvm_fd, type, arg);
+    ret = ioctl(cpu->kvm_fd, type, arg);
     if (ret == -1) {
         ret = -errno;
     }
@@ -1755,12 +1765,12 @@ void kvm_setup_guest_memory(void *start, size_t size)
 }
 
 #ifdef KVM_CAP_SET_GUEST_DEBUG
-struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
+struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
                                                  target_ulong pc)
 {
     struct kvm_sw_breakpoint *bp;
 
-    QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
+    QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
         if (bp->pc == pc) {
             return bp;
         }
@@ -1768,23 +1778,23 @@ struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
     return NULL;
 }
 
-int kvm_sw_breakpoints_active(CPUArchState *env)
+int kvm_sw_breakpoints_active(CPUState *cpu)
 {
-    return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
+    return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
 }
 
 struct kvm_set_guest_debug_data {
     struct kvm_guest_debug dbg;
-    CPUArchState *env;
+    CPUState *cpu;
     int err;
 };
 
 static void kvm_invoke_set_guest_debug(void *data)
 {
     struct kvm_set_guest_debug_data *dbg_data = data;
-    CPUArchState *env = dbg_data->env;
 
-    dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
+    dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
+                                   &dbg_data->dbg);
 }
 
 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
@@ -1797,8 +1807,8 @@ int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
     if (env->singlestep_enabled) {
         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
     }
-    kvm_arch_update_guest_debug(env, &data.dbg);
-    data.env = env;
+    kvm_arch_update_guest_debug(cpu, &data.dbg);
+    data.cpu = cpu;
 
     run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
     return data.err;
@@ -1807,12 +1817,13 @@ int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
                           target_ulong len, int type)
 {
+    CPUState *current_cpu = ENV_GET_CPU(current_env);
     struct kvm_sw_breakpoint *bp;
     CPUArchState *env;
     int err;
 
     if (type == GDB_BREAKPOINT_SW) {
-        bp = kvm_find_sw_breakpoint(current_env, addr);
+        bp = kvm_find_sw_breakpoint(current_cpu, addr);
         if (bp) {
             bp->use_count++;
             return 0;
@@ -1825,13 +1836,13 @@ int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
 
         bp->pc = addr;
         bp->use_count = 1;
-        err = kvm_arch_insert_sw_breakpoint(current_env, bp);
+        err = kvm_arch_insert_sw_breakpoint(current_cpu, bp);
         if (err) {
             g_free(bp);
             return err;
         }
 
-        QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
+        QTAILQ_INSERT_HEAD(&current_cpu->kvm_state->kvm_sw_breakpoints,
                           bp, entry);
     } else {
         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
@@ -1852,12 +1863,13 @@ int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
                           target_ulong len, int type)
 {
+    CPUState *current_cpu = ENV_GET_CPU(current_env);
     struct kvm_sw_breakpoint *bp;
     CPUArchState *env;
     int err;
 
     if (type == GDB_BREAKPOINT_SW) {
-        bp = kvm_find_sw_breakpoint(current_env, addr);
+        bp = kvm_find_sw_breakpoint(current_cpu, addr);
         if (!bp) {
             return -ENOENT;
         }
@@ -1867,12 +1879,12 @@ int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
             return 0;
         }
 
-        err = kvm_arch_remove_sw_breakpoint(current_env, bp);
+        err = kvm_arch_remove_sw_breakpoint(current_cpu, bp);
         if (err) {
             return err;
         }
 
-        QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
+        QTAILQ_REMOVE(&current_cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
         g_free(bp);
     } else {
         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
@@ -1892,15 +1904,18 @@ int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
 
 void kvm_remove_all_breakpoints(CPUArchState *current_env)
 {
+    CPUState *current_cpu = ENV_GET_CPU(current_env);
     struct kvm_sw_breakpoint *bp, *next;
-    KVMState *s = current_env->kvm_state;
+    KVMState *s = current_cpu->kvm_state;
     CPUArchState *env;
+    CPUState *cpu;
 
     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
-        if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
+        if (kvm_arch_remove_sw_breakpoint(current_cpu, bp) != 0) {
             /* Try harder to find a CPU that currently sees the breakpoint. */
             for (env = first_cpu; env != NULL; env = env->next_cpu) {
-                if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
+                cpu = ENV_GET_CPU(env);
+                if (kvm_arch_remove_sw_breakpoint(cpu, bp) == 0) {
                     break;
                 }
             }
@@ -1941,18 +1956,19 @@ void kvm_remove_all_breakpoints(CPUArchState *current_env)
 
 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     struct kvm_signal_mask *sigmask;
     int r;
 
     if (!sigset) {
-        return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
+        return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
     }
 
     sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
 
     sigmask->len = 8;
     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
-    r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
+    r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
     g_free(sigmask);
 
     return r;
@@ -2010,9 +2026,9 @@ int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
     return 0;
 }
 
-int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr)
+int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
-    return kvm_arch_on_sigbus_vcpu(env, code, addr);
+    return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
 }
 
 int kvm_on_sigbus(int code, void *addr)
diff --git a/kvm-stub.c b/kvm-stub.c
index a3455e2..760aadc 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -12,10 +12,10 @@
 
 #include "qemu-common.h"
 #include "hw/hw.h"
-#include "hw/msi.h"
+#include "hw/pci/msi.h"
 #include "cpu.h"
-#include "gdbstub.h"
-#include "kvm.h"
+#include "exec/gdbstub.h"
+#include "sysemu/kvm.h"
 
 KVMState *kvm_state;
 bool kvm_kernel_irqchip;
@@ -24,7 +24,7 @@ bool kvm_irqfds_allowed;
 bool kvm_msi_via_irqfd_allowed;
 bool kvm_gsi_routing_allowed;
 
-int kvm_init_vcpu(CPUArchState *env)
+int kvm_init_vcpu(CPUState *cpu)
 {
     return -ENOSYS;
 }
@@ -112,7 +112,7 @@ int kvm_set_ioeventfd_mmio(int fd, uint32_t adr, uint32_t val, bool assign, uint
     return -ENOSYS;
 }
 
-int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr)
+int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
     return 1;
 }
@@ -131,6 +131,11 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
 {
 }
 
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+{
+    return -ENOSYS;
+}
+
 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
     return -ENOSYS;
diff --git a/alpha.ld b/ldscripts/alpha.ld
index 906d76b..906d76b 100644
--- a/alpha.ld
+++ b/ldscripts/alpha.ld
diff --git a/arm.ld b/ldscripts/arm.ld
index 7f13da9..7f13da9 100644
--- a/arm.ld
+++ b/ldscripts/arm.ld
diff --git a/hppa.ld b/ldscripts/hppa.ld
index 3555b3e..3555b3e 100644
--- a/hppa.ld
+++ b/ldscripts/hppa.ld
diff --git a/i386.ld b/ldscripts/i386.ld
index cc3f160..cc3f160 100644
--- a/i386.ld
+++ b/ldscripts/i386.ld
diff --git a/ia64.ld b/ldscripts/ia64.ld
index 0c37796..0c37796 100644
--- a/ia64.ld
+++ b/ldscripts/ia64.ld
diff --git a/m68k.ld b/ldscripts/m68k.ld
index 0e3d9de..0e3d9de 100644
--- a/m68k.ld
+++ b/ldscripts/m68k.ld
diff --git a/mips.ld b/ldscripts/mips.ld
index 7b610ce..7b610ce 100644
--- a/mips.ld
+++ b/ldscripts/mips.ld
diff --git a/ppc.ld b/ldscripts/ppc.ld
index 2a0dcad..2a0dcad 100644
--- a/ppc.ld
+++ b/ldscripts/ppc.ld
diff --git a/ppc64.ld b/ldscripts/ppc64.ld
index e2dafa0..e2dafa0 100644
--- a/ppc64.ld
+++ b/ldscripts/ppc64.ld
diff --git a/s390.ld b/ldscripts/s390.ld
index a9c5370..a9c5370 100644
--- a/s390.ld
+++ b/ldscripts/s390.ld
diff --git a/sparc.ld b/ldscripts/sparc.ld
index 56efe34..56efe34 100644
--- a/sparc.ld
+++ b/ldscripts/sparc.ld
diff --git a/sparc64.ld b/ldscripts/sparc64.ld
index 9ea4143..9ea4143 100644
--- a/sparc64.ld
+++ b/ldscripts/sparc64.ld
diff --git a/x86_64.ld b/ldscripts/x86_64.ld
index b7a9f4e..b7a9f4e 100644
--- a/x86_64.ld
+++ b/ldscripts/x86_64.ld
diff --git a/libcacard/Makefile b/libcacard/Makefile
index c26aac6..47827a0 100644
--- a/libcacard/Makefile
+++ b/libcacard/Makefile
@@ -1,63 +1,49 @@
--include ../config-host.mak
--include $(SRC_PATH)/rules.mak
--include $(SRC_PATH)/Makefile.objs
-
 libcacard_includedir=$(includedir)/cacard
 
-$(call set-vpath, $(SRC_PATH))
+TOOLS += vscclient$(EXESUF)
 
 # objects linked into a shared library, built with libtool with -fPIC if required
-QEMU_OBJS=$(oslib-obj-y) qemu-timer-common.o $(trace-obj-y) $(stub-obj-y)
-QEMU_OBJS_LIB=$(patsubst %.o,%.lo,$(QEMU_OBJS))
-
-QEMU_CFLAGS+=-I../
-
-libcacard.lib-y=$(patsubst %.o,%.lo,$(libcacard-y))
+libcacard-obj-y = $(stub-obj-y) $(libcacard-y)
+libcacard-obj-y += util/osdep.o util/cutils.o util/qemu-timer-common.o util/error.o
+libcacard-obj-$(CONFIG_WIN32) += util/oslib-win32.o util/qemu-thread-win32.o
+libcacard-obj-$(CONFIG_POSIX) += util/oslib-posix.o util/qemu-thread-posix.o
+libcacard-obj-y += $(filter trace/%, $(util-obj-y))
 
-vscclient: $(libcacard-y) $(QEMU_OBJS) vscclient.o cutils.o
-	$(call quiet-command,$(CC) -o $@ $^ $(libcacard_libs) $(LIBS),"  LINK  $@")
+libcacard-lobj-y=$(patsubst %.o,%.lo,$(libcacard-obj-y))
 
-clean:
-	rm -f *.o */*.o *.d */*.d *.a */*.a *~ */*~ vscclient *.lo */*.lo .libs/* */.libs/* *.la */*.la *.pc
-	rm -Rf .libs */.libs
+# libtool will build the .o files, too
+$(libcacard-obj-y): | $(libcacard-lobj-y)
 
 all: libcacard.la libcacard.pc
-# Dummy command so that make thinks it has done something
-	@true
+
+vscclient$(EXESUF): libcacard/vscclient.o libcacard.la
+	$(call LINK,$^)
 
 #########################################################################
 # Rules for building libcacard standalone library
 
-ifeq ($(LIBTOOL),)
-libcacard.la:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-
-install-libcacard:
-	@echo "libtool is missing, please install and rerun configure"; exit 1
-else
-libcacard.la: $(libcacard.lib-y) $(QEMU_OBJS_LIB)
-	$(call quiet-command,$(LIBTOOL) --mode=link --quiet --tag=CC $(CC) -rpath $(libdir) -o $@ $^ $(libcacard_libs),"  lt LINK $@")
+libcacard.la: LDFLAGS += -rpath $(libdir) -no-undefined \
+	-export-syms $(SRC_PATH)/libcacard/libcacard.syms
+libcacard.la: LIBS += $(libcacard_libs)
+libcacard.la: $(libcacard-lobj-y)
+	$(call LINK,$^)
 
-libcacard_srcpath=$(SRC_PATH)/libcacard
-libcacard.pc: $(libcacard_srcpath)/libcacard.pc.in
+libcacard.pc: $(SRC_PATH)/libcacard/libcacard.pc.in
 	$(call quiet-command,sed -e 's|@LIBDIR@|$(libdir)|' \
 		-e 's|@INCLUDEDIR@|$(libcacard_includedir)|' \
 	    -e 's|@VERSION@|$(shell cat $(SRC_PATH)/VERSION)|' \
-		-e 's|@PREFIX@|$(prefix)|' \
-		< $(libcacard_srcpath)/libcacard.pc.in > libcacard.pc,\
+		-e 's|@PREFIX@|$(prefix)|' $< > libcacard.pc,\
 	"  GEN   $@")
 
 .PHONY: install-libcacard
 
-install-libcacard: libcacard.pc libcacard.la vscclient
+install: install-libcacard
+install-libcacard: libcacard.pc libcacard.la
 	$(INSTALL_DIR) "$(DESTDIR)$(libdir)"
 	$(INSTALL_DIR) "$(DESTDIR)$(libdir)/pkgconfig"
 	$(INSTALL_DIR) "$(DESTDIR)$(libcacard_includedir)"
-	$(INSTALL_DIR) "$(DESTDIR)$(bindir)"
-	$(LIBTOOL) --mode=install $(INSTALL_PROG) vscclient "$(DESTDIR)$(bindir)"
-	$(LIBTOOL) --mode=install $(INSTALL_DATA) libcacard.la "$(DESTDIR)$(libdir)"
-	$(LIBTOOL) --mode=install $(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
-	for inc in *.h; do \
-		$(LIBTOOL) --mode=install $(INSTALL_DATA) $(libcacard_srcpath)/$$inc "$(DESTDIR)$(libcacard_includedir)"; \
+	$(INSTALL_LIB) libcacard.la "$(DESTDIR)$(libdir)"
+	$(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
+	for inc in $(SRC_PATH)/libcacard/*.h; do \
+		$(INSTALL_DATA) $$inc "$(DESTDIR)$(libcacard_includedir)"; \
 	done
-endif
diff --git a/libcacard/event.c b/libcacard/event.c
index 6192376..2d7500f 100644
--- a/libcacard/event.c
+++ b/libcacard/event.c
@@ -6,7 +6,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "vcard.h"
 #include "vreader.h"
diff --git a/libcacard/libcacard.syms b/libcacard/libcacard.syms
new file mode 100644
index 0000000..1697515
--- /dev/null
+++ b/libcacard/libcacard.syms
@@ -0,0 +1,77 @@
+cac_card_init
+cac_is_cac_card
+vcard_add_applet
+vcard_apdu_delete
+vcard_apdu_new
+vcard_applet_get_aid
+vcard_buffer_response_delete
+vcard_buffer_response_new
+vcard_delete_applet
+vcard_emul_delete_key
+vcard_emul_force_card_insert
+vcard_emul_force_card_remove
+vcard_emul_get_atr
+vcard_emul_get_login_count
+vcard_emul_init
+vcard_emul_login
+vcard_emul_options
+vcard_emul_replay_insertion_events
+vcard_emul_reset
+vcard_emul_rsa_op
+vcard_emul_type_from_string
+vcard_emul_type_select
+vcard_emul_usage
+vcard_find_applet
+vcard_free
+vcard_get_atr
+vcard_get_buffer_response
+vcard_get_current_applet_private
+vcard_get_private
+vcard_get_type
+vcard_init
+vcard_make_response
+vcard_new
+vcard_new_applet
+vcard_process_apdu
+vcard_process_applet_apdu
+vcard_reference
+vcard_reset
+vcard_response_delete
+vcard_response_new
+vcard_response_new_bytes
+vcard_response_new_data
+vcard_response_new_status_bytes
+vcard_select_applet
+vcard_set_applet_private
+vcard_set_atr_func
+vcard_set_buffer_response
+vcard_set_type
+vevent_delete
+vevent_get_next_vevent
+vevent_new
+vevent_queue_init
+vevent_queue_vevent
+vevent_wait_next_vevent
+vreader_add_reader
+vreader_card_is_present
+vreader_free
+vreader_get_id
+vreader_get_name
+vreader_get_private
+vreader_get_reader_by_id
+vreader_get_reader_by_name
+vreader_get_reader_list
+vreader_init
+vreader_insert_card
+vreader_list_delete
+vreader_list_get_first
+vreader_list_get_next
+vreader_list_get_reader
+vreader_new
+vreader_power_off
+vreader_power_on
+vreader_queue_card_event
+vreader_reference
+vreader_remove_reader
+vreader_set_id
+vreader_xfr_bytes
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
index 5f565e0..df79476 100644
--- a/libcacard/vcard_emul_nss.c
+++ b/libcacard/vcard_emul_nss.c
@@ -454,7 +454,7 @@ vreader_emul_new(PK11SlotInfo *slot, VCardEmulType type, const char *params)
 
     new_reader_emul->slot = PK11_ReferenceSlot(slot);
     new_reader_emul->default_type = type;
-    new_reader_emul->type_params = strdup(params);
+    new_reader_emul->type_params = g_strdup(params);
     new_reader_emul->present = PR_FALSE;
     new_reader_emul->series = 0;
     new_reader_emul->saved_vcard = NULL;
@@ -997,7 +997,7 @@ vcard_emul_init(const VCardEmulOptions *options)
     /* We should control this with options. For now we mirror out any
      * removable hardware slot */
     default_card_type = options->hw_card_type;
-    default_type_params = strdup(options->hw_type_params);
+    default_type_params = g_strdup(options->hw_type_params);
 
     SECMOD_GetReadLock(module_lock);
     for (mlp = module_list; mlp; mlp = mlp->next) {
diff --git a/libcacard/vreader.c b/libcacard/vreader.c
index 96d2407..f3efc27 100644
--- a/libcacard/vreader.c
+++ b/libcacard/vreader.c
@@ -6,7 +6,7 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 #include "vcard.h"
 #include "vcard_emul.h"
@@ -49,7 +49,7 @@ vreader_new(const char *name, VReaderEmul *private,
     reader = (VReader *)g_malloc(sizeof(VReader));
     qemu_mutex_init(&reader->lock);
     reader->reference_count = 1;
-    reader->name = name ? strdup(name) : NULL;
+    reader->name = g_strdup(name);
     reader->card = NULL;
     reader->id = (vreader_id_t)-1;
     reader->reader_private = private;
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
index b64c93d..9b744f2 100644
--- a/libcacard/vscclient.c
+++ b/libcacard/vscclient.c
@@ -13,8 +13,8 @@
 #include <netdb.h>
 
 #include "qemu-common.h"
-#include "qemu-thread.h"
-#include "qemu_socket.h"
+#include "qemu/thread.h"
+#include "qemu/sockets.h"
 
 #include "vscard_common.h"
 
@@ -503,8 +503,8 @@ main(
         command_line_options = vcard_emul_options(emul_args);
     }
 
-    qemu_host = strdup(argv[argc - 2]);
-    qemu_port = strdup(argv[argc - 1]);
+    qemu_host = g_strdup(argv[argc - 2]);
+    qemu_port = g_strdup(argv[argc - 1]);
     sock = connect_to_qemu(qemu_host, qemu_port);
     if (sock == -1) {
         fprintf(stderr, "error opening socket, exiting.\n");
diff --git a/linux-headers/asm-arm/kvm.h b/linux-headers/asm-arm/kvm.h
index b1c7871..023bfeb 100644
--- a/linux-headers/asm-arm/kvm.h
+++ b/linux-headers/asm-arm/kvm.h
@@ -19,7 +19,7 @@
 #ifndef __ARM_KVM_H__
 #define __ARM_KVM_H__
 
-#include <asm/types.h>
+#include <linux/types.h>
 #include <asm/ptrace.h>
 
 #define __KVM_HAVE_GUEST_DEBUG
@@ -28,6 +28,30 @@
 #define KVM_REG_SIZE(id)						\
 	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
 
+/* Valid for svc_regs, abt_regs, und_regs, irq_regs in struct kvm_regs */
+#define KVM_ARM_SVC_sp		svc_regs[0]
+#define KVM_ARM_SVC_lr		svc_regs[1]
+#define KVM_ARM_SVC_spsr	svc_regs[2]
+#define KVM_ARM_ABT_sp		abt_regs[0]
+#define KVM_ARM_ABT_lr		abt_regs[1]
+#define KVM_ARM_ABT_spsr	abt_regs[2]
+#define KVM_ARM_UND_sp		und_regs[0]
+#define KVM_ARM_UND_lr		und_regs[1]
+#define KVM_ARM_UND_spsr	und_regs[2]
+#define KVM_ARM_IRQ_sp		irq_regs[0]
+#define KVM_ARM_IRQ_lr		irq_regs[1]
+#define KVM_ARM_IRQ_spsr	irq_regs[2]
+
+/* Valid only for fiq_regs in struct kvm_regs */
+#define KVM_ARM_FIQ_r8		fiq_regs[0]
+#define KVM_ARM_FIQ_r9		fiq_regs[1]
+#define KVM_ARM_FIQ_r10		fiq_regs[2]
+#define KVM_ARM_FIQ_fp		fiq_regs[3]
+#define KVM_ARM_FIQ_ip		fiq_regs[4]
+#define KVM_ARM_FIQ_sp		fiq_regs[5]
+#define KVM_ARM_FIQ_lr		fiq_regs[6]
+#define KVM_ARM_FIQ_spsr	fiq_regs[7]
+
 struct kvm_regs {
 	struct pt_regs usr_regs;/* R0_usr - R14_usr, PC, CPSR */
 	__u32 svc_regs[3];	/* SP_svc, LR_svc, SPSR_svc */
@@ -41,11 +65,11 @@ struct kvm_regs {
 #define KVM_ARM_TARGET_CORTEX_A15	0
 #define KVM_ARM_NUM_TARGETS		1
 
-/* KVM_SET_DEVICE_ADDRESS ioctl id encoding */
-#define KVM_DEVICE_TYPE_SHIFT		0
-#define KVM_DEVICE_TYPE_MASK		(0xffff << KVM_DEVICE_TYPE_SHIFT)
-#define KVM_DEVICE_ID_SHIFT		16
-#define KVM_DEVICE_ID_MASK		(0xffff << KVM_DEVICE_ID_SHIFT)
+/* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
+#define KVM_ARM_DEVICE_TYPE_SHIFT	0
+#define KVM_ARM_DEVICE_TYPE_MASK	(0xffff << KVM_ARM_DEVICE_TYPE_SHIFT)
+#define KVM_ARM_DEVICE_ID_SHIFT		16
+#define KVM_ARM_DEVICE_ID_MASK		(0xffff << KVM_ARM_DEVICE_ID_SHIFT)
 
 /* Supported device IDs */
 #define KVM_ARM_DEVICE_VGIC_V2		0
@@ -54,6 +78,11 @@ struct kvm_regs {
 #define KVM_VGIC_V2_ADDR_TYPE_DIST	0
 #define KVM_VGIC_V2_ADDR_TYPE_CPU	1
 
+#define KVM_VGIC_V2_DIST_SIZE		0x1000
+#define KVM_VGIC_V2_CPU_SIZE		0x2000
+
+#define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
+
 struct kvm_vcpu_init {
 	__u32 target;
 	__u32 features[7];
@@ -134,4 +163,18 @@ struct kvm_arch_memory_slot {
 /* Highest supported SPI, from VGIC_NR_IRQS */
 #define KVM_ARM_IRQ_GIC_MAX		127
 
+/* PSCI interface */
+#define KVM_PSCI_FN_BASE		0x95c1ba5e
+#define KVM_PSCI_FN(n)			(KVM_PSCI_FN_BASE + (n))
+
+#define KVM_PSCI_FN_CPU_SUSPEND		KVM_PSCI_FN(0)
+#define KVM_PSCI_FN_CPU_OFF		KVM_PSCI_FN(1)
+#define KVM_PSCI_FN_CPU_ON		KVM_PSCI_FN(2)
+#define KVM_PSCI_FN_MIGRATE		KVM_PSCI_FN(3)
+
+#define KVM_PSCI_RET_SUCCESS		0
+#define KVM_PSCI_RET_NI			((unsigned long)-1)
+#define KVM_PSCI_RET_INVAL		((unsigned long)-2)
+#define KVM_PSCI_RET_DENIED		((unsigned long)-3)
+
 #endif /* __ARM_KVM_H__ */
diff --git a/linux-headers/asm-powerpc/epapr_hcalls.h b/linux-headers/asm-powerpc/epapr_hcalls.h
new file mode 100644
index 0000000..06f7247
--- /dev/null
+++ b/linux-headers/asm-powerpc/epapr_hcalls.h
@@ -0,0 +1,98 @@
+/*
+ * ePAPR hcall interface
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Author: Timur Tabi <timur@freescale.com>
+ *
+ * This file is provided under a dual BSD/GPL license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ASM_POWERPC_EPAPR_HCALLS_H
+#define _ASM_POWERPC_EPAPR_HCALLS_H
+
+#define EV_BYTE_CHANNEL_SEND		1
+#define EV_BYTE_CHANNEL_RECEIVE		2
+#define EV_BYTE_CHANNEL_POLL		3
+#define EV_INT_SET_CONFIG		4
+#define EV_INT_GET_CONFIG		5
+#define EV_INT_SET_MASK			6
+#define EV_INT_GET_MASK			7
+#define EV_INT_IACK			9
+#define EV_INT_EOI			10
+#define EV_INT_SEND_IPI			11
+#define EV_INT_SET_TASK_PRIORITY	12
+#define EV_INT_GET_TASK_PRIORITY	13
+#define EV_DOORBELL_SEND		14
+#define EV_MSGSND			15
+#define EV_IDLE				16
+
+/* vendor ID: epapr */
+#define EV_LOCAL_VENDOR_ID		0	/* for private use */
+#define EV_EPAPR_VENDOR_ID		1
+#define EV_FSL_VENDOR_ID		2	/* Freescale Semiconductor */
+#define EV_IBM_VENDOR_ID		3	/* IBM */
+#define EV_GHS_VENDOR_ID		4	/* Green Hills Software */
+#define EV_ENEA_VENDOR_ID		5	/* Enea */
+#define EV_WR_VENDOR_ID			6	/* Wind River Systems */
+#define EV_AMCC_VENDOR_ID		7	/* Applied Micro Circuits */
+#define EV_KVM_VENDOR_ID		42	/* KVM */
+
+/* The max number of bytes that a byte channel can send or receive per call */
+#define EV_BYTE_CHANNEL_MAX_BYTES	16
+
+
+#define _EV_HCALL_TOKEN(id, num) (((id) << 16) | (num))
+#define EV_HCALL_TOKEN(hcall_num) _EV_HCALL_TOKEN(EV_EPAPR_VENDOR_ID, hcall_num)
+
+/* epapr return codes */
+#define EV_SUCCESS		0
+#define EV_EPERM		1	/* Operation not permitted */
+#define EV_ENOENT		2	/*  Entry Not Found */
+#define EV_EIO			3	/* I/O error occured */
+#define EV_EAGAIN		4	/* The operation had insufficient
+					 * resources to complete and should be
+					 * retried
+					 */
+#define EV_ENOMEM		5	/* There was insufficient memory to
+					 * complete the operation */
+#define EV_EFAULT		6	/* Bad guest address */
+#define EV_ENODEV		7	/* No such device */
+#define EV_EINVAL		8	/* An argument supplied to the hcall
+					   was out of range or invalid */
+#define EV_INTERNAL		9	/* An internal error occured */
+#define EV_CONFIG		10	/* A configuration error was detected */
+#define EV_INVALID_STATE	11	/* The object is in an invalid state */
+#define EV_UNIMPLEMENTED	12	/* Unimplemented hypercall */
+#define EV_BUFFER_OVERFLOW	13	/* Caller-supplied buffer too small */
+
+#endif /* _ASM_POWERPC_EPAPR_HCALLS_H */
diff --git a/linux-headers/asm-powerpc/kvm.h b/linux-headers/asm-powerpc/kvm.h
index b89ae4d..16064d0 100644
--- a/linux-headers/asm-powerpc/kvm.h
+++ b/linux-headers/asm-powerpc/kvm.h
@@ -114,7 +114,10 @@ struct kvm_regs {
 /* Embedded Floating Point (SPE) -- IVOR32-34 if KVM_SREGS_E_IVOR */
 #define KVM_SREGS_E_SPE			(1 << 9)
 
-/* External Proxy (EXP) -- EPR */
+/*
+ * DEPRECATED! USE ONE_REG FOR THIS ONE!
+ * External Proxy (EXP) -- EPR
+ */
 #define KVM_SREGS_EXP			(1 << 10)
 
 /* External PID (E.PD) -- EPSC/EPLC */
@@ -331,6 +334,31 @@ struct kvm_book3e_206_tlb_params {
 	__u32 reserved[8];
 };
 
+/* For KVM_PPC_GET_HTAB_FD */
+struct kvm_get_htab_fd {
+	__u64	flags;
+	__u64	start_index;
+	__u64	reserved[2];
+};
+
+/* Values for kvm_get_htab_fd.flags */
+#define KVM_GET_HTAB_BOLTED_ONLY	((__u64)0x1)
+#define KVM_GET_HTAB_WRITE		((__u64)0x2)
+
+/*
+ * Data read on the file descriptor is formatted as a series of
+ * records, each consisting of a header followed by a series of
+ * `n_valid' HPTEs (16 bytes each), which are all valid.  Following
+ * those valid HPTEs there are `n_invalid' invalid HPTEs, which
+ * are not represented explicitly in the stream.  The same format
+ * is used for writing.
+ */
+struct kvm_get_htab_header {
+	__u32	index;
+	__u16	n_valid;
+	__u16	n_invalid;
+};
+
 #define KVM_REG_PPC_HIOR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x1)
 #define KVM_REG_PPC_IAC1	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x2)
 #define KVM_REG_PPC_IAC2	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0x3)
@@ -386,4 +414,7 @@ struct kvm_book3e_206_tlb_params {
 #define KVM_REG_PPC_VPA_SLB	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x83)
 #define KVM_REG_PPC_VPA_DTL	(KVM_REG_PPC | KVM_REG_SIZE_U128 | 0x84)
 
+#define KVM_REG_PPC_EPCR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x85)
+#define KVM_REG_PPC_EPR		(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0x86)
+
 #endif /* __LINUX_KVM_POWERPC_H */
diff --git a/linux-headers/asm-powerpc/kvm_para.h b/linux-headers/asm-powerpc/kvm_para.h
index ed0e025..7e64f57 100644
--- a/linux-headers/asm-powerpc/kvm_para.h
+++ b/linux-headers/asm-powerpc/kvm_para.h
@@ -17,8 +17,8 @@
  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  */
 
-#ifndef _UAPI__POWERPC_KVM_PARA_H__
-#define _UAPI__POWERPC_KVM_PARA_H__
+#ifndef __POWERPC_KVM_PARA_H__
+#define __POWERPC_KVM_PARA_H__
 
 #include <linux/types.h>
 
@@ -78,7 +78,7 @@ struct kvm_vcpu_arch_shared {
 
 #define KVM_HCALL_TOKEN(num)     _EV_HCALL_TOKEN(EV_KVM_VENDOR_ID, num)
 
-#include <uapi/asm/epapr_hcalls.h>
+#include <asm/epapr_hcalls.h>
 
 #define KVM_FEATURE_MAGIC_PAGE	1
 
@@ -88,4 +88,4 @@ struct kvm_vcpu_arch_shared {
 #define KVM_MAGIC_FEAT_MAS0_TO_SPRG7	(1 << 1)
 
 
-#endif /* _UAPI__POWERPC_KVM_PARA_H__ */
+#endif /* __POWERPC_KVM_PARA_H__ */
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index c819d4c..caca979 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -169,6 +169,8 @@ struct kvm_pit_config {
 #define KVM_EXIT_PAPR_HCALL	  19
 #define KVM_EXIT_S390_UCONTROL	  20
 #define KVM_EXIT_WATCHDOG         21
+#define KVM_EXIT_S390_TSCH        22
+#define KVM_EXIT_EPR              23
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -286,6 +288,19 @@ struct kvm_run {
 			__u64 ret;
 			__u64 args[9];
 		} papr_hcall;
+		/* KVM_EXIT_S390_TSCH */
+		struct {
+			__u16 subchannel_id;
+			__u16 subchannel_nr;
+			__u32 io_int_parm;
+			__u32 io_int_word;
+			__u32 ipb;
+			__u8 dequeued;
+		} s390_tsch;
+		/* KVM_EXIT_EPR */
+		struct {
+			__u32 epr;
+		} epr;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -398,10 +413,20 @@ struct kvm_s390_psw {
 #define KVM_S390_PROGRAM_INT		0xfffe0001u
 #define KVM_S390_SIGP_SET_PREFIX	0xfffe0002u
 #define KVM_S390_RESTART		0xfffe0003u
+#define KVM_S390_MCHK			0xfffe1000u
 #define KVM_S390_INT_VIRTIO		0xffff2603u
 #define KVM_S390_INT_SERVICE		0xffff2401u
 #define KVM_S390_INT_EMERGENCY		0xffff1201u
 #define KVM_S390_INT_EXTERNAL_CALL	0xffff1202u
+/* Anything below 0xfffe0000u is taken by INT_IO */
+#define KVM_S390_INT_IO(ai,cssid,ssid,schid)   \
+	(((schid)) |			       \
+	 ((ssid) << 16) |		       \
+	 ((cssid) << 18) |		       \
+	 ((ai) << 26))
+#define KVM_S390_INT_IO_MIN		0x00000000u
+#define KVM_S390_INT_IO_MAX		0xfffdffffu
+
 
 struct kvm_s390_interrupt {
 	__u32 type;
@@ -635,7 +660,11 @@ struct kvm_ppc_smmu_info {
 #endif
 #define KVM_CAP_IRQFD_RESAMPLE 82
 #define KVM_CAP_PPC_BOOKE_WATCHDOG 83
-#define KVM_CAP_SET_DEVICE_ADDR 84
+#define KVM_CAP_PPC_HTAB_FD 84
+#define KVM_CAP_S390_CSS_SUPPORT 85
+#define KVM_CAP_PPC_EPR 86
+#define KVM_CAP_ARM_PSCI 87
+#define KVM_CAP_ARM_SET_DEVICE_ADDR 88
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -783,7 +812,7 @@ struct kvm_msi {
 	__u8  pad[16];
 };
 
-struct kvm_device_address {
+struct kvm_arm_device_addr {
 	__u64 id;
 	__u64 addr;
 };
@@ -871,8 +900,10 @@ struct kvm_s390_ucas_mapping {
 #define KVM_CREATE_SPAPR_TCE	  _IOW(KVMIO,  0xa8, struct kvm_create_spapr_tce)
 /* Available with KVM_CAP_RMA */
 #define KVM_ALLOCATE_RMA	  _IOR(KVMIO,  0xa9, struct kvm_allocate_rma)
-/* Available with KVM_CAP_SET_DEVICE_ADDR */
-#define KVM_SET_DEVICE_ADDRESS	  _IOW(KVMIO,  0xaa, struct kvm_device_address)
+/* Available with KVM_CAP_PPC_HTAB_FD */
+#define KVM_PPC_GET_HTAB_FD	  _IOW(KVMIO,  0xaa, struct kvm_get_htab_fd)
+/* Available with KVM_CAP_ARM_SET_DEVICE_ADDR */
+#define KVM_ARM_SET_DEVICE_ADDR	  _IOW(KVMIO,  0xab, struct kvm_arm_device_addr)
 
 /*
  * ioctls for vcpu fds
diff --git a/linux-headers/linux/kvm_para.h b/linux-headers/linux/kvm_para.h
index cea2c5c..7bdcf93 100644
--- a/linux-headers/linux/kvm_para.h
+++ b/linux-headers/linux/kvm_para.h
@@ -1,5 +1,5 @@
-#ifndef _UAPI__LINUX_KVM_PARA_H
-#define _UAPI__LINUX_KVM_PARA_H
+#ifndef __LINUX_KVM_PARA_H
+#define __LINUX_KVM_PARA_H
 
 /*
  * This header file provides a method for making a hypercall to the host
@@ -25,4 +25,4 @@
  */
 #include <asm/kvm_para.h>
 
-#endif /* _UAPI__LINUX_KVM_PARA_H */
+#endif /* __LINUX_KVM_PARA_H */
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 4758d1b..f787b72 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -8,8 +8,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
-#ifndef _UAPIVFIO_H
-#define _UAPIVFIO_H
+#ifndef VFIO_H
+#define VFIO_H
 
 #include <linux/types.h>
 #include <linux/ioctl.h>
@@ -365,4 +365,4 @@ struct vfio_iommu_type1_dma_unmap {
 
 #define VFIO_IOMMU_UNMAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 14)
 
-#endif /* _UAPIVFIO_H */
+#endif /* VFIO_H */
diff --git a/linux-headers/linux/virtio_config.h b/linux-headers/linux/virtio_config.h
index b7cda39..4f51d8f 100644
--- a/linux-headers/linux/virtio_config.h
+++ b/linux-headers/linux/virtio_config.h
@@ -1,5 +1,5 @@
-#ifndef _UAPI_LINUX_VIRTIO_CONFIG_H
-#define _UAPI_LINUX_VIRTIO_CONFIG_H
+#ifndef _LINUX_VIRTIO_CONFIG_H
+#define _LINUX_VIRTIO_CONFIG_H
 /* This header, excluding the #ifdef __KERNEL__ part, is BSD licensed so
  * anyone can use the definitions to implement compatible drivers/servers.
  *
@@ -51,4 +51,4 @@
  * suppressed them? */
 #define VIRTIO_F_NOTIFY_ON_EMPTY	24
 
-#endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */
+#endif /* _LINUX_VIRTIO_CONFIG_H */
diff --git a/linux-headers/linux/virtio_ring.h b/linux-headers/linux/virtio_ring.h
index 921694a..1b333e2 100644
--- a/linux-headers/linux/virtio_ring.h
+++ b/linux-headers/linux/virtio_ring.h
@@ -1,5 +1,5 @@
-#ifndef _UAPI_LINUX_VIRTIO_RING_H
-#define _UAPI_LINUX_VIRTIO_RING_H
+#ifndef _LINUX_VIRTIO_RING_H
+#define _LINUX_VIRTIO_RING_H
 /* An interface for efficient virtio implementation, currently for use by KVM
  * and lguest, but hopefully others soon.  Do NOT change this since it will
  * break existing servers and clients.
@@ -160,4 +160,4 @@ static __inline__ int vring_need_event(__u16 event_idx, __u16 new_idx, __u16 old
 	return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old);
 }
 
-#endif /* _UAPI_LINUX_VIRTIO_RING_H */
+#endif /* _LINUX_VIRTIO_RING_H */
diff --git a/linux-user/arm/nwfpe/double_cpdo.c b/linux-user/arm/nwfpe/double_cpdo.c
index 8e9b28f..41c28f3 100644
--- a/linux-user/arm/nwfpe/double_cpdo.c
+++ b/linux-user/arm/nwfpe/double_cpdo.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 
 float64 float64_exp(float64 Fm);
diff --git a/linux-user/arm/nwfpe/extended_cpdo.c b/linux-user/arm/nwfpe/extended_cpdo.c
index 880ce03..48eca3b 100644
--- a/linux-user/arm/nwfpe/extended_cpdo.c
+++ b/linux-user/arm/nwfpe/extended_cpdo.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 
 floatx80 floatx80_exp(floatx80 Fm);
diff --git a/linux-user/arm/nwfpe/fpa11.h b/linux-user/arm/nwfpe/fpa11.h
index 002b3cb..bb9ac65 100644
--- a/linux-user/arm/nwfpe/fpa11.h
+++ b/linux-user/arm/nwfpe/fpa11.h
@@ -43,7 +43,7 @@ extern CPUARMState *user_registers;
 
 /* includes */
 #include "fpsr.h"		/* FP control and status register definitions */
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define		typeNone		0x00
 #define		typeSingle		0x01
diff --git a/linux-user/arm/nwfpe/fpa11_cpdt.c b/linux-user/arm/nwfpe/fpa11_cpdt.c
index 3e7a938..007a3d6 100644
--- a/linux-user/arm/nwfpe/fpa11_cpdt.c
+++ b/linux-user/arm/nwfpe/fpa11_cpdt.c
@@ -20,7 +20,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 //#include "fpmodule.h"
 //#include "fpmodule.inl"
diff --git a/linux-user/arm/nwfpe/fpa11_cprt.c b/linux-user/arm/nwfpe/fpa11_cprt.c
index 8011897..7be93fa 100644
--- a/linux-user/arm/nwfpe/fpa11_cprt.c
+++ b/linux-user/arm/nwfpe/fpa11_cprt.c
@@ -20,7 +20,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 #include "fpa11.inl"
 //#include "fpmodule.h"
diff --git a/linux-user/arm/nwfpe/fpopcode.c b/linux-user/arm/nwfpe/fpopcode.c
index 82ac92f..0dc5c9c 100644
--- a/linux-user/arm/nwfpe/fpopcode.c
+++ b/linux-user/arm/nwfpe/fpopcode.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 #include "fpsr.h"
 //#include "fpmodule.h"
diff --git a/linux-user/arm/nwfpe/single_cpdo.c b/linux-user/arm/nwfpe/single_cpdo.c
index 26168e2..2bfb359 100644
--- a/linux-user/arm/nwfpe/single_cpdo.c
+++ b/linux-user/arm/nwfpe/single_cpdo.c
@@ -19,7 +19,7 @@
 */
 
 #include "fpa11.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "fpopcode.h"
 
 float32 float32_exp(float32 Fm);
diff --git a/linux-user/arm/syscall_nr.h b/linux-user/arm/syscall_nr.h
index 5356395..42d6855 100644
--- a/linux-user/arm/syscall_nr.h
+++ b/linux-user/arm/syscall_nr.h
@@ -182,8 +182,8 @@
 #define TARGET_NR_rt_sigtimedwait		(177)
 #define TARGET_NR_rt_sigqueueinfo		(178)
 #define TARGET_NR_rt_sigsuspend		(179)
-#define TARGET_NR_pread			(180)
-#define TARGET_NR_pwrite			(181)
+#define TARGET_NR_pread64                       (180)
+#define TARGET_NR_pwrite64                      (181)
 #define TARGET_NR_chown			(182)
 #define TARGET_NR_getcwd			(183)
 #define TARGET_NR_capget			(184)
diff --git a/linux-user/cris/syscall.h b/linux-user/cris/syscall.h
index 24f92ba..50e50b4 100644
--- a/linux-user/cris/syscall.h
+++ b/linux-user/cris/syscall.h
@@ -1,3 +1,6 @@
+#ifndef CRIS_SYSCALL_H
+#define CRIS_SYSCALL_H 1
+
 
 #define UNAME_MACHINE "cris"
 
@@ -34,3 +37,5 @@ struct target_pt_regs {
         unsigned long exs;
         unsigned long eda;
 };
+
+#endif
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 1d8bcb4..89db49c 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -14,7 +14,7 @@
 #include <time.h>
 
 #include "qemu.h"
-#include "disas.h"
+#include "disas/disas.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
diff --git a/linux-user/i386/syscall_nr.h b/linux-user/i386/syscall_nr.h
index 74abfca..f080305 100644
--- a/linux-user/i386/syscall_nr.h
+++ b/linux-user/i386/syscall_nr.h
@@ -182,8 +182,8 @@
 #define TARGET_NR_rt_sigtimedwait	177
 #define TARGET_NR_rt_sigqueueinfo	178
 #define TARGET_NR_rt_sigsuspend	179
-#define TARGET_NR_pread		180
-#define TARGET_NR_pwrite		181
+#define TARGET_NR_pread64               180
+#define TARGET_NR_pwrite64              181
 #define TARGET_NR_chown		182
 #define TARGET_NR_getcwd		183
 #define TARGET_NR_capget		184
diff --git a/linux-user/main.c b/linux-user/main.c
index 25e35cd..3df8aa2 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -28,11 +28,11 @@
 
 #include "qemu.h"
 #include "qemu-common.h"
-#include "cache-utils.h"
+#include "qemu/cache-utils.h"
 #include "cpu.h"
 #include "tcg.h"
-#include "qemu-timer.h"
-#include "envlist.h"
+#include "qemu/timer.h"
+#include "qemu/envlist.h"
 #include "elf.h"
 
 #define DEBUG_LOGFILE "/tmp/qemu.log"
@@ -57,7 +57,12 @@ int have_guest_base;
  * This way we will never overlap with our own libraries or binaries or stack
  * or anything else that QEMU maps.
  */
+# ifdef TARGET_MIPS
+/* MIPS only supports 31 bits of virtual address space for user space */
+unsigned long reserved_va = 0x77000000;
+# else
 unsigned long reserved_va = 0xf7000000;
+# endif
 #else
 unsigned long reserved_va;
 #endif
@@ -2933,71 +2938,115 @@ void cpu_loop(CPUAlphaState *env)
 #ifdef TARGET_S390X
 void cpu_loop(CPUS390XState *env)
 {
-    int trapnr;
+    int trapnr, n, sig;
     target_siginfo_t info;
+    target_ulong addr;
 
     while (1) {
-        trapnr = cpu_s390x_exec (env);
-
+        trapnr = cpu_s390x_exec(env);
         switch (trapnr) {
         case EXCP_INTERRUPT:
-            /* just indicate that signals should be handled asap */
+            /* Just indicate that signals should be handled asap.  */
             break;
-        case EXCP_DEBUG:
-            {
-                int sig;
 
-                sig = gdb_handlesig (env, TARGET_SIGTRAP);
-                if (sig) {
-                    info.si_signo = sig;
-                    info.si_errno = 0;
-                    info.si_code = TARGET_TRAP_BRKPT;
-                    queue_signal(env, info.si_signo, &info);
-                }
+        case EXCP_SVC:
+            n = env->int_svc_code;
+            if (!n) {
+                /* syscalls > 255 */
+                n = env->regs[1];
             }
+            env->psw.addr += env->int_svc_ilen;
+            env->regs[2] = do_syscall(env, n, env->regs[2], env->regs[3],
+                                      env->regs[4], env->regs[5],
+                                      env->regs[6], env->regs[7], 0, 0);
             break;
-        case EXCP_SVC:
-            {
-                int n = env->int_svc_code;
-                if (!n) {
-                    /* syscalls > 255 */
-                    n = env->regs[1];
-                }
-                env->psw.addr += env->int_svc_ilc;
-                env->regs[2] = do_syscall(env, n,
-                           env->regs[2],
-                           env->regs[3],
-                           env->regs[4],
-                           env->regs[5],
-                           env->regs[6],
-                           env->regs[7],
-                           0, 0);
+
+        case EXCP_DEBUG:
+            sig = gdb_handlesig(env, TARGET_SIGTRAP);
+            if (sig) {
+                n = TARGET_TRAP_BRKPT;
+                goto do_signal_pc;
             }
             break;
-        case EXCP_ADDR:
-            {
-                info.si_signo = SIGSEGV;
-                info.si_errno = 0;
+        case EXCP_PGM:
+            n = env->int_pgm_code;
+            switch (n) {
+            case PGM_OPERATION:
+            case PGM_PRIVILEGED:
+                sig = SIGILL;
+                n = TARGET_ILL_ILLOPC;
+                goto do_signal_pc;
+            case PGM_PROTECTION:
+            case PGM_ADDRESSING:
+                sig = SIGSEGV;
                 /* XXX: check env->error_code */
-                info.si_code = TARGET_SEGV_MAPERR;
-                info._sifields._sigfault._addr = env->__excp_addr;
-                queue_signal(env, info.si_signo, &info);
+                n = TARGET_SEGV_MAPERR;
+                addr = env->__excp_addr;
+                goto do_signal;
+            case PGM_EXECUTE:
+            case PGM_SPECIFICATION:
+            case PGM_SPECIAL_OP:
+            case PGM_OPERAND:
+            do_sigill_opn:
+                sig = SIGILL;
+                n = TARGET_ILL_ILLOPN;
+                goto do_signal_pc;
+
+            case PGM_FIXPT_OVERFLOW:
+                sig = SIGFPE;
+                n = TARGET_FPE_INTOVF;
+                goto do_signal_pc;
+            case PGM_FIXPT_DIVIDE:
+                sig = SIGFPE;
+                n = TARGET_FPE_INTDIV;
+                goto do_signal_pc;
+
+            case PGM_DATA:
+                n = (env->fpc >> 8) & 0xff;
+                if (n == 0xff) {
+                    /* compare-and-trap */
+                    goto do_sigill_opn;
+                } else {
+                    /* An IEEE exception, simulated or otherwise.  */
+                    if (n & 0x80) {
+                        n = TARGET_FPE_FLTINV;
+                    } else if (n & 0x40) {
+                        n = TARGET_FPE_FLTDIV;
+                    } else if (n & 0x20) {
+                        n = TARGET_FPE_FLTOVF;
+                    } else if (n & 0x10) {
+                        n = TARGET_FPE_FLTUND;
+                    } else if (n & 0x08) {
+                        n = TARGET_FPE_FLTRES;
+                    } else {
+                        /* ??? Quantum exception; BFP, DFP error.  */
+                        goto do_sigill_opn;
+                    }
+                    sig = SIGFPE;
+                    goto do_signal_pc;
+                }
+
+            default:
+                fprintf(stderr, "Unhandled program exception: %#x\n", n);
+                cpu_dump_state(env, stderr, fprintf, 0);
+                exit(1);
             }
             break;
-        case EXCP_SPEC:
-            {
-                fprintf(stderr,"specification exception insn 0x%08x%04x\n", ldl(env->psw.addr), lduw(env->psw.addr + 4));
-                info.si_signo = SIGILL;
-                info.si_errno = 0;
-                info.si_code = TARGET_ILL_ILLOPC;
-                info._sifields._sigfault._addr = env->__excp_addr;
-                queue_signal(env, info.si_signo, &info);
-            }
+
+        do_signal_pc:
+            addr = env->psw.addr;
+        do_signal:
+            info.si_signo = sig;
+            info.si_errno = 0;
+            info.si_code = n;
+            info._sifields._sigfault._addr = addr;
+            queue_signal(env, info.si_signo, &info);
             break;
+
         default:
-            printf ("Unhandled trap: 0x%x\n", trapnr);
+            fprintf(stderr, "Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(env, stderr, fprintf, 0);
-            exit (1);
+            exit(1);
         }
         process_pending_signals (env);
     }
@@ -3491,7 +3540,7 @@ int main(int argc, char **argv, char **envp)
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-#if defined(TARGET_I386) || defined(TARGET_SPARC) || defined(TARGET_PPC)
+#if defined(TARGET_SPARC) || defined(TARGET_PPC)
     cpu_reset(ENV_GET_CPU(env));
 #endif
 
diff --git a/linux-user/microblaze/syscall.h b/linux-user/microblaze/syscall.h
index db1f98a..c3e5c55 100644
--- a/linux-user/microblaze/syscall.h
+++ b/linux-user/microblaze/syscall.h
@@ -1,3 +1,7 @@
+#ifndef MICROBLAZE_SYSCALLS_H
+#define MICROBLAZE_SYSCALLS_H 1
+
+
 #define UNAME_MACHINE "microblaze"
 
 /* We use microblaze_reg_t to keep things similar to the kernel sources.  */
@@ -43,3 +47,5 @@ struct target_pt_regs {
         microblaze_reg_t fsr;
         uint32_t kernel_mode;
 };
+
+#endif
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 5e53dca..b10e957 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -11,14 +11,14 @@
 #include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
-#include "qemu-types.h"
+#include "exec/user/abitypes.h"
 
-#include "thunk.h"
+#include "exec/user/thunk.h"
 #include "syscall_defs.h"
 #include "syscall.h"
 #include "target_signal.h"
-#include "gdbstub.h"
-#include "qemu-queue.h"
+#include "exec/gdbstub.h"
+#include "qemu/queue.h"
 
 #if defined(CONFIG_USE_NPTL)
 #define THREAD __thread
@@ -217,7 +217,7 @@ unsigned long init_guest_space(unsigned long host_start,
                                unsigned long guest_start,
                                bool fixed);
 
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 /* syscall.c */
 int host_to_target_waitstatus(int status);
@@ -287,36 +287,39 @@ static inline int access_ok(int type, abi_ulong addr, abi_ulong size)
                             (type == VERIFY_READ) ? PAGE_READ : (PAGE_READ | PAGE_WRITE)) == 0;
 }
 
-/* NOTE __get_user and __put_user use host pointers and don't check access. */
-/* These are usually used to access struct data members once the
- * struct has been locked - usually with lock_user_struct().
- */
-#define __put_user(x, hptr)\
-({ __typeof(*hptr) pu_ = (x);\
-    switch(sizeof(*hptr)) {\
-    case 1: break;\
-    case 2: pu_ = tswap16(pu_); break; \
-    case 4: pu_ = tswap32(pu_); break; \
-    case 8: pu_ = tswap64(pu_); break; \
-    default: abort();\
-    }\
-    memcpy(hptr, &pu_, sizeof(pu_)); \
-    0;\
-})
-
-#define __get_user(x, hptr) \
-({ __typeof(*hptr) gu_; \
-    memcpy(&gu_, hptr, sizeof(gu_)); \
-    switch(sizeof(*hptr)) {\
-    case 1: break; \
-    case 2: gu_ = tswap16(gu_); break; \
-    case 4: gu_ = tswap32(gu_); break; \
-    case 8: gu_ = tswap64(gu_); break; \
-    default: abort();\
-    }\
-    (x) = gu_; \
-    0;\
-})
+/* NOTE __get_user and __put_user use host pointers and don't check access.
+   These are usually used to access struct data members once the struct has
+   been locked - usually with lock_user_struct.  */
+
+/* Tricky points:
+   - Use __builtin_choose_expr to avoid type promotion from ?:,
+   - Invalid sizes result in a compile time error stemming from
+     the fact that abort has no parameters.
+   - It's easier to use the endian-specific unaligned load/store
+     functions than host-endian unaligned load/store plus tswapN.  */
+
+#define __put_user_e(x, hptr, e)                                        \
+  (__builtin_choose_expr(sizeof(*(hptr)) == 1, stb_p,                   \
+   __builtin_choose_expr(sizeof(*(hptr)) == 2, stw_##e##_p,             \
+   __builtin_choose_expr(sizeof(*(hptr)) == 4, stl_##e##_p,             \
+   __builtin_choose_expr(sizeof(*(hptr)) == 8, stq_##e##_p, abort))))   \
+     ((hptr), (x)), 0)
+
+#define __get_user_e(x, hptr, e)                                        \
+  ((x) = (typeof(*hptr))(                                               \
+   __builtin_choose_expr(sizeof(*(hptr)) == 1, ldub_p,                  \
+   __builtin_choose_expr(sizeof(*(hptr)) == 2, lduw_##e##_p,            \
+   __builtin_choose_expr(sizeof(*(hptr)) == 4, ldl_##e##_p,             \
+   __builtin_choose_expr(sizeof(*(hptr)) == 8, ldq_##e##_p, abort))))   \
+     (hptr)), 0)
+
+#ifdef TARGET_WORDS_BIGENDIAN
+# define __put_user(x, hptr)  __put_user_e(x, hptr, be)
+# define __get_user(x, hptr)  __get_user_e(x, hptr, be)
+#else
+# define __put_user(x, hptr)  __put_user_e(x, hptr, le)
+# define __get_user(x, hptr)  __get_user_e(x, hptr, le)
+#endif
 
 /* put_user()/get_user() take a guest address and check access */
 /* These are usually used to access an atomic data type, such as an int,
diff --git a/linux-user/s390x/syscall.h b/linux-user/s390x/syscall.h
index c2ea151..e4603b7 100644
--- a/linux-user/s390x/syscall.h
+++ b/linux-user/s390x/syscall.h
@@ -16,7 +16,7 @@ struct target_pt_regs {
     target_psw_t psw;
     abi_ulong gprs[TARGET_NUM_GPRS];
     abi_ulong orig_gpr2;
-    unsigned short ilc;
+    unsigned short ilen;
     unsigned short trap;
 };
 
diff --git a/linux-user/signal.c b/linux-user/signal.c
index 95e2ffa..67c2311 100644
--- a/linux-user/signal.c
+++ b/linux-user/signal.c
@@ -607,28 +607,22 @@ int do_sigaction(int sig, const struct target_sigaction *act,
             sig, act, oact);
 #endif
     if (oact) {
-        oact->_sa_handler = tswapal(k->_sa_handler);
-#if defined(TARGET_MIPS) || defined (TARGET_ALPHA)
-        oact->sa_flags = bswap32(k->sa_flags);
-#else
-        oact->sa_flags = tswapal(k->sa_flags);
-#endif
+        __put_user(k->_sa_handler, &oact->_sa_handler);
+        __put_user(k->sa_flags, &oact->sa_flags);
 #if !defined(TARGET_MIPS)
-        oact->sa_restorer = tswapal(k->sa_restorer);
+        __put_user(k->sa_restorer, &oact->sa_restorer);
 #endif
+        /* Not swapped.  */
         oact->sa_mask = k->sa_mask;
     }
     if (act) {
         /* FIXME: This is not threadsafe.  */
-        k->_sa_handler = tswapal(act->_sa_handler);
-#if defined(TARGET_MIPS) || defined (TARGET_ALPHA)
-        k->sa_flags = bswap32(act->sa_flags);
-#else
-        k->sa_flags = tswapal(act->sa_flags);
-#endif
+        __get_user(k->_sa_handler, &act->_sa_handler);
+        __get_user(k->sa_flags, &act->sa_flags);
 #if !defined(TARGET_MIPS)
-        k->sa_restorer = tswapal(act->sa_restorer);
+        __get_user(k->sa_restorer, &act->sa_restorer);
 #endif
+        /* To be swapped in target_to_host_sigset.  */
         k->sa_mask = act->sa_mask;
 
         /* we update the host linux signal state */
@@ -4584,7 +4578,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
     signal = current_exec_domain_sig(sig);
 
-    err |= __put_user(h2g(ka->_sa_handler), &sc->handler);
+    err |= __put_user(ka->_sa_handler, &sc->handler);
     err |= __put_user(set->sig[0], &sc->oldmask);
 #if defined(TARGET_PPC64)
     err |= __put_user(set->sig[0] >> 32, &sc->_unused[3]);
@@ -4606,7 +4600,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
     /* Create a stack frame for the caller of the handler.  */
     newsp = frame_addr - SIGNAL_FRAMESIZE;
-    err |= __put_user(env->gpr[1], (target_ulong *)(uintptr_t) newsp);
+    err |= put_user(env->gpr[1], newsp, target_ulong);
 
     if (err)
         goto sigsegv;
@@ -4614,7 +4608,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     /* Set up registers for signal handler.  */
     env->gpr[1] = newsp;
     env->gpr[3] = signal;
-    env->gpr[4] = (target_ulong) h2g(sc);
+    env->gpr[4] = frame_addr + offsetof(struct target_sigframe, sctx);
     env->nip = (target_ulong) ka->_sa_handler;
     /* Signal handlers are entered in big-endian mode.  */
     env->msr &= ~MSR_LE;
diff --git a/linux-user/sparc/syscall_nr.h b/linux-user/sparc/syscall_nr.h
index f201f9f..061711c 100644
--- a/linux-user/sparc/syscall_nr.h
+++ b/linux-user/sparc/syscall_nr.h
@@ -62,8 +62,8 @@
 #define TARGET_NR_getpagesize         64 /* Common                                      */
 #define TARGET_NR_msync               65 /* Common in newer 1.3.x revs...               */
 #define TARGET_NR_vfork               66 /* Common                                      */
-#define TARGET_NR_pread               67 /* Linux Specific                              */
-#define TARGET_NR_pwrite              68 /* Linux Specific                              */
+#define TARGET_NR_pread64             67 /* Linux Specific */
+#define TARGET_NR_pwrite64            68 /* Linux Specific */
 #define TARGET_NR_geteuid32           69 /* Linux sparc32, sbrk under SunOS             */
 #define TARGET_NR_getegid32           70 /* Linux sparc32, sstk under SunOS             */
 #define TARGET_NR_mmap                71 /* Common                                      */
diff --git a/linux-user/strace.c b/linux-user/strace.c
index 6ec90e8..4e91a6e 100644
--- a/linux-user/strace.c
+++ b/linux-user/strace.c
@@ -682,7 +682,7 @@ print_timeval(abi_ulong tv_addr, int last)
         if (!tv)
             return;
         gemu_log("{" TARGET_ABI_FMT_ld "," TARGET_ABI_FMT_ld "}%s",
-            tv->tv_sec, tv->tv_usec, get_comma(last));
+            tswapal(tv->tv_sec), tswapal(tv->tv_usec), get_comma(last));
         unlock_user(tv, tv_addr, 0);
     } else
         gemu_log("NULL%s", get_comma(last));
diff --git a/linux-user/strace.list b/linux-user/strace.list
index af3c6a0..08f115d 100644
--- a/linux-user/strace.list
+++ b/linux-user/strace.list
@@ -972,9 +972,6 @@
 #ifdef TARGET_NR_prctl
 { TARGET_NR_prctl, "prctl" , NULL, NULL, NULL },
 #endif
-#ifdef TARGET_NR_pread
-{ TARGET_NR_pread, "pread" , NULL, NULL, NULL },
-#endif
 #ifdef TARGET_NR_pread64
 { TARGET_NR_pread64, "pread64" , NULL, NULL, NULL },
 #endif
@@ -993,9 +990,6 @@
 #ifdef TARGET_NR_putpmsg
 { TARGET_NR_putpmsg, "putpmsg" , NULL, NULL, NULL },
 #endif
-#ifdef TARGET_NR_pwrite
-{ TARGET_NR_pwrite, "pwrite" , NULL, NULL, NULL },
-#endif
 #ifdef TARGET_NR_pwrite64
 { TARGET_NR_pwrite64, "pwrite64" , NULL, NULL, NULL },
 #endif
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index e4291ed..9e31ea7 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -28,6 +28,7 @@
 #include <fcntl.h>
 #include <time.h>
 #include <limits.h>
+#include <grp.h>
 #include <sys/types.h>
 #include <sys/ipc.h>
 #include <sys/msg.h>
@@ -35,6 +36,9 @@
 #include <sys/time.h>
 #include <sys/stat.h>
 #include <sys/mount.h>
+#include <sys/file.h>
+#include <sys/fsuid.h>
+#include <sys/personality.h>
 #include <sys/prctl.h>
 #include <sys/resource.h>
 #include <sys/mman.h>
@@ -72,7 +76,7 @@ int __clone2(int (*fn)(void *), void *child_stack_base,
 #include <sys/epoll.h>
 #endif
 #ifdef CONFIG_ATTR
-#include "qemu-xattr.h"
+#include "qemu/xattr.h"
 #endif
 
 #define termios host_termios
@@ -97,6 +101,7 @@ int __clone2(int (*fn)(void *), void *child_stack_base,
 #include <linux/fb.h>
 #include <linux/vt.h>
 #include <linux/dm-ioctl.h>
+#include <linux/reboot.h>
 #include "linux_loop.h"
 #include "cpu-uname.h"
 
@@ -580,12 +585,6 @@ _syscall4(int, sys_prlimit64, pid_t, pid, int, resource,
           struct host_rlimit64 *, old_limit)
 #endif
 
-extern int personality(int);
-extern int flock(int, int);
-extern int setfsuid(int);
-extern int setfsgid(int);
-extern int setgroups(int, gid_t *);
-
 /* ARM EABI and MIPS expect 64bit types aligned even on pairs or registers */
 #ifdef TARGET_ARM
 static inline int regpairs_aligned(void *cpu_env) {
@@ -1491,6 +1490,28 @@ static abi_long do_setsockopt(int sockfd, int level, int optname,
         break;
     case TARGET_SOL_SOCKET:
         switch (optname) {
+        case TARGET_SO_RCVTIMEO:
+        {
+                struct timeval tv;
+
+                optname = SO_RCVTIMEO;
+
+set_timeout:
+                if (optlen != sizeof(struct target_timeval)) {
+                    return -TARGET_EINVAL;
+                }
+
+                if (copy_from_user_timeval(&tv, optval_addr)) {
+                    return -TARGET_EFAULT;
+                }
+
+                ret = get_errno(setsockopt(sockfd, SOL_SOCKET, optname,
+                                &tv, sizeof(tv)));
+                return ret;
+        }
+        case TARGET_SO_SNDTIMEO:
+                optname = SO_SNDTIMEO;
+                goto set_timeout;
             /* Options with 'int' argument.  */
         case TARGET_SO_DEBUG:
 		optname = SO_DEBUG;
@@ -1542,12 +1563,6 @@ static abi_long do_setsockopt(int sockfd, int level, int optname,
         case TARGET_SO_RCVLOWAT:
 		optname = SO_RCVLOWAT;
 		break;
-        case TARGET_SO_RCVTIMEO:
-		optname = SO_RCVTIMEO;
-		break;
-        case TARGET_SO_SNDTIMEO:
-		optname = SO_SNDTIMEO;
-		break;
             break;
         default:
             goto unimplemented;
@@ -2899,7 +2914,7 @@ static inline abi_long do_msgrcv(int msqid, abi_long msgp,
         return -TARGET_EFAULT;
 
     host_mb = g_malloc(msgsz+sizeof(long));
-    ret = get_errno(msgrcv(msqid, host_mb, msgsz, tswapal(msgtyp), msgflg));
+    ret = get_errno(msgrcv(msqid, host_mb, msgsz, msgtyp, msgflg));
 
     if (ret > 0) {
         abi_ulong target_mtext_addr = msgp + sizeof(abi_ulong);
@@ -3191,7 +3206,7 @@ static abi_long do_ipc(unsigned int call, int first,
                     break;
                 }
 
-                ret = do_msgrcv(first, tmp->msgp, second, tmp->msgtyp, third);
+                ret = do_msgrcv(first, tswapal(tmp->msgp), second, tswapal(tmp->msgtyp), third);
 
                 unlock_user_struct(tmp, ptr, 0);
                 break;
@@ -4514,6 +4529,16 @@ static int target_to_host_fcntl_cmd(int cmd)
     return -TARGET_EINVAL;
 }
 
+#define TRANSTBL_CONVERT(a) { -1, TARGET_##a, -1, a }
+static const bitmask_transtbl flock_tbl[] = {
+    TRANSTBL_CONVERT(F_RDLCK),
+    TRANSTBL_CONVERT(F_WRLCK),
+    TRANSTBL_CONVERT(F_UNLCK),
+    TRANSTBL_CONVERT(F_EXLCK),
+    TRANSTBL_CONVERT(F_SHLCK),
+    { 0, 0, 0, 0 }
+};
+
 static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
 {
     struct flock fl;
@@ -4530,7 +4555,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_GETLK:
         if (!lock_user_struct(VERIFY_READ, target_fl, arg, 1))
             return -TARGET_EFAULT;
-        fl.l_type = tswap16(target_fl->l_type);
+        fl.l_type =
+                  target_to_host_bitmask(tswap16(target_fl->l_type), flock_tbl);
         fl.l_whence = tswap16(target_fl->l_whence);
         fl.l_start = tswapal(target_fl->l_start);
         fl.l_len = tswapal(target_fl->l_len);
@@ -4540,7 +4566,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
         if (ret == 0) {
             if (!lock_user_struct(VERIFY_WRITE, target_fl, arg, 0))
                 return -TARGET_EFAULT;
-            target_fl->l_type = tswap16(fl.l_type);
+            target_fl->l_type =
+                          host_to_target_bitmask(tswap16(fl.l_type), flock_tbl);
             target_fl->l_whence = tswap16(fl.l_whence);
             target_fl->l_start = tswapal(fl.l_start);
             target_fl->l_len = tswapal(fl.l_len);
@@ -4553,7 +4580,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_SETLKW:
         if (!lock_user_struct(VERIFY_READ, target_fl, arg, 1))
             return -TARGET_EFAULT;
-        fl.l_type = tswap16(target_fl->l_type);
+        fl.l_type =
+                  target_to_host_bitmask(tswap16(target_fl->l_type), flock_tbl);
         fl.l_whence = tswap16(target_fl->l_whence);
         fl.l_start = tswapal(target_fl->l_start);
         fl.l_len = tswapal(target_fl->l_len);
@@ -4565,7 +4593,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_GETLK64:
         if (!lock_user_struct(VERIFY_READ, target_fl64, arg, 1))
             return -TARGET_EFAULT;
-        fl64.l_type = tswap16(target_fl64->l_type) >> 1;
+        fl64.l_type =
+           target_to_host_bitmask(tswap16(target_fl64->l_type), flock_tbl) >> 1;
         fl64.l_whence = tswap16(target_fl64->l_whence);
         fl64.l_start = tswap64(target_fl64->l_start);
         fl64.l_len = tswap64(target_fl64->l_len);
@@ -4575,7 +4604,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
         if (ret == 0) {
             if (!lock_user_struct(VERIFY_WRITE, target_fl64, arg, 0))
                 return -TARGET_EFAULT;
-            target_fl64->l_type = tswap16(fl64.l_type) >> 1;
+            target_fl64->l_type =
+                   host_to_target_bitmask(tswap16(fl64.l_type), flock_tbl) >> 1;
             target_fl64->l_whence = tswap16(fl64.l_whence);
             target_fl64->l_start = tswap64(fl64.l_start);
             target_fl64->l_len = tswap64(fl64.l_len);
@@ -4587,7 +4617,8 @@ static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
     case TARGET_F_SETLKW64:
         if (!lock_user_struct(VERIFY_READ, target_fl64, arg, 1))
             return -TARGET_EFAULT;
-        fl64.l_type = tswap16(target_fl64->l_type) >> 1;
+        fl64.l_type =
+           target_to_host_bitmask(tswap16(target_fl64->l_type), flock_tbl) >> 1;
         fl64.l_whence = tswap16(target_fl64->l_whence);
         fl64.l_start = tswap64(target_fl64->l_start);
         fl64.l_len = tswap64(target_fl64->l_len);
@@ -5188,7 +5219,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
                         NULL, NULL, 0);
           }
           thread_env = NULL;
-          object_delete(OBJECT(ENV_GET_CPU(cpu_env)));
+          object_unref(OBJECT(ENV_GET_CPU(cpu_env)));
           g_free(ts);
           pthread_exit(NULL);
       }
@@ -6213,8 +6244,11 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             ret = get_errno(settimeofday(&tv, NULL));
         }
         break;
-#if defined(TARGET_NR_select) && !defined(TARGET_S390X) && !defined(TARGET_S390)
+#if defined(TARGET_NR_select)
     case TARGET_NR_select:
+#if defined(TARGET_S390X) || defined(TARGET_ALPHA)
+        ret = do_select(arg1, arg2, arg3, arg4, arg5);
+#else
         {
             struct target_sel_arg_struct *sel;
             abi_ulong inp, outp, exp, tvp;
@@ -6230,6 +6264,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user_struct(sel, arg1, 0);
             ret = do_select(nsel, inp, outp, exp, tvp);
         }
+#endif
         break;
 #endif
 #ifdef TARGET_NR_pselect6
@@ -6417,10 +6452,17 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
 #endif
     case TARGET_NR_reboot:
-        if (!(p = lock_user_string(arg4)))
-            goto efault;
-        ret = reboot(arg1, arg2, arg3, p);
-        unlock_user(p, arg4, 0);
+        if (arg3 == LINUX_REBOOT_CMD_RESTART2) {
+           /* arg4 must be ignored in all other cases */
+           p = lock_user_string(arg4);
+           if (!p) {
+              goto efault;
+           }
+           ret = get_errno(reboot(arg1, arg2, arg3, p));
+           unlock_user(p, arg4, 0);
+        } else {
+           ret = get_errno(reboot(arg1, arg2, arg3, NULL));
+        }
         break;
 #ifdef TARGET_NR_readdir
     case TARGET_NR_readdir:
@@ -7153,12 +7195,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif /* TARGET_NR_getdents64 */
-#if defined(TARGET_NR__newselect) || defined(TARGET_S390X)
-#ifdef TARGET_S390X
-    case TARGET_NR_select:
-#else
+#if defined(TARGET_NR__newselect)
     case TARGET_NR__newselect:
-#endif
         ret = do_select(arg1, arg2, arg3, arg4, arg5);
         break;
 #endif
@@ -7449,24 +7487,6 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         goto unimplemented;
 #endif
 #endif
-#ifdef TARGET_NR_pread
-    case TARGET_NR_pread:
-        if (regpairs_aligned(cpu_env))
-            arg4 = arg5;
-        if (!(p = lock_user(VERIFY_WRITE, arg2, arg3, 0)))
-            goto efault;
-        ret = get_errno(pread(arg1, p, arg3, arg4));
-        unlock_user(p, arg2, ret);
-        break;
-    case TARGET_NR_pwrite:
-        if (regpairs_aligned(cpu_env))
-            arg4 = arg5;
-        if (!(p = lock_user(VERIFY_READ, arg2, arg3, 1)))
-            goto efault;
-        ret = get_errno(pwrite(arg1, p, arg3, arg4));
-        unlock_user(p, arg2, 0);
-        break;
-#endif
 #ifdef TARGET_NR_pread64
     case TARGET_NR_pread64:
         if (regpairs_aligned(cpu_env)) {
diff --git a/linux-user/syscall_defs.h b/linux-user/syscall_defs.h
index a98cbf7..92c01a9 100644
--- a/linux-user/syscall_defs.h
+++ b/linux-user/syscall_defs.h
@@ -4,6 +4,10 @@
    most of them stay the same, so we handle it by putting ifdefs if
    necessary */
 
+#ifndef SYSCALL_DEFS_H
+#define SYSCALL_DEFS_H 1
+
+
 #include "syscall_nr.h"
 
 #define SOCKOP_socket           1
@@ -540,7 +544,7 @@ int do_sigaction(int sig, const struct target_sigaction *act,
 struct target_old_sigaction {
     abi_ulong _sa_handler;
     abi_ulong sa_mask;
-    abi_ulong sa_flags;
+    int32_t sa_flags;
 };
 
 struct target_rt_sigaction {
@@ -2013,6 +2017,12 @@ struct target_statfs64 {
 #define TARGET_F_SETLKW        9
 #define TARGET_F_SETOWN        5       /*  for sockets. */
 #define TARGET_F_GETOWN        6       /*  for sockets. */
+
+#define TARGET_F_RDLCK         1
+#define TARGET_F_WRLCK         2
+#define TARGET_F_UNLCK         8
+#define TARGET_F_EXLCK         16
+#define TARGET_F_SHLCK         32
 #elif defined(TARGET_MIPS)
 #define TARGET_F_GETLK         14
 #define TARGET_F_SETLK         6
@@ -2027,6 +2037,18 @@ struct target_statfs64 {
 #define TARGET_F_GETOWN        9       /*  for sockets. */
 #endif
 
+#ifndef TARGET_F_RDLCK
+#define TARGET_F_RDLCK         0
+#define TARGET_F_WRLCK         1
+#define TARGET_F_UNLCK         2
+#endif
+
+#ifndef TARGET_F_EXLCK
+#define TARGET_F_EXLCK         4
+#define TARGET_F_SHLCK         8
+#endif
+
+
 #define TARGET_F_SETSIG        10      /*  for sockets. */
 #define TARGET_F_GETSIG        11      /*  for sockets. */
 
@@ -2425,3 +2447,5 @@ struct target_ucred {
     uint32_t uid;
     uint32_t gid;
 };
+
+#endif
diff --git a/linux-user/unicore32/syscall_nr.h b/linux-user/unicore32/syscall_nr.h
index 9c72d84..486b8c4 100644
--- a/linux-user/unicore32/syscall_nr.h
+++ b/linux-user/unicore32/syscall_nr.h
@@ -187,8 +187,8 @@
 #define TARGET_NR_rt_sigtimedwait               177
 #define TARGET_NR_rt_sigqueueinfo               178
 #define TARGET_NR_rt_sigsuspend                 179
-#define TARGET_NR_pread                         180
-#define TARGET_NR_pwrite                        181
+#define TARGET_NR_pread64                       180
+#define TARGET_NR_pwrite64                      181
 #define TARGET_NR_chown                         182
 #define TARGET_NR_getcwd                        183
 #define TARGET_NR_capget                        184
diff --git a/main-loop.c b/main-loop.c
index c87624e..6f52ac3 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -23,14 +23,14 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "slirp/slirp.h"
-#include "main-loop.h"
-#include "qemu-aio.h"
+#include "qemu/main-loop.h"
+#include "block/aio.h"
 
 #ifndef _WIN32
 
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 
 /* If we have signalfd, we mask out the signals we want to handle and then
  * use signalfd to listen for them.  We rely on whatever the current signal
@@ -330,7 +330,7 @@ void qemu_fd_register(int fd)
 static int os_host_main_loop_wait(uint32_t timeout)
 {
     GMainContext *context = g_main_context_default();
-    int ret, i;
+    int select_ret, g_poll_ret, ret, i;
     PollingEntry *pe;
     WaitObjects *w = &wait_objects;
     gint poll_timeout;
@@ -345,13 +345,6 @@ static int os_host_main_loop_wait(uint32_t timeout)
         return ret;
     }
 
-    if (nfds >= 0) {
-        ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv0);
-        if (ret != 0) {
-            timeout = 0;
-        }
-    }
-
     g_main_context_prepare(context, &max_priority);
     n_poll_fds = g_main_context_query(context, max_priority, &poll_timeout,
                                       poll_fds, ARRAY_SIZE(poll_fds));
@@ -367,9 +360,9 @@ static int os_host_main_loop_wait(uint32_t timeout)
     }
 
     qemu_mutex_unlock_iothread();
-    ret = g_poll(poll_fds, n_poll_fds + w->num, poll_timeout);
+    g_poll_ret = g_poll(poll_fds, n_poll_fds + w->num, poll_timeout);
     qemu_mutex_lock_iothread();
-    if (ret > 0) {
+    if (g_poll_ret > 0) {
         for (i = 0; i < w->num; i++) {
             w->revents[i] = poll_fds[n_poll_fds + i].revents;
         }
@@ -384,12 +377,18 @@ static int os_host_main_loop_wait(uint32_t timeout)
         g_main_context_dispatch(context);
     }
 
-    /* If an edge-triggered socket event occurred, select will return a
-     * positive result on the next iteration.  We do not need to do anything
-     * here.
+    /* Call select after g_poll to avoid a useless iteration and therefore
+     * improve socket latency.
      */
 
-    return ret;
+    if (nfds >= 0) {
+        select_ret = select(nfds + 1, &rfds, &wfds, &xfds, &tv0);
+        if (select_ret != 0) {
+            timeout = 0;
+        }
+    }
+
+    return select_ret || g_poll_ret;
 }
 #endif
 
@@ -432,11 +431,6 @@ QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque)
     return aio_bh_new(qemu_aio_context, cb, opaque);
 }
 
-void qemu_aio_flush(void)
-{
-    aio_flush(qemu_aio_context);
-}
-
 bool qemu_aio_wait(void)
 {
     return aio_poll(qemu_aio_context, true);
diff --git a/memory.c b/memory.c
index 7419853..cd7d5e0 100644
--- a/memory.c
+++ b/memory.c
@@ -13,14 +13,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "memory.h"
-#include "exec-memory.h"
-#include "ioport.h"
-#include "bitops.h"
-#include "kvm.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "exec/ioport.h"
+#include "qemu/bitops.h"
+#include "sysemu/kvm.h"
 #include <assert.h>
 
-#include "memory-internal.h"
+#include "exec/memory-internal.h"
 
 static unsigned memory_region_transaction_depth;
 static bool memory_region_update_pending;
@@ -855,7 +855,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr,
     }
 
     if (!mr->ops->read) {
-        return mr->ops->old_mmio.read[bitops_ffsl(size)](mr->opaque, addr);
+        return mr->ops->old_mmio.read[bitops_ctzl(size)](mr->opaque, addr);
     }
 
     /* FIXME: support unaligned access */
@@ -908,7 +908,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr,
     adjust_endianness(mr, &data, size);
 
     if (!mr->ops->write) {
-        mr->ops->old_mmio.write[bitops_ffsl(size)](mr->opaque, addr, data);
+        mr->ops->old_mmio.write[bitops_ctzl(size)](mr->opaque, addr, data);
         return;
     }
 
@@ -1081,6 +1081,22 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
     return cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, -1);
 }
 
+bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
+                                        hwaddr size, unsigned client)
+{
+    bool ret;
+    assert(mr->terminates);
+    ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size,
+                                        1 << client);
+    if (ret) {
+        cpu_physical_memory_reset_dirty(mr->ram_addr + addr,
+                                        mr->ram_addr + addr + size,
+                                        1 << client);
+    }
+    return ret;
+}
+
+
 void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 {
     AddressSpace *as;
diff --git a/memory_mapping-stub.c b/memory_mapping-stub.c
index 76be34d..24d5d67 100644
--- a/memory_mapping-stub.c
+++ b/memory_mapping-stub.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "memory_mapping.h"
+#include "exec/cpu-all.h"
+#include "sysemu/memory_mapping.h"
 
 int qemu_get_guest_memory_mapping(MemoryMappingList *list)
 {
diff --git a/memory_mapping.c b/memory_mapping.c
index a82e190..ff45b3a 100644
--- a/memory_mapping.c
+++ b/memory_mapping.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "memory_mapping.h"
+#include "exec/cpu-all.h"
+#include "sysemu/memory_mapping.h"
 
 static void memory_mapping_list_add_mapping_sorted(MemoryMappingList *list,
                                                    MemoryMapping *mapping)
@@ -200,7 +200,7 @@ int qemu_get_guest_memory_mapping(MemoryMappingList *list)
      * If the guest doesn't use paging, the virtual address is equal to physical
      * address.
      */
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         offset = block->offset;
         length = block->length;
         create_new_memory_mapping(list, offset, offset, length);
@@ -213,7 +213,7 @@ void qemu_get_guest_simple_memory_mapping(MemoryMappingList *list)
 {
     RAMBlock *block;
 
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
+    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
         create_new_memory_mapping(list, block->offset, 0, block->length);
     }
 }
diff --git a/migration-exec.c b/migration-exec.c
index 2b6fcb4..a051a6e 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -16,11 +16,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 #include <sys/types.h>
 #include <sys/wait.h>
 
@@ -70,7 +69,6 @@ void exec_start_outgoing_migration(MigrationState *s, const char *command, Error
 
     s->fd = fileno(f);
     assert(s->fd != -1);
-    socket_set_nonblock(s->fd);
 
     s->opaque = qemu_popen(f, "w");
 
diff --git a/migration-fd.c b/migration-fd.c
index 5fe28e0..a99e0e3 100644
--- a/migration-fd.c
+++ b/migration-fd.c
@@ -14,13 +14,11 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "monitor.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 
 //#define DEBUG_MIGRATION_FD
 
@@ -77,7 +75,6 @@ void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **
         return;
     }
 
-    fcntl(s->fd, F_SETFL, O_NONBLOCK);
     s->get_error = fd_errno;
     s->write = fd_write;
     s->close = fd_close;
diff --git a/migration-tcp.c b/migration-tcp.c
index 5e855fe..e78a296 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -14,11 +14,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 
 //#define DEBUG_MIGRATION_TCP
 
@@ -61,6 +60,7 @@ static void tcp_wait_for_connect(int fd, void *opaque)
     } else {
         DPRINTF("migrate connect success\n");
         s->fd = fd;
+        socket_set_block(s->fd);
         migrate_fd_connect(s);
     }
 }
diff --git a/migration-unix.c b/migration-unix.c
index dba72b4..218835a 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -14,11 +14,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "migration.h"
-#include "qemu-char.h"
-#include "buffered_file.h"
-#include "block.h"
+#include "qemu/sockets.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "block/block.h"
 
 //#define DEBUG_MIGRATION_UNIX
 
@@ -61,6 +60,7 @@ static void unix_wait_for_connect(int fd, void *opaque)
     } else {
         DPRINTF("migrate connect success\n");
         s->fd = fd;
+        socket_set_block(s->fd);
         migrate_fd_connect(s);
     }
 }
diff --git a/migration.c b/migration.c
index 73ce170..b1ebb01 100644
--- a/migration.c
+++ b/migration.c
@@ -14,13 +14,14 @@
  */
 
 #include "qemu-common.h"
-#include "migration.h"
-#include "monitor.h"
-#include "buffered_file.h"
-#include "sysemu.h"
-#include "block.h"
-#include "qemu_socket.h"
-#include "block-migration.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
+#include "migration/qemu-file.h"
+#include "sysemu/sysemu.h"
+#include "block/block.h"
+#include "qemu/sockets.h"
+#include "migration/block.h"
+#include "qemu/thread.h"
 #include "qmp-commands.h"
 
 //#define DEBUG_MIGRATION
@@ -43,6 +44,11 @@ enum {
 
 #define MAX_THROTTLE  (32 << 20)      /* Migration speed throttling */
 
+/* Amount of time to allocate to each "chunk" of bandwidth-throttled
+ * data. */
+#define BUFFER_DELAY     100
+#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
+
 /* Migration XBZRLE default cache size */
 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
 
@@ -79,7 +85,7 @@ void qemu_start_incoming_migration(const char *uri, Error **errp)
         fd_start_incoming_migration(p, errp);
 #endif
     else {
-        error_setg(errp, "unknown migration protocol: %s\n", uri);
+        error_setg(errp, "unknown migration protocol: %s", uri);
     }
 }
 
@@ -89,7 +95,6 @@ static void process_incoming_migration_co(void *opaque)
     int ret;
 
     ret = qemu_loadvm_state(f);
-    qemu_set_fd_handler(qemu_get_fd(f), NULL, NULL, NULL);
     qemu_fclose(f);
     if (ret < 0) {
         fprintf(stderr, "load of migration failed\n");
@@ -109,12 +114,6 @@ static void process_incoming_migration_co(void *opaque)
     }
 }
 
-static void enter_migration_coroutine(void *opaque)
-{
-    Coroutine *co = opaque;
-    qemu_coroutine_enter(co, NULL);
-}
-
 void process_incoming_migration(QEMUFile *f)
 {
     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
@@ -122,7 +121,6 @@ void process_incoming_migration(QEMUFile *f)
 
     assert(fd != -1);
     socket_set_nonblock(fd);
-    qemu_set_fd_handler(fd, enter_migration_coroutine, NULL, co);
     qemu_coroutine_enter(co, f);
 }
 
@@ -272,7 +270,7 @@ static int migrate_fd_cleanup(MigrationState *s)
         s->file = NULL;
     }
 
-    migrate_fd_close(s);
+    assert(s->fd == -1);
     return ret;
 }
 
@@ -296,20 +294,8 @@ static void migrate_fd_completed(MigrationState *s)
     notifier_list_notify(&migration_state_notifiers, s);
 }
 
-static void migrate_fd_put_notify(void *opaque)
-{
-    MigrationState *s = opaque;
-    int ret;
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-    ret = qemu_file_put_notify(s->file);
-    if (ret) {
-        migrate_fd_error(s);
-    }
-}
-
-ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
-                              size_t size)
+static ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
+                                     size_t size)
 {
     ssize_t ret;
 
@@ -324,51 +310,9 @@ ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
     if (ret == -1)
         ret = -(s->get_error(s));
 
-    if (ret == -EAGAIN) {
-        qemu_set_fd_handler2(s->fd, NULL, NULL, migrate_fd_put_notify, s);
-    }
-
     return ret;
 }
 
-void migrate_fd_put_ready(MigrationState *s)
-{
-    int ret;
-
-    if (s->state != MIG_STATE_ACTIVE) {
-        DPRINTF("put_ready returning because of non-active state\n");
-        return;
-    }
-
-    DPRINTF("iterate\n");
-    ret = qemu_savevm_state_iterate(s->file);
-    if (ret < 0) {
-        migrate_fd_error(s);
-    } else if (ret == 1) {
-        int old_vm_running = runstate_is_running();
-        int64_t start_time, end_time;
-
-        DPRINTF("done iterating\n");
-        start_time = qemu_get_clock_ms(rt_clock);
-        qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-        vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
-
-        if (qemu_savevm_state_complete(s->file) < 0) {
-            migrate_fd_error(s);
-        } else {
-            migrate_fd_completed(s);
-        }
-        end_time = qemu_get_clock_ms(rt_clock);
-        s->total_time = end_time - s->total_time;
-        s->downtime = end_time - start_time;
-        if (s->state != MIG_STATE_COMPLETED) {
-            if (old_vm_running) {
-                vm_start();
-            }
-        }
-    }
-}
-
 static void migrate_fd_cancel(MigrationState *s)
 {
     if (s->state != MIG_STATE_ACTIVE)
@@ -378,39 +322,15 @@ static void migrate_fd_cancel(MigrationState *s)
 
     s->state = MIG_STATE_CANCELLED;
     notifier_list_notify(&migration_state_notifiers, s);
-    qemu_savevm_state_cancel(s->file);
+    qemu_savevm_state_cancel();
 
     migrate_fd_cleanup(s);
 }
 
-int migrate_fd_wait_for_unfreeze(MigrationState *s)
-{
-    int ret;
-
-    DPRINTF("wait for unfreeze\n");
-    if (s->state != MIG_STATE_ACTIVE)
-        return -EINVAL;
-
-    do {
-        fd_set wfds;
-
-        FD_ZERO(&wfds);
-        FD_SET(s->fd, &wfds);
-
-        ret = select(s->fd + 1, NULL, &wfds, NULL, NULL);
-    } while (ret == -1 && (s->get_error(s)) == EINTR);
-
-    if (ret == -1) {
-        return -s->get_error(s);
-    }
-    return 0;
-}
-
 int migrate_fd_close(MigrationState *s)
 {
     int rc = 0;
     if (s->fd != -1) {
-        qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
         rc = s->close(s);
         s->fd = -1;
     }
@@ -443,23 +363,6 @@ bool migration_has_failed(MigrationState *s)
             s->state == MIG_STATE_ERROR);
 }
 
-void migrate_fd_connect(MigrationState *s)
-{
-    int ret;
-
-    s->state = MIG_STATE_ACTIVE;
-    s->file = qemu_fopen_ops_buffered(s);
-
-    DPRINTF("beginning savevm\n");
-    ret = qemu_savevm_state_begin(s->file, &s->params);
-    if (ret < 0) {
-        DPRINTF("failed, %d\n", ret);
-        migrate_fd_error(s);
-        return;
-    }
-    migrate_fd_put_ready(s);
-}
-
 static MigrationState *migrate_init(const MigrationParams *params)
 {
     MigrationState *s = migrate_get_current();
@@ -544,8 +447,6 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
         error_propagate(errp, local_err);
         return;
     }
-
-    notifier_list_notify(&migration_state_notifiers, s);
 }
 
 void qmp_migrate_cancel(Error **errp)
@@ -609,3 +510,277 @@ int64_t migrate_xbzrle_cache_size(void)
 
     return s->xbzrle_cache_size;
 }
+
+/* migration thread support */
+
+
+static ssize_t buffered_flush(MigrationState *s)
+{
+    size_t offset = 0;
+    ssize_t ret = 0;
+
+    DPRINTF("flushing %zu byte(s) of data\n", s->buffer_size);
+
+    while (s->bytes_xfer < s->xfer_limit && offset < s->buffer_size) {
+        size_t to_send = MIN(s->buffer_size - offset, s->xfer_limit - s->bytes_xfer);
+        ret = migrate_fd_put_buffer(s, s->buffer + offset, to_send);
+        if (ret <= 0) {
+            DPRINTF("error flushing data, %zd\n", ret);
+            break;
+        } else {
+            DPRINTF("flushed %zd byte(s)\n", ret);
+            offset += ret;
+            s->bytes_xfer += ret;
+        }
+    }
+
+    DPRINTF("flushed %zu of %zu byte(s)\n", offset, s->buffer_size);
+    memmove(s->buffer, s->buffer + offset, s->buffer_size - offset);
+    s->buffer_size -= offset;
+
+    if (ret < 0) {
+        return ret;
+    }
+    return offset;
+}
+
+static int buffered_put_buffer(void *opaque, const uint8_t *buf,
+                               int64_t pos, int size)
+{
+    MigrationState *s = opaque;
+    ssize_t error;
+
+    DPRINTF("putting %d bytes at %" PRId64 "\n", size, pos);
+
+    error = qemu_file_get_error(s->file);
+    if (error) {
+        DPRINTF("flush when error, bailing: %s\n", strerror(-error));
+        return error;
+    }
+
+    if (size <= 0) {
+        return size;
+    }
+
+    if (size > (s->buffer_capacity - s->buffer_size)) {
+        DPRINTF("increasing buffer capacity from %zu by %zu\n",
+                s->buffer_capacity, size + 1024);
+
+        s->buffer_capacity += size + 1024;
+
+        s->buffer = g_realloc(s->buffer, s->buffer_capacity);
+    }
+
+    memcpy(s->buffer + s->buffer_size, buf, size);
+    s->buffer_size += size;
+
+    return size;
+}
+
+static int buffered_close(void *opaque)
+{
+    MigrationState *s = opaque;
+    ssize_t ret = 0;
+    int ret2;
+
+    DPRINTF("closing\n");
+
+    s->xfer_limit = INT_MAX;
+    while (!qemu_file_get_error(s->file) && s->buffer_size) {
+        ret = buffered_flush(s);
+        if (ret < 0) {
+            break;
+        }
+    }
+
+    ret2 = migrate_fd_close(s);
+    if (ret >= 0) {
+        ret = ret2;
+    }
+    s->complete = true;
+    return ret;
+}
+
+static int buffered_get_fd(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    return s->fd;
+}
+
+/*
+ * The meaning of the return values is:
+ *   0: We can continue sending
+ *   1: Time to stop
+ *   negative: There has been an error
+ */
+static int buffered_rate_limit(void *opaque)
+{
+    MigrationState *s = opaque;
+    int ret;
+
+    ret = qemu_file_get_error(s->file);
+    if (ret) {
+        return ret;
+    }
+
+    if (s->bytes_xfer >= s->xfer_limit) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static int64_t buffered_set_rate_limit(void *opaque, int64_t new_rate)
+{
+    MigrationState *s = opaque;
+    if (qemu_file_get_error(s->file)) {
+        goto out;
+    }
+    if (new_rate > SIZE_MAX) {
+        new_rate = SIZE_MAX;
+    }
+
+    s->xfer_limit = new_rate / XFER_LIMIT_RATIO;
+
+out:
+    return s->xfer_limit;
+}
+
+static int64_t buffered_get_rate_limit(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    return s->xfer_limit;
+}
+
+static void *buffered_file_thread(void *opaque)
+{
+    MigrationState *s = opaque;
+    int64_t initial_time = qemu_get_clock_ms(rt_clock);
+    int64_t max_size = 0;
+    bool last_round = false;
+    int ret;
+
+    qemu_mutex_lock_iothread();
+    DPRINTF("beginning savevm\n");
+    ret = qemu_savevm_state_begin(s->file, &s->params);
+    if (ret < 0) {
+        DPRINTF("failed, %d\n", ret);
+        qemu_mutex_unlock_iothread();
+        goto out;
+    }
+    qemu_mutex_unlock_iothread();
+
+    while (true) {
+        int64_t current_time = qemu_get_clock_ms(rt_clock);
+        uint64_t pending_size;
+
+        qemu_mutex_lock_iothread();
+        if (s->state != MIG_STATE_ACTIVE) {
+            DPRINTF("put_ready returning because of non-active state\n");
+            qemu_mutex_unlock_iothread();
+            break;
+        }
+        if (s->complete) {
+            qemu_mutex_unlock_iothread();
+            break;
+        }
+        if (s->bytes_xfer < s->xfer_limit) {
+            DPRINTF("iterate\n");
+            pending_size = qemu_savevm_state_pending(s->file, max_size);
+            DPRINTF("pending size %lu max %lu\n", pending_size, max_size);
+            if (pending_size && pending_size >= max_size) {
+                ret = qemu_savevm_state_iterate(s->file);
+                if (ret < 0) {
+                    qemu_mutex_unlock_iothread();
+                    break;
+                }
+            } else {
+                int old_vm_running = runstate_is_running();
+                int64_t start_time, end_time;
+
+                DPRINTF("done iterating\n");
+                start_time = qemu_get_clock_ms(rt_clock);
+                qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+                if (old_vm_running) {
+                    vm_stop(RUN_STATE_FINISH_MIGRATE);
+                } else {
+                    vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+                }
+                ret = qemu_savevm_state_complete(s->file);
+                if (ret < 0) {
+                    qemu_mutex_unlock_iothread();
+                    break;
+                } else {
+                    migrate_fd_completed(s);
+                }
+                end_time = qemu_get_clock_ms(rt_clock);
+                s->total_time = end_time - s->total_time;
+                s->downtime = end_time - start_time;
+                if (s->state != MIG_STATE_COMPLETED) {
+                    if (old_vm_running) {
+                        vm_start();
+                    }
+                }
+                last_round = true;
+            }
+        }
+        qemu_mutex_unlock_iothread();
+        if (current_time >= initial_time + BUFFER_DELAY) {
+            uint64_t transferred_bytes = s->bytes_xfer;
+            uint64_t time_spent = current_time - initial_time;
+            double bandwidth = transferred_bytes / time_spent;
+            max_size = bandwidth * migrate_max_downtime() / 1000000;
+
+            DPRINTF("transferred %" PRIu64 " time_spent %" PRIu64
+                    " bandwidth %g max_size %" PRId64 "\n",
+                    transferred_bytes, time_spent, bandwidth, max_size);
+
+            s->bytes_xfer = 0;
+            initial_time = current_time;
+        }
+        if (!last_round && (s->bytes_xfer >= s->xfer_limit)) {
+            /* usleep expects microseconds */
+            g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
+        }
+        ret = buffered_flush(s);
+        if (ret < 0) {
+            break;
+        }
+    }
+
+out:
+    if (ret < 0) {
+        migrate_fd_error(s);
+    }
+    g_free(s->buffer);
+    return NULL;
+}
+
+static const QEMUFileOps buffered_file_ops = {
+    .get_fd =         buffered_get_fd,
+    .put_buffer =     buffered_put_buffer,
+    .close =          buffered_close,
+    .rate_limit =     buffered_rate_limit,
+    .get_rate_limit = buffered_get_rate_limit,
+    .set_rate_limit = buffered_set_rate_limit,
+};
+
+void migrate_fd_connect(MigrationState *s)
+{
+    s->state = MIG_STATE_ACTIVE;
+    s->bytes_xfer = 0;
+    s->buffer = NULL;
+    s->buffer_size = 0;
+    s->buffer_capacity = 0;
+
+    s->xfer_limit = s->bandwidth_limit / XFER_LIMIT_RATIO;
+    s->complete = false;
+
+    s->file = qemu_fopen_ops(s, &buffered_file_ops);
+
+    qemu_thread_create(&s->thread, buffered_file_thread, s,
+                       QEMU_THREAD_DETACHED);
+    notifier_list_notify(&migration_state_notifiers, s);
+}
diff --git a/monitor.c b/monitor.c
index c0e32d6..20bd19b 100644
--- a/monitor.c
+++ b/monitor.c
@@ -27,35 +27,35 @@
 #include "hw/usb.h"
 #include "hw/pcmcia.h"
 #include "hw/pc.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/watchdog.h"
 #include "hw/loader.h"
-#include "gdbstub.h"
-#include "net.h"
+#include "exec/gdbstub.h"
+#include "net/net.h"
 #include "net/slirp.h"
-#include "qemu-char.h"
+#include "char/char.h"
 #include "ui/qemu-spice.h"
-#include "sysemu.h"
-#include "monitor.h"
-#include "readline.h"
-#include "console.h"
-#include "blockdev.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "monitor/readline.h"
+#include "ui/console.h"
+#include "sysemu/blockdev.h"
 #include "audio/audio.h"
-#include "disas.h"
-#include "balloon.h"
-#include "qemu-timer.h"
-#include "migration.h"
-#include "kvm.h"
-#include "acl.h"
-#include "qint.h"
-#include "qfloat.h"
-#include "qlist.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qjson.h"
-#include "json-streamer.h"
-#include "json-parser.h"
-#include "osdep.h"
+#include "disas/disas.h"
+#include "sysemu/balloon.h"
+#include "qemu/timer.h"
+#include "migration/migration.h"
+#include "sysemu/kvm.h"
+#include "qemu/acl.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "trace/control.h"
@@ -63,10 +63,10 @@
 #include "trace/simple.h"
 #endif
 #include "ui/qemu-spice.h"
-#include "memory.h"
+#include "exec/memory.h"
 #include "qmp-commands.h"
 #include "hmp.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 /* for pic/irq_info */
 #if defined(TARGET_SPARC)
@@ -123,13 +123,17 @@ typedef struct mon_cmd_t {
     const char *help;
     void (*user_print)(Monitor *mon, const QObject *data);
     union {
-        void (*info)(Monitor *mon);
         void (*cmd)(Monitor *mon, const QDict *qdict);
         int  (*cmd_new)(Monitor *mon, const QDict *params, QObject **ret_data);
         int  (*cmd_async)(Monitor *mon, const QDict *params,
                           MonitorCompletion *cb, void *opaque);
     } mhandler;
     int flags;
+    /* @sub_table is a list of 2nd level of commands. If it do not exist,
+     * mhandler should be used. If it exist, sub_table[?].mhandler should be
+     * used, and mhandler of 1st level plays the role of help function.
+     */
+    struct mon_cmd_t *sub_table;
 } mon_cmd_t;
 
 /* file descriptors passed via SCM_RIGHTS */
@@ -270,6 +274,7 @@ static void monitor_puts(Monitor *mon, const char *str)
     char c;
 
     for(;;) {
+        assert(mon->outbuf_index < sizeof(mon->outbuf) - 1);
         c = *str++;
         if (c == '\0')
             break;
@@ -806,28 +811,8 @@ static void user_async_cmd_handler(Monitor *mon, const mon_cmd_t *cmd,
     }
 }
 
-static void do_info(Monitor *mon, const QDict *qdict)
+static void do_info_help(Monitor *mon, const QDict *qdict)
 {
-    const mon_cmd_t *cmd;
-    const char *item = qdict_get_try_str(qdict, "item");
-
-    if (!item) {
-        goto help;
-    }
-
-    for (cmd = info_cmds; cmd->name != NULL; cmd++) {
-        if (compare_cmd(item, cmd->name))
-            break;
-    }
-
-    if (cmd->name == NULL) {
-        goto help;
-    }
-
-    cmd->mhandler.info(mon);
-    return;
-
-help:
     help_cmd(mon, "info");
 }
 
@@ -871,9 +856,11 @@ EventInfoList *qmp_query_events(Error **errp)
 int monitor_set_cpu(int cpu_index)
 {
     CPUArchState *env;
+    CPUState *cpu;
 
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
-        if (env->cpu_index == cpu_index) {
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
+        if (cpu->cpu_index == cpu_index) {
             cur_mon->mon_cpu = env;
             return 0;
         }
@@ -892,22 +879,23 @@ static CPUArchState *mon_get_cpu(void)
 
 int monitor_get_cpu_index(void)
 {
-    return mon_get_cpu()->cpu_index;
+    CPUState *cpu = ENV_GET_CPU(mon_get_cpu());
+    return cpu->cpu_index;
 }
 
-static void do_info_registers(Monitor *mon)
+static void do_info_registers(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
     env = mon_get_cpu();
     cpu_dump_state(env, (FILE *)mon, monitor_fprintf, CPU_DUMP_FPU);
 }
 
-static void do_info_jit(Monitor *mon)
+static void do_info_jit(Monitor *mon, const QDict *qdict)
 {
     dump_exec_info((FILE *)mon, monitor_fprintf);
 }
 
-static void do_info_history(Monitor *mon)
+static void do_info_history(Monitor *mon, const QDict *qdict)
 {
     int i;
     const char *str;
@@ -926,7 +914,7 @@ static void do_info_history(Monitor *mon)
 
 #if defined(TARGET_PPC)
 /* XXX: not implemented in other targets */
-static void do_info_cpu_stats(Monitor *mon)
+static void do_info_cpu_stats(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
 
@@ -935,7 +923,7 @@ static void do_info_cpu_stats(Monitor *mon)
 }
 #endif
 
-static void do_trace_print_events(Monitor *mon)
+static void do_trace_print_events(Monitor *mon, const QDict *qdict)
 {
     trace_print_events((FILE *)mon, &monitor_fprintf);
 }
@@ -1487,7 +1475,7 @@ static void tlb_info_64(Monitor *mon, CPUArchState *env)
 }
 #endif
 
-static void tlb_info(Monitor *mon)
+static void tlb_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
 
@@ -1710,7 +1698,7 @@ static void mem_info_64(Monitor *mon, CPUArchState *env)
 }
 #endif
 
-static void mem_info(Monitor *mon)
+static void mem_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env;
 
@@ -1749,7 +1737,7 @@ static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
                    tlb->d, tlb->wt);
 }
 
-static void tlb_info(Monitor *mon)
+static void tlb_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env = mon_get_cpu();
     int i;
@@ -1765,7 +1753,7 @@ static void tlb_info(Monitor *mon)
 #endif
 
 #if defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_XTENSA)
-static void tlb_info(Monitor *mon)
+static void tlb_info(Monitor *mon, const QDict *qdict)
 {
     CPUArchState *env1 = mon_get_cpu();
 
@@ -1773,22 +1761,24 @@ static void tlb_info(Monitor *mon)
 }
 #endif
 
-static void do_info_mtree(Monitor *mon)
+static void do_info_mtree(Monitor *mon, const QDict *qdict)
 {
     mtree_info((fprintf_function)monitor_printf, mon);
 }
 
-static void do_info_numa(Monitor *mon)
+static void do_info_numa(Monitor *mon, const QDict *qdict)
 {
     int i;
     CPUArchState *env;
+    CPUState *cpu;
 
     monitor_printf(mon, "%d nodes\n", nb_numa_nodes);
     for (i = 0; i < nb_numa_nodes; i++) {
         monitor_printf(mon, "node %d cpus:", i);
         for (env = first_cpu; env != NULL; env = env->next_cpu) {
-            if (env->numa_node == i) {
-                monitor_printf(mon, " %d", env->cpu_index);
+            cpu = ENV_GET_CPU(env);
+            if (cpu->numa_node == i) {
+                monitor_printf(mon, " %d", cpu->cpu_index);
             }
         }
         monitor_printf(mon, "\n");
@@ -1802,7 +1792,7 @@ static void do_info_numa(Monitor *mon)
 int64_t qemu_time;
 int64_t dev_time;
 
-static void do_info_profile(Monitor *mon)
+static void do_info_profile(Monitor *mon, const QDict *qdict)
 {
     int64_t total;
     total = qemu_time;
@@ -1816,7 +1806,7 @@ static void do_info_profile(Monitor *mon)
     dev_time = 0;
 }
 #else
-static void do_info_profile(Monitor *mon)
+static void do_info_profile(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "Internal profiler not compiled\n");
 }
@@ -1825,7 +1815,7 @@ static void do_info_profile(Monitor *mon)
 /* Capture support */
 static QLIST_HEAD (capture_list_head, CaptureState) capture_head;
 
-static void do_info_capture(Monitor *mon)
+static void do_info_capture(Monitor *mon, const QDict *qdict)
 {
     int i;
     CaptureState *s;
@@ -1990,6 +1980,7 @@ static void do_inject_mce(Monitor *mon, const QDict *qdict)
 {
     X86CPU *cpu;
     CPUX86State *cenv;
+    CPUState *cs;
     int cpu_index = qdict_get_int(qdict, "cpu_index");
     int bank = qdict_get_int(qdict, "bank");
     uint64_t status = qdict_get_int(qdict, "status");
@@ -2003,7 +1994,8 @@ static void do_inject_mce(Monitor *mon, const QDict *qdict)
     }
     for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
         cpu = x86_env_get_cpu(cenv);
-        if (cenv->cpu_index == cpu_index) {
+        cs = CPU(cpu);
+        if (cs->cpu_index == cpu_index) {
             cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
                                flags);
             break;
@@ -2421,12 +2413,6 @@ int monitor_handle_fd_param(Monitor *mon, const char *fdname)
     return fd;
 }
 
-/* mon_cmds and info_cmds would be sorted at runtime */
-static mon_cmd_t mon_cmds[] = {
-#include "hmp-commands.h"
-    { NULL, NULL, },
-};
-
 /* Please update hmp-commands.hx when adding or changing commands */
 static mon_cmd_t info_cmds[] = {
     {
@@ -2434,63 +2420,63 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the version of QEMU",
-        .mhandler.info = hmp_info_version,
+        .mhandler.cmd = hmp_info_version,
     },
     {
         .name       = "network",
         .args_type  = "",
         .params     = "",
         .help       = "show the network state",
-        .mhandler.info = do_info_network,
+        .mhandler.cmd = do_info_network,
     },
     {
         .name       = "chardev",
         .args_type  = "",
         .params     = "",
         .help       = "show the character devices",
-        .mhandler.info = hmp_info_chardev,
+        .mhandler.cmd = hmp_info_chardev,
     },
     {
         .name       = "block",
         .args_type  = "",
         .params     = "",
         .help       = "show the block devices",
-        .mhandler.info = hmp_info_block,
+        .mhandler.cmd = hmp_info_block,
     },
     {
         .name       = "blockstats",
         .args_type  = "",
         .params     = "",
         .help       = "show block device statistics",
-        .mhandler.info = hmp_info_blockstats,
+        .mhandler.cmd = hmp_info_blockstats,
     },
     {
         .name       = "block-jobs",
         .args_type  = "",
         .params     = "",
         .help       = "show progress of ongoing block device operations",
-        .mhandler.info = hmp_info_block_jobs,
+        .mhandler.cmd = hmp_info_block_jobs,
     },
     {
         .name       = "registers",
         .args_type  = "",
         .params     = "",
         .help       = "show the cpu registers",
-        .mhandler.info = do_info_registers,
+        .mhandler.cmd = do_info_registers,
     },
     {
         .name       = "cpus",
         .args_type  = "",
         .params     = "",
         .help       = "show infos for each CPU",
-        .mhandler.info = hmp_info_cpus,
+        .mhandler.cmd = hmp_info_cpus,
     },
     {
         .name       = "history",
         .args_type  = "",
         .params     = "",
         .help       = "show the command line history",
-        .mhandler.info = do_info_history,
+        .mhandler.cmd = do_info_history,
     },
 #if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
     defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
@@ -2500,11 +2486,11 @@ static mon_cmd_t info_cmds[] = {
         .params     = "",
         .help       = "show the interrupts statistics (if available)",
 #ifdef TARGET_SPARC
-        .mhandler.info = sun4m_irq_info,
+        .mhandler.cmd = sun4m_irq_info,
 #elif defined(TARGET_LM32)
-        .mhandler.info = lm32_irq_info,
+        .mhandler.cmd = lm32_irq_info,
 #else
-        .mhandler.info = irq_info,
+        .mhandler.cmd = irq_info,
 #endif
     },
     {
@@ -2513,11 +2499,11 @@ static mon_cmd_t info_cmds[] = {
         .params     = "",
         .help       = "show i8259 (PIC) state",
 #ifdef TARGET_SPARC
-        .mhandler.info = sun4m_pic_info,
+        .mhandler.cmd = sun4m_pic_info,
 #elif defined(TARGET_LM32)
-        .mhandler.info = lm32_do_pic_info,
+        .mhandler.cmd = lm32_do_pic_info,
 #else
-        .mhandler.info = pic_info,
+        .mhandler.cmd = pic_info,
 #endif
     },
 #endif
@@ -2526,7 +2512,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show PCI info",
-        .mhandler.info = hmp_info_pci,
+        .mhandler.cmd = hmp_info_pci,
     },
 #if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
     defined(TARGET_PPC) || defined(TARGET_XTENSA)
@@ -2535,7 +2521,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show virtual to physical memory mappings",
-        .mhandler.info = tlb_info,
+        .mhandler.cmd = tlb_info,
     },
 #endif
 #if defined(TARGET_I386)
@@ -2544,7 +2530,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the active virtual memory mappings",
-        .mhandler.info = mem_info,
+        .mhandler.cmd = mem_info,
     },
 #endif
     {
@@ -2552,91 +2538,91 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show memory tree",
-        .mhandler.info = do_info_mtree,
+        .mhandler.cmd = do_info_mtree,
     },
     {
         .name       = "jit",
         .args_type  = "",
         .params     = "",
         .help       = "show dynamic compiler info",
-        .mhandler.info = do_info_jit,
+        .mhandler.cmd = do_info_jit,
     },
     {
         .name       = "kvm",
         .args_type  = "",
         .params     = "",
         .help       = "show KVM information",
-        .mhandler.info = hmp_info_kvm,
+        .mhandler.cmd = hmp_info_kvm,
     },
     {
         .name       = "numa",
         .args_type  = "",
         .params     = "",
         .help       = "show NUMA information",
-        .mhandler.info = do_info_numa,
+        .mhandler.cmd = do_info_numa,
     },
     {
         .name       = "usb",
         .args_type  = "",
         .params     = "",
         .help       = "show guest USB devices",
-        .mhandler.info = usb_info,
+        .mhandler.cmd = usb_info,
     },
     {
         .name       = "usbhost",
         .args_type  = "",
         .params     = "",
         .help       = "show host USB devices",
-        .mhandler.info = usb_host_info,
+        .mhandler.cmd = usb_host_info,
     },
     {
         .name       = "profile",
         .args_type  = "",
         .params     = "",
         .help       = "show profiling information",
-        .mhandler.info = do_info_profile,
+        .mhandler.cmd = do_info_profile,
     },
     {
         .name       = "capture",
         .args_type  = "",
         .params     = "",
         .help       = "show capture information",
-        .mhandler.info = do_info_capture,
+        .mhandler.cmd = do_info_capture,
     },
     {
         .name       = "snapshots",
         .args_type  = "",
         .params     = "",
         .help       = "show the currently saved VM snapshots",
-        .mhandler.info = do_info_snapshots,
+        .mhandler.cmd = do_info_snapshots,
     },
     {
         .name       = "status",
         .args_type  = "",
         .params     = "",
         .help       = "show the current VM status (running|paused)",
-        .mhandler.info = hmp_info_status,
+        .mhandler.cmd = hmp_info_status,
     },
     {
         .name       = "pcmcia",
         .args_type  = "",
         .params     = "",
         .help       = "show guest PCMCIA status",
-        .mhandler.info = pcmcia_info,
+        .mhandler.cmd = pcmcia_info,
     },
     {
         .name       = "mice",
         .args_type  = "",
         .params     = "",
         .help       = "show which guest mouse is receiving events",
-        .mhandler.info = hmp_info_mice,
+        .mhandler.cmd = hmp_info_mice,
     },
     {
         .name       = "vnc",
         .args_type  = "",
         .params     = "",
         .help       = "show the vnc server status",
-        .mhandler.info = hmp_info_vnc,
+        .mhandler.cmd = hmp_info_vnc,
     },
 #if defined(CONFIG_SPICE)
     {
@@ -2644,7 +2630,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the spice server status",
-        .mhandler.info = hmp_info_spice,
+        .mhandler.cmd = hmp_info_spice,
     },
 #endif
     {
@@ -2652,14 +2638,14 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show the current VM name",
-        .mhandler.info = hmp_info_name,
+        .mhandler.cmd = hmp_info_name,
     },
     {
         .name       = "uuid",
         .args_type  = "",
         .params     = "",
         .help       = "show the current VM UUID",
-        .mhandler.info = hmp_info_uuid,
+        .mhandler.cmd = hmp_info_uuid,
     },
 #if defined(TARGET_PPC)
     {
@@ -2667,7 +2653,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show CPU statistics",
-        .mhandler.info = do_info_cpu_stats,
+        .mhandler.cmd = do_info_cpu_stats,
     },
 #endif
 #if defined(CONFIG_SLIRP)
@@ -2676,7 +2662,7 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show user network stack connection states",
-        .mhandler.info = do_info_usernet,
+        .mhandler.cmd = do_info_usernet,
     },
 #endif
     {
@@ -2684,62 +2670,68 @@ static mon_cmd_t info_cmds[] = {
         .args_type  = "",
         .params     = "",
         .help       = "show migration status",
-        .mhandler.info = hmp_info_migrate,
+        .mhandler.cmd = hmp_info_migrate,
     },
     {
         .name       = "migrate_capabilities",
         .args_type  = "",
         .params     = "",
         .help       = "show current migration capabilities",
-        .mhandler.info = hmp_info_migrate_capabilities,
+        .mhandler.cmd = hmp_info_migrate_capabilities,
     },
     {
         .name       = "migrate_cache_size",
         .args_type  = "",
         .params     = "",
         .help       = "show current migration xbzrle cache size",
-        .mhandler.info = hmp_info_migrate_cache_size,
+        .mhandler.cmd = hmp_info_migrate_cache_size,
     },
     {
         .name       = "balloon",
         .args_type  = "",
         .params     = "",
         .help       = "show balloon information",
-        .mhandler.info = hmp_info_balloon,
+        .mhandler.cmd = hmp_info_balloon,
     },
     {
         .name       = "qtree",
         .args_type  = "",
         .params     = "",
         .help       = "show device tree",
-        .mhandler.info = do_info_qtree,
+        .mhandler.cmd = do_info_qtree,
     },
     {
         .name       = "qdm",
         .args_type  = "",
         .params     = "",
         .help       = "show qdev device model list",
-        .mhandler.info = do_info_qdm,
+        .mhandler.cmd = do_info_qdm,
     },
     {
         .name       = "roms",
         .args_type  = "",
         .params     = "",
         .help       = "show roms",
-        .mhandler.info = do_info_roms,
+        .mhandler.cmd = do_info_roms,
     },
     {
         .name       = "trace-events",
         .args_type  = "",
         .params     = "",
         .help       = "show available trace-events & their state",
-        .mhandler.info = do_trace_print_events,
+        .mhandler.cmd = do_trace_print_events,
     },
     {
         .name       = NULL,
     },
 };
 
+/* mon_cmds and info_cmds would be sorted at runtime */
+static mon_cmd_t mon_cmds[] = {
+#include "hmp-commands.h"
+    { NULL, NULL, },
+};
+
 static const mon_cmd_t qmp_cmds[] = {
 #include "qmp-commands-old.h"
     { /* NULL */ },
@@ -3534,18 +3526,27 @@ static const mon_cmd_t *search_dispatch_table(const mon_cmd_t *disp_table,
     return NULL;
 }
 
-static const mon_cmd_t *monitor_find_command(const char *cmdname)
-{
-    return search_dispatch_table(mon_cmds, cmdname);
-}
-
 static const mon_cmd_t *qmp_find_cmd(const char *cmdname)
 {
     return search_dispatch_table(qmp_cmds, cmdname);
 }
 
+/*
+ * Parse @cmdline according to command table @table.
+ * If @cmdline is blank, return NULL.
+ * If it can't be parsed, report to @mon, and return NULL.
+ * Else, insert command arguments into @qdict, and return the command.
+ * If sub-command table exist, and if @cmdline contains addtional string for
+ * sub-command, this function will try search sub-command table. if no
+ * addtional string for sub-command exist, this function will return the found
+ * one in @table.
+ * Do not assume the returned command points into @table!  It doesn't
+ * when the command is a sub-command.
+ */
 static const mon_cmd_t *monitor_parse_command(Monitor *mon,
                                               const char *cmdline,
+                                              int start,
+                                              mon_cmd_t *table,
                                               QDict *qdict)
 {
     const char *p, *typestr;
@@ -3556,20 +3557,35 @@ static const mon_cmd_t *monitor_parse_command(Monitor *mon,
     char *key;
 
 #ifdef DEBUG
-    monitor_printf(mon, "command='%s'\n", cmdline);
+    monitor_printf(mon, "command='%s', start='%d'\n", cmdline, start);
 #endif
 
     /* extract the command name */
-    p = get_command_name(cmdline, cmdname, sizeof(cmdname));
+    p = get_command_name(cmdline + start, cmdname, sizeof(cmdname));
     if (!p)
         return NULL;
 
-    cmd = monitor_find_command(cmdname);
+    cmd = search_dispatch_table(table, cmdname);
     if (!cmd) {
-        monitor_printf(mon, "unknown command: '%s'\n", cmdname);
+        monitor_printf(mon, "unknown command: '%.*s'\n",
+                       (int)(p - cmdline), cmdline);
         return NULL;
     }
 
+    /* filter out following useless space */
+    while (qemu_isspace(*p)) {
+        p++;
+    }
+    /* search sub command */
+    if (cmd->sub_table != NULL) {
+        /* check if user set additional command */
+        if (*p == '\0') {
+            return cmd;
+        }
+        return monitor_parse_command(mon, cmdline, p - cmdline,
+                                     cmd->sub_table, qdict);
+    }
+
     /* parse the parameters */
     typestr = cmd->args_type;
     for(;;) {
@@ -3925,7 +3941,7 @@ static void handle_user_command(Monitor *mon, const char *cmdline)
 
     qdict = qdict_new();
 
-    cmd = monitor_parse_command(mon, cmdline, qdict);
+    cmd = monitor_parse_command(mon, cmdline, 0, mon_cmds, qdict);
     if (!cmd)
         goto out;
 
@@ -4790,3 +4806,25 @@ int monitor_read_block_device_key(Monitor *mon, const char *device,
 
     return monitor_read_bdrv_key_start(mon, bs, completion_cb, opaque);
 }
+
+QemuOptsList qemu_mon_opts = {
+    .name = "mon",
+    .implied_opt_name = "chardev",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_mon_opts.head),
+    .desc = {
+        {
+            .name = "mode",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "chardev",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "default",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "pretty",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
diff --git a/nbd.c b/nbd.c
index 01976e8..0698a02 100644
--- a/nbd.c
+++ b/nbd.c
@@ -16,10 +16,10 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "nbd.h"
-#include "block.h"
+#include "block/nbd.h"
+#include "block/block.h"
 
-#include "qemu-coroutine.h"
+#include "block/coroutine.h"
 
 #include <errno.h>
 #include <string.h>
@@ -36,8 +36,8 @@
 #include <linux/fs.h>
 #endif
 
-#include "qemu_socket.h"
-#include "qemu-queue.h"
+#include "qemu/sockets.h"
+#include "qemu/queue.h"
 
 //#define DEBUG_NBD
 
diff --git a/net/Makefile.objs b/net/Makefile.objs
index cf04187..a08cd14 100644
--- a/net/Makefile.objs
+++ b/net/Makefile.objs
@@ -1,4 +1,4 @@
-common-obj-y = queue.o checksum.o util.o hub.o
+common-obj-y = net.o queue.o checksum.o util.o hub.o
 common-obj-y += socket.o
 common-obj-y += dump.o
 common-obj-$(CONFIG_POSIX) += tap.o
diff --git a/net/clients.h b/net/clients.h
index c58cc60..7793294 100644
--- a/net/clients.h
+++ b/net/clients.h
@@ -24,7 +24,7 @@
 #ifndef QEMU_NET_CLIENTS_H
 #define QEMU_NET_CLIENTS_H
 
-#include "net.h"
+#include "net/net.h"
 #include "qapi-types.h"
 
 int net_init_dump(const NetClientOptions *opts, const char *name,
diff --git a/net/dump.c b/net/dump.c
index e0a5d74..4119721 100644
--- a/net/dump.c
+++ b/net/dump.c
@@ -24,9 +24,9 @@
 
 #include "clients.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-log.h"
-#include "qemu-timer.h"
+#include "qemu/error-report.h"
+#include "qemu/log.h"
+#include "qemu/timer.h"
 #include "hub.h"
 
 typedef struct DumpState {
diff --git a/net/hub.c b/net/hub.c
index be41301..a24c9d1 100644
--- a/net/hub.c
+++ b/net/hub.c
@@ -12,11 +12,11 @@
  *
  */
 
-#include "monitor.h"
-#include "net.h"
+#include "monitor/monitor.h"
+#include "net/net.h"
 #include "clients.h"
 #include "hub.h"
-#include "iov.h"
+#include "qemu/iov.h"
 
 /*
  * A hub broadcasts incoming packets to all its ports except the source port.
@@ -256,7 +256,7 @@ void net_hub_info(Monitor *mon)
 /**
  * Get the hub id that a client is connected to
  *
- * @id              Pointer for hub id output, may be NULL
+ * @id: Pointer for hub id output, may be NULL
  */
 int net_hub_id_for_client(NetClientState *nc, int *id)
 {
diff --git a/net/hub.h b/net/hub.h
index 4cbfdb1..583ada8 100644
--- a/net/hub.h
+++ b/net/hub.h
@@ -20,8 +20,6 @@
 NetClientState *net_hub_add_port(int hub_id, const char *name);
 NetClientState *net_hub_find_client_by_name(int hub_id, const char *name);
 void net_hub_info(Monitor *mon);
-int net_hub_id_for_client(NetClientState *nc, int *id);
 void net_hub_check_clients(void);
-NetClientState *net_hub_port_find(int hub_id);
 
 #endif /* NET_HUB_H */
diff --git a/net.c b/net/net.c
index e8ae13e..be03a8d 100644
--- a/net.c
+++ b/net/net.c
@@ -23,21 +23,22 @@
  */
 #include "config-host.h"
 
-#include "net.h"
-#include "net/clients.h"
-#include "net/hub.h"
+#include "net/net.h"
+#include "clients.h"
+#include "hub.h"
 #include "net/slirp.h"
-#include "net/util.h"
+#include "util.h"
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 #include "qemu-common.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
+#include "qemu/config-file.h"
 #include "qmp-commands.h"
 #include "hw/qdev.h"
-#include "iov.h"
+#include "qemu/iov.h"
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/qapi-dealloc-visitor.h"
+#include "qapi/dealloc-visitor.h"
 
 /* Net bridge is currently not supported for W32. */
 #if !defined(_WIN32)
@@ -181,17 +182,18 @@ static char *assign_name(NetClientState *nc1, const char *model)
     return g_strdup(buf);
 }
 
-NetClientState *qemu_new_net_client(NetClientInfo *info,
-                                    NetClientState *peer,
-                                    const char *model,
-                                    const char *name)
+static void qemu_net_client_destructor(NetClientState *nc)
 {
-    NetClientState *nc;
-
-    assert(info->size >= sizeof(NetClientState));
-
-    nc = g_malloc0(info->size);
+    g_free(nc);
+}
 
+static void qemu_net_client_setup(NetClientState *nc,
+                                  NetClientInfo *info,
+                                  NetClientState *peer,
+                                  const char *model,
+                                  const char *name,
+                                  NetClientDestructor *destructor)
+{
     nc->info = info;
     nc->model = g_strdup(model);
     if (name) {
@@ -208,6 +210,21 @@ NetClientState *qemu_new_net_client(NetClientInfo *info,
     QTAILQ_INSERT_TAIL(&net_clients, nc, next);
 
     nc->send_queue = qemu_new_net_queue(nc);
+    nc->destructor = destructor;
+}
+
+NetClientState *qemu_new_net_client(NetClientInfo *info,
+                                    NetClientState *peer,
+                                    const char *model,
+                                    const char *name)
+{
+    NetClientState *nc;
+
+    assert(info->size >= sizeof(NetClientState));
+
+    nc = g_malloc0(info->size);
+    qemu_net_client_setup(nc, info, peer, model, name,
+                          qemu_net_client_destructor);
 
     return nc;
 }
@@ -219,20 +236,53 @@ NICState *qemu_new_nic(NetClientInfo *info,
                        void *opaque)
 {
     NetClientState *nc;
+    NetClientState **peers = conf->peers.ncs;
     NICState *nic;
+    int i;
 
     assert(info->type == NET_CLIENT_OPTIONS_KIND_NIC);
     assert(info->size >= sizeof(NICState));
 
-    nc = qemu_new_net_client(info, conf->peer, model, name);
+    nc = qemu_new_net_client(info, peers[0], model, name);
+    nc->queue_index = 0;
 
-    nic = DO_UPCAST(NICState, nc, nc);
+    nic = qemu_get_nic(nc);
     nic->conf = conf;
     nic->opaque = opaque;
 
+    for (i = 1; i < conf->queues; i++) {
+        qemu_net_client_setup(&nic->ncs[i], info, peers[i], model, nc->name,
+                              NULL);
+        nic->ncs[i].queue_index = i;
+    }
+
     return nic;
 }
 
+NetClientState *qemu_get_subqueue(NICState *nic, int queue_index)
+{
+    return &nic->ncs[queue_index];
+}
+
+NetClientState *qemu_get_queue(NICState *nic)
+{
+    return qemu_get_subqueue(nic, 0);
+}
+
+NICState *qemu_get_nic(NetClientState *nc)
+{
+    NetClientState *nc0 = nc - nc->queue_index;
+
+    return DO_UPCAST(NICState, ncs[0], nc0);
+}
+
+void *qemu_get_nic_opaque(NetClientState *nc)
+{
+    NICState *nic = qemu_get_nic(nc);
+
+    return nic->opaque;
+}
+
 static void qemu_cleanup_net_client(NetClientState *nc)
 {
     QTAILQ_REMOVE(&net_clients, nc, next);
@@ -252,37 +302,72 @@ static void qemu_free_net_client(NetClientState *nc)
     }
     g_free(nc->name);
     g_free(nc->model);
-    g_free(nc);
+    if (nc->destructor) {
+        nc->destructor(nc);
+    }
 }
 
 void qemu_del_net_client(NetClientState *nc)
 {
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    int queues, i;
+
+    /* If the NetClientState belongs to a multiqueue backend, we will change all
+     * other NetClientStates also.
+     */
+    queues = qemu_find_net_clients_except(nc->name, ncs,
+                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          MAX_QUEUE_NUM);
+    assert(queues != 0);
+
     /* If there is a peer NIC, delete and cleanup client, but do not free. */
     if (nc->peer && nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
-        NICState *nic = DO_UPCAST(NICState, nc, nc->peer);
+        NICState *nic = qemu_get_nic(nc->peer);
         if (nic->peer_deleted) {
             return;
         }
         nic->peer_deleted = true;
-        /* Let NIC know peer is gone. */
-        nc->peer->link_down = true;
+
+        for (i = 0; i < queues; i++) {
+            ncs[i]->peer->link_down = true;
+        }
+
         if (nc->peer->info->link_status_changed) {
             nc->peer->info->link_status_changed(nc->peer);
         }
-        qemu_cleanup_net_client(nc);
+
+        for (i = 0; i < queues; i++) {
+            qemu_cleanup_net_client(ncs[i]);
+        }
+
         return;
     }
 
+    assert(nc->info->type != NET_CLIENT_OPTIONS_KIND_NIC);
+
+    for (i = 0; i < queues; i++) {
+        qemu_cleanup_net_client(ncs[i]);
+        qemu_free_net_client(ncs[i]);
+    }
+}
+
+void qemu_del_nic(NICState *nic)
+{
+    int i, queues = MAX(nic->conf->queues, 1);
+
     /* If this is a peer NIC and peer has already been deleted, free it now. */
-    if (nc->peer && nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
-        NICState *nic = DO_UPCAST(NICState, nc, nc);
-        if (nic->peer_deleted) {
-            qemu_free_net_client(nc->peer);
+    if (nic->peer_deleted) {
+        for (i = 0; i < queues; i++) {
+            qemu_free_net_client(qemu_get_subqueue(nic, i)->peer);
         }
     }
 
-    qemu_cleanup_net_client(nc);
-    qemu_free_net_client(nc);
+    for (i = queues - 1; i >= 0; i--) {
+        NetClientState *nc = qemu_get_subqueue(nic, i);
+
+        qemu_cleanup_net_client(nc);
+        qemu_free_net_client(nc);
+    }
 }
 
 void qemu_foreach_nic(qemu_nic_foreach func, void *opaque)
@@ -291,7 +376,9 @@ void qemu_foreach_nic(qemu_nic_foreach func, void *opaque)
 
     QTAILQ_FOREACH(nc, &net_clients, next) {
         if (nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
-            func(DO_UPCAST(NICState, nc, nc), opaque);
+            if (nc->queue_index == 0) {
+                func(qemu_get_nic(nc), opaque);
+            }
         }
     }
 }
@@ -481,6 +568,27 @@ NetClientState *qemu_find_netdev(const char *id)
     return NULL;
 }
 
+int qemu_find_net_clients_except(const char *id, NetClientState **ncs,
+                                 NetClientOptionsKind type, int max)
+{
+    NetClientState *nc;
+    int ret = 0;
+
+    QTAILQ_FOREACH(nc, &net_clients, next) {
+        if (nc->info->type == type) {
+            continue;
+        }
+        if (!strcmp(nc->name, id)) {
+            if (ret < max) {
+                ncs[ret] = nc;
+            }
+            ret++;
+        }
+    }
+
+    return ret;
+}
+
 static int nic_get_free_idx(void)
 {
     int index;
@@ -565,9 +673,7 @@ static int net_init_nic(const NetClientOptions *opts, const char *name,
         assert(peer);
         nd->netdev = peer;
     }
-    if (name) {
-        nd->name = g_strdup(name);
-    }
+    nd->name = g_strdup(name);
     if (nic->has_model) {
         nd->model = g_strdup(nic->model);
     }
@@ -847,11 +953,13 @@ void qmp_netdev_del(const char *id, Error **errp)
 
 void print_net_client(Monitor *mon, NetClientState *nc)
 {
-    monitor_printf(mon, "%s: type=%s,%s\n", nc->name,
-                   NetClientOptionsKind_lookup[nc->info->type], nc->info_str);
+    monitor_printf(mon, "%s: index=%d,type=%s,%s\n", nc->name,
+                   nc->queue_index,
+                   NetClientOptionsKind_lookup[nc->info->type],
+                   nc->info_str);
 }
 
-void do_info_network(Monitor *mon)
+void do_info_network(Monitor *mon, const QDict *qdict)
 {
     NetClientState *nc, *peer;
     NetClientOptionsKind type;
@@ -879,20 +987,23 @@ void do_info_network(Monitor *mon)
 
 void qmp_set_link(const char *name, bool up, Error **errp)
 {
-    NetClientState *nc = NULL;
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    NetClientState *nc;
+    int queues, i;
 
-    QTAILQ_FOREACH(nc, &net_clients, next) {
-        if (!strcmp(nc->name, name)) {
-            goto done;
-        }
-    }
-done:
-    if (!nc) {
+    queues = qemu_find_net_clients_except(name, ncs,
+                                          NET_CLIENT_OPTIONS_KIND_MAX,
+                                          MAX_QUEUE_NUM);
+
+    if (queues == 0) {
         error_set(errp, QERR_DEVICE_NOT_FOUND, name);
         return;
     }
+    nc = ncs[0];
 
-    nc->link_down = !up;
+    for (i = 0; i < queues; i++) {
+        ncs[i]->link_down = !up;
+    }
 
     if (nc->info->link_status_changed) {
         nc->info->link_status_changed(nc);
@@ -912,10 +1023,18 @@ done:
 
 void net_cleanup(void)
 {
-    NetClientState *nc, *next_vc;
+    NetClientState *nc;
 
-    QTAILQ_FOREACH_SAFE(nc, &net_clients, next, next_vc) {
-        qemu_del_net_client(nc);
+    /* We may del multiple entries during qemu_del_net_client(),
+     * so QTAILQ_FOREACH_SAFE() is also not safe here.
+     */
+    while (!QTAILQ_EMPTY(&net_clients)) {
+        nc = QTAILQ_FIRST(&net_clients);
+        if (nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
+            qemu_del_nic(qemu_get_nic(nc));
+        } else {
+            qemu_del_net_client(nc);
+        }
     }
 }
 
@@ -1053,3 +1172,29 @@ unsigned compute_mcast_idx(const uint8_t *ep)
     }
     return crc >> 26;
 }
+
+QemuOptsList qemu_netdev_opts = {
+    .name = "netdev",
+    .implied_opt_name = "type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_netdev_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any params
+         * validation will happen later
+         */
+        { /* end of list */ }
+    },
+};
+
+QemuOptsList qemu_net_opts = {
+    .name = "net",
+    .implied_opt_name = "type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_net_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any params
+         * validation will happen later
+         */
+        { /* end of list */ }
+    },
+};
diff --git a/net/queue.c b/net/queue.c
index 254f280..6eaf5b6 100644
--- a/net/queue.c
+++ b/net/queue.c
@@ -22,8 +22,8 @@
  */
 
 #include "net/queue.h"
-#include "qemu-queue.h"
-#include "net.h"
+#include "qemu/queue.h"
+#include "net/net.h"
 
 /* The delivery handler may only return zero if it will call
  * qemu_net_queue_flush() when it determines that it is once again able
diff --git a/net/slirp.c b/net/slirp.c
index afb52c3..4df550f 100644
--- a/net/slirp.c
+++ b/net/slirp.c
@@ -29,12 +29,13 @@
 #include <pwd.h>
 #include <sys/wait.h>
 #endif
-#include "net.h"
+#include "net/net.h"
 #include "clients.h"
 #include "hub.h"
-#include "monitor.h"
-#include "qemu_socket.h"
+#include "monitor/monitor.h"
+#include "qemu/sockets.h"
 #include "slirp/libslirp.h"
+#include "char/char.h"
 
 static int get_str_sep(char *buf, int buf_size, const char **pp, int sep)
 {
@@ -669,7 +670,7 @@ static int slirp_guestfwd(SlirpState *s, const char *config_str,
     return -1;
 }
 
-void do_info_usernet(Monitor *mon)
+void do_info_usernet(Monitor *mon, const QDict *qdict)
 {
     SlirpState *s;
 
diff --git a/net/socket.c b/net/socket.c
index c01323d..396dc8c 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -23,15 +23,14 @@
  */
 #include "config-host.h"
 
-#include "net.h"
+#include "net/net.h"
 #include "clients.h"
-#include "monitor.h"
-#include "qemu-char.h"
+#include "monitor/monitor.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-option.h"
-#include "qemu_socket.h"
-#include "iov.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
 
 typedef struct NetSocketState {
     NetClientState nc;
diff --git a/net/tap-aix.c b/net/tap-aix.c
index f27c177..804d164 100644
--- a/net/tap-aix.c
+++ b/net/tap-aix.c
@@ -22,10 +22,11 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 #include <stdio.h>
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required)
 {
     fprintf(stderr, "no tap on AIX\n");
     return -1;
@@ -59,3 +60,19 @@ void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
 }
+
+int tap_fd_enable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_disable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_get_ifname(int fd, char *ifname)
+{
+    return -1;
+}
+
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index a3b717d..bcdb268 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -22,10 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 #include "qemu-common.h"
-#include "sysemu.h"
-#include "qemu-error.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
 
 #ifdef __NetBSD__
 #include <sys/ioctl.h>
@@ -33,7 +33,8 @@
 #include <net/if_tap.h>
 #endif
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required)
 {
     int fd;
 #ifdef TAPGIFNAME
@@ -145,3 +146,18 @@ void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
 }
+
+int tap_fd_enable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_disable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_get_ifname(int fd, char *ifname)
+{
+    return -1;
+}
diff --git a/net/tap-haiku.c b/net/tap-haiku.c
index 34739d1..e5ce436 100644
--- a/net/tap-haiku.c
+++ b/net/tap-haiku.c
@@ -22,10 +22,11 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap_int.h"
 #include <stdio.h>
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required)
 {
     fprintf(stderr, "no tap on Haiku\n");
     return -1;
@@ -59,3 +60,18 @@ void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
 }
+
+int tap_fd_enable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_disable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_get_ifname(int fd, char *ifname)
+{
+    return -1;
+}
diff --git a/net/tap-linux.c b/net/tap-linux.c
index c6521be..a953189 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -23,22 +23,25 @@
  * THE SOFTWARE.
  */
 
+#include "tap_int.h"
+#include "tap-linux.h"
 #include "net/tap.h"
-#include "net/tap-linux.h"
 
 #include <net/if.h>
 #include <sys/ioctl.h>
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 #define PATH_NET_TUN "/dev/net/tun"
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required)
 {
     struct ifreq ifr;
     int fd, ret;
+    int len = sizeof(struct virtio_net_hdr);
 
     TFR(fd = open(PATH_NET_TUN, O_RDWR));
     if (fd < 0) {
@@ -65,6 +68,27 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required
             close(fd);
             return -1;
         }
+        /*
+         * Make sure vnet header size has the default value: for a persistent
+         * tap it might have been modified e.g. by another instance of qemu.
+         * Ignore errors since old kernels do not support this ioctl: in this
+         * case the header size implicitly has the correct value.
+         */
+        ioctl(fd, TUNSETVNETHDRSZ, &len);
+    }
+
+    if (mq_required) {
+        unsigned int features;
+
+        if ((ioctl(fd, TUNGETFEATURES, &features) != 0) ||
+            !(features & IFF_MULTI_QUEUE)) {
+            error_report("multiqueue required, but no kernel "
+                         "support for IFF_MULTI_QUEUE available");
+            close(fd);
+            return -1;
+        } else {
+            ifr.ifr_flags |= IFF_MULTI_QUEUE;
+        }
     }
 
     if (ifname[0] != '\0')
@@ -155,7 +179,7 @@ int tap_probe_vnet_hdr_len(int fd, int len)
     if (ioctl(fd, TUNSETVNETHDRSZ, &orig) == -1) {
         fprintf(stderr, "TUNGETVNETHDRSZ ioctl() failed: %s. Exiting.\n",
                 strerror(errno));
-        assert(0);
+        abort();
         return -errno;
     }
     return 1;
@@ -166,7 +190,7 @@ void tap_fd_set_vnet_hdr_len(int fd, int len)
     if (ioctl(fd, TUNSETVNETHDRSZ, &len) == -1) {
         fprintf(stderr, "TUNSETVNETHDRSZ ioctl() failed: %s. Exiting.\n",
                 strerror(errno));
-        assert(0);
+        abort();
     }
 }
 
@@ -200,3 +224,53 @@ void tap_fd_set_offload(int fd, int csum, int tso4,
         }
     }
 }
+
+/* Enable a specific queue of tap. */
+int tap_fd_enable(int fd)
+{
+    struct ifreq ifr;
+    int ret;
+
+    memset(&ifr, 0, sizeof(ifr));
+
+    ifr.ifr_flags = IFF_ATTACH_QUEUE;
+    ret = ioctl(fd, TUNSETQUEUE, (void *) &ifr);
+
+    if (ret != 0) {
+        error_report("could not enable queue");
+    }
+
+    return ret;
+}
+
+/* Disable a specific queue of tap/ */
+int tap_fd_disable(int fd)
+{
+    struct ifreq ifr;
+    int ret;
+
+    memset(&ifr, 0, sizeof(ifr));
+
+    ifr.ifr_flags = IFF_DETACH_QUEUE;
+    ret = ioctl(fd, TUNSETQUEUE, (void *) &ifr);
+
+    if (ret != 0) {
+        error_report("could not disable queue");
+    }
+
+    return ret;
+}
+
+int tap_fd_get_ifname(int fd, char *ifname)
+{
+    struct ifreq ifr;
+
+    if (ioctl(fd, TUNGETIFF, &ifr) != 0) {
+        error_report("TUNGETIFF ioctl() failed: %s",
+                     strerror(errno));
+        return -1;
+    }
+
+    pstrcpy(ifname, sizeof(ifr.ifr_name), ifr.ifr_name);
+    return 0;
+}
diff --git a/net/tap-linux.h b/net/tap-linux.h
index 659e981..65087e1 100644
--- a/net/tap-linux.h
+++ b/net/tap-linux.h
@@ -13,8 +13,8 @@
  *  GNU General Public License for more details.
  */
 
-#ifndef QEMU_TAP_H
-#define QEMU_TAP_H
+#ifndef QEMU_TAP_LINUX_H
+#define QEMU_TAP_LINUX_H
 
 #include <stdint.h>
 #ifdef __linux__
@@ -29,6 +29,7 @@
 #define TUNSETSNDBUF   _IOW('T', 212, int)
 #define TUNGETVNETHDRSZ _IOR('T', 215, int)
 #define TUNSETVNETHDRSZ _IOW('T', 216, int)
+#define TUNSETQUEUE  _IOW('T', 217, int)
 
 #endif
 
@@ -36,6 +37,9 @@
 #define IFF_TAP		0x0002
 #define IFF_NO_PI	0x1000
 #define IFF_VNET_HDR	0x4000
+#define IFF_MULTI_QUEUE 0x0100
+#define IFF_ATTACH_QUEUE 0x0200
+#define IFF_DETACH_QUEUE 0x0400
 
 /* Features for GSO (TUNSETOFFLOAD). */
 #define TUN_F_CSUM	0x01	/* You can hand me unchecksummed packets. */
@@ -44,20 +48,4 @@
 #define TUN_F_TSO_ECN	0x08	/* I can handle TSO with ECN bits. */
 #define TUN_F_UFO	0x10	/* I can handle UFO packets */
 
-struct virtio_net_hdr
-{
-    uint8_t flags;
-    uint8_t gso_type;
-    uint16_t hdr_len;
-    uint16_t gso_size;
-    uint16_t csum_start;
-    uint16_t csum_offset;
-};
-
-struct virtio_net_hdr_mrg_rxbuf
-{
-    struct virtio_net_hdr hdr;
-    uint16_t num_buffers;   /* Number of merged rx buffers */
-};
-
 #endif /* QEMU_TAP_H */
diff --git a/net/tap-solaris.c b/net/tap-solaris.c
index 5d6ac42..9c7278f 100644
--- a/net/tap-solaris.c
+++ b/net/tap-solaris.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
-#include "sysemu.h"
+#include "tap_int.h"
+#include "sysemu/sysemu.h"
 
 #include <sys/stat.h>
 #include <sys/ethernet.h>
@@ -38,7 +38,7 @@
 #include <net/if.h>
 #include <syslog.h>
 #include <stropts.h>
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen)
 {
@@ -173,7 +173,8 @@ static int tap_alloc(char *dev, size_t dev_size)
     return tap_fd;
 }
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required)
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required)
 {
     char  dev[10]="";
     int fd;
@@ -225,3 +226,18 @@ void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
 }
+
+int tap_fd_enable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_disable(int fd)
+{
+    return -1;
+}
+
+int tap_fd_get_ifname(int fd, char *ifname)
+{
+    return -1;
+}
diff --git a/net/tap-win32.c b/net/tap-win32.c
index 8d2d32b..91e9e84 100644
--- a/net/tap-win32.c
+++ b/net/tap-win32.c
@@ -26,13 +26,14 @@
  *  distribution); if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tap.h"
+#include "tap_int.h"
 
 #include "qemu-common.h"
 #include "clients.h"            /* net_init_tap */
-#include "net.h"
-#include "sysemu.h"
-#include "qemu-error.h"
+#include "net/net.h"
+#include "net/tap.h"            /* tap_has_ufo, ... */
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
 #include <stdio.h>
 #include <windows.h>
 #include <winioctl.h>
@@ -565,7 +566,7 @@ static void tap_win32_free_buffer(tap_win32_overlapped_t *overlapped,
 }
 
 static int tap_win32_open(tap_win32_overlapped_t **phandle,
-                          const char *prefered_name)
+                          const char *preferred_name)
 {
     char device_path[256];
     char device_guid[0x100];
@@ -581,8 +582,9 @@ static int tap_win32_open(tap_win32_overlapped_t **phandle,
     DWORD version_len;
     DWORD idThread;
 
-    if (prefered_name != NULL)
-        snprintf(name_buffer, sizeof(name_buffer), "%s", prefered_name);
+    if (preferred_name != NULL) {
+        snprintf(name_buffer, sizeof(name_buffer), "%s", preferred_name);
+    }
 
     rc = get_device_guid(device_guid, sizeof(device_guid), name_buffer, sizeof(name_buffer));
     if (rc)
@@ -720,9 +722,9 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
     return 0;
 }
 
-int tap_has_ufo(NetClientState *nc)
+bool tap_has_ufo(NetClientState *nc)
 {
-    return 0;
+    return false;
 }
 
 int tap_has_vnet_hdr(NetClientState *nc)
@@ -739,7 +741,7 @@ void tap_fd_set_vnet_hdr_len(int fd, int len)
 {
 }
 
-void tap_using_vnet_hdr(NetClientState *nc, int using_vnet_hdr)
+void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr)
 {
 }
 
@@ -760,5 +762,15 @@ int tap_has_vnet_hdr_len(NetClientState *nc, int len)
 
 void tap_set_vnet_hdr_len(NetClientState *nc, int len)
 {
-    assert(0);
+    abort();
+}
+
+int tap_enable(NetClientState *nc)
+{
+    abort();
+}
+
+int tap_disable(NetClientState *nc)
+{
+    abort();
 }
diff --git a/net/tap.c b/net/tap.c
index 1abfd44..48c254e 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include "tap.h"
+#include "tap_int.h"
 
 #include "config-host.h"
 
@@ -33,15 +33,14 @@
 #include <sys/socket.h>
 #include <net/if.h>
 
-#include "net.h"
+#include "net/net.h"
 #include "clients.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "qemu-char.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
 #include "qemu-common.h"
-#include "qemu-error.h"
+#include "qemu/error-report.h"
 
-#include "net/tap-linux.h"
+#include "net/tap.h"
 
 #include "hw/vhost_net.h"
 
@@ -56,10 +55,11 @@ typedef struct TAPState {
     char down_script[1024];
     char down_script_arg[128];
     uint8_t buf[TAP_BUFSIZE];
-    unsigned int read_poll : 1;
-    unsigned int write_poll : 1;
-    unsigned int using_vnet_hdr : 1;
-    unsigned int has_ufo: 1;
+    bool read_poll;
+    bool write_poll;
+    bool using_vnet_hdr;
+    bool has_ufo;
+    bool enabled;
     VHostNetState *vhost_net;
     unsigned host_vnet_hdr_len;
 } TAPState;
@@ -73,21 +73,21 @@ static void tap_writable(void *opaque);
 static void tap_update_fd_handler(TAPState *s)
 {
     qemu_set_fd_handler2(s->fd,
-                         s->read_poll  ? tap_can_send : NULL,
-                         s->read_poll  ? tap_send     : NULL,
-                         s->write_poll ? tap_writable : NULL,
+                         s->read_poll && s->enabled ? tap_can_send : NULL,
+                         s->read_poll && s->enabled ? tap_send     : NULL,
+                         s->write_poll && s->enabled ? tap_writable : NULL,
                          s);
 }
 
-static void tap_read_poll(TAPState *s, int enable)
+static void tap_read_poll(TAPState *s, bool enable)
 {
-    s->read_poll = !!enable;
+    s->read_poll = enable;
     tap_update_fd_handler(s);
 }
 
-static void tap_write_poll(TAPState *s, int enable)
+static void tap_write_poll(TAPState *s, bool enable)
 {
-    s->write_poll = !!enable;
+    s->write_poll = enable;
     tap_update_fd_handler(s);
 }
 
@@ -95,7 +95,7 @@ static void tap_writable(void *opaque)
 {
     TAPState *s = opaque;
 
-    tap_write_poll(s, 0);
+    tap_write_poll(s, false);
 
     qemu_flush_queued_packets(&s->nc);
 }
@@ -109,7 +109,7 @@ static ssize_t tap_write_packet(TAPState *s, const struct iovec *iov, int iovcnt
     } while (len == -1 && errno == EINTR);
 
     if (len == -1 && errno == EAGAIN) {
-        tap_write_poll(s, 1);
+        tap_write_poll(s, true);
         return 0;
     }
 
@@ -187,7 +187,7 @@ ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen)
 static void tap_send_completed(NetClientState *nc, ssize_t len)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
-    tap_read_poll(s, 1);
+    tap_read_poll(s, true);
 }
 
 static void tap_send(void *opaque)
@@ -210,12 +210,12 @@ static void tap_send(void *opaque)
 
         size = qemu_send_packet_async(&s->nc, buf, size, tap_send_completed);
         if (size == 0) {
-            tap_read_poll(s, 0);
+            tap_read_poll(s, false);
         }
     } while (size > 0 && qemu_can_send_packet(&s->nc));
 }
 
-int tap_has_ufo(NetClientState *nc)
+bool tap_has_ufo(NetClientState *nc)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
@@ -254,12 +254,10 @@ void tap_set_vnet_hdr_len(NetClientState *nc, int len)
     s->host_vnet_hdr_len = len;
 }
 
-void tap_using_vnet_hdr(NetClientState *nc, int using_vnet_hdr)
+void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
-    using_vnet_hdr = using_vnet_hdr != 0;
-
     assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
     assert(!!s->host_vnet_hdr_len == using_vnet_hdr);
 
@@ -291,8 +289,8 @@ static void tap_cleanup(NetClientState *nc)
     if (s->down_script[0])
         launch_script(s->down_script, s->down_script_arg, s->fd);
 
-    tap_read_poll(s, 0);
-    tap_write_poll(s, 0);
+    tap_read_poll(s, false);
+    tap_write_poll(s, false);
     close(s->fd);
     s->fd = -1;
 }
@@ -338,8 +336,9 @@ static TAPState *net_tap_fd_init(NetClientState *peer,
 
     s->fd = fd;
     s->host_vnet_hdr_len = vnet_hdr ? sizeof(struct virtio_net_hdr) : 0;
-    s->using_vnet_hdr = 0;
+    s->using_vnet_hdr = false;
     s->has_ufo = tap_probe_has_ufo(s->fd);
+    s->enabled = true;
     tap_set_offload(&s->nc, 0, 0, 0, 0, 0);
     /*
      * Make sure host header length is set correctly in tap:
@@ -348,7 +347,7 @@ static TAPState *net_tap_fd_init(NetClientState *peer,
     if (tap_probe_vnet_hdr_len(s->fd, s->host_vnet_hdr_len)) {
         tap_fd_set_vnet_hdr_len(s->fd, s->host_vnet_hdr_len);
     }
-    tap_read_poll(s, 1);
+    tap_read_poll(s, true);
     s->vhost_net = NULL;
     return s;
 }
@@ -559,17 +558,10 @@ int net_init_bridge(const NetClientOptions *opts, const char *name,
 
 static int net_tap_init(const NetdevTapOptions *tap, int *vnet_hdr,
                         const char *setup_script, char *ifname,
-                        size_t ifname_sz)
+                        size_t ifname_sz, int mq_required)
 {
     int fd, vnet_hdr_required;
 
-    if (tap->has_ifname) {
-        pstrcpy(ifname, ifname_sz, tap->ifname);
-    } else {
-        assert(ifname_sz > 0);
-        ifname[0] = '\0';
-    }
-
     if (tap->has_vnet_hdr) {
         *vnet_hdr = tap->vnet_hdr;
         vnet_hdr_required = *vnet_hdr;
@@ -578,7 +570,8 @@ static int net_tap_init(const NetdevTapOptions *tap, int *vnet_hdr,
         vnet_hdr_required = 0;
     }
 
-    TFR(fd = tap_open(ifname, ifname_sz, vnet_hdr, vnet_hdr_required));
+    TFR(fd = tap_open(ifname, ifname_sz, vnet_hdr, vnet_hdr_required,
+                      mq_required));
     if (fd < 0) {
         return -1;
     }
@@ -594,27 +587,118 @@ static int net_tap_init(const NetdevTapOptions *tap, int *vnet_hdr,
     return fd;
 }
 
+#define MAX_TAP_QUEUES 1024
+
+static int net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
+                            const char *model, const char *name,
+                            const char *ifname, const char *script,
+                            const char *downscript, const char *vhostfdname,
+                            int vnet_hdr, int fd)
+{
+    TAPState *s;
+
+    s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
+    if (!s) {
+        close(fd);
+        return -1;
+    }
+
+    if (tap_set_sndbuf(s->fd, tap) < 0) {
+        return -1;
+    }
+
+    if (tap->has_fd || tap->has_fds) {
+        snprintf(s->nc.info_str, sizeof(s->nc.info_str), "fd=%d", fd);
+    } else if (tap->has_helper) {
+        snprintf(s->nc.info_str, sizeof(s->nc.info_str), "helper=%s",
+                 tap->helper);
+    } else {
+        snprintf(s->nc.info_str, sizeof(s->nc.info_str),
+                 "ifname=%s,script=%s,downscript=%s", ifname, script,
+                 downscript);
+
+        if (strcmp(downscript, "no") != 0) {
+            snprintf(s->down_script, sizeof(s->down_script), "%s", downscript);
+            snprintf(s->down_script_arg, sizeof(s->down_script_arg),
+                     "%s", ifname);
+        }
+    }
+
+    if (tap->has_vhost ? tap->vhost :
+        vhostfdname || (tap->has_vhostforce && tap->vhostforce)) {
+        int vhostfd;
+
+        if (tap->has_vhostfd) {
+            vhostfd = monitor_handle_fd_param(cur_mon, vhostfdname);
+            if (vhostfd == -1) {
+                return -1;
+            }
+        } else {
+            vhostfd = -1;
+        }
+
+        s->vhost_net = vhost_net_init(&s->nc, vhostfd,
+                                      tap->has_vhostforce && tap->vhostforce);
+        if (!s->vhost_net) {
+            error_report("vhost-net requested but could not be initialized");
+            return -1;
+        }
+    } else if (tap->has_vhostfd || tap->has_vhostfds) {
+        error_report("vhostfd= is not valid without vhost");
+        return -1;
+    }
+
+    return 0;
+}
+
+static int get_fds(char *str, char *fds[], int max)
+{
+    char *ptr = str, *this;
+    size_t len = strlen(str);
+    int i = 0;
+
+    while (i < max && ptr < str + len) {
+        this = strchr(ptr, ':');
+
+        if (this == NULL) {
+            fds[i] = g_strdup(ptr);
+        } else {
+            fds[i] = g_strndup(ptr, this - ptr);
+        }
+
+        i++;
+        if (this == NULL) {
+            break;
+        } else {
+            ptr = this + 1;
+        }
+    }
+
+    return i;
+}
+
 int net_init_tap(const NetClientOptions *opts, const char *name,
                  NetClientState *peer)
 {
     const NetdevTapOptions *tap;
-
-    int fd, vnet_hdr = 0;
-    const char *model;
-    TAPState *s;
-
+    int fd, vnet_hdr = 0, i = 0, queues;
     /* for the no-fd, no-helper case */
     const char *script = NULL; /* suppress wrong "uninit'd use" gcc warning */
+    const char *downscript = NULL;
+    const char *vhostfdname;
     char ifname[128];
 
     assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
     tap = opts->tap;
+    queues = tap->has_queues ? tap->queues : 1;
+    vhostfdname = tap->has_vhostfd ? tap->vhostfd : NULL;
 
     if (tap->has_fd) {
         if (tap->has_ifname || tap->has_script || tap->has_downscript ||
-            tap->has_vnet_hdr || tap->has_helper) {
+            tap->has_vnet_hdr || tap->has_helper || tap->has_queues ||
+            tap->has_fds) {
             error_report("ifname=, script=, downscript=, vnet_hdr=, "
-                         "and helper= are invalid with fd=");
+                         "helper=, queues=, and fds= are invalid with fd=");
             return -1;
         }
 
@@ -627,13 +711,61 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
 
         vnet_hdr = tap_probe_vnet_hdr(fd);
 
-        model = "tap";
+        if (net_init_tap_one(tap, peer, "tap", name, NULL,
+                             script, downscript,
+                             vhostfdname, vnet_hdr, fd)) {
+            return -1;
+        }
+    } else if (tap->has_fds) {
+        char *fds[MAX_TAP_QUEUES];
+        char *vhost_fds[MAX_TAP_QUEUES];
+        int nfds, nvhosts;
+
+        if (tap->has_ifname || tap->has_script || tap->has_downscript ||
+            tap->has_vnet_hdr || tap->has_helper || tap->has_queues ||
+            tap->has_fd) {
+            error_report("ifname=, script=, downscript=, vnet_hdr=, "
+                         "helper=, queues=, and fd= are invalid with fds=");
+            return -1;
+        }
 
+        nfds = get_fds(tap->fds, fds, MAX_TAP_QUEUES);
+        if (tap->has_vhostfds) {
+            nvhosts = get_fds(tap->vhostfds, vhost_fds, MAX_TAP_QUEUES);
+            if (nfds != nvhosts) {
+                error_report("The number of fds passed does not match the "
+                             "number of vhostfds passed");
+                return -1;
+            }
+        }
+
+        for (i = 0; i < nfds; i++) {
+            fd = monitor_handle_fd_param(cur_mon, fds[i]);
+            if (fd == -1) {
+                return -1;
+            }
+
+            fcntl(fd, F_SETFL, O_NONBLOCK);
+
+            if (i == 0) {
+                vnet_hdr = tap_probe_vnet_hdr(fd);
+            } else if (vnet_hdr != tap_probe_vnet_hdr(fd)) {
+                error_report("vnet_hdr not consistent across given tap fds");
+                return -1;
+            }
+
+            if (net_init_tap_one(tap, peer, "tap", name, ifname,
+                                 script, downscript,
+                                 tap->has_vhostfds ? vhost_fds[i] : NULL,
+                                 vnet_hdr, fd)) {
+                return -1;
+            }
+        }
     } else if (tap->has_helper) {
         if (tap->has_ifname || tap->has_script || tap->has_downscript ||
-            tap->has_vnet_hdr) {
+            tap->has_vnet_hdr || tap->has_queues || tap->has_fds) {
             error_report("ifname=, script=, downscript=, and vnet_hdr= "
-                         "are invalid with helper=");
+                         "queues=, and fds= are invalid with helper=");
             return -1;
         }
 
@@ -643,74 +775,45 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
         }
 
         fcntl(fd, F_SETFL, O_NONBLOCK);
-
         vnet_hdr = tap_probe_vnet_hdr(fd);
 
-        model = "bridge";
-
-    } else {
-        script = tap->has_script ? tap->script : DEFAULT_NETWORK_SCRIPT;
-        fd = net_tap_init(tap, &vnet_hdr, script, ifname, sizeof ifname);
-        if (fd == -1) {
+        if (net_init_tap_one(tap, peer, "bridge", name, ifname,
+                             script, downscript, vhostfdname,
+                             vnet_hdr, fd)) {
             return -1;
         }
-
-        model = "tap";
-    }
-
-    s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
-    if (!s) {
-        close(fd);
-        return -1;
-    }
-
-    if (tap_set_sndbuf(s->fd, tap) < 0) {
-        return -1;
-    }
-
-    if (tap->has_fd) {
-        snprintf(s->nc.info_str, sizeof(s->nc.info_str), "fd=%d", fd);
-    } else if (tap->has_helper) {
-        snprintf(s->nc.info_str, sizeof(s->nc.info_str), "helper=%s",
-                 tap->helper);
     } else {
-        const char *downscript;
-
+        script = tap->has_script ? tap->script : DEFAULT_NETWORK_SCRIPT;
         downscript = tap->has_downscript ? tap->downscript :
-                                           DEFAULT_NETWORK_DOWN_SCRIPT;
+            DEFAULT_NETWORK_DOWN_SCRIPT;
 
-        snprintf(s->nc.info_str, sizeof(s->nc.info_str),
-                 "ifname=%s,script=%s,downscript=%s", ifname, script,
-                 downscript);
-
-        if (strcmp(downscript, "no") != 0) {
-            snprintf(s->down_script, sizeof(s->down_script), "%s", downscript);
-            snprintf(s->down_script_arg, sizeof(s->down_script_arg), "%s", ifname);
+        if (tap->has_ifname) {
+            pstrcpy(ifname, sizeof ifname, tap->ifname);
+        } else {
+            ifname[0] = '\0';
         }
-    }
-
-    if (tap->has_vhost ? tap->vhost :
-        tap->has_vhostfd || (tap->has_vhostforce && tap->vhostforce)) {
-        int vhostfd;
 
-        if (tap->has_vhostfd) {
-            vhostfd = monitor_handle_fd_param(cur_mon, tap->vhostfd);
-            if (vhostfd == -1) {
+        for (i = 0; i < queues; i++) {
+            fd = net_tap_init(tap, &vnet_hdr, i >= 1 ? "no" : script,
+                              ifname, sizeof ifname, queues > 1);
+            if (fd == -1) {
                 return -1;
             }
-        } else {
-            vhostfd = -1;
-        }
 
-        s->vhost_net = vhost_net_init(&s->nc, vhostfd,
-                                      tap->has_vhostforce && tap->vhostforce);
-        if (!s->vhost_net) {
-            error_report("vhost-net requested but could not be initialized");
-            return -1;
+            if (queues > 1 && i == 0 && !tap->has_ifname) {
+                if (tap_fd_get_ifname(fd, ifname)) {
+                    error_report("Fail to get ifname");
+                    return -1;
+                }
+            }
+
+            if (net_init_tap_one(tap, peer, "tap", name, ifname,
+                                 i >= 1 ? "no" : script,
+                                 i >= 1 ? "no" : downscript,
+                                 vhostfdname, vnet_hdr, fd)) {
+                return -1;
+            }
         }
-    } else if (tap->has_vhostfd) {
-        error_report("vhostfd= is not valid without vhost");
-        return -1;
     }
 
     return 0;
@@ -722,3 +825,38 @@ VHostNetState *tap_get_vhost_net(NetClientState *nc)
     assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
     return s->vhost_net;
 }
+
+int tap_enable(NetClientState *nc)
+{
+    TAPState *s = DO_UPCAST(TAPState, nc, nc);
+    int ret;
+
+    if (s->enabled) {
+        return 0;
+    } else {
+        ret = tap_fd_enable(s->fd);
+        if (ret == 0) {
+            s->enabled = true;
+            tap_update_fd_handler(s);
+        }
+        return ret;
+    }
+}
+
+int tap_disable(NetClientState *nc)
+{
+    TAPState *s = DO_UPCAST(TAPState, nc, nc);
+    int ret;
+
+    if (s->enabled == 0) {
+        return 0;
+    } else {
+        ret = tap_fd_disable(s->fd);
+        if (ret == 0) {
+            qemu_purge_queued_packets(nc);
+            s->enabled = false;
+            tap_update_fd_handler(s);
+        }
+        return ret;
+    }
+}
diff --git a/net/tap.h b/net/tap_int.h
index d44d83a..86bb224 100644
--- a/net/tap.h
+++ b/net/tap_int.h
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef QEMU_NET_TAP_H
-#define QEMU_NET_TAP_H
+#ifndef QEMU_TAP_H
+#define QEMU_TAP_H
 
 #include "qemu-common.h"
 #include "qapi-types.h"
@@ -32,27 +32,19 @@
 #define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
 #define DEFAULT_NETWORK_DOWN_SCRIPT "/etc/qemu-ifdown"
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required);
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required);
 
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen);
 
-int tap_has_ufo(NetClientState *nc);
-int tap_has_vnet_hdr(NetClientState *nc);
-int tap_has_vnet_hdr_len(NetClientState *nc, int len);
-void tap_using_vnet_hdr(NetClientState *nc, int using_vnet_hdr);
-void tap_set_offload(NetClientState *nc, int csum, int tso4, int tso6, int ecn, int ufo);
-void tap_set_vnet_hdr_len(NetClientState *nc, int len);
-
 int tap_set_sndbuf(int fd, const NetdevTapOptions *tap);
 int tap_probe_vnet_hdr(int fd);
 int tap_probe_vnet_hdr_len(int fd, int len);
 int tap_probe_has_ufo(int fd);
 void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo);
 void tap_fd_set_vnet_hdr_len(int fd, int len);
+int tap_fd_enable(int fd);
+int tap_fd_disable(int fd);
+int tap_fd_get_ifname(int fd, char *ifname);
 
-int tap_get_fd(NetClientState *nc);
-
-struct vhost_net;
-struct vhost_net *tap_get_vhost_net(NetClientState *nc);
-
-#endif /* QEMU_NET_TAP_H */
+#endif /* QEMU_TAP_H */
diff --git a/net/util.c b/net/util.c
index 1e9afbc..7e95076 100644
--- a/net/util.c
+++ b/net/util.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "net/util.h"
+#include "util.h"
 #include <errno.h>
 #include <stdlib.h>
 
diff --git a/net/vde.c b/net/vde.c
index 275bda9..4dea32d 100644
--- a/net/vde.c
+++ b/net/vde.c
@@ -25,11 +25,11 @@
 
 #include <libvdeplug.h>
 
-#include "net.h"
+#include "net/net.h"
 #include "clients.h"
-#include "qemu-char.h"
 #include "qemu-common.h"
-#include "qemu-option.h"
+#include "qemu/option.h"
+#include "qemu/main-loop.h"
 
 typedef struct VDEState {
     NetClientState nc;
diff --git a/os-posix.c b/os-posix.c
index 488e480..5c64518 100644
--- a/os-posix.c
+++ b/os-posix.c
@@ -36,7 +36,7 @@
 
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "net/slirp.h"
 #include "qemu-options.h"
 
diff --git a/os-win32.c b/os-win32.c
index 13892ba..9673a81 100644
--- a/os-win32.c
+++ b/os-win32.c
@@ -30,7 +30,7 @@
 #include <errno.h>
 #include <sys/time.h>
 #include "config-host.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "qemu-options.h"
 
 /***********************************************************/
diff --git a/page_cache.c b/page_cache.c
index 0294f7e..ba5640b 100644
--- a/page_cache.c
+++ b/page_cache.c
@@ -24,7 +24,7 @@
 #include <strings.h>
 
 #include "qemu-common.h"
-#include "qemu/page_cache.h"
+#include "migration/page_cache.h"
 
 #ifdef DEBUG_CACHE
 #define DPRINTF(fmt, ...) \
diff --git a/pc-bios/README b/pc-bios/README
index 3037130..eff3de7 100644
--- a/pc-bios/README
+++ b/pc-bios/README
@@ -17,7 +17,7 @@
 - SLOF (Slimline Open Firmware) is a free IEEE 1275 Open Firmware
   implementation for certain IBM POWER hardware.  The sources are at
   https://github.com/dgibson/SLOF, and the image currently in qemu is
-  built from git tag qemu-slof-20120731.
+  built from git tag qemu-slof-20121018.
 
 - sgabios (the Serial Graphics Adapter option ROM) provides a means for
   legacy x86 software to communicate with an attached serial console as
diff --git a/pc-bios/acpi-dsdt.aml b/pc-bios/acpi-dsdt.aml
index bb3dd83..75dfd1e 100644
--- a/pc-bios/acpi-dsdt.aml
+++ b/pc-bios/acpi-dsdt.aml
diff --git a/pc-bios/q35-acpi-dsdt.aml b/pc-bios/q35-acpi-dsdt.aml
new file mode 100644
index 0000000..cf7b085
--- /dev/null
+++ b/pc-bios/q35-acpi-dsdt.aml
diff --git a/pci-ids.txt b/pci-ids.txt
deleted file mode 100644
index 73125a8..0000000
--- a/pci-ids.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-
-PCI IDs for qemu
-================
-
-Red Hat, Inc. donates a part of its device ID range to qemu, to be used for
-virtual devices.  The vendor ID is 1af4 (formerly Qumranet ID).
-
-The 1000 -> 10ff device ID range is used for VirtIO devices.
-
-The 1100 device ID is used as PCI Subsystem ID for existing hardware
-devices emulated by qemu.
-
-All other device IDs are reserved.
-
-
-VirtIO Device IDs
------------------
-
-1af4:1000  network device
-1af4:1001  block device
-1af4:1002  balloon device
-1af4:1003  console device
-
-1af4:1004  Reserved.
-   to      Contact Gerd Hoffmann <kraxel@redhat.com> to get a
-1af4:10ef  device ID assigned for your new virtio device.
-
-1af4:10f0  Available for experimental usage without registration.  Must get
-   to      official ID when the code leaves the test lab (i.e. when seeking
-1af4:10ff  upstream merge or shipping a distro/product) to avoid conflicts.
-
diff --git a/qapi-schema.json b/qapi-schema.json
index 5dfa052..7275b5d 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -325,6 +325,73 @@
 { 'command': 'query-chardev', 'returns': ['ChardevInfo'] }
 
 ##
+# @DataFormat:
+#
+# An enumeration of data format.
+#
+# @utf8: Data is a UTF-8 string (RFC 3629)
+#
+# @base64: Data is Base64 encoded binary (RFC 3548)
+#
+# Since: 1.4
+##
+{ 'enum': 'DataFormat'
+  'data': [ 'utf8', 'base64' ] }
+
+##
+# @ringbuf-write:
+#
+# Write to a ring buffer character device.
+#
+# @device: the ring buffer character device name
+#
+# @data: data to write
+#
+# @format: #optional data encoding (default 'utf8').
+#          - base64: data must be base64 encoded text.  Its binary
+#            decoding gets written.
+#            Bug: invalid base64 is currently not rejected.
+#            Whitespace *is* invalid.
+#          - utf8: data's UTF-8 encoding is written
+#          - data itself is always Unicode regardless of format, like
+#            any other string.
+#
+# Returns: Nothing on success
+#
+# Since: 1.4
+##
+{ 'command': 'ringbuf-write',
+  'data': {'device': 'str', 'data': 'str',
+           '*format': 'DataFormat'} }
+
+##
+# @ringbuf-read:
+#
+# Read from a ring buffer character device.
+#
+# @device: the ring buffer character device name
+#
+# @size: how many bytes to read at most
+#
+# @format: #optional data encoding (default 'utf8').
+#          - base64: the data read is returned in base64 encoding.
+#          - utf8: the data read is interpreted as UTF-8.
+#            Bug: can screw up when the buffer contains invalid UTF-8
+#            sequences, NUL characters, after the ring buffer lost
+#            data, and when reading stops because the size limit is
+#            reached.
+#          - The return value is always Unicode regardless of format,
+#            like any other string.
+#
+# Returns: data read from the device
+#
+# Since: 1.4
+##
+{ 'command': 'ringbuf-read',
+  'data': {'device': 'str', 'size': 'int', '*format': 'DataFormat'},
+  'returns': 'str' }
+
+##
 # @CommandInfo:
 #
 # Information about a QMP command
@@ -667,10 +734,12 @@
 #
 # @count: number of dirty bytes according to the dirty bitmap
 #
+# @granularity: granularity of the dirty bitmap in bytes (since 1.4)
+#
 # Since: 1.3
 ##
 { 'type': 'BlockDirtyInfo',
-  'data': {'count': 'int'} }
+  'data': {'count': 'int', 'granularity': 'int'} }
 
 ##
 # @BlockInfo:
@@ -977,28 +1046,10 @@
 #
 # @actual: the number of bytes the balloon currently contains
 #
-# @mem_swapped_in: #optional number of pages swapped in within the guest
-#
-# @mem_swapped_out: #optional number of pages swapped out within the guest
-#
-# @major_page_faults: #optional number of major page faults within the guest
-#
-# @minor_page_faults: #optional number of minor page faults within the guest
-#
-# @free_mem: #optional amount of memory (in bytes) free in the guest
-#
-# @total_mem: #optional amount of memory (in bytes) visible to the guest
-#
 # Since: 0.14.0
 #
-# Notes: all current versions of QEMU do not fill out optional information in
-#        this structure.
 ##
-{ 'type': 'BalloonInfo',
-  'data': {'actual': 'int', '*mem_swapped_in': 'int',
-           '*mem_swapped_out': 'int', '*major_page_faults': 'int',
-           '*minor_page_faults': 'int', '*free_mem': 'int',
-           '*total_mem': 'int'} }
+{ 'type': 'BalloonInfo', 'data': {'actual': 'int' } }
 
 ##
 # @query-balloon:
@@ -1634,6 +1685,14 @@
 #        (all the disk, only the sectors allocated in the topmost image, or
 #        only new I/O).
 #
+# @granularity: #optional granularity of the dirty bitmap, default is 64K
+#               if the image format doesn't have clusters, 4K if the clusters
+#               are smaller than that, else the cluster size.  Must be a
+#               power of 2 between 512 and 64M (since 1.4).
+#
+# @buf-size: #optional maximum amount of data in flight from source to
+#            target (since 1.4).
+#
 # @on-source-error: #optional the action to take on an error on the source,
 #                   default 'report'.  'stop' and 'enospc' can only be used
 #                   if the block device supports io-status (see BlockInfo).
@@ -1650,7 +1709,8 @@
 { 'command': 'drive-mirror',
   'data': { 'device': 'str', 'target': 'str', '*format': 'str',
             'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
-            '*speed': 'int', '*on-source-error': 'BlockdevOnError',
+            '*speed': 'int', '*granularity': 'uint32',
+            '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
             '*on-target-error': 'BlockdevOnError' } }
 
 ##
@@ -2466,6 +2526,7 @@
   'data': {
     '*ifname':     'str',
     '*fd':         'str',
+    '*fds':        'str',
     '*script':     'str',
     '*downscript': 'str',
     '*helper':     'str',
@@ -2473,7 +2534,9 @@
     '*vnet_hdr':   'bool',
     '*vhost':      'bool',
     '*vhostfd':    'str',
-    '*vhostforce': 'bool' } }
+    '*vhostfds':   'str',
+    '*vhostforce': 'bool',
+    '*queues':     'uint32'} }
 
 ##
 # @NetdevSocketOptions
@@ -3017,3 +3080,107 @@
 # Since: 1.3.0
 ##
 { 'command': 'nbd-server-stop' }
+
+##
+# @ChardevFile:
+#
+# Configuration info for file chardevs.
+#
+# @in:  #optional The name of the input file
+# @out: The name of the output file
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevFile', 'data': { '*in' : 'str',
+                                   'out' : 'str' } }
+
+##
+# @ChardevHostdev:
+#
+# Configuration info for device chardevs.
+#
+# @device: The name of the special file for the device,
+#          i.e. /dev/ttyS0 on Unix or COM1: on Windows
+# @type: What kind of device this is.
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevHostdev', 'data': { 'device' : 'str' } }
+
+##
+# @ChardevSocket:
+#
+# Configuration info for socket chardevs.
+#
+# @addr: socket address to listen on (server=true)
+#        or connect to (server=false)
+# @server: #optional create server socket (default: true)
+# @wait: #optional wait for connect (not used for server
+#        sockets, default: false)
+# @nodelay: #optional set TCP_NODELAY socket option (default: false)
+# @telnet: #optional enable telnet protocol (default: false)
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevSocket', 'data': { 'addr'     : 'SocketAddress',
+                                     '*server'  : 'bool',
+                                     '*wait'    : 'bool',
+                                     '*nodelay' : 'bool',
+                                     '*telnet'  : 'bool' } }
+
+##
+# @ChardevBackend:
+#
+# Configuration info for the new chardev backend.
+#
+# Since: 1.4
+##
+{ 'type': 'ChardevDummy', 'data': { } }
+
+{ 'union': 'ChardevBackend', 'data': { 'file'   : 'ChardevFile',
+                                       'serial' : 'ChardevHostdev',
+                                       'parallel': 'ChardevHostdev',
+                                       'socket' : 'ChardevSocket',
+                                       'pty'    : 'ChardevDummy',
+                                       'null'   : 'ChardevDummy' } }
+
+##
+# @ChardevReturn:
+#
+# Return info about the chardev backend just created.
+#
+# @pty: #optional name of the slave pseudoterminal device, present if
+#       and only if a chardev of type 'pty' was created
+#
+# Since: 1.4
+##
+{ 'type' : 'ChardevReturn', 'data': { '*pty' : 'str' } }
+
+##
+# @chardev-add:
+#
+# Add a character device backend
+#
+# @id: the chardev's ID, must be unique
+# @backend: backend type and parameters
+#
+# Returns: ChardevReturn.
+#
+# Since: 1.4
+##
+{ 'command': 'chardev-add', 'data': {'id'      : 'str',
+                                     'backend' : 'ChardevBackend' },
+  'returns': 'ChardevReturn' }
+
+##
+# @chardev-remove:
+#
+# Remove a character device backend
+#
+# @id: the chardev's ID, must exist and not be in use
+#
+# Returns: Nothing on success
+#
+# Since: 1.4
+##
+{ 'command': 'chardev-remove', 'data': {'id': 'str'} }
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index f9bd3b9..1f9c973 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,5 +1,5 @@
-qapi-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
-qapi-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
-qapi-obj-y += string-input-visitor.o string-output-visitor.o
+util-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
+util-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
+util-obj-y += string-input-visitor.o string-output-visitor.o
 
-common-obj-y += opts-visitor.o
+util-obj-y += opts-visitor.o
diff --git a/qapi/opts-visitor.c b/qapi/opts-visitor.c
index e048b6c..174bd8b 100644
--- a/qapi/opts-visitor.c
+++ b/qapi/opts-visitor.c
@@ -10,10 +10,12 @@
  *
  */
 
-#include "opts-visitor.h"
-#include "qemu-queue.h"
-#include "qemu-option-internal.h"
-#include "qapi-visit-impl.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/opts-visitor.h"
+#include "qemu/queue.h"
+#include "qemu/option_int.h"
+#include "qapi/visitor-impl.h"
 
 
 struct OptsVisitor
diff --git a/qapi/qapi-dealloc-visitor.c b/qapi/qapi-dealloc-visitor.c
index 75214e7..1334de3 100644
--- a/qapi/qapi-dealloc-visitor.c
+++ b/qapi/qapi-dealloc-visitor.c
@@ -11,10 +11,11 @@
  *
  */
 
-#include "qapi-dealloc-visitor.h"
-#include "qemu-queue.h"
+#include "qapi/dealloc-visitor.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
+#include "qapi/visitor-impl.h"
 
 typedef struct StackEntry
 {
diff --git a/qapi/qapi-types-core.h b/qapi/qapi-types-core.h
deleted file mode 100644
index f781fc3..0000000
--- a/qapi/qapi-types-core.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Core Definitions for QAPI-generated Types
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef QAPI_TYPES_CORE_H
-#define QAPI_TYPES_CORE_H
-
-#include "qemu-common.h"
-#include "error.h"
-#include "qerror.h"
-
-#endif
diff --git a/qapi/qapi-visit-core.c b/qapi/qapi-visit-core.c
index 7a82b63..401ee6e 100644
--- a/qapi/qapi-visit-core.c
+++ b/qapi/qapi-visit-core.c
@@ -11,8 +11,10 @@
  *
  */
 
-#include "qapi/qapi-visit-core.h"
-#include "qapi/qapi-visit-impl.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/visitor.h"
+#include "qapi/visitor-impl.h"
 
 void visit_start_handle(Visitor *v, void **obj, const char *kind,
                         const char *name, Error **errp)
diff --git a/qapi/qapi-visit-impl.h b/qapi/qapi-visit-impl.h
deleted file mode 100644
index 0f3a189..0000000
--- a/qapi/qapi-visit-impl.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Core Definitions for QAPI Visitor implementations
- *
- * Copyright (C) 2012 Red Hat, Inc.
- *
- * Author: Paolo Bonizni <pbonzini@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-#ifndef QAPI_VISITOR_IMPL_H
-#define QAPI_VISITOR_IMPL_H
-
-#include "qapi/qapi-types-core.h"
-#include "qapi/qapi-visit-core.h"
-
-void input_type_enum(Visitor *v, int *obj, const char *strings[],
-                     const char *kind, const char *name, Error **errp);
-void output_type_enum(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name, Error **errp);
-
-#endif
diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c
index 4085994..921de33 100644
--- a/qapi/qmp-dispatch.c
+++ b/qapi/qmp-dispatch.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qemu-objects.h"
-#include "qapi/qmp-core.h"
-#include "json-parser.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/dispatch.h"
+#include "qapi/qmp/json-parser.h"
 #include "qapi-types.h"
-#include "error.h"
-#include "qerror.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 
 static QDict *qmp_dispatch_check_obj(const QObject *request, Error **errp)
 {
diff --git a/qapi/qmp-input-visitor.c b/qapi/qmp-input-visitor.c
index 107d8d3..67fb127 100644
--- a/qapi/qmp-input-visitor.c
+++ b/qapi/qmp-input-visitor.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qmp-input-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qemu-queue.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qemu-objects.h"
-#include "qerror.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/qerror.h"
 
 #define QIV_STACK_SIZE 1024
 
diff --git a/qapi/qmp-output-visitor.c b/qapi/qmp-output-visitor.c
index 2bce9d5..74a5684 100644
--- a/qapi/qmp-output-visitor.c
+++ b/qapi/qmp-output-visitor.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qmp-output-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qemu-queue.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
-#include "qemu-objects.h"
-#include "qerror.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/qerror.h"
 
 typedef struct QStackEntry
 {
diff --git a/qapi/qmp-registry.c b/qapi/qmp-registry.c
index 5414613..28bbbe8 100644
--- a/qapi/qmp-registry.c
+++ b/qapi/qmp-registry.c
@@ -12,7 +12,9 @@
  *
  */
 
-#include "qapi/qmp-core.h"
+#include <glib.h>
+#include <string.h>
+#include "qapi/qmp/dispatch.h"
 
 static QTAILQ_HEAD(QmpCommandList, QmpCommand) qmp_commands =
     QTAILQ_HEAD_INITIALIZER(qmp_commands);
@@ -90,7 +92,7 @@ char **qmp_get_command_list(void)
     list_head = list = g_malloc0(count * sizeof(char *));
 
     QTAILQ_FOREACH(cmd, &qmp_commands, node) {
-        *list = strdup(cmd->name);
+        *list = g_strdup(cmd->name);
         list++;
     }
 
diff --git a/qapi/string-input-visitor.c b/qapi/string-input-visitor.c
index 497eb9a..8f1bc41 100644
--- a/qapi/string-input-visitor.c
+++ b/qapi/string-input-visitor.c
@@ -11,9 +11,9 @@
  */
 
 #include "qemu-common.h"
-#include "string-input-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qerror.h"
+#include "qapi/string-input-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qapi/qmp/qerror.h"
 
 struct StringInputVisitor
 {
diff --git a/qapi/string-output-visitor.c b/qapi/string-output-visitor.c
index 34e525e..921653d 100644
--- a/qapi/string-output-visitor.c
+++ b/qapi/string-output-visitor.c
@@ -11,9 +11,9 @@
  */
 
 #include "qemu-common.h"
-#include "string-output-visitor.h"
-#include "qapi/qapi-visit-impl.h"
-#include "qerror.h"
+#include "qapi/string-output-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qapi/qmp/qerror.h"
 
 struct StringOutputVisitor
 {
diff --git a/qemu-bridge-helper.c b/qemu-bridge-helper.c
index 652eec9..287bfd5 100644
--- a/qemu-bridge-helper.c
+++ b/qemu-bridge-helper.c
@@ -39,7 +39,7 @@
 #include <linux/if_bridge.h>
 #endif
 
-#include "qemu-queue.h"
+#include "qemu/queue.h"
 
 #include "net/tap-linux.h"
 
diff --git a/qemu-char.c b/qemu-char.c
index 242b799..e4b0f53 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -22,12 +22,11 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "net.h"
-#include "monitor.h"
-#include "console.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "hw/usb.h"
 #include "hw/baum.h"
 #include "hw/msmouse.h"
@@ -95,7 +94,7 @@
 #endif
 #endif
 
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "ui/qemu-spice.h"
 
 #define READ_BUF_LEN 4096
@@ -773,6 +772,10 @@ static CharDriverState *qemu_chr_open_stdio(QemuOpts *opts)
     if (stdio_nb_clients >= STDIO_MAX_CLIENTS) {
         return NULL;
     }
+    if (is_daemonized()) {
+        error_report("cannot use stdio with -daemonize");
+        return NULL;
+    }
     if (stdio_nb_clients == 0) {
         old_fd0_flags = fcntl(0, F_GETFL);
         tcgetattr (0, &oldtty);
@@ -853,6 +856,8 @@ static void cfmakeraw (struct termios *termios_p)
     || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
     || defined(__GLIBC__)
 
+#define HAVE_CHARDEV_TTY 1
+
 typedef struct {
     int fd;
     int connected;
@@ -981,6 +986,7 @@ static CharDriverState *qemu_chr_open_pty(QemuOpts *opts)
     CharDriverState *chr;
     PtyCharDriver *s;
     struct termios tty;
+    const char *label;
     int master_fd, slave_fd, len;
 #if defined(__OpenBSD__) || defined(__DragonFly__)
     char pty_name[PATH_MAX];
@@ -1006,7 +1012,13 @@ static CharDriverState *qemu_chr_open_pty(QemuOpts *opts)
     chr->filename = g_malloc(len);
     snprintf(chr->filename, len, "pty:%s", q_ptsname(master_fd));
     qemu_opt_set(opts, "path", q_ptsname(master_fd));
-    fprintf(stderr, "char device redirected to %s\n", q_ptsname(master_fd));
+
+    label = qemu_opts_id(opts);
+    fprintf(stderr, "char device redirected to %s%s%s%s\n",
+            q_ptsname(master_fd),
+            label ? " (label " : "",
+            label ? label      : "",
+            label ? ")"        : "");
 
     s = g_malloc0(sizeof(PtyCharDriver));
     chr->opaque = s;
@@ -1218,30 +1230,34 @@ static void qemu_chr_close_tty(CharDriverState *chr)
     }
 }
 
-static CharDriverState *qemu_chr_open_tty(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_tty_fd(int fd)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
-    int fd;
 
-    TFR(fd = qemu_open(filename, O_RDWR | O_NONBLOCK));
-    if (fd < 0) {
-        return NULL;
-    }
     tty_serial_init(fd, 115200, 'N', 8, 1);
     chr = qemu_chr_open_fd(fd, fd);
     chr->chr_ioctl = tty_serial_ioctl;
     chr->chr_close = qemu_chr_close_tty;
     return chr;
 }
-#else  /* ! __linux__ && ! __sun__ */
-static CharDriverState *qemu_chr_open_pty(QemuOpts *opts)
+
+static CharDriverState *qemu_chr_open_tty(QemuOpts *opts)
 {
-    return NULL;
+    const char *filename = qemu_opt_get(opts, "path");
+    int fd;
+
+    TFR(fd = qemu_open(filename, O_RDWR | O_NONBLOCK));
+    if (fd < 0) {
+        return NULL;
+    }
+    return qemu_chr_open_tty_fd(fd);
 }
 #endif /* __linux__ || __sun__ */
 
 #if defined(__linux__)
+
+#define HAVE_CHARDEV_PARPORT 1
+
 typedef struct {
     int fd;
     int mode;
@@ -1351,17 +1367,10 @@ static void pp_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_pp_fd(int fd)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
     ParallelCharDriver *drv;
-    int fd;
-
-    TFR(fd = qemu_open(filename, O_RDWR));
-    if (fd < 0) {
-        return NULL;
-    }
 
     if (ioctl(fd, PPCLAIM) < 0) {
         close(fd);
@@ -1385,6 +1394,9 @@ static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
 #endif /* __linux__ */
 
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
+
+#define HAVE_CHARDEV_PARPORT 1
+
 static int pp_ioctl(CharDriverState *chr, int cmd, void *arg)
 {
     int fd = (int)(intptr_t)chr->opaque;
@@ -1422,16 +1434,9 @@ static int pp_ioctl(CharDriverState *chr, int cmd, void *arg)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_pp_fd(int fd)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
-    int fd;
-
-    fd = qemu_open(filename, O_RDWR);
-    if (fd < 0) {
-        return NULL;
-    }
 
     chr = g_malloc0(sizeof(CharDriverState));
     chr->opaque = (void *)(intptr_t)fd;
@@ -1653,9 +1658,8 @@ static int win_chr_poll(void *opaque)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_win(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_win_path(const char *filename)
 {
-    const char *filename = qemu_opt_get(opts, "path");
     CharDriverState *chr;
     WinCharState *s;
 
@@ -1674,6 +1678,11 @@ static CharDriverState *qemu_chr_open_win(QemuOpts *opts)
     return chr;
 }
 
+static CharDriverState *qemu_chr_open_win(QemuOpts *opts)
+{
+    return qemu_chr_open_win_path(qemu_opt_get(opts, "path"));
+}
+
 static int win_chr_pipe_poll(void *opaque)
 {
     CharDriverState *chr = opaque;
@@ -2429,10 +2438,88 @@ static void tcp_chr_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
+static CharDriverState *qemu_chr_open_socket_fd(int fd, bool do_nodelay,
+                                                bool is_listen, bool is_telnet,
+                                                bool is_waitconnect,
+                                                Error **errp)
 {
     CharDriverState *chr = NULL;
     TCPCharDriver *s = NULL;
+    char host[NI_MAXHOST], serv[NI_MAXSERV];
+    const char *left = "", *right = "";
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    memset(&ss, 0, ss_len);
+    if (getsockname(fd, (struct sockaddr *) &ss, &ss_len) != 0) {
+        error_setg(errp, "getsockname: %s", strerror(errno));
+        return NULL;
+    }
+
+    chr = g_malloc0(sizeof(CharDriverState));
+    s = g_malloc0(sizeof(TCPCharDriver));
+
+    s->connected = 0;
+    s->fd = -1;
+    s->listen_fd = -1;
+    s->msgfd = -1;
+
+    chr->filename = g_malloc(256);
+    switch (ss.ss_family) {
+#ifndef _WIN32
+    case AF_UNIX:
+        s->is_unix = 1;
+        snprintf(chr->filename, 256, "unix:%s%s",
+                 ((struct sockaddr_un *)(&ss))->sun_path,
+                 is_listen ? ",server" : "");
+        break;
+#endif
+    case AF_INET6:
+        left  = "[";
+        right = "]";
+        /* fall through */
+    case AF_INET:
+        s->do_nodelay = do_nodelay;
+        getnameinfo((struct sockaddr *) &ss, ss_len, host, sizeof(host),
+                    serv, sizeof(serv), NI_NUMERICHOST | NI_NUMERICSERV);
+        snprintf(chr->filename, 256, "%s:%s:%s%s%s%s",
+                 is_telnet ? "telnet" : "tcp",
+                 left, host, right, serv,
+                 is_listen ? ",server" : "");
+        break;
+    }
+
+    chr->opaque = s;
+    chr->chr_write = tcp_chr_write;
+    chr->chr_close = tcp_chr_close;
+    chr->get_msgfd = tcp_get_msgfd;
+    chr->chr_add_client = tcp_chr_add_client;
+
+    if (is_listen) {
+        s->listen_fd = fd;
+        qemu_set_fd_handler2(s->listen_fd, NULL, tcp_chr_accept, NULL, chr);
+        if (is_telnet) {
+            s->do_telnetopt = 1;
+        }
+    } else {
+        s->connected = 1;
+        s->fd = fd;
+        socket_set_nodelay(fd);
+        tcp_chr_connect(chr);
+    }
+
+    if (is_listen && is_waitconnect) {
+        printf("QEMU waiting for connection on: %s\n",
+               chr->filename);
+        tcp_chr_accept(chr);
+        socket_set_nonblock(s->listen_fd);
+    }
+    return chr;
+}
+
+static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
+{
+    CharDriverState *chr = NULL;
     Error *local_err = NULL;
     int fd = -1;
     int is_listen;
@@ -2449,9 +2536,6 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
     if (!is_listen)
         is_waitconnect = 0;
 
-    chr = g_malloc0(sizeof(CharDriverState));
-    s = g_malloc0(sizeof(TCPCharDriver));
-
     if (is_unix) {
         if (is_listen) {
             fd = unix_listen_opts(opts, &local_err);
@@ -2472,56 +2556,14 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
     if (!is_waitconnect)
         socket_set_nonblock(fd);
 
-    s->connected = 0;
-    s->fd = -1;
-    s->listen_fd = -1;
-    s->msgfd = -1;
-    s->is_unix = is_unix;
-    s->do_nodelay = do_nodelay && !is_unix;
-
-    chr->opaque = s;
-    chr->chr_write = tcp_chr_write;
-    chr->chr_close = tcp_chr_close;
-    chr->get_msgfd = tcp_get_msgfd;
-    chr->chr_add_client = tcp_chr_add_client;
-
-    if (is_listen) {
-        s->listen_fd = fd;
-        qemu_set_fd_handler2(s->listen_fd, NULL, tcp_chr_accept, NULL, chr);
-        if (is_telnet)
-            s->do_telnetopt = 1;
-
-    } else {
-        s->connected = 1;
-        s->fd = fd;
-        socket_set_nodelay(fd);
-        tcp_chr_connect(chr);
-    }
-
-    /* for "info chardev" monitor command */
-    chr->filename = g_malloc(256);
-    if (is_unix) {
-        snprintf(chr->filename, 256, "unix:%s%s",
-                 qemu_opt_get(opts, "path"),
-                 qemu_opt_get_bool(opts, "server", 0) ? ",server" : "");
-    } else if (is_telnet) {
-        snprintf(chr->filename, 256, "telnet:%s:%s%s",
-                 qemu_opt_get(opts, "host"), qemu_opt_get(opts, "port"),
-                 qemu_opt_get_bool(opts, "server", 0) ? ",server" : "");
-    } else {
-        snprintf(chr->filename, 256, "tcp:%s:%s%s",
-                 qemu_opt_get(opts, "host"), qemu_opt_get(opts, "port"),
-                 qemu_opt_get_bool(opts, "server", 0) ? ",server" : "");
-    }
-
-    if (is_listen && is_waitconnect) {
-        printf("QEMU waiting for connection on: %s\n",
-               chr->filename);
-        tcp_chr_accept(chr);
-        socket_set_nonblock(s->listen_fd);
+    chr = qemu_chr_open_socket_fd(fd, do_nodelay, is_listen, is_telnet,
+                                  is_waitconnect, &local_err);
+    if (error_is_set(&local_err)) {
+        goto fail;
     }
     return chr;
 
+
  fail:
     if (local_err) {
         qerror_report_err(local_err);
@@ -2530,8 +2572,10 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
     if (fd >= 0) {
         closesocket(fd);
     }
-    g_free(s);
-    g_free(chr);
+    if (chr) {
+        g_free(chr->opaque);
+        g_free(chr);
+    }
     return NULL;
 }
 
@@ -2599,6 +2643,191 @@ size_t qemu_chr_mem_osize(const CharDriverState *chr)
     return d->outbuf_size;
 }
 
+/*********************************************************/
+/* Ring buffer chardev */
+
+typedef struct {
+    size_t size;
+    size_t prod;
+    size_t cons;
+    uint8_t *cbuf;
+} RingBufCharDriver;
+
+static size_t ringbuf_count(const CharDriverState *chr)
+{
+    const RingBufCharDriver *d = chr->opaque;
+
+    return d->prod - d->cons;
+}
+
+static int ringbuf_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
+{
+    RingBufCharDriver *d = chr->opaque;
+    int i;
+
+    if (!buf || (len < 0)) {
+        return -1;
+    }
+
+    for (i = 0; i < len; i++ ) {
+        d->cbuf[d->prod++ & (d->size - 1)] = buf[i];
+        if (d->prod - d->cons > d->size) {
+            d->cons = d->prod - d->size;
+        }
+    }
+
+    return 0;
+}
+
+static int ringbuf_chr_read(CharDriverState *chr, uint8_t *buf, int len)
+{
+    RingBufCharDriver *d = chr->opaque;
+    int i;
+
+    for (i = 0; i < len && d->cons != d->prod; i++) {
+        buf[i] = d->cbuf[d->cons++ & (d->size - 1)];
+    }
+
+    return i;
+}
+
+static void ringbuf_chr_close(struct CharDriverState *chr)
+{
+    RingBufCharDriver *d = chr->opaque;
+
+    g_free(d->cbuf);
+    g_free(d);
+    chr->opaque = NULL;
+}
+
+static CharDriverState *qemu_chr_open_ringbuf(QemuOpts *opts)
+{
+    CharDriverState *chr;
+    RingBufCharDriver *d;
+
+    chr = g_malloc0(sizeof(CharDriverState));
+    d = g_malloc(sizeof(*d));
+
+    d->size = qemu_opt_get_size(opts, "size", 0);
+    if (d->size == 0) {
+        d->size = 65536;
+    }
+
+    /* The size must be power of 2 */
+    if (d->size & (d->size - 1)) {
+        error_report("size of ringbuf device must be power of two");
+        goto fail;
+    }
+
+    d->prod = 0;
+    d->cons = 0;
+    d->cbuf = g_malloc0(d->size);
+
+    chr->opaque = d;
+    chr->chr_write = ringbuf_chr_write;
+    chr->chr_close = ringbuf_chr_close;
+
+    return chr;
+
+fail:
+    g_free(d);
+    g_free(chr);
+    return NULL;
+}
+
+static bool chr_is_ringbuf(const CharDriverState *chr)
+{
+    return chr->chr_write == ringbuf_chr_write;
+}
+
+void qmp_ringbuf_write(const char *device, const char *data,
+                       bool has_format, enum DataFormat format,
+                       Error **errp)
+{
+    CharDriverState *chr;
+    const uint8_t *write_data;
+    int ret;
+    size_t write_count;
+
+    chr = qemu_chr_find(device);
+    if (!chr) {
+        error_setg(errp, "Device '%s' not found", device);
+        return;
+    }
+
+    if (!chr_is_ringbuf(chr)) {
+        error_setg(errp,"%s is not a ringbuf device", device);
+        return;
+    }
+
+    if (has_format && (format == DATA_FORMAT_BASE64)) {
+        write_data = g_base64_decode(data, &write_count);
+    } else {
+        write_data = (uint8_t *)data;
+        write_count = strlen(data);
+    }
+
+    ret = ringbuf_chr_write(chr, write_data, write_count);
+
+    if (write_data != (uint8_t *)data) {
+        g_free((void *)write_data);
+    }
+
+    if (ret < 0) {
+        error_setg(errp, "Failed to write to device %s", device);
+        return;
+    }
+}
+
+char *qmp_ringbuf_read(const char *device, int64_t size,
+                       bool has_format, enum DataFormat format,
+                       Error **errp)
+{
+    CharDriverState *chr;
+    uint8_t *read_data;
+    size_t count;
+    char *data;
+
+    chr = qemu_chr_find(device);
+    if (!chr) {
+        error_setg(errp, "Device '%s' not found", device);
+        return NULL;
+    }
+
+    if (!chr_is_ringbuf(chr)) {
+        error_setg(errp,"%s is not a ringbuf device", device);
+        return NULL;
+    }
+
+    if (size <= 0) {
+        error_setg(errp, "size must be greater than zero");
+        return NULL;
+    }
+
+    count = ringbuf_count(chr);
+    size = size > count ? count : size;
+    read_data = g_malloc(size + 1);
+
+    ringbuf_chr_read(chr, read_data, size);
+
+    if (has_format && (format == DATA_FORMAT_BASE64)) {
+        data = g_base64_encode(read_data, size);
+        g_free(read_data);
+    } else {
+        /*
+         * FIXME should read only complete, valid UTF-8 characters up
+         * to @size bytes.  Invalid sequences should be replaced by a
+         * suitable replacement character.  Except when (and only
+         * when) ring buffer lost characters since last read, initial
+         * continuation characters should be dropped.
+         */
+        read_data[size] = 0;
+        data = (char *)read_data;
+    }
+
+    return data;
+}
+
 QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename)
 {
     char host[65], port[33], width[8], height[8];
@@ -2727,6 +2956,22 @@ fail:
     return NULL;
 }
 
+#ifdef HAVE_CHARDEV_PARPORT
+
+static CharDriverState *qemu_chr_open_pp(QemuOpts *opts)
+{
+    const char *filename = qemu_opt_get(opts, "path");
+    int fd;
+
+    fd = qemu_open(filename, O_RDWR);
+    if (fd < 0) {
+        return NULL;
+    }
+    return qemu_chr_open_pp_fd(fd);
+}
+
+#endif
+
 static const struct {
     const char *name;
     CharDriverState *(*open)(QemuOpts *opts);
@@ -2736,6 +2981,7 @@ static const struct {
     { .name = "udp",       .open = qemu_chr_open_udp },
     { .name = "msmouse",   .open = qemu_chr_open_msmouse },
     { .name = "vc",        .open = text_console_init },
+    { .name = "memory",    .open = qemu_chr_open_ringbuf },
 #ifdef _WIN32
     { .name = "file",      .open = qemu_chr_open_win_file_out },
     { .name = "pipe",      .open = qemu_chr_open_win_pipe },
@@ -2745,57 +2991,60 @@ static const struct {
 #else
     { .name = "file",      .open = qemu_chr_open_file_out },
     { .name = "pipe",      .open = qemu_chr_open_pipe },
-    { .name = "pty",       .open = qemu_chr_open_pty },
     { .name = "stdio",     .open = qemu_chr_open_stdio },
 #endif
 #ifdef CONFIG_BRLAPI
     { .name = "braille",   .open = chr_baum_init },
 #endif
-#if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
-    || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
-    || defined(__FreeBSD_kernel__)
+#ifdef HAVE_CHARDEV_TTY
     { .name = "tty",       .open = qemu_chr_open_tty },
+    { .name = "serial",    .open = qemu_chr_open_tty },
+    { .name = "pty",       .open = qemu_chr_open_pty },
 #endif
-#if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__) \
-    || defined(__FreeBSD_kernel__)
+#ifdef HAVE_CHARDEV_PARPORT
+    { .name = "parallel",  .open = qemu_chr_open_pp },
     { .name = "parport",   .open = qemu_chr_open_pp },
 #endif
 #ifdef CONFIG_SPICE
     { .name = "spicevmc",     .open = qemu_chr_open_spice },
+#if SPICE_SERVER_VERSION >= 0x000c02
+    { .name = "spiceport",    .open = qemu_chr_open_spice_port },
+#endif
 #endif
 };
 
 CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
-                                    void (*init)(struct CharDriverState *s))
+                                    void (*init)(struct CharDriverState *s),
+                                    Error **errp)
 {
     CharDriverState *chr;
     int i;
 
     if (qemu_opts_id(opts) == NULL) {
-        fprintf(stderr, "chardev: no id specified\n");
-        return NULL;
+        error_setg(errp, "chardev: no id specified");
+        goto err;
     }
 
     if (qemu_opt_get(opts, "backend") == NULL) {
-        fprintf(stderr, "chardev: \"%s\" missing backend\n",
-                qemu_opts_id(opts));
-        return NULL;
+        error_setg(errp, "chardev: \"%s\" missing backend",
+                   qemu_opts_id(opts));
+        goto err;
     }
     for (i = 0; i < ARRAY_SIZE(backend_table); i++) {
         if (strcmp(backend_table[i].name, qemu_opt_get(opts, "backend")) == 0)
             break;
     }
     if (i == ARRAY_SIZE(backend_table)) {
-        fprintf(stderr, "chardev: backend \"%s\" not found\n",
-                qemu_opt_get(opts, "backend"));
-        return NULL;
+        error_setg(errp, "chardev: backend \"%s\" not found",
+                   qemu_opt_get(opts, "backend"));
+        goto err;
     }
 
     chr = backend_table[i].open(opts);
     if (!chr) {
-        fprintf(stderr, "chardev: opening backend \"%s\" failed\n",
-                qemu_opt_get(opts, "backend"));
-        return NULL;
+        error_setg(errp, "chardev: opening backend \"%s\" failed",
+                   qemu_opt_get(opts, "backend"));
+        goto err;
     }
 
     if (!chr->filename)
@@ -2816,7 +3065,12 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
         chr->avail_connections = 1;
     }
     chr->label = g_strdup(qemu_opts_id(opts));
+    chr->opts = opts;
     return chr;
+
+err:
+    qemu_opts_del(opts);
+    return NULL;
 }
 
 CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
@@ -2824,6 +3078,7 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*in
     const char *p;
     CharDriverState *chr;
     QemuOpts *opts;
+    Error *err = NULL;
 
     if (strstart(filename, "chardev:", &p)) {
         return qemu_chr_find(p);
@@ -2833,11 +3088,14 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*in
     if (!opts)
         return NULL;
 
-    chr = qemu_chr_new_from_opts(opts, init);
+    chr = qemu_chr_new_from_opts(opts, init, &err);
+    if (error_is_set(&err)) {
+        fprintf(stderr, "%s\n", error_get_pretty(err));
+        error_free(err);
+    }
     if (chr && qemu_opt_get_bool(opts, "mux", 0)) {
         monitor_init(chr, MONITOR_USE_READLINE);
     }
-    qemu_opts_del(opts);
     return chr;
 }
 
@@ -2865,10 +3123,14 @@ void qemu_chr_fe_close(struct CharDriverState *chr)
 void qemu_chr_delete(CharDriverState *chr)
 {
     QTAILQ_REMOVE(&chardevs, chr, next);
-    if (chr->chr_close)
+    if (chr->chr_close) {
         chr->chr_close(chr);
+    }
     g_free(chr->filename);
     g_free(chr->label);
+    if (chr->opts) {
+        qemu_opts_del(chr->opts);
+    }
     g_free(chr);
 }
 
@@ -2911,3 +3173,284 @@ CharDriverState *qemu_char_get_next_serial(void)
     return serial_hds[next_serial++];
 }
 
+QemuOptsList qemu_chardev_opts = {
+    .name = "chardev",
+    .implied_opt_name = "backend",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_chardev_opts.head),
+    .desc = {
+        {
+            .name = "backend",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "host",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "port",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "localaddr",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "localport",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "to",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "wait",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "server",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "delay",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "telnet",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "width",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "height",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "cols",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "rows",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "mux",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "signal",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "name",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "debug",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "size",
+            .type = QEMU_OPT_SIZE,
+        },
+        { /* end of list */ }
+    },
+};
+
+#ifdef _WIN32
+
+static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+{
+    HANDLE out;
+
+    if (file->in) {
+        error_setg(errp, "input file not supported");
+        return NULL;
+    }
+
+    out = CreateFile(file->out, GENERIC_WRITE, FILE_SHARE_READ, NULL,
+                     OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (out == INVALID_HANDLE_VALUE) {
+        error_setg(errp, "open %s failed", file->out);
+        return NULL;
+    }
+    return qemu_chr_open_win_file(out);
+}
+
+static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+                                                Error **errp)
+{
+    return qemu_chr_open_win_path(serial->device);
+}
+
+static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
+                                                  Error **errp)
+{
+    error_setg(errp, "character device backend type 'parallel' not supported");
+    return NULL;
+}
+
+#else /* WIN32 */
+
+static int qmp_chardev_open_file_source(char *src, int flags,
+                                        Error **errp)
+{
+    int fd = -1;
+
+    TFR(fd = qemu_open(src, flags, 0666));
+    if (fd == -1) {
+        error_setg(errp, "open %s: %s", src, strerror(errno));
+    }
+    return fd;
+}
+
+static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+{
+    int flags, in = -1, out = -1;
+
+    flags = O_WRONLY | O_TRUNC | O_CREAT | O_BINARY;
+    out = qmp_chardev_open_file_source(file->out, flags, errp);
+    if (error_is_set(errp)) {
+        return NULL;
+    }
+
+    if (file->in) {
+        flags = O_RDONLY;
+        in = qmp_chardev_open_file_source(file->in, flags, errp);
+        if (error_is_set(errp)) {
+            qemu_close(out);
+            return NULL;
+        }
+    }
+
+    return qemu_chr_open_fd(in, out);
+}
+
+static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+                                                Error **errp)
+{
+#ifdef HAVE_CHARDEV_TTY
+    int fd;
+
+    fd = qmp_chardev_open_file_source(serial->device, O_RDWR, errp);
+    if (error_is_set(errp)) {
+        return NULL;
+    }
+    socket_set_nonblock(fd);
+    return qemu_chr_open_tty_fd(fd);
+#else
+    error_setg(errp, "character device backend type 'serial' not supported");
+    return NULL;
+#endif
+}
+
+static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
+                                                  Error **errp)
+{
+#ifdef HAVE_CHARDEV_PARPORT
+    int fd;
+
+    fd = qmp_chardev_open_file_source(parallel->device, O_RDWR, errp);
+    if (error_is_set(errp)) {
+        return NULL;
+    }
+    return qemu_chr_open_pp_fd(fd);
+#else
+    error_setg(errp, "character device backend type 'parallel' not supported");
+    return NULL;
+#endif
+}
+
+#endif /* WIN32 */
+
+static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
+                                                Error **errp)
+{
+    SocketAddress *addr = sock->addr;
+    bool do_nodelay     = sock->has_nodelay ? sock->nodelay : false;
+    bool is_listen      = sock->has_server  ? sock->server  : true;
+    bool is_telnet      = sock->has_telnet  ? sock->telnet  : false;
+    bool is_waitconnect = sock->has_wait    ? sock->wait    : false;
+    int fd;
+
+    if (is_listen) {
+        fd = socket_listen(addr, errp);
+    } else {
+        fd = socket_connect(addr, errp, NULL, NULL);
+    }
+    if (error_is_set(errp)) {
+        return NULL;
+    }
+    return qemu_chr_open_socket_fd(fd, do_nodelay, is_listen,
+                                   is_telnet, is_waitconnect, errp);
+}
+
+ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
+                               Error **errp)
+{
+    ChardevReturn *ret = g_new0(ChardevReturn, 1);
+    CharDriverState *chr = NULL;
+
+    chr = qemu_chr_find(id);
+    if (chr) {
+        error_setg(errp, "Chardev '%s' already exists", id);
+        g_free(ret);
+        return NULL;
+    }
+
+    switch (backend->kind) {
+    case CHARDEV_BACKEND_KIND_FILE:
+        chr = qmp_chardev_open_file(backend->file, errp);
+        break;
+    case CHARDEV_BACKEND_KIND_SERIAL:
+        chr = qmp_chardev_open_serial(backend->serial, errp);
+        break;
+    case CHARDEV_BACKEND_KIND_PARALLEL:
+        chr = qmp_chardev_open_parallel(backend->parallel, errp);
+        break;
+    case CHARDEV_BACKEND_KIND_SOCKET:
+        chr = qmp_chardev_open_socket(backend->socket, errp);
+        break;
+#ifdef HAVE_CHARDEV_TTY
+    case CHARDEV_BACKEND_KIND_PTY:
+    {
+        /* qemu_chr_open_pty sets "path" in opts */
+        QemuOpts *opts;
+        opts = qemu_opts_create_nofail(qemu_find_opts("chardev"));
+        chr = qemu_chr_open_pty(opts);
+        ret->pty = g_strdup(qemu_opt_get(opts, "path"));
+        ret->has_pty = true;
+        qemu_opts_del(opts);
+        break;
+    }
+#endif
+    case CHARDEV_BACKEND_KIND_NULL:
+        chr = qemu_chr_open_null(NULL);
+        break;
+    default:
+        error_setg(errp, "unknown chardev backend (%d)", backend->kind);
+        break;
+    }
+
+    if (chr == NULL && !error_is_set(errp)) {
+        error_setg(errp, "Failed to create chardev");
+    }
+    if (chr) {
+        chr->label = g_strdup(id);
+        chr->avail_connections = 1;
+        QTAILQ_INSERT_TAIL(&chardevs, chr, next);
+        return ret;
+    } else {
+        g_free(ret);
+        return NULL;
+    }
+}
+
+void qmp_chardev_remove(const char *id, Error **errp)
+{
+    CharDriverState *chr;
+
+    chr = qemu_chr_find(id);
+    if (NULL == chr) {
+        error_setg(errp, "Chardev '%s' not found", id);
+        return;
+    }
+    if (chr->chr_can_read || chr->chr_read ||
+        chr->chr_event || chr->handler_opaque) {
+        error_setg(errp, "Chardev '%s' is busy", id);
+        return;
+    }
+    qemu_chr_delete(chr);
+}
diff --git a/qemu-config.c b/qemu-config.c
deleted file mode 100644
index 10d1ba4..0000000
--- a/qemu-config.c
+++ /dev/null
@@ -1,939 +0,0 @@
-#include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
-#include "hw/qdev.h"
-#include "error.h"
-
-static QemuOptsList qemu_drive_opts = {
-    .name = "drive",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_drive_opts.head),
-    .desc = {
-        {
-            .name = "bus",
-            .type = QEMU_OPT_NUMBER,
-            .help = "bus number",
-        },{
-            .name = "unit",
-            .type = QEMU_OPT_NUMBER,
-            .help = "unit number (i.e. lun for scsi)",
-        },{
-            .name = "if",
-            .type = QEMU_OPT_STRING,
-            .help = "interface (ide, scsi, sd, mtd, floppy, pflash, virtio)",
-        },{
-            .name = "index",
-            .type = QEMU_OPT_NUMBER,
-            .help = "index number",
-        },{
-            .name = "cyls",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of cylinders (ide disk geometry)",
-        },{
-            .name = "heads",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of heads (ide disk geometry)",
-        },{
-            .name = "secs",
-            .type = QEMU_OPT_NUMBER,
-            .help = "number of sectors (ide disk geometry)",
-        },{
-            .name = "trans",
-            .type = QEMU_OPT_STRING,
-            .help = "chs translation (auto, lba. none)",
-        },{
-            .name = "media",
-            .type = QEMU_OPT_STRING,
-            .help = "media type (disk, cdrom)",
-        },{
-            .name = "snapshot",
-            .type = QEMU_OPT_BOOL,
-            .help = "enable/disable snapshot mode",
-        },{
-            .name = "file",
-            .type = QEMU_OPT_STRING,
-            .help = "disk image",
-        },{
-            .name = "cache",
-            .type = QEMU_OPT_STRING,
-            .help = "host cache usage (none, writeback, writethrough, "
-                    "directsync, unsafe)",
-        },{
-            .name = "aio",
-            .type = QEMU_OPT_STRING,
-            .help = "host AIO implementation (threads, native)",
-        },{
-            .name = "format",
-            .type = QEMU_OPT_STRING,
-            .help = "disk format (raw, qcow2, ...)",
-        },{
-            .name = "serial",
-            .type = QEMU_OPT_STRING,
-            .help = "disk serial number",
-        },{
-            .name = "rerror",
-            .type = QEMU_OPT_STRING,
-            .help = "read error action",
-        },{
-            .name = "werror",
-            .type = QEMU_OPT_STRING,
-            .help = "write error action",
-        },{
-            .name = "addr",
-            .type = QEMU_OPT_STRING,
-            .help = "pci address (virtio only)",
-        },{
-            .name = "readonly",
-            .type = QEMU_OPT_BOOL,
-            .help = "open drive file as read-only",
-        },{
-            .name = "iops",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit total I/O operations per second",
-        },{
-            .name = "iops_rd",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit read operations per second",
-        },{
-            .name = "iops_wr",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit write operations per second",
-        },{
-            .name = "bps",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit total bytes per second",
-        },{
-            .name = "bps_rd",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit read bytes per second",
-        },{
-            .name = "bps_wr",
-            .type = QEMU_OPT_NUMBER,
-            .help = "limit write bytes per second",
-        },{
-            .name = "copy-on-read",
-            .type = QEMU_OPT_BOOL,
-            .help = "copy read data from backing file into image file",
-        },{
-            .name = "boot",
-            .type = QEMU_OPT_BOOL,
-            .help = "(deprecated, ignored)",
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_iscsi_opts = {
-    .name = "iscsi",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_iscsi_opts.head),
-    .desc = {
-        {
-            .name = "user",
-            .type = QEMU_OPT_STRING,
-            .help = "username for CHAP authentication to target",
-        },{
-            .name = "password",
-            .type = QEMU_OPT_STRING,
-            .help = "password for CHAP authentication to target",
-        },{
-            .name = "header-digest",
-            .type = QEMU_OPT_STRING,
-            .help = "HeaderDigest setting. "
-                    "{CRC32C|CRC32C-NONE|NONE-CRC32C|NONE}",
-        },{
-            .name = "initiator-name",
-            .type = QEMU_OPT_STRING,
-            .help = "Initiator iqn name to use when connecting",
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_chardev_opts = {
-    .name = "chardev",
-    .implied_opt_name = "backend",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_chardev_opts.head),
-    .desc = {
-        {
-            .name = "backend",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "host",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "port",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "localaddr",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "localport",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "to",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "wait",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "server",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "delay",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "telnet",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "width",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "height",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "cols",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "rows",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "mux",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "signal",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "name",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "debug",
-            .type = QEMU_OPT_NUMBER,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_fsdev_opts = {
-    .name = "fsdev",
-    .implied_opt_name = "fsdriver",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_fsdev_opts.head),
-    .desc = {
-        {
-            .name = "fsdriver",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "security_model",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "writeout",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "readonly",
-            .type = QEMU_OPT_BOOL,
-
-        }, {
-            .name = "socket",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "sock_fd",
-            .type = QEMU_OPT_NUMBER,
-        },
-
-        { /*End of list */ }
-    },
-};
-
-QemuOptsList qemu_virtfs_opts = {
-    .name = "virtfs",
-    .implied_opt_name = "fsdriver",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_virtfs_opts.head),
-    .desc = {
-        {
-            .name = "fsdriver",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "mount_tag",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "security_model",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "writeout",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "readonly",
-            .type = QEMU_OPT_BOOL,
-        }, {
-            .name = "socket",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "sock_fd",
-            .type = QEMU_OPT_NUMBER,
-        },
-
-        { /*End of list */ }
-    },
-};
-
-static QemuOptsList qemu_device_opts = {
-    .name = "device",
-    .implied_opt_name = "driver",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_device_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any
-         * sanity checking will happen later
-         * when setting device properties
-         */
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_netdev_opts = {
-    .name = "netdev",
-    .implied_opt_name = "type",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_netdev_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any params
-         * validation will happen later
-         */
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_net_opts = {
-    .name = "net",
-    .implied_opt_name = "type",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_net_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any params
-         * validation will happen later
-         */
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_rtc_opts = {
-    .name = "rtc",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_rtc_opts.head),
-    .desc = {
-        {
-            .name = "base",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "clock",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "driftfix",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_global_opts = {
-    .name = "global",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_global_opts.head),
-    .desc = {
-        {
-            .name = "driver",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "property",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "value",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_sandbox_opts = {
-    .name = "sandbox",
-    .implied_opt_name = "enable",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_sandbox_opts.head),
-    .desc = {
-        {
-            .name = "enable",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_mon_opts = {
-    .name = "mon",
-    .implied_opt_name = "chardev",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_mon_opts.head),
-    .desc = {
-        {
-            .name = "mode",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "chardev",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "default",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "pretty",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_trace_opts = {
-    .name = "trace",
-    .implied_opt_name = "trace",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
-    .desc = {
-        {
-            .name = "events",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "file",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_cpudef_opts = {
-    .name = "cpudef",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_cpudef_opts.head),
-    .desc = {
-        {
-            .name = "name",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "level",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "vendor",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "family",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "model",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "stepping",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "feature_edx",      /* cpuid 0000_0001.edx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "feature_ecx",      /* cpuid 0000_0001.ecx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "extfeature_edx",   /* cpuid 8000_0001.edx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "extfeature_ecx",   /* cpuid 8000_0001.ecx */
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "xlevel",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "model_id",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "vendor_override",
-            .type = QEMU_OPT_NUMBER,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_spice_opts = {
-    .name = "spice",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_spice_opts.head),
-    .desc = {
-        {
-            .name = "port",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "tls-port",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "addr",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "password",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "disable-ticketing",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "disable-copy-paste",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "sasl",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "x509-dir",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-key-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-key-password",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-cert-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-cacert-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "x509-dh-key-file",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "tls-ciphers",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "tls-channel",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "plaintext-channel",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "image-compression",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "jpeg-wan-compression",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "zlib-glz-wan-compression",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "streaming-video",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "agent-mouse",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "playback-compression",
-            .type = QEMU_OPT_BOOL,
-        }, {
-            .name = "seamless-migration",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end of list */ }
-    },
-};
-
-QemuOptsList qemu_option_rom_opts = {
-    .name = "option-rom",
-    .implied_opt_name = "romfile",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_option_rom_opts.head),
-    .desc = {
-        {
-            .name = "bootindex",
-            .type = QEMU_OPT_NUMBER,
-        }, {
-            .name = "romfile",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_machine_opts = {
-    .name = "machine",
-    .implied_opt_name = "type",
-    .merge_lists = true,
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_machine_opts.head),
-    .desc = {
-        {
-            .name = "type",
-            .type = QEMU_OPT_STRING,
-            .help = "emulated machine"
-        }, {
-            .name = "accel",
-            .type = QEMU_OPT_STRING,
-            .help = "accelerator list",
-        }, {
-            .name = "kernel_irqchip",
-            .type = QEMU_OPT_BOOL,
-            .help = "use KVM in-kernel irqchip",
-        }, {
-            .name = "kvm_shadow_mem",
-            .type = QEMU_OPT_SIZE,
-            .help = "KVM shadow MMU size",
-        }, {
-            .name = "kernel",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel image file",
-        }, {
-            .name = "initrd",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux initial ramdisk file",
-        }, {
-            .name = "append",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel command line",
-        }, {
-            .name = "dtb",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel device tree file",
-        }, {
-            .name = "dumpdtb",
-            .type = QEMU_OPT_STRING,
-            .help = "Dump current dtb to a file and quit",
-        }, {
-            .name = "phandle_start",
-            .type = QEMU_OPT_STRING,
-            .help = "The first phandle ID we may generate dynamically",
-        }, {
-            .name = "dt_compatible",
-            .type = QEMU_OPT_STRING,
-            .help = "Overrides the \"compatible\" property of the dt root node",
-        }, {
-            .name = "dump-guest-core",
-            .type = QEMU_OPT_BOOL,
-            .help = "Include guest memory in  a core dump",
-        }, {
-            .name = "mem-merge",
-            .type = QEMU_OPT_BOOL,
-            .help = "enable/disable memory merge support",
-        },{
-            .name = "usb",
-            .type = QEMU_OPT_BOOL,
-            .help = "Set on/off to enable/disable usb",
-        },
-        { /* End of list */ }
-    },
-};
-
-QemuOptsList qemu_boot_opts = {
-    .name = "boot-opts",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_boot_opts.head),
-    .desc = {
-        /* the three names below are not used now */
-        {
-            .name = "order",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "once",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "menu",
-            .type = QEMU_OPT_STRING,
-        /* following are really used */
-        }, {
-            .name = "splash",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "splash-time",
-            .type = QEMU_OPT_STRING,
-        }, {
-            .name = "reboot-timeout",
-            .type = QEMU_OPT_STRING,
-        },
-        { /*End of list */ }
-    },
-};
-
-static QemuOptsList qemu_add_fd_opts = {
-    .name = "add-fd",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_add_fd_opts.head),
-    .desc = {
-        {
-            .name = "fd",
-            .type = QEMU_OPT_NUMBER,
-            .help = "file descriptor of which a duplicate is added to fd set",
-        },{
-            .name = "set",
-            .type = QEMU_OPT_NUMBER,
-            .help = "ID of the fd set to add fd to",
-        },{
-            .name = "opaque",
-            .type = QEMU_OPT_STRING,
-            .help = "free-form string used to describe fd",
-        },
-        { /* end of list */ }
-    },
-};
-
-static QemuOptsList qemu_object_opts = {
-    .name = "object",
-    .implied_opt_name = "qom-type",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
-    .desc = {
-        { }
-    },
-};
-
-static QemuOptsList *vm_config_groups[32] = {
-    &qemu_drive_opts,
-    &qemu_chardev_opts,
-    &qemu_device_opts,
-    &qemu_netdev_opts,
-    &qemu_net_opts,
-    &qemu_rtc_opts,
-    &qemu_global_opts,
-    &qemu_mon_opts,
-    &qemu_cpudef_opts,
-    &qemu_trace_opts,
-    &qemu_option_rom_opts,
-    &qemu_machine_opts,
-    &qemu_boot_opts,
-    &qemu_iscsi_opts,
-    &qemu_sandbox_opts,
-    &qemu_add_fd_opts,
-    &qemu_object_opts,
-    NULL,
-};
-
-static QemuOptsList *find_list(QemuOptsList **lists, const char *group,
-                               Error **errp)
-{
-    int i;
-
-    for (i = 0; lists[i] != NULL; i++) {
-        if (strcmp(lists[i]->name, group) == 0)
-            break;
-    }
-    if (lists[i] == NULL) {
-        error_set(errp, QERR_INVALID_OPTION_GROUP, group);
-    }
-    return lists[i];
-}
-
-QemuOptsList *qemu_find_opts(const char *group)
-{
-    QemuOptsList *ret;
-    Error *local_err = NULL;
-
-    ret = find_list(vm_config_groups, group, &local_err);
-    if (error_is_set(&local_err)) {
-        error_report("%s\n", error_get_pretty(local_err));
-        error_free(local_err);
-    }
-
-    return ret;
-}
-
-QemuOptsList *qemu_find_opts_err(const char *group, Error **errp)
-{
-    return find_list(vm_config_groups, group, errp);
-}
-
-void qemu_add_opts(QemuOptsList *list)
-{
-    int entries, i;
-
-    entries = ARRAY_SIZE(vm_config_groups);
-    entries--; /* keep list NULL terminated */
-    for (i = 0; i < entries; i++) {
-        if (vm_config_groups[i] == NULL) {
-            vm_config_groups[i] = list;
-            return;
-        }
-    }
-    fprintf(stderr, "ran out of space in vm_config_groups");
-    abort();
-}
-
-int qemu_set_option(const char *str)
-{
-    char group[64], id[64], arg[64];
-    QemuOptsList *list;
-    QemuOpts *opts;
-    int rc, offset;
-
-    rc = sscanf(str, "%63[^.].%63[^.].%63[^=]%n", group, id, arg, &offset);
-    if (rc < 3 || str[offset] != '=') {
-        error_report("can't parse: \"%s\"", str);
-        return -1;
-    }
-
-    list = qemu_find_opts(group);
-    if (list == NULL) {
-        return -1;
-    }
-
-    opts = qemu_opts_find(list, id);
-    if (!opts) {
-        error_report("there is no %s \"%s\" defined",
-                     list->name, id);
-        return -1;
-    }
-
-    if (qemu_opt_set(opts, arg, str+offset+1) == -1) {
-        return -1;
-    }
-    return 0;
-}
-
-int qemu_global_option(const char *str)
-{
-    char driver[64], property[64];
-    QemuOpts *opts;
-    int rc, offset;
-
-    rc = sscanf(str, "%63[^.].%63[^=]%n", driver, property, &offset);
-    if (rc < 2 || str[offset] != '=') {
-        error_report("can't parse: \"%s\"", str);
-        return -1;
-    }
-
-    opts = qemu_opts_create(&qemu_global_opts, NULL, 0, NULL);
-    qemu_opt_set(opts, "driver", driver);
-    qemu_opt_set(opts, "property", property);
-    qemu_opt_set(opts, "value", str+offset+1);
-    return 0;
-}
-
-struct ConfigWriteData {
-    QemuOptsList *list;
-    FILE *fp;
-};
-
-static int config_write_opt(const char *name, const char *value, void *opaque)
-{
-    struct ConfigWriteData *data = opaque;
-
-    fprintf(data->fp, "  %s = \"%s\"\n", name, value);
-    return 0;
-}
-
-static int config_write_opts(QemuOpts *opts, void *opaque)
-{
-    struct ConfigWriteData *data = opaque;
-    const char *id = qemu_opts_id(opts);
-
-    if (id) {
-        fprintf(data->fp, "[%s \"%s\"]\n", data->list->name, id);
-    } else {
-        fprintf(data->fp, "[%s]\n", data->list->name);
-    }
-    qemu_opt_foreach(opts, config_write_opt, data, 0);
-    fprintf(data->fp, "\n");
-    return 0;
-}
-
-void qemu_config_write(FILE *fp)
-{
-    struct ConfigWriteData data = { .fp = fp };
-    QemuOptsList **lists = vm_config_groups;
-    int i;
-
-    fprintf(fp, "# qemu config file\n\n");
-    for (i = 0; lists[i] != NULL; i++) {
-        data.list = lists[i];
-        qemu_opts_foreach(data.list, config_write_opts, &data, 0);
-    }
-}
-
-int qemu_config_parse(FILE *fp, QemuOptsList **lists, const char *fname)
-{
-    char line[1024], group[64], id[64], arg[64], value[1024];
-    Location loc;
-    QemuOptsList *list = NULL;
-    Error *local_err = NULL;
-    QemuOpts *opts = NULL;
-    int res = -1, lno = 0;
-
-    loc_push_none(&loc);
-    while (fgets(line, sizeof(line), fp) != NULL) {
-        loc_set_file(fname, ++lno);
-        if (line[0] == '\n') {
-            /* skip empty lines */
-            continue;
-        }
-        if (line[0] == '#') {
-            /* comment */
-            continue;
-        }
-        if (sscanf(line, "[%63s \"%63[^\"]\"]", group, id) == 2) {
-            /* group with id */
-            list = find_list(lists, group, &local_err);
-            if (error_is_set(&local_err)) {
-                error_report("%s\n", error_get_pretty(local_err));
-                error_free(local_err);
-                goto out;
-            }
-            opts = qemu_opts_create(list, id, 1, NULL);
-            continue;
-        }
-        if (sscanf(line, "[%63[^]]]", group) == 1) {
-            /* group without id */
-            list = find_list(lists, group, &local_err);
-            if (error_is_set(&local_err)) {
-                error_report("%s\n", error_get_pretty(local_err));
-                error_free(local_err);
-                goto out;
-            }
-            opts = qemu_opts_create(list, NULL, 0, NULL);
-            continue;
-        }
-        if (sscanf(line, " %63s = \"%1023[^\"]\"", arg, value) == 2) {
-            /* arg = value */
-            if (opts == NULL) {
-                error_report("no group defined");
-                goto out;
-            }
-            if (qemu_opt_set(opts, arg, value) != 0) {
-                goto out;
-            }
-            continue;
-        }
-        error_report("parse error");
-        goto out;
-    }
-    if (ferror(fp)) {
-        error_report("error reading file");
-        goto out;
-    }
-    res = 0;
-out:
-    loc_pop(&loc);
-    return res;
-}
-
-int qemu_read_config_file(const char *filename)
-{
-    FILE *f = fopen(filename, "r");
-    int ret;
-
-    if (f == NULL) {
-        return -errno;
-    }
-
-    ret = qemu_config_parse(f, vm_config_groups, filename);
-    fclose(f);
-
-    if (ret == 0) {
-        return 0;
-    } else {
-        return -EINVAL;
-    }
-}
diff --git a/qemu-coroutine-io.c b/qemu-coroutine-io.c
index 5734965..e8ad1a4 100644
--- a/qemu-coroutine-io.c
+++ b/qemu-coroutine-io.c
@@ -23,9 +23,9 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu_socket.h"
-#include "qemu-coroutine.h"
-#include "iov.h"
+#include "qemu/sockets.h"
+#include "block/coroutine.h"
+#include "qemu/iov.h"
 
 ssize_t coroutine_fn
 qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
diff --git a/qemu-coroutine-lock.c b/qemu-coroutine-lock.c
index 9dda3f8..97ef01c 100644
--- a/qemu-coroutine-lock.c
+++ b/qemu-coroutine-lock.c
@@ -23,10 +23,10 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-coroutine.h"
-#include "qemu-coroutine-int.h"
-#include "qemu-queue.h"
-#include "qemu-aio.h"
+#include "block/coroutine.h"
+#include "block/coroutine_int.h"
+#include "qemu/queue.h"
+#include "block/aio.h"
 #include "trace.h"
 
 static QTAILQ_HEAD(, Coroutine) unlock_bh_queue =
diff --git a/qemu-coroutine-sleep.c b/qemu-coroutine-sleep.c
index d7083ee..169ce5c 100644
--- a/qemu-coroutine-sleep.c
+++ b/qemu-coroutine-sleep.c
@@ -11,8 +11,8 @@
  *
  */
 
-#include "qemu-coroutine.h"
-#include "qemu-timer.h"
+#include "block/coroutine.h"
+#include "qemu/timer.h"
 
 typedef struct CoSleepCB {
     QEMUTimer *ts;
diff --git a/qemu-coroutine.c b/qemu-coroutine.c
index 600be26..0f6e268 100644
--- a/qemu-coroutine.c
+++ b/qemu-coroutine.c
@@ -14,8 +14,8 @@
 
 #include "trace.h"
 #include "qemu-common.h"
-#include "qemu-coroutine.h"
-#include "qemu-coroutine-int.h"
+#include "block/coroutine.h"
+#include "block/coroutine_int.h"
 
 Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
 {
diff --git a/qemu-img.c b/qemu-img.c
index e29e01b..85d3740 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -23,13 +23,13 @@
  */
 #include "qapi-visit.h"
 #include "qapi/qmp-output-visitor.h"
-#include "qjson.h"
+#include "qapi/qmp/qjson.h"
 #include "qemu-common.h"
-#include "qemu-option.h"
-#include "qemu-error.h"
-#include "osdep.h"
-#include "sysemu.h"
-#include "block_int.h"
+#include "qemu/option.h"
+#include "qemu/error-report.h"
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
+#include "block/block_int.h"
 #include <getopt.h>
 #include <stdio.h>
 
@@ -294,13 +294,14 @@ static int add_old_style_options(const char *fmt, QEMUOptionParameter *list,
 
 static int img_create(int argc, char **argv)
 {
-    int c, ret = 0;
+    int c;
     uint64_t img_size = -1;
     const char *fmt = "raw";
     const char *base_fmt = NULL;
     const char *filename;
     const char *base_filename = NULL;
     char *options = NULL;
+    Error *local_err = NULL;
 
     for(;;) {
         c = getopt(argc, argv, "F:b:f:he6o:");
@@ -347,26 +348,30 @@ static int img_create(int argc, char **argv)
         char *end;
         sval = strtosz_suffix(argv[optind++], &end, STRTOSZ_DEFSUFFIX_B);
         if (sval < 0 || *end) {
-            error_report("Invalid image size specified! You may use k, M, G or "
-                  "T suffixes for ");
-            error_report("kilobytes, megabytes, gigabytes and terabytes.");
-            ret = -1;
-            goto out;
+            if (sval == -ERANGE) {
+                error_report("Image size must be less than 8 EiB!");
+            } else {
+                error_report("Invalid image size specified! You may use k, M, "
+                      "G or T suffixes for ");
+                error_report("kilobytes, megabytes, gigabytes and terabytes.");
+            }
+            return 1;
         }
         img_size = (uint64_t)sval;
     }
 
     if (options && is_help_option(options)) {
-        ret = print_block_option_help(filename, fmt);
-        goto out;
+        return print_block_option_help(filename, fmt);
     }
 
-    ret = bdrv_img_create(filename, fmt, base_filename, base_fmt,
-                          options, img_size, BDRV_O_FLAGS);
-out:
-    if (ret) {
+    bdrv_img_create(filename, fmt, base_filename, base_fmt,
+                    options, img_size, BDRV_O_FLAGS, &local_err);
+    if (error_is_set(&local_err)) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
         return 1;
     }
+
     return 0;
 }
 
@@ -1933,7 +1938,7 @@ static int img_resize(int argc, char **argv)
     }
 
     /* Parse size */
-    param = qemu_opts_create(&resize_options, NULL, 0, NULL);
+    param = qemu_opts_create_nofail(&resize_options);
     if (qemu_opt_set(param, BLOCK_OPT_SIZE, size)) {
         /* Error message already printed when size parsing fails */
         ret = -1;
diff --git a/qemu-io.c b/qemu-io.c
index 92cdb2a..6188093 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -15,8 +15,8 @@
 #include <libgen.h>
 
 #include "qemu-common.h"
-#include "main-loop.h"
-#include "block_int.h"
+#include "qemu/main-loop.h"
+#include "block/block_int.h"
 #include "cmd.h"
 #include "trace/control.h"
 
@@ -265,6 +265,18 @@ static int do_co_write_zeroes(int64_t offset, int count, int *total)
     }
 }
 
+static int do_write_compressed(char *buf, int64_t offset, int count, int *total)
+{
+    int ret;
+
+    ret = bdrv_write_compressed(bs, offset >> 9, (uint8_t *)buf, count >> 9);
+    if (ret < 0) {
+        return ret;
+    }
+    *total = count;
+    return 1;
+}
+
 static int do_load_vmstate(char *buf, int64_t offset, int count, int *total)
 {
     *total = bdrv_load_vmstate(bs, (uint8_t *)buf, offset, count);
@@ -687,6 +699,7 @@ static void write_help(void)
 " Writes into a segment of the currently open file, using a buffer\n"
 " filled with a set pattern (0xcdcdcdcd).\n"
 " -b, -- write to the VM state rather than the virtual disk\n"
+" -c, -- write compressed data with bdrv_write_compressed\n"
 " -p, -- use bdrv_pwrite to write the file\n"
 " -P, -- use different pattern to fill file\n"
 " -C, -- report statistics in a machine parsable format\n"
@@ -703,7 +716,7 @@ static const cmdinfo_t write_cmd = {
     .cfunc      = write_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-bCpqz] [-P pattern ] off len",
+    .args       = "[-bcCpqz] [-P pattern ] off len",
     .oneline    = "writes a number of bytes at a specified offset",
     .help       = write_help,
 };
@@ -712,6 +725,7 @@ static int write_f(int argc, char **argv)
 {
     struct timeval t1, t2;
     int Cflag = 0, pflag = 0, qflag = 0, bflag = 0, Pflag = 0, zflag = 0;
+    int cflag = 0;
     int c, cnt;
     char *buf = NULL;
     int64_t offset;
@@ -720,11 +734,14 @@ static int write_f(int argc, char **argv)
     int total = 0;
     int pattern = 0xcd;
 
-    while ((c = getopt(argc, argv, "bCpP:qz")) != EOF) {
+    while ((c = getopt(argc, argv, "bcCpP:qz")) != EOF) {
         switch (c) {
         case 'b':
             bflag = 1;
             break;
+        case 'c':
+            cflag = 1;
+            break;
         case 'C':
             Cflag = 1;
             break;
@@ -801,6 +818,8 @@ static int write_f(int argc, char **argv)
         cnt = do_save_vmstate(buf, offset, count, &total);
     } else if (zflag) {
         cnt = do_co_write_zeroes(offset, count, &total);
+    } else if (cflag) {
+        cnt = do_write_compressed(buf, offset, count, &total);
     } else {
         cnt = do_write(buf, offset, count, &total);
     }
@@ -1652,6 +1671,67 @@ static const cmdinfo_t map_cmd = {
        .oneline        = "prints the allocated areas of a file",
 };
 
+static int break_f(int argc, char **argv)
+{
+    int ret;
+
+    ret = bdrv_debug_breakpoint(bs, argv[1], argv[2]);
+    if (ret < 0) {
+        printf("Could not set breakpoint: %s\n", strerror(-ret));
+    }
+
+    return 0;
+}
+
+static const cmdinfo_t break_cmd = {
+       .name           = "break",
+       .argmin         = 2,
+       .argmax         = 2,
+       .cfunc          = break_f,
+       .args           = "event tag",
+       .oneline        = "sets a breakpoint on event and tags the stopped "
+                         "request as tag",
+};
+
+static int resume_f(int argc, char **argv)
+{
+    int ret;
+
+    ret = bdrv_debug_resume(bs, argv[1]);
+    if (ret < 0) {
+        printf("Could not resume request: %s\n", strerror(-ret));
+    }
+
+    return 0;
+}
+
+static const cmdinfo_t resume_cmd = {
+       .name           = "resume",
+       .argmin         = 1,
+       .argmax         = 1,
+       .cfunc          = resume_f,
+       .args           = "tag",
+       .oneline        = "resumes the request tagged as tag",
+};
+
+static int wait_break_f(int argc, char **argv)
+{
+    while (!bdrv_debug_is_suspended(bs, argv[1])) {
+        qemu_aio_wait();
+    }
+
+    return 0;
+}
+
+static const cmdinfo_t wait_break_cmd = {
+       .name           = "wait_break",
+       .argmin         = 1,
+       .argmax         = 1,
+       .cfunc          = wait_break_f,
+       .args           = "tag",
+       .oneline        = "waits for the suspension of a request",
+};
+
 static int abort_f(int argc, char **argv)
 {
     abort();
@@ -1915,6 +1995,9 @@ int main(int argc, char **argv)
     add_command(&discard_cmd);
     add_command(&alloc_cmd);
     add_command(&map_cmd);
+    add_command(&break_cmd);
+    add_command(&resume_cmd);
+    add_command(&wait_break_cmd);
     add_command(&abort_cmd);
 
     add_args_command(init_args_command);
diff --git a/qemu-log.c b/qemu-log.c
index a4c3d1f..30c9ab0 100644
--- a/qemu-log.c
+++ b/qemu-log.c
@@ -18,13 +18,15 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #ifdef WIN32
-static const char *logfilename = "qemu.log";
+#define DEFAULT_LOGFILENAME "qemu.log"
 #else
-static const char *logfilename = "/tmp/qemu.log";
+#define DEFAULT_LOGFILENAME "/tmp/qemu.log"
 #endif
+
+static char *logfilename;
 FILE *qemu_logfile;
 int qemu_loglevel;
 static int log_append = 0;
@@ -54,11 +56,13 @@ void qemu_log_mask(int mask, const char *fmt, ...)
 /* enable or disable low levels log */
 void qemu_set_log(int log_flags, bool use_own_buffers)
 {
+    const char *fname = logfilename ?: DEFAULT_LOGFILENAME;
+
     qemu_loglevel = log_flags;
     if (qemu_loglevel && !qemu_logfile) {
-        qemu_logfile = fopen(logfilename, log_append ? "a" : "w");
+        qemu_logfile = fopen(fname, log_append ? "a" : "w");
         if (!qemu_logfile) {
-            perror(logfilename);
+            perror(fname);
             _exit(1);
         }
         /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
@@ -84,7 +88,8 @@ void qemu_set_log(int log_flags, bool use_own_buffers)
 
 void cpu_set_log_filename(const char *filename)
 {
-    logfilename = strdup(filename);
+    g_free(logfilename);
+    logfilename = g_strdup(filename);
     if (qemu_logfile) {
         fclose(qemu_logfile);
         qemu_logfile = NULL;
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 80f08d8..0a6091b 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -17,8 +17,8 @@
  */
 
 #include "qemu-common.h"
-#include "block.h"
-#include "nbd.h"
+#include "block/block.h"
+#include "block/nbd.h"
 
 #include <stdarg.h>
 #include <stdio.h>
diff --git a/qemu-nbd.texi b/qemu-nbd.texi
index 6955d90..3e57200 100644
--- a/qemu-nbd.texi
+++ b/qemu-nbd.texi
@@ -29,7 +29,12 @@ Export QEMU disk image using NBD protocol.
 @item -s, --snapshot
   use snapshot file
 @item -n, --nocache
-  disable host cache
+@itemx --cache=@var{cache}
+  set cache mode to be used with the file.  See the documentation of
+  the emulator's @code{-drive cache=...} option for allowed values.
+@item --aio=@var{aio}
+  choose asynchronous I/O mode between @samp{threads} (the default)
+  and @samp{native} (Linux only).
 @item -c, --connect=@var{dev}
   connect @var{filename} to NBD device @var{dev}
 @item -d, --disconnect
diff --git a/qemu-options.hx b/qemu-options.hx
index de43b1b..9d7131a 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -1096,6 +1096,14 @@ client is specified by the @var{display}. For reverse network
 connections (@var{host}:@var{d},@code{reverse}), the @var{d} argument
 is a TCP port number, not a display number.
 
+@item websocket
+
+Opens an additional TCP listening port dedicated to VNC Websocket connections.
+By defintion the Websocket port is 5700+@var{display}. If @var{host} is
+specified connections will only be allowed from this host.
+As an alternative the Websocket port could be specified by using
+@code{websocket}=@var{port}.
+
 @item password
 
 Require that password based authentication is used for client connections.
@@ -1331,7 +1339,7 @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
     "                connect the host TAP network interface to VLAN 'n'\n"
 #else
     "-net tap[,vlan=n][,name=str][,fd=h][,ifname=name][,script=file][,downscript=dfile][,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off][,vhostfd=h][,vhostforce=on|off]\n"
-    "                connect the host TAP network interface to VLAN 'n' \n"
+    "                connect the host TAP network interface to VLAN 'n'\n"
     "                use network scripts 'file' (default=" DEFAULT_NETWORK_SCRIPT ")\n"
     "                to configure it and 'dfile' (default=" DEFAULT_NETWORK_DOWN_SCRIPT ")\n"
     "                to deconfigure it\n"
@@ -1728,6 +1736,7 @@ DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
     "-chardev msmouse,id=id[,mux=on|off]\n"
     "-chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]]\n"
     "         [,mux=on|off]\n"
+    "-chardev ringbuf,id=id[,size=size]\n"
     "-chardev file,id=id,path=path[,mux=on|off]\n"
     "-chardev pipe,id=id,path=path[,mux=on|off]\n"
 #ifdef _WIN32
@@ -1742,13 +1751,16 @@ DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
 #endif
 #if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
         || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
+    "-chardev serial,id=id,path=path[,mux=on|off]\n"
     "-chardev tty,id=id,path=path[,mux=on|off]\n"
 #endif
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__)
+    "-chardev parallel,id=id,path=path[,mux=on|off]\n"
     "-chardev parport,id=id,path=path[,mux=on|off]\n"
 #endif
 #if defined(CONFIG_SPICE)
     "-chardev spicevmc,id=id,name=name[,debug=debug]\n"
+    "-chardev spiceport,id=id,name=name[,debug=debug]\n"
 #endif
     , QEMU_ARCH_ALL
 )
@@ -1766,6 +1778,7 @@ Backend is one of:
 @option{udp},
 @option{msmouse},
 @option{vc},
+@option{ringbuf},
 @option{file},
 @option{pipe},
 @option{console},
@@ -1774,8 +1787,10 @@ Backend is one of:
 @option{stdio},
 @option{braille},
 @option{tty},
+@option{parallel},
 @option{parport},
 @option{spicevmc}.
+@option{spiceport}.
 The specific backend will determine the applicable options.
 
 All devices must have an id, which can be any string up to 127 characters long.
@@ -1872,6 +1887,11 @@ the console, in pixels.
 @option{cols} and @option{rows} specify that the console be sized to fit a text
 console with the given dimensions.
 
+@item -chardev ringbuf ,id=@var{id} [,size=@var{size}]
+
+Create a ring buffer with fixed size @option{size}.
+@var{size} must be a power of two, and defaults to @code{64K}).
+
 @item -chardev file ,id=@var{id} ,path=@var{path}
 
 Log all traffic received from the guest to a file.
@@ -1908,8 +1928,8 @@ take any options.
 
 Send traffic from the guest to a serial device on the host.
 
-@option{serial} is
-only available on Windows hosts.
+On Unix hosts serial will actually accept any tty device,
+not only serial lines.
 
 @option{path} specifies the name of the serial device to open.
 
@@ -1935,16 +1955,15 @@ Connect to a local BrlAPI server. @option{braille} does not take any options.
 
 @item -chardev tty ,id=@var{id} ,path=@var{path}
 
-Connect to a local tty device.
-
 @option{tty} is only available on Linux, Sun, FreeBSD, NetBSD, OpenBSD and
-DragonFlyBSD hosts.
+DragonFlyBSD hosts.  It is an alias for @option{serial}.
 
 @option{path} specifies the path to the tty. @option{path} is required.
 
+@item -chardev parallel ,id=@var{id} ,path=@var{path}
 @item -chardev parport ,id=@var{id} ,path=@var{path}
 
-@option{parport} is only available on Linux, FreeBSD and DragonFlyBSD hosts.
+@option{parallel} is only available on Linux, FreeBSD and DragonFlyBSD hosts.
 
 Connect to a local parallel port.
 
@@ -1961,6 +1980,17 @@ required.
 
 Connect to a spice virtual machine channel, such as vdiport.
 
+@item -chardev spiceport ,id=@var{id} ,debug=@var{debug}, name=@var{name}
+
+@option{spiceport} is only available when spice support is built in.
+
+@option{debug} debug level for spicevmc
+
+@option{name} name of spice port to connect to
+
+Connect to a spice port, allowing a Spice client to handle the traffic
+identified by a name (preferably a fqdn).
+
 @end table
 ETEXI
 
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index 2a71d6f..031da1d 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -14,7 +14,7 @@
  */
 #include <stdio.h>
 #include <seccomp.h>
-#include "qemu-seccomp.h"
+#include "sysemu/seccomp.h"
 
 struct QemuSeccompSyscall {
     int32_t num;
diff --git a/qemu-timer.c b/qemu-timer.c
index 0d2bb94..8fb5c75 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -22,14 +22,13 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu.h"
-#include "net.h"
-#include "monitor.h"
-#include "console.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
 
 #include "hw/hw.h"
 
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #ifdef CONFIG_POSIX
 #include <pthread.h>
 #endif
@@ -478,7 +477,7 @@ static void host_alarm_handler(int host_signum)
 
 #if defined(__linux__)
 
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 
 static int dynticks_start_timer(struct qemu_alarm_timer *t)
 {
diff --git a/qemu-tool.c b/qemu-tool.c
deleted file mode 100644
index b46631e..0000000
--- a/qemu-tool.c
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Compatibility for qemu-img/qemu-nbd
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "monitor.h"
-#include "qemu-timer.h"
-#include "qemu-log.h"
-#include "migration.h"
-#include "main-loop.h"
-#include "sysemu.h"
-#include "qemu_socket.h"
-#include "slirp/libslirp.h"
-
-#include <sys/time.h>
-
-struct QEMUBH
-{
-    QEMUBHFunc *cb;
-    void *opaque;
-};
-
-const char *qemu_get_vm_name(void)
-{
-    return NULL;
-}
-
-Monitor *cur_mon;
-
-void vm_stop(RunState state)
-{
-    abort();
-}
-
-int monitor_cur_is_qmp(void)
-{
-    return 0;
-}
-
-void monitor_set_error(Monitor *mon, QError *qerror)
-{
-}
-
-void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
-{
-}
-
-void monitor_printf(Monitor *mon, const char *fmt, ...)
-{
-}
-
-void monitor_print_filename(Monitor *mon, const char *filename)
-{
-}
-
-void monitor_protocol_event(MonitorEvent event, QObject *data)
-{
-}
-
-int64_t cpu_get_clock(void)
-{
-    return get_clock_realtime();
-}
-
-int64_t cpu_get_icount(void)
-{
-    abort();
-}
-
-void qemu_mutex_lock_iothread(void)
-{
-}
-
-void qemu_mutex_unlock_iothread(void)
-{
-}
-
-int use_icount;
-
-void qemu_clock_warp(QEMUClock *clock)
-{
-}
-
-void slirp_update_timeout(uint32_t *timeout)
-{
-}
-
-void slirp_select_fill(int *pnfds, fd_set *readfds,
-                       fd_set *writefds, fd_set *xfds)
-{
-}
-
-void slirp_select_poll(fd_set *readfds, fd_set *writefds,
-                       fd_set *xfds, int select_error)
-{
-}
-
-void migrate_add_blocker(Error *reason)
-{
-}
-
-void migrate_del_blocker(Error *reason)
-{
-}
diff --git a/qemu-user.c b/qemu-user.c
deleted file mode 100644
index 08ccb0f..0000000
--- a/qemu-user.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Stubs for QEMU user emulation
- *
- * Copyright (c) 2012 SUSE LINUX Products GmbH
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see
- * <http://www.gnu.org/licenses/gpl-2.0.html>
- */
-
-#include "qemu-common.h"
-#include "monitor.h"
-
-Monitor *cur_mon;
-
-int monitor_cur_is_qmp(void)
-{
-    return 0;
-}
-
-void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
-{
-}
-
-void monitor_set_error(Monitor *mon, QError *qerror)
-{
-}
diff --git a/qga/Makefile.objs b/qga/Makefile.objs
index cd3e135..b8d7cd0 100644
--- a/qga/Makefile.objs
+++ b/qga/Makefile.objs
@@ -1,4 +1,4 @@
-qga-obj-y = commands.o guest-agent-command-state.o
+qga-obj-y = commands.o guest-agent-command-state.o main.o
 qga-obj-$(CONFIG_POSIX) += commands-posix.o channel-posix.o
 qga-obj-$(CONFIG_WIN32) += commands-win32.o channel-win32.o service-win32.o
 qga-obj-y += qapi-generated/qga-qapi-types.o qapi-generated/qga-qapi-visit.o
diff --git a/qga/channel-posix.c b/qga/channel-posix.c
index d152827..e65dda3 100644
--- a/qga/channel-posix.c
+++ b/qga/channel-posix.c
@@ -1,6 +1,12 @@
 #include <glib.h>
 #include <termios.h>
-#include "qemu_socket.h"
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/sockets.h"
 #include "qga/channel.h"
 
 #ifdef CONFIG_SOLARIS
@@ -40,6 +46,7 @@ static gboolean ga_channel_listen_accept(GIOChannel *channel,
     ret = ga_channel_client_add(c, client_fd);
     if (ret) {
         g_warning("error setting up connection");
+        close(client_fd);
         goto out;
     }
     accepted = true;
@@ -134,19 +141,21 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
                            );
         if (fd == -1) {
             g_critical("error opening channel: %s", strerror(errno));
-            exit(EXIT_FAILURE);
+            return false;
         }
 #ifdef CONFIG_SOLARIS
         ret = ioctl(fd, I_SETSIG, S_OUTPUT | S_INPUT | S_HIPRI);
         if (ret == -1) {
             g_critical("error setting event mask for channel: %s",
                        strerror(errno));
-            exit(EXIT_FAILURE);
+            close(fd);
+            return false;
         }
 #endif
         ret = ga_channel_client_add(c, fd);
         if (ret) {
             g_critical("error adding channel to main loop");
+            close(fd);
             return false;
         }
         break;
@@ -156,7 +165,7 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
         int fd = qemu_open(path, O_RDWR | O_NOCTTY | O_NONBLOCK);
         if (fd == -1) {
             g_critical("error opening channel: %s", strerror(errno));
-            exit(EXIT_FAILURE);
+            return false;
         }
         tcgetattr(fd, &tio);
         /* set up serial port for non-canonical, dumb byte streaming */
@@ -176,7 +185,9 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
         tcsetattr(fd, TCSANOW, &tio);
         ret = ga_channel_client_add(c, fd);
         if (ret) {
-            g_error("error adding channel to main loop");
+            g_critical("error adding channel to main loop");
+            close(fd);
+            return false;
         }
         break;
     }
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index 726930a..7a0202e 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -17,9 +17,9 @@
 #include <sys/wait.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
-#include "qerror.h"
-#include "qemu-queue.h"
-#include "host-utils.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/queue.h"
+#include "qemu/host-utils.h"
 
 #ifndef CONFIG_HAS_ENVIRON
 #ifdef __APPLE__
@@ -46,10 +46,29 @@ extern char **environ;
 #endif
 #endif
 
+static void ga_wait_child(pid_t pid, int *status, Error **err)
+{
+    pid_t rpid;
+
+    *status = 0;
+
+    do {
+        rpid = waitpid(pid, status, 0);
+    } while (rpid == -1 && errno == EINTR);
+
+    if (rpid == -1) {
+        error_setg_errno(err, errno, "failed to wait for child (pid: %d)", pid);
+        return;
+    }
+
+    g_assert(rpid == pid);
+}
+
 void qmp_guest_shutdown(bool has_mode, const char *mode, Error **err)
 {
     const char *shutdown_flag;
-    pid_t rpid, pid;
+    Error *local_err = NULL;
+    pid_t pid;
     int status;
 
     slog("guest-shutdown called, mode: %s", mode);
@@ -60,8 +79,8 @@ void qmp_guest_shutdown(bool has_mode, const char *mode, Error **err)
     } else if (strcmp(mode, "reboot") == 0) {
         shutdown_flag = "-r";
     } else {
-        error_set(err, QERR_INVALID_PARAMETER_VALUE, "mode",
-                  "halt|powerdown|reboot");
+        error_setg(err,
+                   "mode is invalid (valid values are: halt|powerdown|reboot");
         return;
     }
 
@@ -77,18 +96,27 @@ void qmp_guest_shutdown(bool has_mode, const char *mode, Error **err)
                "hypervisor initiated shutdown", (char*)NULL, environ);
         _exit(EXIT_FAILURE);
     } else if (pid < 0) {
-        goto exit_err;
+        error_setg_errno(err, errno, "failed to create child process");
+        return;
     }
 
-    do {
-        rpid = waitpid(pid, &status, 0);
-    } while (rpid == -1 && errno == EINTR);
-    if (rpid == pid && WIFEXITED(status) && !WEXITSTATUS(status)) {
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return;
+    }
+
+    if (!WIFEXITED(status)) {
+        error_setg(err, "child process has terminated abnormally");
         return;
     }
 
-exit_err:
-    error_set(err, QERR_UNDEFINED_ERROR);
+    if (WEXITSTATUS(status)) {
+        error_setg(err, "child process has failed to shutdown");
+        return;
+    }
+
+    /* succeded */
 }
 
 typedef struct GuestFileHandle {
@@ -111,7 +139,7 @@ static void guest_file_handle_add(FILE *fh)
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
 }
 
-static GuestFileHandle *guest_file_handle_find(int64_t id)
+static GuestFileHandle *guest_file_handle_find(int64_t id, Error **err)
 {
     GuestFileHandle *gfh;
 
@@ -122,6 +150,7 @@ static GuestFileHandle *guest_file_handle_find(int64_t id)
         }
     }
 
+    error_setg(err, "handle '%" PRId64 "' has not been found", id);
     return NULL;
 }
 
@@ -137,7 +166,8 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode, E
     slog("guest-file-open called, filepath: %s, mode: %s", path, mode);
     fh = fopen(path, mode);
     if (!fh) {
-        error_set(err, QERR_OPEN_FILE_FAILED, path);
+        error_setg_errno(err, errno, "failed to open file '%s' (mode: '%s')",
+                         path, mode);
         return -1;
     }
 
@@ -148,7 +178,8 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode, E
     ret = fcntl(fd, F_GETFL);
     ret = fcntl(fd, F_SETFL, ret | O_NONBLOCK);
     if (ret == -1) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fcntl() failed");
+        error_setg_errno(err, errno, "failed to make file '%s' non-blocking",
+                         path);
         fclose(fh);
         return -1;
     }
@@ -160,18 +191,17 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode, E
 
 void qmp_guest_file_close(int64_t handle, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     int ret;
 
     slog("guest-file-close called, handle: %ld", handle);
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return;
     }
 
     ret = fclose(gfh->fh);
-    if (ret == -1) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fclose() failed");
+    if (ret == EOF) {
+        error_setg_errno(err, errno, "failed to close handle");
         return;
     }
 
@@ -182,21 +212,21 @@ void qmp_guest_file_close(int64_t handle, Error **err)
 struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
                                           int64_t count, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     GuestFileRead *read_data = NULL;
     guchar *buf;
     FILE *fh;
     size_t read_count;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return NULL;
     }
 
     if (!has_count) {
         count = QGA_READ_COUNT_DEFAULT;
     } else if (count < 0) {
-        error_set(err, QERR_INVALID_PARAMETER, "count");
+        error_setg(err, "value '%" PRId64 "' is invalid for argument count",
+                   count);
         return NULL;
     }
 
@@ -204,8 +234,8 @@ struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
     buf = g_malloc0(count+1);
     read_count = fread(buf, 1, count, fh);
     if (ferror(fh)) {
+        error_setg_errno(err, errno, "failed to read file");
         slog("guest-file-read failed, handle: %ld", handle);
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fread() failed");
     } else {
         buf[read_count] = 0;
         read_data = g_malloc0(sizeof(GuestFileRead));
@@ -228,11 +258,10 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
     guchar *buf;
     gsize buf_len;
     int write_count;
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     FILE *fh;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return NULL;
     }
 
@@ -242,15 +271,16 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
     if (!has_count) {
         count = buf_len;
     } else if (count < 0 || count > buf_len) {
+        error_setg(err, "value '%" PRId64 "' is invalid for argument count",
+                   count);
         g_free(buf);
-        error_set(err, QERR_INVALID_PARAMETER, "count");
         return NULL;
     }
 
     write_count = fwrite(buf, 1, count, fh);
     if (ferror(fh)) {
+        error_setg_errno(err, errno, "failed to write to file");
         slog("guest-file-write failed, handle: %ld", handle);
-        error_set(err, QERR_QGA_COMMAND_FAILED, "fwrite() error");
     } else {
         write_data = g_malloc0(sizeof(GuestFileWrite));
         write_data->count = write_count;
@@ -265,20 +295,19 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
 struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
                                           int64_t whence, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     GuestFileSeek *seek_data = NULL;
     FILE *fh;
     int ret;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return NULL;
     }
 
     fh = gfh->fh;
     ret = fseek(fh, offset, whence);
     if (ret == -1) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, strerror(errno));
+        error_setg_errno(err, errno, "failed to seek file");
     } else {
         seek_data = g_malloc0(sizeof(GuestFileRead));
         seek_data->position = ftell(fh);
@@ -291,19 +320,18 @@ struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
 
 void qmp_guest_file_flush(int64_t handle, Error **err)
 {
-    GuestFileHandle *gfh = guest_file_handle_find(handle);
+    GuestFileHandle *gfh = guest_file_handle_find(handle, err);
     FILE *fh;
     int ret;
 
     if (!gfh) {
-        error_set(err, QERR_FD_NOT_FOUND, "handle");
         return;
     }
 
     fh = gfh->fh;
     ret = fflush(fh);
     if (ret == EOF) {
-        error_set(err, QERR_QGA_COMMAND_FAILED, strerror(errno));
+        error_setg_errno(err, errno, "failed to flush file");
     }
 }
 
@@ -343,7 +371,7 @@ static void free_fs_mount_list(FsMountList *mounts)
 /*
  * Walk the mount table and build a list of local file systems
  */
-static int build_fs_mount_list(FsMountList *mounts)
+static void build_fs_mount_list(FsMountList *mounts, Error **err)
 {
     struct mntent *ment;
     FsMount *mount;
@@ -352,8 +380,8 @@ static int build_fs_mount_list(FsMountList *mounts)
 
     fp = setmntent(mtab, "r");
     if (!fp) {
-        g_warning("fsfreeze: unable to read mtab");
-        return -1;
+        error_setg(err, "failed to open mtab file: '%s'", mtab);
+        return;
     }
 
     while ((ment = getmntent(fp))) {
@@ -377,13 +405,71 @@ static int build_fs_mount_list(FsMountList *mounts)
     }
 
     endmntent(fp);
-
-    return 0;
 }
 #endif
 
 #if defined(CONFIG_FSFREEZE)
 
+typedef enum {
+    FSFREEZE_HOOK_THAW = 0,
+    FSFREEZE_HOOK_FREEZE,
+} FsfreezeHookArg;
+
+const char *fsfreeze_hook_arg_string[] = {
+    "thaw",
+    "freeze",
+};
+
+static void execute_fsfreeze_hook(FsfreezeHookArg arg, Error **err)
+{
+    int status;
+    pid_t pid;
+    const char *hook;
+    const char *arg_str = fsfreeze_hook_arg_string[arg];
+    Error *local_err = NULL;
+
+    hook = ga_fsfreeze_hook(ga_state);
+    if (!hook) {
+        return;
+    }
+    if (access(hook, X_OK) != 0) {
+        error_setg_errno(err, errno, "can't access fsfreeze hook '%s'", hook);
+        return;
+    }
+
+    slog("executing fsfreeze hook with arg '%s'", arg_str);
+    pid = fork();
+    if (pid == 0) {
+        setsid();
+        reopen_fd_to_null(0);
+        reopen_fd_to_null(1);
+        reopen_fd_to_null(2);
+
+        execle(hook, hook, arg_str, NULL, environ);
+        _exit(EXIT_FAILURE);
+    } else if (pid < 0) {
+        error_setg_errno(err, errno, "failed to create child process");
+        return;
+    }
+
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return;
+    }
+
+    if (!WIFEXITED(status)) {
+        error_setg(err, "fsfreeze hook has terminated abnormally");
+        return;
+    }
+
+    status = WEXITSTATUS(status);
+    if (status) {
+        error_setg(err, "fsfreeze hook has failed with status %d", status);
+        return;
+    }
+}
+
 /*
  * Return status of freeze/thaw
  */
@@ -405,15 +491,22 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
     int ret = 0, i = 0;
     FsMountList mounts;
     struct FsMount *mount;
+    Error *local_err = NULL;
     int fd;
-    char err_msg[512];
 
     slog("guest-fsfreeze called");
 
+    execute_fsfreeze_hook(FSFREEZE_HOOK_FREEZE, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return -1;
+    }
+
     QTAILQ_INIT(&mounts);
-    ret = build_fs_mount_list(&mounts);
-    if (ret < 0) {
-        return ret;
+    build_fs_mount_list(&mounts, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        return -1;
     }
 
     /* cannot risk guest agent blocking itself on a write in this state */
@@ -422,9 +515,7 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
     QTAILQ_FOREACH(mount, &mounts, next) {
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
-            sprintf(err_msg, "failed to open %s, %s", mount->dirname,
-                    strerror(errno));
-            error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+            error_setg_errno(err, errno, "failed to open %s", mount->dirname);
             goto error;
         }
 
@@ -440,9 +531,8 @@ int64_t qmp_guest_fsfreeze_freeze(Error **err)
         ret = ioctl(fd, FIFREEZE);
         if (ret == -1) {
             if (errno != EOPNOTSUPP) {
-                sprintf(err_msg, "failed to freeze %s, %s",
-                        mount->dirname, strerror(errno));
-                error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(err, errno, "failed to freeze %s",
+                                 mount->dirname);
                 close(fd);
                 goto error;
             }
@@ -470,12 +560,12 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
     FsMountList mounts;
     FsMount *mount;
     int fd, i = 0, logged;
+    Error *local_err = NULL;
 
     QTAILQ_INIT(&mounts);
-    ret = build_fs_mount_list(&mounts);
-    if (ret) {
-        error_set(err, QERR_QGA_COMMAND_FAILED,
-                  "failed to enumerate filesystems");
+    build_fs_mount_list(&mounts, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
         return 0;
     }
 
@@ -513,18 +603,22 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
 
     ga_unset_frozen(ga_state);
     free_fs_mount_list(&mounts);
+
+    execute_fsfreeze_hook(FSFREEZE_HOOK_THAW, err);
+
     return i;
 }
 
 static void guest_fsfreeze_cleanup(void)
 {
-    int64_t ret;
     Error *err = NULL;
 
     if (ga_is_frozen(ga_state) == GUEST_FSFREEZE_STATUS_FROZEN) {
-        ret = qmp_guest_fsfreeze_thaw(&err);
-        if (ret < 0 || err) {
-            slog("failed to clean up frozen filesystems");
+        qmp_guest_fsfreeze_thaw(&err);
+        if (err) {
+            slog("failed to clean up frozen filesystems: %s",
+                 error_get_pretty(err));
+            error_free(err);
         }
     }
 }
@@ -540,7 +634,7 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
     FsMountList mounts;
     struct FsMount *mount;
     int fd;
-    char err_msg[512];
+    Error *local_err = NULL;
     struct fstrim_range r = {
         .start = 0,
         .len = -1,
@@ -550,17 +644,16 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
     slog("guest-fstrim called");
 
     QTAILQ_INIT(&mounts);
-    ret = build_fs_mount_list(&mounts);
-    if (ret < 0) {
+    build_fs_mount_list(&mounts, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
         return;
     }
 
     QTAILQ_FOREACH(mount, &mounts, next) {
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
-            sprintf(err_msg, "failed to open %s, %s", mount->dirname,
-                    strerror(errno));
-            error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+            error_setg_errno(err, errno, "failed to open %s", mount->dirname);
             goto error;
         }
 
@@ -573,9 +666,8 @@ void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
         ret = ioctl(fd, FITRIM, &r);
         if (ret == -1) {
             if (errno != ENOTTY && errno != EOPNOTSUPP) {
-                sprintf(err_msg, "failed to trim %s, %s",
-                        mount->dirname, strerror(errno));
-                error_set(err, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(err, errno, "failed to trim %s",
+                                 mount->dirname);
                 close(fd);
                 goto error;
             }
@@ -596,8 +688,9 @@ error:
 static void bios_supports_mode(const char *pmutils_bin, const char *pmutils_arg,
                                const char *sysfile_str, Error **err)
 {
+    Error *local_err = NULL;
     char *pmutils_path;
-    pid_t pid, rpid;
+    pid_t pid;
     int status;
 
     pmutils_path = g_find_program_in_path(pmutils_bin);
@@ -642,38 +735,46 @@ static void bios_supports_mode(const char *pmutils_bin, const char *pmutils_arg,
         }
 
         _exit(SUSPEND_NOT_SUPPORTED);
+    } else if (pid < 0) {
+        error_setg_errno(err, errno, "failed to create child process");
+        goto out;
     }
 
-    g_free(pmutils_path);
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        goto out;
+    }
 
-    if (pid < 0) {
-        goto undef_err;
+    if (!WIFEXITED(status)) {
+        error_setg(err, "child process has terminated abnormally");
+        goto out;
     }
 
-    do {
-        rpid = waitpid(pid, &status, 0);
-    } while (rpid == -1 && errno == EINTR);
-    if (rpid == pid && WIFEXITED(status)) {
-        switch (WEXITSTATUS(status)) {
-        case SUSPEND_SUPPORTED:
-            return;
-        case SUSPEND_NOT_SUPPORTED:
-            error_set(err, QERR_UNSUPPORTED);
-            return;
-        default:
-            goto undef_err;
-        }
+    switch (WEXITSTATUS(status)) {
+    case SUSPEND_SUPPORTED:
+        goto out;
+    case SUSPEND_NOT_SUPPORTED:
+        error_setg(err,
+                   "the requested suspend mode is not supported by the guest");
+        goto out;
+    default:
+        error_setg(err,
+                   "the helper program '%s' returned an unexpected exit status"
+                   " code (%d)", pmutils_path, WEXITSTATUS(status));
+        goto out;
     }
 
-undef_err:
-    error_set(err, QERR_UNDEFINED_ERROR);
+out:
+    g_free(pmutils_path);
 }
 
 static void guest_suspend(const char *pmutils_bin, const char *sysfile_str,
                           Error **err)
 {
+    Error *local_err = NULL;
     char *pmutils_path;
-    pid_t rpid, pid;
+    pid_t pid;
     int status;
 
     pmutils_path = g_find_program_in_path(pmutils_bin);
@@ -711,23 +812,29 @@ static void guest_suspend(const char *pmutils_bin, const char *sysfile_str,
         }
 
         _exit(EXIT_SUCCESS);
+    } else if (pid < 0) {
+        error_setg_errno(err, errno, "failed to create child process");
+        goto out;
     }
 
-    g_free(pmutils_path);
+    ga_wait_child(pid, &status, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(err, local_err);
+        goto out;
+    }
 
-    if (pid < 0) {
-        goto exit_err;
+    if (!WIFEXITED(status)) {
+        error_setg(err, "child process has terminated abnormally");
+        goto out;
     }
 
-    do {
-        rpid = waitpid(pid, &status, 0);
-    } while (rpid == -1 && errno == EINTR);
-    if (rpid == pid && WIFEXITED(status) && !WEXITSTATUS(status)) {
-        return;
+    if (WEXITSTATUS(status)) {
+        error_setg(err, "child process has failed to suspend");
+        goto out;
     }
 
-exit_err:
-    error_set(err, QERR_UNDEFINED_ERROR);
+out:
+    g_free(pmutils_path);
 }
 
 void qmp_guest_suspend_disk(Error **err)
@@ -780,12 +887,9 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
 {
     GuestNetworkInterfaceList *head = NULL, *cur_item = NULL;
     struct ifaddrs *ifap, *ifa;
-    char err_msg[512];
 
     if (getifaddrs(&ifap) < 0) {
-        snprintf(err_msg, sizeof(err_msg),
-                 "getifaddrs failed: %s", strerror(errno));
-        error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+        error_setg_errno(errp, errno, "getifaddrs failed");
         goto error;
     }
 
@@ -821,53 +925,43 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
             /* we haven't obtained HW address yet */
             sock = socket(PF_INET, SOCK_STREAM, 0);
             if (sock == -1) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "failed to create socket: %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno, "failed to create socket");
                 goto error;
             }
 
             memset(&ifr, 0, sizeof(ifr));
             pstrcpy(ifr.ifr_name, IF_NAMESIZE, info->value->name);
             if (ioctl(sock, SIOCGIFHWADDR, &ifr) == -1) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "failed to get MAC address of %s: %s",
-                         ifa->ifa_name,
-                         strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno,
+                                 "failed to get MAC address of %s",
+                                 ifa->ifa_name);
+                close(sock);
                 goto error;
             }
 
+            close(sock);
             mac_addr = (unsigned char *) &ifr.ifr_hwaddr.sa_data;
 
-            if (asprintf(&info->value->hardware_address,
-                         "%02x:%02x:%02x:%02x:%02x:%02x",
-                         (int) mac_addr[0], (int) mac_addr[1],
-                         (int) mac_addr[2], (int) mac_addr[3],
-                         (int) mac_addr[4], (int) mac_addr[5]) == -1) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "failed to format MAC: %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
-                goto error;
-            }
+            info->value->hardware_address =
+                g_strdup_printf("%02x:%02x:%02x:%02x:%02x:%02x",
+                                (int) mac_addr[0], (int) mac_addr[1],
+                                (int) mac_addr[2], (int) mac_addr[3],
+                                (int) mac_addr[4], (int) mac_addr[5]);
 
             info->value->has_hardware_address = true;
-            close(sock);
         }
 
         if (ifa->ifa_addr &&
             ifa->ifa_addr->sa_family == AF_INET) {
             /* interface with IPv4 address */
-            address_item = g_malloc0(sizeof(*address_item));
-            address_item->value = g_malloc0(sizeof(*address_item->value));
             p = &((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
             if (!inet_ntop(AF_INET, p, addr4, sizeof(addr4))) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "inet_ntop failed : %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno, "inet_ntop failed");
                 goto error;
             }
 
+            address_item = g_malloc0(sizeof(*address_item));
+            address_item->value = g_malloc0(sizeof(*address_item->value));
             address_item->value->ip_address = g_strdup(addr4);
             address_item->value->ip_address_type = GUEST_IP_ADDRESS_TYPE_IPV4;
 
@@ -880,16 +974,14 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
         } else if (ifa->ifa_addr &&
                    ifa->ifa_addr->sa_family == AF_INET6) {
             /* interface with IPv6 address */
-            address_item = g_malloc0(sizeof(*address_item));
-            address_item->value = g_malloc0(sizeof(*address_item->value));
             p = &((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr;
             if (!inet_ntop(AF_INET6, p, addr6, sizeof(addr6))) {
-                snprintf(err_msg, sizeof(err_msg),
-                         "inet_ntop failed : %s", strerror(errno));
-                error_set(errp, QERR_QGA_COMMAND_FAILED, err_msg);
+                error_setg_errno(errp, errno, "inet_ntop failed");
                 goto error;
             }
 
+            address_item = g_malloc0(sizeof(*address_item));
+            address_item->value = g_malloc0(sizeof(*address_item->value));
             address_item->value->ip_address = g_strdup(addr6);
             address_item->value->ip_address_type = GUEST_IP_ADDRESS_TYPE_IPV6;
 
diff --git a/qga/commands-win32.c b/qga/commands-win32.c
index 5bd8fb2..7e8ecb3 100644
--- a/qga/commands-win32.c
+++ b/qga/commands-win32.c
@@ -16,7 +16,7 @@
 #include <powrprof.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 #ifndef SHTDN_REASON_FLAG_PLANNED
 #define SHTDN_REASON_FLAG_PLANNED 0x80000000
diff --git a/qga/commands.c b/qga/commands.c
index 46b0b08..528b082 100644
--- a/qga/commands.c
+++ b/qga/commands.c
@@ -13,7 +13,7 @@
 #include <glib.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 /* Note: in some situations, like with the fsfreeze, logging may be
  * temporarilly disabled. if it is necessary that a command be able
@@ -61,7 +61,7 @@ struct GuestAgentInfo *qmp_guest_info(Error **err)
 
     while (*cmd_list) {
         cmd_info = g_malloc0(sizeof(GuestAgentCommandInfo));
-        cmd_info->name = strdup(*cmd_list);
+        cmd_info->name = g_strdup(*cmd_list);
         cmd_info->enabled = qmp_command_is_enabled(cmd_info->name);
 
         cmd_info_list = g_malloc0(sizeof(GuestAgentCommandInfoList));
diff --git a/qga/guest-agent-core.h b/qga/guest-agent-core.h
index 49a7abe..3354598 100644
--- a/qga/guest-agent-core.h
+++ b/qga/guest-agent-core.h
@@ -10,7 +10,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include "qapi/qmp-core.h"
+#include "qapi/qmp/dispatch.h"
 #include "qemu-common.h"
 
 #define QGA_READ_COUNT_DEFAULT 4096
@@ -34,6 +34,7 @@ void ga_set_response_delimited(GAState *s);
 bool ga_is_frozen(GAState *s);
 void ga_set_frozen(GAState *s);
 void ga_unset_frozen(GAState *s);
+const char *ga_fsfreeze_hook(GAState *s);
 
 #ifndef _WIN32
 void reopen_fd_to_null(int fd);
diff --git a/qemu-ga.c b/qga/main.c
index 9b59a52..db281a5 100644
--- a/qemu-ga.c
+++ b/qga/main.c
@@ -20,20 +20,26 @@
 #include <sys/wait.h>
 #include <sys/stat.h>
 #endif
-#include "json-streamer.h"
-#include "json-parser.h"
-#include "qint.h"
-#include "qjson.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qjson.h"
 #include "qga/guest-agent-core.h"
-#include "module.h"
+#include "qemu/module.h"
 #include "signal.h"
-#include "qerror.h"
-#include "qapi/qmp-core.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/dispatch.h"
 #include "qga/channel.h"
 #ifdef _WIN32
 #include "qga/service-win32.h"
 #include <windows.h>
 #endif
+#ifdef __linux__
+#include <linux/fs.h>
+#ifdef FIFREEZE
+#define CONFIG_FSFREEZE
+#endif
+#endif
 
 #ifndef _WIN32
 #define QGA_VIRTIO_PATH_DEFAULT "/dev/virtio-ports/org.qemu.guest_agent.0"
@@ -42,6 +48,9 @@
 #endif
 #define QGA_STATEDIR_DEFAULT CONFIG_QEMU_LOCALSTATEDIR "/run"
 #define QGA_PIDFILE_DEFAULT QGA_STATEDIR_DEFAULT "/qemu-ga.pid"
+#ifdef CONFIG_FSFREEZE
+#define QGA_FSFREEZE_HOOK_DEFAULT CONFIG_QEMU_CONFDIR "/fsfreeze-hook"
+#endif
 #define QGA_SENTINEL_BYTE 0xFF
 
 struct GAState {
@@ -64,6 +73,9 @@ struct GAState {
         const char *log_filepath;
         const char *pid_filepath;
     } deferred_options;
+#ifdef CONFIG_FSFREEZE
+    const char *fsfreeze_hook;
+#endif
 };
 
 struct GAState *ga_state;
@@ -153,6 +165,16 @@ static void usage(const char *cmd)
 "                    %s)\n"
 "  -l, --logfile     set logfile path, logs to stderr by default\n"
 "  -f, --pidfile     specify pidfile (default is %s)\n"
+#ifdef CONFIG_FSFREEZE
+"  -F, --fsfreeze-hook\n"
+"                    enable fsfreeze hook. Accepts an optional argument that\n"
+"                    specifies script to run on freeze/thaw. Script will be\n"
+"                    called with 'freeze'/'thaw' arguments accordingly.\n"
+"                    (default is %s)\n"
+"                    If using -F with an argument, do not follow -F with a\n"
+"                    space.\n"
+"                    (for example: -F/var/run/fsfreezehook.sh)\n"
+#endif
 "  -t, --statedir    specify dir to store state information (absolute paths\n"
 "                    only, default is %s)\n"
 "  -v, --verbose     log extra debugging information\n"
@@ -167,6 +189,9 @@ static void usage(const char *cmd)
 "\n"
 "Report bugs to <mdroth@linux.vnet.ibm.com>\n"
     , cmd, QEMU_VERSION, QGA_VIRTIO_PATH_DEFAULT, QGA_PIDFILE_DEFAULT,
+#ifdef CONFIG_FSFREEZE
+    QGA_FSFREEZE_HOOK_DEFAULT,
+#endif
     QGA_STATEDIR_DEFAULT);
 }
 
@@ -236,13 +261,26 @@ void ga_set_response_delimited(GAState *s)
     s->delimit_response = true;
 }
 
+static FILE *ga_open_logfile(const char *logfile)
+{
+    FILE *f;
+
+    f = fopen(logfile, "a");
+    if (!f) {
+        return NULL;
+    }
+
+    qemu_set_cloexec(fileno(f));
+    return f;
+}
+
 #ifndef _WIN32
 static bool ga_open_pidfile(const char *pidfile)
 {
     int pidfd;
     char pidstr[32];
 
-    pidfd = open(pidfile, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR);
+    pidfd = qemu_open(pidfile, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR);
     if (pidfd == -1 || lockf(pidfd, F_TLOCK, 0)) {
         g_critical("Cannot lock pid file, %s", strerror(errno));
         if (pidfd != -1) {
@@ -251,7 +289,7 @@ static bool ga_open_pidfile(const char *pidfile)
         return false;
     }
 
-    if (ftruncate(pidfd, 0) || lseek(pidfd, 0, SEEK_SET)) {
+    if (ftruncate(pidfd, 0)) {
         g_critical("Failed to truncate pid file");
         goto fail;
     }
@@ -261,10 +299,12 @@ static bool ga_open_pidfile(const char *pidfile)
         goto fail;
     }
 
+    /* keep pidfile open & locked forever */
     return true;
 
 fail:
     unlink(pidfile);
+    close(pidfd);
     return false;
 }
 #else /* _WIN32 */
@@ -377,7 +417,7 @@ void ga_unset_frozen(GAState *s)
      * in a frozen state at start up, do it now
      */
     if (s->deferred_options.log_filepath) {
-        s->log_file = fopen(s->deferred_options.log_filepath, "a");
+        s->log_file = ga_open_logfile(s->deferred_options.log_filepath);
         if (!s->log_file) {
             s->log_file = stderr;
         }
@@ -401,6 +441,13 @@ void ga_unset_frozen(GAState *s)
     }
 }
 
+#ifdef CONFIG_FSFREEZE
+const char *ga_fsfreeze_hook(GAState *s)
+{
+    return s->fsfreeze_hook;
+}
+#endif
+
 static void become_daemon(const char *pidfile)
 {
 #ifndef _WIN32
@@ -573,6 +620,7 @@ static gboolean channel_event_cb(GIOCondition condition, gpointer data)
         if (!s->virtio) {
             return false;
         }
+        /* fall through */
     case G_IO_STATUS_AGAIN:
         /* virtio causes us to spin here when no process is attached to
          * host-side chardev. sleep a bit to mitigate this
@@ -678,10 +726,13 @@ VOID WINAPI service_main(DWORD argc, TCHAR *argv[])
 
 int main(int argc, char **argv)
 {
-    const char *sopt = "hVvdm:p:l:f:b:s:t:";
+    const char *sopt = "hVvdm:p:l:f:F::b:s:t:";
     const char *method = NULL, *path = NULL;
     const char *log_filepath = NULL;
     const char *pid_filepath = QGA_PIDFILE_DEFAULT;
+#ifdef CONFIG_FSFREEZE
+    const char *fsfreeze_hook = NULL;
+#endif
     const char *state_dir = QGA_STATEDIR_DEFAULT;
 #ifdef _WIN32
     const char *service = NULL;
@@ -691,6 +742,9 @@ int main(int argc, char **argv)
         { "version", 0, NULL, 'V' },
         { "logfile", 1, NULL, 'l' },
         { "pidfile", 1, NULL, 'f' },
+#ifdef CONFIG_FSFREEZE
+        { "fsfreeze-hook", 2, NULL, 'F' },
+#endif
         { "verbose", 0, NULL, 'v' },
         { "method", 1, NULL, 'm' },
         { "path", 1, NULL, 'p' },
@@ -723,6 +777,11 @@ int main(int argc, char **argv)
         case 'f':
             pid_filepath = optarg;
             break;
+#ifdef CONFIG_FSFREEZE
+        case 'F':
+            fsfreeze_hook = optarg ? optarg : QGA_FSFREEZE_HOOK_DEFAULT;
+            break;
+#endif
         case 't':
              state_dir = optarg;
              break;
@@ -786,6 +845,9 @@ int main(int argc, char **argv)
     s = g_malloc0(sizeof(GAState));
     s->log_level = log_level;
     s->log_file = stderr;
+#ifdef CONFIG_FSFREEZE
+    s->fsfreeze_hook = fsfreeze_hook;
+#endif
     g_log_set_default_handler(ga_log, s);
     g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
     ga_enable_logging(s);
@@ -838,7 +900,7 @@ int main(int argc, char **argv)
             become_daemon(pid_filepath);
         }
         if (log_filepath) {
-            FILE *log_file = fopen(log_filepath, "a");
+            FILE *log_file = ga_open_logfile(log_filepath);
             if (!log_file) {
                 g_critical("unable to open specified log file: %s",
                            strerror(errno));
diff --git a/qapi-schema-guest.json b/qga/qapi-schema.json
index ed0eb69..d91d903 100644
--- a/qapi-schema-guest.json
+++ b/qga/qapi-schema.json
@@ -31,7 +31,7 @@
 #
 # Since: 1.1
 # ##
-{ 'command': 'guest-sync-delimited'
+{ 'command': 'guest-sync-delimited',
   'data':    { 'id': 'int' },
   'returns': 'int' }
 
@@ -69,7 +69,7 @@
 #
 # Since: 0.15.0
 ##
-{ 'command': 'guest-sync'
+{ 'command': 'guest-sync',
   'data':    { 'id': 'int' },
   'returns': 'int' }
 
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 5c692d0..799adea 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -466,6 +466,73 @@ Note: inject-nmi fails when the guest doesn't support injecting.
 EQMP
 
     {
+        .name       = "ringbuf-write",
+        .args_type  = "device:s,data:s,format:s?",
+        .mhandler.cmd_new = qmp_marshal_input_ringbuf_write,
+    },
+
+SQMP
+ringbuf-write
+-------------
+
+Write to a ring buffer character device.
+
+Arguments:
+
+- "device": ring buffer character device name (json-string)
+- "data": data to write (json-string)
+- "format": data format (json-string, optional)
+          - Possible values: "utf8" (default), "base64"
+            Bug: invalid base64 is currently not rejected.
+            Whitespace *is* invalid.
+
+Example:
+
+-> { "execute": "ringbuf-write",
+                "arguments": { "device": "foo",
+                               "data": "abcdefgh",
+                               "format": "utf8" } }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "ringbuf-read",
+        .args_type  = "device:s,size:i,format:s?",
+        .mhandler.cmd_new = qmp_marshal_input_ringbuf_read,
+    },
+
+SQMP
+ringbuf-read
+-------------
+
+Read from a ring buffer character device.
+
+Arguments:
+
+- "device": ring buffer character device name (json-string)
+- "size": how many bytes to read at most (json-int)
+          - Number of data bytes, not number of characters in encoded data
+- "format": data format (json-string, optional)
+          - Possible values: "utf8" (default), "base64"
+          - Naturally, format "utf8" works only when the ring buffer
+            contains valid UTF-8 text.  Invalid UTF-8 sequences get
+            replaced.  Bug: replacement doesn't work.  Bug: can screw
+            up on encountering NUL characters, after the ring buffer
+            lost data, and when reading stops because the size limit
+            is reached.
+
+Example:
+
+-> { "execute": "ringbuf-read",
+                "arguments": { "device": "foo",
+                               "size": 1000,
+                               "format": "utf8" } }
+<- {"return": "abcdefgh"}
+
+EQMP
+
+    {
         .name       = "xen-save-devices-state",
         .args_type  = "filename:F",
     .mhandler.cmd_new = qmp_marshal_input_xen_save_devices_state,
@@ -938,7 +1005,8 @@ EQMP
     {
         .name       = "drive-mirror",
         .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
-                      "on-source-error:s?,on-target-error:s?",
+                      "on-source-error:s?,on-target-error:s?,"
+                      "granularity:i?,buf-size:i?",
         .mhandler.cmd_new = qmp_marshal_input_drive_mirror,
     },
 
@@ -962,6 +1030,9 @@ Arguments:
   file/device (NewImageMode, optional, default 'absolute-paths')
 - "speed": maximum speed of the streaming job, in bytes per second
   (json-int)
+- "granularity": granularity of the dirty bitmap, in bytes (json-int, optional)
+- "buf_size": maximum amount of data in flight from source to target, in bytes
+  (json-int, default 10M)
 - "sync": what parts of the disk image should be copied to the destination;
   possibilities include "full" for all the disk, "top" for only the sectors
   allocated in the topmost image, or "none" to only replicate new I/O
@@ -971,6 +1042,10 @@ Arguments:
 - "on-target-error": the action to take on an error on the target
   (BlockdevOnError, default 'report')
 
+The default value of the granularity is the image cluster size clamped
+between 4096 and 65536, if the image format defines one.  If the format
+does not define a cluster size, the default value of the granularity
+is 65536.
 
 
 Example:
@@ -1585,7 +1660,7 @@ Each json-object contain the following:
          - Possible values: "unknown"
 - "removable": true if the device is removable, false otherwise (json-bool)
 - "locked": true if the device is locked, false otherwise (json-bool)
-- "tray-open": only present if removable, true if the device has a tray,
+- "tray_open": only present if removable, true if the device has a tray,
                and it is open (json-bool)
 - "inserted": only present if the device is inserted, it is a json-object
    containing the following:
@@ -2527,10 +2602,8 @@ Arguments:
 Example:
 
 -> { "execute": "query-migrate-capabilities" }
-<- { "return": {
-        "capabilities" :  [ { "capability" : "xbzrle", "state" : false } ]
-     }
-   }
+<- { "return": [ { "state": false, "capability": "xbzrle" } ] }
+
 EQMP
 
     {
@@ -2549,13 +2622,6 @@ Make an asynchronous request for balloon info. When the request completes a
 json-object will be returned containing the following data:
 
 - "actual": current balloon value in bytes (json-int)
-- "mem_swapped_in": Amount of memory swapped in bytes (json-int, optional)
-- "mem_swapped_out": Amount of memory swapped out in bytes (json-int, optional)
-- "major_page_faults": Number of major faults (json-int, optional)
-- "minor_page_faults": Number of minor faults (json-int, optional)
-- "free_mem": Total amount of free and unused memory in
-              bytes (json-int, optional)
-- "total_mem": Total amount of available memory in bytes (json-int, optional)
 
 Example:
 
@@ -2563,12 +2629,6 @@ Example:
 <- {
       "return":{
          "actual":1073741824,
-         "mem_swapped_in":0,
-         "mem_swapped_out":0,
-         "major_page_faults":142,
-         "minor_page_faults":239245,
-         "free_mem":1014185984,
-         "total_mem":1044668416
       }
    }
 
@@ -2654,3 +2714,64 @@ EQMP
         .args_type  = "",
         .mhandler.cmd_new = qmp_marshal_input_query_target,
     },
+
+    {
+        .name       = "chardev-add",
+        .args_type  = "id:s,backend:q",
+        .mhandler.cmd_new = qmp_marshal_input_chardev_add,
+    },
+
+SQMP
+chardev-add
+----------------
+
+Add a chardev.
+
+Arguments:
+
+- "id": the chardev's ID, must be unique (json-string)
+- "backend": chardev backend type + parameters
+
+Examples:
+
+-> { "execute" : "chardev-add",
+     "arguments" : { "id" : "foo",
+                     "backend" : { "type" : "null", "data" : {} } } }
+<- { "return": {} }
+
+-> { "execute" : "chardev-add",
+     "arguments" : { "id" : "bar",
+                     "backend" : { "type" : "file",
+                                   "data" : { "out" : "/tmp/bar.log" } } } }
+<- { "return": {} }
+
+-> { "execute" : "chardev-add",
+     "arguments" : { "id" : "baz",
+                     "backend" : { "type" : "pty", "data" : {} } } }
+<- { "return": { "pty" : "/dev/pty/42" } }
+
+EQMP
+
+    {
+        .name       = "chardev-remove",
+        .args_type  = "id:s",
+        .mhandler.cmd_new = qmp_marshal_input_chardev_remove,
+    },
+
+
+SQMP
+chardev-remove
+--------------
+
+Remove a chardev.
+
+Arguments:
+
+- "id": the chardev's ID, must exist and not be in use (json-string)
+
+Example:
+
+-> { "execute": "chardev-remove", "arguments": { "id" : "foo" } }
+<- { "return": {} }
+
+EQMP
diff --git a/qmp.c b/qmp.c
index e3a7f0b..55b056b 100644
--- a/qmp.c
+++ b/qmp.c
@@ -14,15 +14,16 @@
  */
 
 #include "qemu-common.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "qmp-commands.h"
+#include "char/char.h"
 #include "ui/qemu-spice.h"
 #include "ui/vnc.h"
-#include "kvm.h"
-#include "arch_init.h"
+#include "sysemu/kvm.h"
+#include "sysemu/arch_init.h"
 #include "hw/qdev.h"
-#include "blockdev.h"
-#include "qemu/qom-qobject.h"
+#include "sysemu/blockdev.h"
+#include "qom/qom-qobject.h"
 
 NameInfo *qmp_query_name(Error **errp)
 {
diff --git a/qobject/Makefile.objs b/qobject/Makefile.objs
new file mode 100644
index 0000000..c9ff59c
--- /dev/null
+++ b/qobject/Makefile.objs
@@ -0,0 +1,3 @@
+util-obj-y = qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
+util-obj-y += qjson.o json-lexer.o json-streamer.o json-parser.o
+util-obj-y += qerror.o
diff --git a/json-lexer.c b/qobject/json-lexer.c
index 3cd3285..440df60 100644
--- a/json-lexer.c
+++ b/qobject/json-lexer.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qstring.h"
-#include "qlist.h"
-#include "qdict.h"
-#include "qint.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qint.h"
 #include "qemu-common.h"
-#include "json-lexer.h"
+#include "qapi/qmp/json-lexer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 
diff --git a/json-parser.c b/qobject/json-parser.c
index 457291b..05279c1 100644
--- a/json-parser.c
+++ b/qobject/json-parser.c
@@ -14,15 +14,15 @@
 #include <stdarg.h>
 
 #include "qemu-common.h"
-#include "qstring.h"
-#include "qint.h"
-#include "qdict.h"
-#include "qlist.h"
-#include "qfloat.h"
-#include "qbool.h"
-#include "json-parser.h"
-#include "json-lexer.h"
-#include "qerror.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/qerror.h"
 
 typedef struct JSONParserContext
 {
diff --git a/json-streamer.c b/qobject/json-streamer.c
index c255c78..1b2f9b1 100644
--- a/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -11,12 +11,12 @@
  *
  */
 
-#include "qlist.h"
-#include "qint.h"
-#include "qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
 #include "qemu-common.h"
-#include "json-lexer.h"
-#include "json-streamer.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 #define MAX_NESTING (1ULL << 10)
diff --git a/qbool.c b/qobject/qbool.c
index 590cd71..a3d2afa 100644
--- a/qbool.c
+++ b/qobject/qbool.c
@@ -11,8 +11,8 @@
  *
  */
 
-#include "qbool.h"
-#include "qobject.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
 static void qbool_destroy_obj(QObject *obj);
diff --git a/qdict.c b/qobject/qdict.c
index 4bf308b..7543ccc 100644
--- a/qdict.c
+++ b/qobject/qdict.c
@@ -10,13 +10,13 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qint.h"
-#include "qfloat.h"
-#include "qdict.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qobject.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
 
 static void qdict_destroy_obj(QObject *obj);
diff --git a/qerror.c b/qobject/qerror.c
index 0818504..3aee1cf 100644
--- a/qerror.c
+++ b/qobject/qerror.c
@@ -10,9 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "monitor.h"
-#include "qjson.h"
-#include "qerror.h"
+#include "monitor/monitor.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu-common.h"
 
 static void qerror_destroy_obj(QObject *obj);
diff --git a/qfloat.c b/qobject/qfloat.c
index 98338f3..7de0992 100644
--- a/qfloat.c
+++ b/qobject/qfloat.c
@@ -11,8 +11,8 @@
  *
  */
 
-#include "qfloat.h"
-#include "qobject.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
 static void qfloat_destroy_obj(QObject *obj);
diff --git a/qint.c b/qobject/qint.c
index ee51804..86b9b04 100644
--- a/qint.c
+++ b/qobject/qint.c
@@ -10,8 +10,8 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qint.h"
-#include "qobject.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
 static void qint_destroy_obj(QObject *obj);
diff --git a/qjson.c b/qobject/qjson.c
index f9c8e77..83a6b4f 100644
--- a/qjson.c
+++ b/qobject/qjson.c
@@ -11,15 +11,15 @@
  *
  */
 
-#include "json-lexer.h"
-#include "json-parser.h"
-#include "json-streamer.h"
-#include "qjson.h"
-#include "qint.h"
-#include "qlist.h"
-#include "qbool.h"
-#include "qfloat.h"
-#include "qdict.h"
+#include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/json-streamer.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qdict.h"
 
 typedef struct JSONParsingState
 {
diff --git a/qlist.c b/qobject/qlist.c
index b48ec5b..1ced0de 100644
--- a/qlist.c
+++ b/qobject/qlist.c
@@ -10,9 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qlist.h"
-#include "qobject.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qobject.h"
+#include "qemu/queue.h"
 #include "qemu-common.h"
 
 static void qlist_destroy_obj(QObject *obj);
diff --git a/qstring.c b/qobject/qstring.c
index b7e12e4..5f7376c 100644
--- a/qstring.c
+++ b/qobject/qstring.c
@@ -10,8 +10,8 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include "qobject.h"
-#include "qstring.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
 static void qstring_destroy_obj(QObject *obj);
diff --git a/qom/Makefile.objs b/qom/Makefile.objs
index 5ef060a..6a93ac7 100644
--- a/qom/Makefile.objs
+++ b/qom/Makefile.objs
@@ -1,4 +1,2 @@
-qom-obj-y = object.o container.o qom-qobject.o
-qom-obj-twice-y = cpu.o
-common-obj-y = $(qom-obj-twice-y)
-user-obj-y = $(qom-obj-twice-y)
+common-obj-y = object.o container.o qom-qobject.o
+common-obj-y += cpu.o
diff --git a/qom/container.c b/qom/container.c
index 4ca8b5c..62b1648 100644
--- a/qom/container.c
+++ b/qom/container.c
@@ -10,11 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu/object.h"
-#include "module.h"
+#include "qom/object.h"
+#include "qemu/module.h"
 #include <assert.h>
 
-static TypeInfo container_info = {
+static const TypeInfo container_info = {
     .name          = "container",
     .instance_size = sizeof(Object),
     .parent        = TYPE_OBJECT,
diff --git a/qom/cpu.c b/qom/cpu.c
index 5b36046..8fb538b 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -18,7 +18,7 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "qemu-common.h"
 
 void cpu_reset(CPUState *cpu)
@@ -34,16 +34,31 @@ static void cpu_common_reset(CPUState *cpu)
 {
 }
 
+ObjectClass *cpu_class_by_name(const char *typename, const char *cpu_model)
+{
+    CPUClass *cc = CPU_CLASS(object_class_by_name(typename));
+
+    return cc->class_by_name(cpu_model);
+}
+
+static ObjectClass *cpu_common_class_by_name(const char *cpu_model)
+{
+    return NULL;
+}
+
 static void cpu_class_init(ObjectClass *klass, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(klass);
     CPUClass *k = CPU_CLASS(klass);
 
+    k->class_by_name = cpu_common_class_by_name;
     k->reset = cpu_common_reset;
+    dc->no_user = 1;
 }
 
-static TypeInfo cpu_type_info = {
+static const TypeInfo cpu_type_info = {
     .name = TYPE_CPU,
-    .parent = TYPE_OBJECT,
+    .parent = TYPE_DEVICE,
     .instance_size = sizeof(CPUState),
     .abstract = true,
     .class_size = sizeof(CPUClass),
diff --git a/qom/object.c b/qom/object.c
index 0739aa2..563e45b 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -10,19 +10,20 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu/object.h"
+#include "qom/object.h"
 #include "qemu-common.h"
-#include "qapi/qapi-visit-core.h"
+#include "qapi/visitor.h"
 #include "qapi/string-input-visitor.h"
 #include "qapi/string-output-visitor.h"
+#include "qapi/qmp/qerror.h"
 
 /* TODO: replace QObject with a simpler visitor to avoid a dependency
  * of the QOM core on QObject?  */
-#include "qemu/qom-qobject.h"
-#include "qobject.h"
-#include "qbool.h"
-#include "qint.h"
-#include "qstring.h"
+#include "qom/qom-qobject.h"
+#include "qapi/qmp/qobject.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
 
 #define MAX_INTERFACES 32
 
@@ -360,12 +361,14 @@ static void object_property_del_child(Object *obj, Object *child, Error **errp)
 
 void object_unparent(Object *obj)
 {
+    object_ref(obj);
     if (obj->parent) {
         object_property_del_child(obj->parent, obj, NULL);
     }
     if (obj->class->unparent) {
         (obj->class->unparent)(obj);
     }
+    object_unref(obj);
 }
 
 static void object_deinit(Object *obj, TypeImpl *type)
@@ -414,13 +417,6 @@ Object *object_new(const char *typename)
     return object_new_with_type(ti);
 }
 
-void object_delete(Object *obj)
-{
-    object_unparent(obj);
-    g_assert(obj->ref == 1);
-    object_unref(obj);
-}
-
 Object *object_dynamic_cast(Object *obj, const char *typename)
 {
     if (obj && object_class_dynamic_cast(object_get_class(obj), typename)) {
@@ -500,6 +496,11 @@ ObjectClass *object_get_class(Object *obj)
     return obj->class;
 }
 
+bool object_class_is_abstract(ObjectClass *klass)
+{
+    return klass->type->abstract;
+}
+
 const char *object_class_get_name(ObjectClass *klass)
 {
     return klass->type->name;
@@ -1016,7 +1017,7 @@ gchar *object_get_canonical_path(Object *obj)
     return newpath;
 }
 
-Object *object_resolve_path_component(Object *parent, gchar *part)
+Object *object_resolve_path_component(Object *parent, const gchar *part)
 {
     ObjectProperty *prop = object_property_find(parent, part, NULL);
     if (prop == NULL) {
diff --git a/qom/qom-qobject.c b/qom/qom-qobject.c
index 0689914..6384b8e 100644
--- a/qom/qom-qobject.c
+++ b/qom/qom-qobject.c
@@ -10,9 +10,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu/object.h"
-#include "qemu/qom-qobject.h"
-#include "qapi/qapi-visit-core.h"
+#include "qom/object.h"
+#include "qom/qom-qobject.h"
+#include "qapi/visitor.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
 
diff --git a/qtest.c b/qtest.c
index fbfab4e..4663a38 100644
--- a/qtest.c
+++ b/qtest.c
@@ -11,20 +11,20 @@
  *
  */
 
-#include "qtest.h"
+#include "sysemu/qtest.h"
 #include "hw/qdev.h"
-#include "qemu-char.h"
-#include "ioport.h"
-#include "memory.h"
+#include "char/char.h"
+#include "exec/ioport.h"
+#include "exec/memory.h"
 #include "hw/irq.h"
-#include "sysemu.h"
-#include "cpus.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/cpus.h"
 
 #define MAX_IRQ 256
 
 const char *qtest_chrdev;
 const char *qtest_log;
-int qtest_allowed = 0;
+bool qtest_allowed;
 
 static DeviceState *irq_intercept_dev;
 static FILE *qtest_log_fp;
@@ -282,8 +282,8 @@ static void qtest_process_command(CharDriverState *chr, gchar **words)
         uint8_t *data;
 
         g_assert(words[1] && words[2]);
-        addr = strtoul(words[1], NULL, 0);
-        len = strtoul(words[2], NULL, 0);
+        addr = strtoull(words[1], NULL, 0);
+        len = strtoull(words[2], NULL, 0);
 
         data = g_malloc(len);
         cpu_physical_memory_read(addr, data, len);
@@ -302,8 +302,8 @@ static void qtest_process_command(CharDriverState *chr, gchar **words)
         size_t data_len;
 
         g_assert(words[1] && words[2] && words[3]);
-        addr = strtoul(words[1], NULL, 0);
-        len = strtoul(words[2], NULL, 0);
+        addr = strtoull(words[1], NULL, 0);
+        len = strtoull(words[2], NULL, 0);
 
         data_len = strlen(words[3]);
         if (data_len < 3) {
diff --git a/readline.c b/readline.c
index 540cd8a..d6e04d4 100644
--- a/readline.c
+++ b/readline.c
@@ -21,8 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "readline.h"
-#include "monitor.h"
+#include "monitor/readline.h"
+#include "monitor/monitor.h"
 
 #define IS_NORM 0
 #define IS_ESC  1
@@ -247,14 +247,14 @@ static void readline_hist_add(ReadLineState *rs, const char *cmdline)
     }
     if (idx == READLINE_MAX_CMDS) {
 	/* Need to get one free slot */
-	free(rs->history[0]);
-	memcpy(rs->history, &rs->history[1],
-	       (READLINE_MAX_CMDS - 1) * sizeof(char *));
+        g_free(rs->history[0]);
+	memmove(rs->history, &rs->history[1],
+	        (READLINE_MAX_CMDS - 1) * sizeof(char *));
 	rs->history[READLINE_MAX_CMDS - 1] = NULL;
 	idx = READLINE_MAX_CMDS - 1;
     }
     if (new_entry == NULL)
-	new_entry = strdup(cmdline);
+        new_entry = g_strdup(cmdline);
     rs->history[idx] = new_entry;
     rs->hist_entry = -1;
 }
diff --git a/rules.mak b/rules.mak
index d0b04e4..edc2552 100644
--- a/rules.mak
+++ b/rules.mak
@@ -14,24 +14,42 @@ MAKEFLAGS += -rR
 # Flags for dependency generation
 QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(*D)/$(*F).d
 
+# Same as -I$(SRC_PATH) -I., but for the nested source/object directories
+QEMU_CFLAGS += -I$(<D) -I$(@D)
+
 %.o: %.c
 	$(call quiet-command,$(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  CC    $(TARGET_DIR)$@")
 
 ifeq ($(LIBTOOL),)
-%.lo: %.c
-	@echo "missing libtool. please install and rerun configure"; exit 1
+LIBTOOL = /bin/false
+LINK = $(call quiet-command,$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
+       $(sort $(filter %.o, $1)) $(filter-out %.o, $1) $(version-obj-y) \
+       $(LIBS),"  LINK  $(TARGET_DIR)$@")
 else
+LIBTOOL += $(if $(V),,--quiet)
 %.lo: %.c
-	$(call quiet-command,$(LIBTOOL) --mode=compile --quiet --tag=CC $(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  lt CC $@")
+	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC $(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  lt CC $@")
+%.lo: %.dtrace
+	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC dtrace -o $@ -G -s $<, " lt GEN $(TARGET_DIR)$@")
+
+LINK = $(call quiet-command,\
+       $(if $(filter %.lo %.la,$^),$(LIBTOOL) --mode=link --tag=CC \
+       )$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
+       $(sort $(filter %.o, $1)) $(filter-out %.o, $1) $(version-obj-y) \
+       $(LIBS),$(if $(filter %.lo %.la,$^),"lt LINK ", "  LINK  ")"$(TARGET_DIR)$@")
 endif
 
-%.o: %.S
-	$(call quiet-command,$(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  AS    $(TARGET_DIR)$@")
+%.asm: %.S
+	$(call quiet-command,$(CPP) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -o $@ $<,"  CPP   $(TARGET_DIR)$@")
+
+%.o: %.asm
+	$(call quiet-command,$(AS) $(ASFLAGS) -o $@ $<,"  AS    $(TARGET_DIR)$@")
 
 %.o: %.m
 	$(call quiet-command,$(OBJCC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  OBJC  $(TARGET_DIR)$@")
 
-LINK = $(call quiet-command,$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $(sort $(filter %.o, $1)) $(filter-out %.o, $1) $(LIBS),"  LINK  $(TARGET_DIR)$@")
+%.o: %.dtrace
+	$(call quiet-command,dtrace -o $@ -G -s $<, "  GEN   $(TARGET_DIR)$@")
 
 %$(EXESUF): %.o
 	$(call LINK,$^)
@@ -64,12 +82,16 @@ TRACETOOL=$(PYTHON) $(SRC_PATH)/scripts/tracetool.py
 
 # Generate timestamp files for .h include files
 
-%.h: %.h-timestamp
-	@test -f $@ || cp $< $@
+config-%.h: config-%.h-timestamp
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+
+config-%.h-timestamp: config-%.mak
+	$(call quiet-command, sh $(SRC_PATH)/scripts/create_config < $< > $@, "  GEN   $(TARGET_DIR)config-$*.h")
 
-%.h-timestamp: %.mak
-	$(call quiet-command, sh $(SRC_PATH)/scripts/create_config < $< > $@, "  GEN   $*.h")
-	@cmp $@ $*.h >/dev/null 2>&1 || cp $@ $*.h
+.PHONY: clean-timestamp
+clean-timestamp:
+	rm -f *.timestamp
+clean: clean-timestamp
 
 # will delete the target of a rule if commands exit with a nonzero exit status
 .DELETE_ON_ERROR:
diff --git a/savevm.c b/savevm.c
index 5d04d59..a8a53ef 100644
--- a/savevm.c
+++ b/savevm.c
@@ -21,72 +21,24 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
-#include <zlib.h>
-
-/* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
-
-#ifndef _WIN32
-#include <sys/times.h>
-#include <sys/wait.h>
-#include <termios.h>
-#include <sys/mman.h>
-#include <sys/ioctl.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <net/if.h>
-#include <arpa/inet.h>
-#include <dirent.h>
-#include <netdb.h>
-#include <sys/select.h>
-#ifdef CONFIG_BSD
-#include <sys/stat.h>
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
-#include <libutil.h>
-#else
-#include <util.h>
-#endif
-#ifdef __linux__
-#include <pty.h>
-#include <malloc.h>
-#include <linux/rtc.h>
-#endif
-#endif
-#endif
-
-#ifdef _WIN32
-#include <windows.h>
-#include <malloc.h>
-#include <sys/timeb.h>
-#include <mmsystem.h>
-#define getopt_long_only getopt_long
-#define memalign(align, size) malloc(size)
-#endif
 
+#include "config-host.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/qdev.h"
-#include "net.h"
-#include "monitor.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "net/net.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
 #include "audio/audio.h"
-#include "migration.h"
-#include "qemu_socket.h"
-#include "qemu-queue.h"
-#include "qemu-timer.h"
-#include "cpus.h"
-#include "memory.h"
+#include "migration/migration.h"
+#include "qemu/sockets.h"
+#include "qemu/queue.h"
+#include "sysemu/cpus.h"
+#include "exec/memory.h"
 #include "qmp-commands.h"
 #include "trace.h"
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 #define SELF_ANNOUNCE_ROUNDS 5
 
@@ -129,7 +81,7 @@ static void qemu_announce_self_iter(NICState *nic, void *opaque)
 
     len = announce_self_create(buf, nic->conf->macaddr.a);
 
-    qemu_send_packet_raw(&nic->nc, buf, len);
+    qemu_send_packet_raw(qemu_get_queue(nic), buf, len);
 }
 
 
@@ -188,6 +140,34 @@ typedef struct QEMUFileSocket
     QEMUFile *file;
 } QEMUFileSocket;
 
+typedef struct {
+    Coroutine *co;
+    int fd;
+} FDYieldUntilData;
+
+static void fd_coroutine_enter(void *opaque)
+{
+    FDYieldUntilData *data = opaque;
+    qemu_set_fd_handler(data->fd, NULL, NULL, NULL);
+    qemu_coroutine_enter(data->co, NULL);
+}
+
+/**
+ * Yield until a file descriptor becomes readable
+ *
+ * Note that this function clobbers the handlers for the file descriptor.
+ */
+static void coroutine_fn yield_until_fd_readable(int fd)
+{
+    FDYieldUntilData data;
+
+    assert(qemu_in_coroutine());
+    data.co = qemu_coroutine_self();
+    data.fd = fd;
+    qemu_set_fd_handler(fd, fd_coroutine_enter, NULL, &data);
+    qemu_coroutine_yield();
+}
+
 static int socket_get_fd(void *opaque)
 {
     QEMUFileSocket *s = opaque;
@@ -206,8 +186,7 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
             break;
         }
         if (socket_error() == EAGAIN) {
-            assert(qemu_in_coroutine());
-            qemu_coroutine_yield();
+            yield_until_fd_readable(s->fd);
         } else if (socket_error() != EINTR) {
             break;
         }
@@ -253,8 +232,7 @@ static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
             break;
         }
         if (errno == EAGAIN) {
-            assert(qemu_in_coroutine());
-            qemu_coroutine_yield();
+            yield_until_fd_readable(fileno(fp));
         } else if (errno != EINTR) {
             break;
         }
@@ -467,7 +445,9 @@ int qemu_file_get_error(QEMUFile *f)
 
 static void qemu_file_set_error(QEMUFile *f, int ret)
 {
-    f->last_error = ret;
+    if (f->last_error == 0) {
+        f->last_error = ret;
+    }
 }
 
 /** Flushes QEMUFile buffer
@@ -556,11 +536,6 @@ int qemu_fclose(QEMUFile *f)
     return ret;
 }
 
-int qemu_file_put_notify(QEMUFile *f)
-{
-    return f->ops->put_buffer(f->opaque, NULL, 0, 0);
-}
-
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
 {
     int l;
@@ -698,9 +673,14 @@ int qemu_get_byte(QEMUFile *f)
     return result;
 }
 
-static int64_t qemu_ftell(QEMUFile *f)
+int64_t qemu_ftell(QEMUFile *f)
 {
-    return f->buf_offset - f->buf_size + f->buf_index;
+    /* buf_offset excludes buffer for writing but includes it for reading */
+    if (f->is_write) {
+        return f->buf_offset + f->buf_index;
+    } else {
+        return f->buf_offset - f->buf_size + f->buf_index;
+    }
 }
 
 int qemu_file_rate_limit(QEMUFile *f)
@@ -1641,13 +1621,13 @@ int qemu_savevm_state_begin(QEMUFile *f,
 
         ret = se->ops->save_live_setup(f, se->opaque);
         if (ret < 0) {
-            qemu_savevm_state_cancel(f);
+            qemu_savevm_state_cancel();
             return ret;
         }
     }
     ret = qemu_file_get_error(f);
     if (ret != 0) {
-        qemu_savevm_state_cancel(f);
+        qemu_savevm_state_cancel();
     }
 
     return ret;
@@ -1698,7 +1678,7 @@ int qemu_savevm_state_iterate(QEMUFile *f)
     }
     ret = qemu_file_get_error(f);
     if (ret != 0) {
-        qemu_savevm_state_cancel(f);
+        qemu_savevm_state_cancel();
     }
     return ret;
 }
@@ -1759,7 +1739,26 @@ int qemu_savevm_state_complete(QEMUFile *f)
     return qemu_file_get_error(f);
 }
 
-void qemu_savevm_state_cancel(QEMUFile *f)
+uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
+{
+    SaveStateEntry *se;
+    uint64_t ret = 0;
+
+    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
+        if (!se->ops || !se->ops->save_live_pending) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        ret += se->ops->save_live_pending(f, se->opaque, max_size);
+    }
+    return ret;
+}
+
+void qemu_savevm_state_cancel(void)
 {
     SaveStateEntry *se;
 
@@ -2118,13 +2117,8 @@ void do_savevm(Monitor *mon, const QDict *qdict)
     QEMUFile *f;
     int saved_vm_running;
     uint64_t vm_state_size;
-#ifdef _WIN32
-    struct _timeb tb;
-    struct tm *ptm;
-#else
-    struct timeval tv;
+    qemu_timeval tv;
     struct tm tm;
-#endif
     const char *name = qdict_get_try_str(qdict, "name");
 
     /* Verify if there is a device that doesn't support snapshots and is writable */
@@ -2154,15 +2148,9 @@ void do_savevm(Monitor *mon, const QDict *qdict)
     memset(sn, 0, sizeof(*sn));
 
     /* fill auxiliary fields */
-#ifdef _WIN32
-    _ftime(&tb);
-    sn->date_sec = tb.time;
-    sn->date_nsec = tb.millitm * 1000000;
-#else
-    gettimeofday(&tv, NULL);
+    qemu_gettimeofday(&tv);
     sn->date_sec = tv.tv_sec;
     sn->date_nsec = tv.tv_usec * 1000;
-#endif
     sn->vm_clock_nsec = qemu_get_clock_ns(vm_clock);
 
     if (name) {
@@ -2174,15 +2162,9 @@ void do_savevm(Monitor *mon, const QDict *qdict)
             pstrcpy(sn->name, sizeof(sn->name), name);
         }
     } else {
-#ifdef _WIN32
-        time_t t = tb.time;
-        ptm = localtime(&t);
-        strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", ptm);
-#else
         /* cast below needed for OpenBSD where tv_sec is still 'long' */
         localtime_r((const time_t *)&tv.tv_sec, &tm);
         strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm);
-#endif
     }
 
     /* Delete old snapshots of the same name */
@@ -2358,7 +2340,7 @@ void do_delvm(Monitor *mon, const QDict *qdict)
     }
 }
 
-void do_info_snapshots(Monitor *mon)
+void do_info_snapshots(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
     QEMUSnapshotInfo *sn_tab, *sn, s, *sn_info = &s;
@@ -2437,162 +2419,3 @@ void vmstate_register_ram_global(MemoryRegion *mr)
 {
     vmstate_register_ram(mr, NULL);
 }
-
-/*
-  page = zrun nzrun
-       | zrun nzrun page
-
-  zrun = length
-
-  nzrun = length byte...
-
-  length = uleb128 encoded integer
- */
-int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
-                         uint8_t *dst, int dlen)
-{
-    uint32_t zrun_len = 0, nzrun_len = 0;
-    int d = 0, i = 0;
-    long res, xor;
-    uint8_t *nzrun_start = NULL;
-
-    g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) %
-               sizeof(long)));
-
-    while (i < slen) {
-        /* overflow */
-        if (d + 2 > dlen) {
-            return -1;
-        }
-
-        /* not aligned to sizeof(long) */
-        res = (slen - i) % sizeof(long);
-        while (res && old_buf[i] == new_buf[i]) {
-            zrun_len++;
-            i++;
-            res--;
-        }
-
-        /* word at a time for speed */
-        if (!res) {
-            while (i < slen &&
-                   (*(long *)(old_buf + i)) == (*(long *)(new_buf + i))) {
-                i += sizeof(long);
-                zrun_len += sizeof(long);
-            }
-
-            /* go over the rest */
-            while (i < slen && old_buf[i] == new_buf[i]) {
-                zrun_len++;
-                i++;
-            }
-        }
-
-        /* buffer unchanged */
-        if (zrun_len == slen) {
-            return 0;
-        }
-
-        /* skip last zero run */
-        if (i == slen) {
-            return d;
-        }
-
-        d += uleb128_encode_small(dst + d, zrun_len);
-
-        zrun_len = 0;
-        nzrun_start = new_buf + i;
-
-        /* overflow */
-        if (d + 2 > dlen) {
-            return -1;
-        }
-        /* not aligned to sizeof(long) */
-        res = (slen - i) % sizeof(long);
-        while (res && old_buf[i] != new_buf[i]) {
-            i++;
-            nzrun_len++;
-            res--;
-        }
-
-        /* word at a time for speed, use of 32-bit long okay */
-        if (!res) {
-            /* truncation to 32-bit long okay */
-            long mask = (long)0x0101010101010101ULL;
-            while (i < slen) {
-                xor = *(long *)(old_buf + i) ^ *(long *)(new_buf + i);
-                if ((xor - mask) & ~xor & (mask << 7)) {
-                    /* found the end of an nzrun within the current long */
-                    while (old_buf[i] != new_buf[i]) {
-                        nzrun_len++;
-                        i++;
-                    }
-                    break;
-                } else {
-                    i += sizeof(long);
-                    nzrun_len += sizeof(long);
-                }
-            }
-        }
-
-        d += uleb128_encode_small(dst + d, nzrun_len);
-        /* overflow */
-        if (d + nzrun_len > dlen) {
-            return -1;
-        }
-        memcpy(dst + d, nzrun_start, nzrun_len);
-        d += nzrun_len;
-        nzrun_len = 0;
-    }
-
-    return d;
-}
-
-int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen)
-{
-    int i = 0, d = 0;
-    int ret;
-    uint32_t count = 0;
-
-    while (i < slen) {
-
-        /* zrun */
-        if ((slen - i) < 2) {
-            return -1;
-        }
-
-        ret = uleb128_decode_small(src + i, &count);
-        if (ret < 0 || (i && !count)) {
-            return -1;
-        }
-        i += ret;
-        d += count;
-
-        /* overflow */
-        if (d > dlen) {
-            return -1;
-        }
-
-        /* nzrun */
-        if ((slen - i) < 2) {
-            return -1;
-        }
-
-        ret = uleb128_decode_small(src + i, &count);
-        if (ret < 0 || !count) {
-            return -1;
-        }
-        i += ret;
-
-        /* overflow */
-        if (d + count > dlen || i + count > slen) {
-            return -1;
-        }
-
-        memcpy(dst + d, src + i, count);
-        d += count;
-        i += count;
-    }
-
-    return d;
-}
diff --git a/scripts/feature_to_c.sh b/scripts/feature_to_c.sh
index b62da8a..888548e 100644
--- a/scripts/feature_to_c.sh
+++ b/scripts/feature_to_c.sh
@@ -38,7 +38,7 @@ for input; do
   ${AWK:-awk} 'BEGIN { n = 0
       printf "#include \"config.h\"\n"
       printf "#include \"qemu-common.h\"\n"
-      printf "#include \"gdbstub.h\"\n"
+      printf "#include \"exec/gdbstub.h\"\n"
       print "static const char '$arrayname'[] = {"
       for (i = 0; i < 255; i++)
         _ord_[sprintf("%c", i)] = i
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index d9c48e0..bf5342a 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -83,6 +83,8 @@ push(@signature_tags, "Signed-off-by:");
 push(@signature_tags, "Reviewed-by:");
 push(@signature_tags, "Acked-by:");
 
+my $signature_pattern = "\(" . join("|", @signature_tags) . "\)";
+
 # rfc822 email address - preloaded methods go here.
 my $rfc822_lwsp = "(?:(?:\\r\\n)?[ \\t])";
 my $rfc822_char = '[\\000-\\377]';
@@ -95,7 +97,7 @@ my %VCS_cmds_git = (
     "execute_cmd" => \&git_execute_cmd,
     "available" => '(which("git") ne "") && (-d ".git")',
     "find_signers_cmd" =>
-	"git log --no-color --since=\$email_git_since " .
+	"git log --no-color --follow --since=\$email_git_since " .
 	    '--format="GitCommit: %H%n' .
 		      'GitAuthor: %an <%ae>%n' .
 		      'GitDate: %aD%n' .
@@ -328,7 +330,8 @@ sub read_mailmap {
 	# name1 <mail1> <mail2>
 	# name1 <mail1> name2 <mail2>
 	# (see man git-shortlog)
-	if (/^(.+)<(.+)>$/) {
+
+	if (/^([^<]+)<([^>]+)>$/) {
 	    my $real_name = $1;
 	    my $address = $2;
 
@@ -336,13 +339,13 @@ sub read_mailmap {
 	    ($real_name, $address) = parse_email("$real_name <$address>");
 	    $mailmap->{names}->{$address} = $real_name;
 
-	} elsif (/^<([^\s]+)>\s*<([^\s]+)>$/) {
+	} elsif (/^<([^>]+)>\s*<([^>]+)>$/) {
 	    my $real_address = $1;
 	    my $wrong_address = $2;
 
 	    $mailmap->{addresses}->{$wrong_address} = $real_address;
 
-	} elsif (/^(.+)<([^\s]+)>\s*<([^\s]+)>$/) {
+	} elsif (/^(.+)<([^>]+)>\s*<([^>]+)>$/) {
 	    my $real_name = $1;
 	    my $real_address = $2;
 	    my $wrong_address = $3;
@@ -353,7 +356,7 @@ sub read_mailmap {
 	    $mailmap->{names}->{$wrong_address} = $real_name;
 	    $mailmap->{addresses}->{$wrong_address} = $real_address;
 
-	} elsif (/^(.+)<([^\s]+)>\s*([^\s].*)<([^\s]+)>$/) {
+	} elsif (/^(.+)<([^>]+)>\s*(.+)\s*<([^>]+)>$/) {
 	    my $real_name = $1;
 	    my $real_address = $2;
 	    my $wrong_name = $3;
@@ -472,7 +475,6 @@ my @subsystem = ();
 my @status = ();
 my %deduplicate_name_hash = ();
 my %deduplicate_address_hash = ();
-my $signature_pattern;
 
 my @maintainers = get_maintainers();
 
@@ -920,7 +922,7 @@ sub get_maintainer_role {
     my $start = find_starting_index($index);
     my $end = find_ending_index($index);
 
-    my $role;
+    my $role = "unknown";
     my $subsystem = $typevalue[$start];
     if (length($subsystem) > 20) {
 	$subsystem = substr($subsystem, 0, 17);
@@ -1016,8 +1018,13 @@ sub add_categories {
 		    if ($email_list) {
 			if (!$hash_list_to{lc($list_address)}) {
 			    $hash_list_to{lc($list_address)} = 1;
-			    push(@list_to, [$list_address,
-					    "open list${list_role}"]);
+			    if ($list_additional =~ m/moderated/) {
+				push(@list_to, [$list_address,
+						"moderated list${list_role}"]);
+			    } else {
+				push(@list_to, [$list_address,
+						"open list${list_role}"]);
+			    }
 			}
 		    }
 		}
diff --git a/scripts/kvm/vmxcap b/scripts/kvm/vmxcap
index cbe6440..0b23f77 100755
--- a/scripts/kvm/vmxcap
+++ b/scripts/kvm/vmxcap
@@ -147,6 +147,7 @@ controls = [
             5: 'Enable VPID',
             6: 'WBINVD exiting',
             7: 'Unrestricted guest',
+            9: 'Virtual interrupt delivery',
             10: 'PAUSE-loop exiting',
             11: 'RDRAND exiting',
             12: 'Enable INVPCID',
diff --git a/scripts/make_device_config.sh b/scripts/make_device_config.sh
index 5d14885..0778fe2 100644
--- a/scripts/make_device_config.sh
+++ b/scripts/make_device_config.sh
@@ -25,4 +25,4 @@ done
 process_includes $src > $dest
 
 cat $src $all_includes | grep -v '^include' > $dest
-echo "$1: $all_includes" > $dep
+echo "`basename $1`: $all_includes" > $dep
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index 3c4678d..e06332b 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -342,8 +342,8 @@ def gen_command_decl_prologue(header, guard, prefix=""):
 #define %(guard)s
 
 #include "%(prefix)sqapi-types.h"
-#include "qdict.h"
-#include "error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/error.h"
 
 ''',
                  header=basename(header), guard=guardname(header), prefix=prefix)
@@ -366,12 +366,15 @@ def gen_command_def_prologue(prefix="", proxy=False):
  *
  */
 
-#include "qemu-objects.h"
-#include "qapi/qmp-core.h"
-#include "qapi/qapi-visit-core.h"
+#include "qemu-common.h"
+#include "qemu/module.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/types.h"
+#include "qapi/qmp/dispatch.h"
+#include "qapi/visitor.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/qmp-input-visitor.h"
-#include "qapi/qapi-dealloc-visitor.h"
+#include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
 
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index 6bc2391..9e19920 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -248,7 +248,7 @@ fdef.write(mcgen('''
  *
  */
 
-#include "qapi/qapi-dealloc-visitor.h"
+#include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
 
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index a360de7..a276540 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -298,6 +298,7 @@ fdef.write(mcgen('''
  *
  */
 
+#include "qemu-common.h"
 #include "%(header)s"
 ''',
                  header=basename(h_file)))
@@ -321,7 +322,7 @@ fdecl.write(mcgen('''
 #ifndef %(guard)s
 #define %(guard)s
 
-#include "qapi/qapi-visit-core.h"
+#include "qapi/visitor.h"
 #include "%(prefix)sqapi-types.h"
 ''',
                   prefix=prefix, guard=guardname(h_file)))
diff --git a/scripts/qemu-guest-agent/fsfreeze-hook b/scripts/qemu-guest-agent/fsfreeze-hook
new file mode 100755
index 0000000..c27b29f
--- /dev/null
+++ b/scripts/qemu-guest-agent/fsfreeze-hook
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script is executed when a guest agent receives fsfreeze-freeze and
+# fsfreeze-thaw command, if it is specified in --fsfreeze-hook (-F)
+# option of qemu-ga or placed in default path (/etc/qemu/fsfreeze-hook).
+# When the agent receives fsfreeze-freeze request, this script is issued with
+# "freeze" argument before the filesystem is frozen. And for fsfreeze-thaw
+# request, it is issued with "thaw" argument after filesystem is thawed.
+
+LOGFILE=/var/log/qga-fsfreeze-hook.log
+FSFREEZE_D=$(dirname -- "$0")/fsfreeze-hook.d
+
+# Check whether file $1 is a backup or rpm-generated file and should be ignored
+is_ignored_file() {
+    case "$1" in
+        *~ | *.bak | *.orig | *.rpmnew | *.rpmorig | *.rpmsave | *.sample)
+            return 0 ;;
+    esac
+    return 1
+}
+
+# Iterate executables in directory "fsfreeze-hook.d" with the specified args
+[ ! -d "$FSFREEZE_D" ] && exit 0
+for file in "$FSFREEZE_D"/* ; do
+    is_ignored_file "$file" && continue
+    [ -x "$file" ] || continue
+    printf "$(date): execute $file $@\n" >>$LOGFILE
+    "$file" "$@" >>$LOGFILE 2>&1
+    STATUS=$?
+    printf "$(date): $file finished with status=$STATUS\n" >>$LOGFILE
+done
+
+exit 0
diff --git a/scripts/qemu-guest-agent/fsfreeze-hook.d/mysql-flush.sh.sample b/scripts/qemu-guest-agent/fsfreeze-hook.d/mysql-flush.sh.sample
new file mode 100755
index 0000000..2b4fa3a
--- /dev/null
+++ b/scripts/qemu-guest-agent/fsfreeze-hook.d/mysql-flush.sh.sample
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Flush MySQL tables to the disk before the filesystem is frozen.
+# At the same time, this keeps a read lock in order to avoid write accesses
+# from the other clients until the filesystem is thawed.
+
+MYSQL="/usr/bin/mysql"
+MYSQL_OPTS="-uroot" #"-prootpassword"
+FIFO=/var/run/mysql-flush.fifo
+
+# Check mysql is installed and the server running
+[ -x "$MYSQL" ] && "$MYSQL" $MYSQL_OPTS < /dev/null || exit 0
+
+flush_and_wait() {
+    printf "FLUSH TABLES WITH READ LOCK \\G\n"
+    trap 'printf "$(date): $0 is killed\n">&2' HUP INT QUIT ALRM TERM
+    read < $FIFO
+    printf "UNLOCK TABLES \\G\n"
+    rm -f $FIFO
+}
+
+case "$1" in
+    freeze)
+        mkfifo $FIFO || exit 1
+        flush_and_wait | "$MYSQL" $MYSQL_OPTS &
+        # wait until every block is flushed
+        while [ "$(echo 'SHOW STATUS LIKE "Key_blocks_not_flushed"' |\
+                 "$MYSQL" $MYSQL_OPTS | tail -1 | cut -f 2)" -gt 0 ]; do
+            sleep 1
+        done
+        # for InnoDB, wait until every log is flushed
+        INNODB_STATUS=$(mktemp /tmp/mysql-flush.XXXXXX)
+        [ $? -ne 0 ] && exit 2
+        trap "rm -f $INNODB_STATUS; exit 1" HUP INT QUIT ALRM TERM
+        while :; do
+            printf "SHOW ENGINE INNODB STATUS \\G" |\
+                "$MYSQL" $MYSQL_OPTS > $INNODB_STATUS
+            LOG_CURRENT=$(grep 'Log sequence number' $INNODB_STATUS |\
+                          tr -s ' ' | cut -d' ' -f4)
+            LOG_FLUSHED=$(grep 'Log flushed up to' $INNODB_STATUS |\
+                          tr -s ' ' | cut -d' ' -f5)
+            [ "$LOG_CURRENT" = "$LOG_FLUSHED" ] && break
+            sleep 1
+        done
+        rm -f $INNODB_STATUS
+        ;;
+
+    thaw)
+        [ ! -p $FIFO ] && exit 1
+        echo > $FIFO
+        ;;
+
+    *)
+        exit 1
+        ;;
+esac
diff --git a/scripts/tracetool/backend/dtrace.py b/scripts/tracetool/backend/dtrace.py
index 23c43e2..ad5eb3b 100644
--- a/scripts/tracetool/backend/dtrace.py
+++ b/scripts/tracetool/backend/dtrace.py
@@ -37,7 +37,7 @@ def c(events):
 
 
 def h(events):
-    out('#include "trace-dtrace.h"',
+    out('#include "trace/generated-tracers-dtrace.h"',
         '')
 
     for e in events:
diff --git a/scripts/tracetool/format/h.py b/scripts/tracetool/format/h.py
index 6ffb3c2..9a58de1 100644
--- a/scripts/tracetool/format/h.py
+++ b/scripts/tracetool/format/h.py
@@ -19,8 +19,8 @@ from tracetool import out
 def begin(events):
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
-        '#ifndef TRACE_H',
-        '#define TRACE_H',
+        '#ifndef TRACE__GENERATED_TRACERS_H',
+        '#define TRACE__GENERATED_TRACERS_H',
         '',
         '#include "qemu-common.h"')
 
@@ -32,7 +32,7 @@ def end(events):
             enabled = 1
         out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled))
     out('',
-        '#endif /* TRACE_H */')
+        '#endif /* TRACE__GENERATED_TRACERS_H */')
 
 def nop(events):
     for e in events:
diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh
index 4c7b566..120a694 100755
--- a/scripts/update-linux-headers.sh
+++ b/scripts/update-linux-headers.sh
@@ -54,6 +54,9 @@ for arch in $ARCHLIST; do
     if [ $arch = x86 ]; then
         cp "$tmpdir/include/asm/hyperv.h" "$output/linux-headers/asm-x86"
     fi
+    if [ $arch = powerpc ]; then
+        cp "$tmpdir/include/asm/epapr_hcalls.h" "$output/linux-headers/asm-powerpc/"
+    fi
 done
 
 rm -rf "$output/linux-headers/linux"
diff --git a/slirp/bootp.h b/slirp/bootp.h
index 30c30ab..ec3b687 100644
--- a/slirp/bootp.h
+++ b/slirp/bootp.h
@@ -1,4 +1,6 @@
 /* bootp/dhcp defines */
+#ifndef SLIRP_BOOTP_H
+#define SLIRP_BOOTP_H 1
 
 #define BOOTP_SERVER	67
 #define BOOTP_CLIENT	68
@@ -120,3 +122,5 @@ typedef struct {
 #define NB_BOOTP_CLIENTS 16
 
 void bootp_input(struct mbuf *m);
+
+#endif
diff --git a/slirp/if.c b/slirp/if.c
index 533295d..dcd5faf 100644
--- a/slirp/if.c
+++ b/slirp/if.c
@@ -6,7 +6,7 @@
  */
 
 #include <slirp.h>
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 static void
 ifs_insque(struct mbuf *ifm, struct mbuf *ifmhead)
diff --git a/slirp/ip_icmp.c b/slirp/ip_icmp.c
index d571fd0..9f1cb08 100644
--- a/slirp/ip_icmp.c
+++ b/slirp/ip_icmp.c
@@ -352,7 +352,7 @@ icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
 
   ip->ip_ttl = MAXTTL;
   ip->ip_p = IPPROTO_ICMP;
-  ip->ip_dst = ip->ip_src;    /* ip adresses */
+  ip->ip_dst = ip->ip_src;    /* ip addresses */
   ip->ip_src = m->slirp->vhost_addr;
 
   (void ) ip_output((struct socket *)NULL, m);
diff --git a/slirp/ip_input.c b/slirp/ip_input.c
index 6f4cff8..880bdfd 100644
--- a/slirp/ip_input.c
+++ b/slirp/ip_input.c
@@ -39,7 +39,7 @@
  */
 
 #include <slirp.h>
-#include <osdep.h>
+#include <qemu/osdep.h>
 #include "ip_icmp.h"
 
 static struct ip *ip_reass(Slirp *slirp, struct ip *ip, struct ipq *fp);
diff --git a/slirp/main.h b/slirp/main.h
index 1f3b84d..66e4f92 100644
--- a/slirp/main.h
+++ b/slirp/main.h
@@ -4,6 +4,8 @@
  * Please read the file COPYRIGHT for the
  * terms and conditions of the copyright.
  */
+#ifndef SLIRP_MAIN_H
+#define SLIRP_MAIN_H 1
 
 #ifdef HAVE_SYS_SELECT_H
 #include <sys/select.h>
@@ -45,3 +47,5 @@ extern int tcp_keepintvl;
 
 int if_encap(Slirp *slirp, struct mbuf *ifm);
 ssize_t slirp_send(struct socket *so, const void *buf, size_t len, int flags);
+
+#endif
diff --git a/slirp/misc.c b/slirp/misc.c
index 664532a..d4df972 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -8,7 +8,7 @@
 #include <slirp.h>
 #include <libslirp.h>
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 #ifdef DEBUG
 int slirp_debug = DBG_CALL|DBG_MISC|DBG_ERROR;
@@ -242,7 +242,7 @@ strdup(str)
 }
 #endif
 
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 void lprint(const char *format, ...)
 {
diff --git a/slirp/sbuf.c b/slirp/sbuf.c
index 637f8fe..08ec2b4 100644
--- a/slirp/sbuf.c
+++ b/slirp/sbuf.c
@@ -6,7 +6,7 @@
  */
 
 #include <slirp.h>
-#include <main-loop.h>
+#include <qemu/main-loop.h>
 
 static void sbappendsb(struct sbuf *sb, struct mbuf *m);
 
diff --git a/slirp/slirp.c b/slirp/slirp.c
index 3395d50..0e6e232 100644
--- a/slirp/slirp.c
+++ b/slirp/slirp.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
+#include "qemu/timer.h"
+#include "char/char.h"
 #include "slirp.h"
 #include "hw/hw.h"
 
@@ -225,12 +225,8 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
         pstrcpy(slirp->client_hostname, sizeof(slirp->client_hostname),
                 vhostname);
     }
-    if (tftp_path) {
-        slirp->tftp_prefix = g_strdup(tftp_path);
-    }
-    if (bootfile) {
-        slirp->bootp_filename = g_strdup(bootfile);
-    }
+    slirp->tftp_prefix = g_strdup(tftp_path);
+    slirp->bootp_filename = g_strdup(bootfile);
     slirp->vdhcp_startaddr = vdhcp_start;
     slirp->vnameserver_addr = vnameserver;
 
diff --git a/slirp/slirp.h b/slirp/slirp.h
index 0107b07..fe0e65d 100644
--- a/slirp/slirp.h
+++ b/slirp/slirp.h
@@ -133,8 +133,8 @@ void free(void *ptr);
 
 #include "debug.h"
 
-#include "qemu-queue.h"
-#include "qemu_socket.h"
+#include "qemu/queue.h"
+#include "qemu/sockets.h"
 
 #include "libslirp.h"
 #include "ip.h"
@@ -215,7 +215,6 @@ struct Slirp {
     char client_hostname[33];
 
     int restricted;
-    struct timeval tt;
     struct ex_list *exec_list;
 
     /* mbuf states */
diff --git a/slirp/tftp.h b/slirp/tftp.h
index 51704e4..87adeb5 100644
--- a/slirp/tftp.h
+++ b/slirp/tftp.h
@@ -1,4 +1,6 @@
 /* tftp defines */
+#ifndef SLIRP_TFTP_H
+#define SLIRP_TFTP_H 1
 
 #define TFTP_SESSIONS_MAX 3
 
@@ -43,3 +45,5 @@ struct tftp_session {
 };
 
 void tftp_input(struct mbuf *m);
+
+#endif
diff --git a/spice-qemu-char.c b/spice-qemu-char.c
index 09aa22d..a4d7de8 100644
--- a/spice-qemu-char.c
+++ b/spice-qemu-char.c
@@ -1,10 +1,12 @@
 #include "config-host.h"
 #include "trace.h"
 #include "ui/qemu-spice.h"
+#include "char/char.h"
 #include <spice.h>
 #include <spice-experimental.h>
+#include <spice/protocol.h>
 
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 #define dprintf(_scd, _level, _fmt, ...)                                \
     do {                                                                \
@@ -14,8 +16,6 @@
         }                                                               \
     } while (0)
 
-#define VMC_MAX_HOST_WRITE    2048
-
 typedef struct SpiceCharDriver {
     CharDriverState*      chr;
     SpiceCharDeviceInstance     sin;
@@ -25,8 +25,12 @@ typedef struct SpiceCharDriver {
     uint8_t               *datapos;
     ssize_t               bufsize, datalen;
     uint32_t              debug;
+    QLIST_ENTRY(SpiceCharDriver) next;
 } SpiceCharDriver;
 
+static QLIST_HEAD(, SpiceCharDriver) spice_chars =
+    QLIST_HEAD_INITIALIZER(spice_chars);
+
 static int vmc_write(SpiceCharDeviceInstance *sin, const uint8_t *buf, int len)
 {
     SpiceCharDriver *scd = container_of(sin, SpiceCharDriver, sin);
@@ -35,8 +39,8 @@ static int vmc_write(SpiceCharDeviceInstance *sin, const uint8_t *buf, int len)
     uint8_t* p = (uint8_t*)buf;
 
     while (len > 0) {
-        last_out = MIN(len, VMC_MAX_HOST_WRITE);
-        if (qemu_chr_be_can_write(scd->chr) < last_out) {
+        last_out = MIN(len, qemu_chr_be_can_write(scd->chr));
+        if (last_out <= 0) {
             break;
         }
         qemu_chr_be_write(scd->chr, p, last_out);
@@ -69,6 +73,27 @@ static int vmc_read(SpiceCharDeviceInstance *sin, uint8_t *buf, int len)
     return bytes;
 }
 
+#if SPICE_SERVER_VERSION >= 0x000c02
+static void vmc_event(SpiceCharDeviceInstance *sin, uint8_t event)
+{
+    SpiceCharDriver *scd = container_of(sin, SpiceCharDriver, sin);
+    int chr_event;
+
+    switch (event) {
+    case SPICE_PORT_EVENT_BREAK:
+        chr_event = CHR_EVENT_BREAK;
+        break;
+    default:
+        dprintf(scd, 2, "%s: unknown %d\n", __func__, event);
+        return;
+    }
+
+    dprintf(scd, 2, "%s: %d\n", __func__, event);
+    trace_spice_vmc_event(chr_event);
+    qemu_chr_be_event(scd->chr, chr_event);
+}
+#endif
+
 static void vmc_state(SpiceCharDeviceInstance *sin, int connected)
 {
     SpiceCharDriver *scd = container_of(sin, SpiceCharDriver, sin);
@@ -105,6 +130,9 @@ static SpiceCharDeviceInterface vmc_interface = {
     .state              = vmc_state,
     .write              = vmc_write,
     .read               = vmc_read,
+#if SPICE_SERVER_VERSION >= 0x000c02
+    .event              = vmc_event,
+#endif
 };
 
 
@@ -156,6 +184,7 @@ static void spice_chr_close(struct CharDriverState *chr)
 
     printf("%s\n", __func__);
     vmc_unregister_interface(s);
+    QLIST_REMOVE(s, next);
     g_free(s);
 }
 
@@ -188,13 +217,34 @@ static void print_allowed_subtypes(void)
     fprintf(stderr, "\n");
 }
 
-CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
+static CharDriverState *chr_open(QemuOpts *opts, const char *subtype)
 {
     CharDriverState *chr;
     SpiceCharDriver *s;
-    const char* name = qemu_opt_get(opts, "name");
     uint32_t debug = qemu_opt_get_number(opts, "debug", 0);
-    const char** psubtype = spice_server_char_device_recognized_subtypes();
+
+    chr = g_malloc0(sizeof(CharDriverState));
+    s = g_malloc0(sizeof(SpiceCharDriver));
+    s->chr = chr;
+    s->debug = debug;
+    s->active = false;
+    s->sin.subtype = subtype;
+    chr->opaque = s;
+    chr->chr_write = spice_chr_write;
+    chr->chr_close = spice_chr_close;
+    chr->chr_guest_open = spice_chr_guest_open;
+    chr->chr_guest_close = spice_chr_guest_close;
+
+    QLIST_INSERT_HEAD(&spice_chars, s, next);
+
+    return chr;
+}
+
+CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
+{
+    CharDriverState *chr;
+    const char *name = qemu_opt_get(opts, "name");
+    const char **psubtype = spice_server_char_device_recognized_subtypes();
     const char *subtype = NULL;
 
     if (name == NULL) {
@@ -214,17 +264,7 @@ CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
         return NULL;
     }
 
-    chr = g_malloc0(sizeof(CharDriverState));
-    s = g_malloc0(sizeof(SpiceCharDriver));
-    s->chr = chr;
-    s->debug = debug;
-    s->active = false;
-    s->sin.subtype = subtype;
-    chr->opaque = s;
-    chr->chr_write = spice_chr_write;
-    chr->chr_close = spice_chr_close;
-    chr->chr_guest_open = spice_chr_guest_open;
-    chr->chr_guest_close = spice_chr_guest_close;
+    chr = chr_open(opts, subtype);
 
 #if SPICE_SERVER_VERSION < 0x000901
     /* See comment in vmc_state() */
@@ -235,3 +275,35 @@ CharDriverState *qemu_chr_open_spice(QemuOpts *opts)
 
     return chr;
 }
+
+#if SPICE_SERVER_VERSION >= 0x000c02
+CharDriverState *qemu_chr_open_spice_port(QemuOpts *opts)
+{
+    CharDriverState *chr;
+    SpiceCharDriver *s;
+    const char *name = qemu_opt_get(opts, "name");
+
+    if (name == NULL) {
+        fprintf(stderr, "spice-qemu-char: missing name parameter\n");
+        return NULL;
+    }
+
+    chr = chr_open(opts, "port");
+    s = chr->opaque;
+    s->sin.portname = name;
+
+    return chr;
+}
+
+void qemu_spice_register_ports(void)
+{
+    SpiceCharDriver *s;
+
+    QLIST_FOREACH(s, &spice_chars, next) {
+        if (s->sin.portname == NULL) {
+            continue;
+        }
+        vmc_register_interface(s);
+    }
+}
+#endif
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index 035b29a..a260394 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -1,8 +1,24 @@
 stub-obj-y += arch-query-cpu-def.o
+stub-obj-y += clock-warp.o
+stub-obj-y += cpu-get-clock.o
+stub-obj-y += cpu-get-icount.o
 stub-obj-y += fdset-add-fd.o
 stub-obj-y += fdset-find-fd.o
 stub-obj-y += fdset-get-fd.o
 stub-obj-y += fdset-remove-fd.o
 stub-obj-y += get-fd.o
+stub-obj-y += get-vm-name.o
+stub-obj-y += iothread-lock.o
+stub-obj-y += migr-blocker.o
+stub-obj-y += mon-is-qmp.o
+stub-obj-y += mon-printf.o
+stub-obj-y += mon-print-filename.o
+stub-obj-y += mon-protocol-event.o
+stub-obj-y += mon-set-error.o
+stub-obj-y += reset.o
 stub-obj-y += set-fd-handler.o
+stub-obj-y += slirp.o
+stub-obj-y += sysbus.o
+stub-obj-y += vm-stop.o
+stub-obj-y += vmstate.o
 stub-obj-$(CONFIG_WIN32) += fd-register.o
diff --git a/stubs/arch-query-cpu-def.c b/stubs/arch-query-cpu-def.c
index 47b5246..fa67895 100644
--- a/stubs/arch-query-cpu-def.c
+++ b/stubs/arch-query-cpu-def.c
@@ -1,6 +1,6 @@
 #include "qemu-common.h"
-#include "arch_init.h"
-#include "qerror.h"
+#include "sysemu/arch_init.h"
+#include "qapi/qmp/qerror.h"
 
 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
 {
diff --git a/stubs/clock-warp.c b/stubs/clock-warp.c
new file mode 100644
index 0000000..b64c462
--- /dev/null
+++ b/stubs/clock-warp.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "qemu/timer.h"
+
+void qemu_clock_warp(QEMUClock *clock)
+{
+}
+
diff --git a/stubs/cpu-get-clock.c b/stubs/cpu-get-clock.c
new file mode 100644
index 0000000..5b34c97
--- /dev/null
+++ b/stubs/cpu-get-clock.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "qemu/timer.h"
+
+int64_t cpu_get_clock(void)
+{
+    return get_clock_realtime();
+}
diff --git a/stubs/cpu-get-icount.c b/stubs/cpu-get-icount.c
new file mode 100644
index 0000000..d685859
--- /dev/null
+++ b/stubs/cpu-get-icount.c
@@ -0,0 +1,9 @@
+#include "qemu-common.h"
+#include "qemu/timer.h"
+
+int use_icount;
+
+int64_t cpu_get_icount(void)
+{
+    abort();
+}
diff --git a/stubs/fd-register.c b/stubs/fd-register.c
index 813b6dd..d0c34fd 100644
--- a/stubs/fd-register.c
+++ b/stubs/fd-register.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "main-loop.h"
+#include "qemu/main-loop.h"
 
 void qemu_fd_register(int fd)
 {
diff --git a/stubs/fdset-add-fd.c b/stubs/fdset-add-fd.c
index 09fe2a8..ee16437 100644
--- a/stubs/fdset-add-fd.c
+++ b/stubs/fdset-add-fd.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
 {
diff --git a/stubs/fdset-find-fd.c b/stubs/fdset-find-fd.c
index f82baa0..4f18344 100644
--- a/stubs/fdset-find-fd.c
+++ b/stubs/fdset-find-fd.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 int monitor_fdset_dup_fd_find(int dup_fd)
 {
diff --git a/stubs/fdset-get-fd.c b/stubs/fdset-get-fd.c
index 4106cf9..7112c15 100644
--- a/stubs/fdset-get-fd.c
+++ b/stubs/fdset-get-fd.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 int monitor_fdset_get_fd(int64_t fdset_id, int flags)
 {
diff --git a/stubs/fdset-remove-fd.c b/stubs/fdset-remove-fd.c
index 861b312..b3886d9 100644
--- a/stubs/fdset-remove-fd.c
+++ b/stubs/fdset-remove-fd.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 int monitor_fdset_dup_fd_remove(int dupfd)
 {
diff --git a/stubs/get-fd.c b/stubs/get-fd.c
index 3561ab6..9f2c65c 100644
--- a/stubs/get-fd.c
+++ b/stubs/get-fd.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 int monitor_get_fd(Monitor *mon, const char *name, Error **errp)
 {
diff --git a/stubs/get-vm-name.c b/stubs/get-vm-name.c
new file mode 100644
index 0000000..e5f619f
--- /dev/null
+++ b/stubs/get-vm-name.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+
+const char *qemu_get_vm_name(void)
+{
+    return NULL;
+}
+
diff --git a/stubs/iothread-lock.c b/stubs/iothread-lock.c
new file mode 100644
index 0000000..5d8aca1
--- /dev/null
+++ b/stubs/iothread-lock.c
@@ -0,0 +1,10 @@
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+
+void qemu_mutex_lock_iothread(void)
+{
+}
+
+void qemu_mutex_unlock_iothread(void)
+{
+}
diff --git a/stubs/migr-blocker.c b/stubs/migr-blocker.c
new file mode 100644
index 0000000..300df6e
--- /dev/null
+++ b/stubs/migr-blocker.c
@@ -0,0 +1,10 @@
+#include "qemu-common.h"
+#include "migration/migration.h"
+
+void migrate_add_blocker(Error *reason)
+{
+}
+
+void migrate_del_blocker(Error *reason)
+{
+}
diff --git a/stubs/mon-is-qmp.c b/stubs/mon-is-qmp.c
new file mode 100644
index 0000000..1f0a8fd
--- /dev/null
+++ b/stubs/mon-is-qmp.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+int monitor_cur_is_qmp(void)
+{
+    return 0;
+}
diff --git a/stubs/mon-print-filename.c b/stubs/mon-print-filename.c
new file mode 100644
index 0000000..9c93964
--- /dev/null
+++ b/stubs/mon-print-filename.c
@@ -0,0 +1,6 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+void monitor_print_filename(Monitor *mon, const char *filename)
+{
+}
diff --git a/stubs/mon-printf.c b/stubs/mon-printf.c
new file mode 100644
index 0000000..0ce2ca6
--- /dev/null
+++ b/stubs/mon-printf.c
@@ -0,0 +1,10 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+void monitor_printf(Monitor *mon, const char *fmt, ...)
+{
+}
+
+void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
+{
+}
diff --git a/stubs/mon-protocol-event.c b/stubs/mon-protocol-event.c
new file mode 100644
index 0000000..0946e94
--- /dev/null
+++ b/stubs/mon-protocol-event.c
@@ -0,0 +1,6 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+void monitor_protocol_event(MonitorEvent event, QObject *data)
+{
+}
diff --git a/stubs/mon-set-error.c b/stubs/mon-set-error.c
new file mode 100644
index 0000000..d0411f9
--- /dev/null
+++ b/stubs/mon-set-error.c
@@ -0,0 +1,8 @@
+#include "qemu-common.h"
+#include "monitor/monitor.h"
+
+Monitor *cur_mon;
+
+void monitor_set_error(Monitor *mon, QError *qerror)
+{
+}
diff --git a/stubs/reset.c b/stubs/reset.c
new file mode 100644
index 0000000..ad28725
--- /dev/null
+++ b/stubs/reset.c
@@ -0,0 +1,13 @@
+#include "hw/hw.h"
+
+/* Stub functions for binaries that never call qemu_devices_reset(),
+ * and don't need to keep track of the reset handler list.
+ */
+
+void qemu_register_reset(QEMUResetHandler *func, void *opaque)
+{
+}
+
+void qemu_unregister_reset(QEMUResetHandler *func, void *opaque)
+{
+}
diff --git a/stubs/set-fd-handler.c b/stubs/set-fd-handler.c
index 4807b5d..fc874d3 100644
--- a/stubs/set-fd-handler.c
+++ b/stubs/set-fd-handler.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "main-loop.h"
+#include "qemu/main-loop.h"
 
 int qemu_set_fd_handler2(int fd,
                          IOCanReadHandler *fd_read_poll,
diff --git a/stubs/slirp.c b/stubs/slirp.c
new file mode 100644
index 0000000..9a3309a
--- /dev/null
+++ b/stubs/slirp.c
@@ -0,0 +1,17 @@
+#include "qemu-common.h"
+#include "slirp/slirp.h"
+
+void slirp_update_timeout(uint32_t *timeout)
+{
+}
+
+void slirp_select_fill(int *pnfds, fd_set *readfds,
+                       fd_set *writefds, fd_set *xfds)
+{
+}
+
+void slirp_select_poll(fd_set *readfds, fd_set *writefds,
+                       fd_set *xfds, int select_error)
+{
+}
+
diff --git a/stubs/sysbus.c b/stubs/sysbus.c
new file mode 100644
index 0000000..e134965
--- /dev/null
+++ b/stubs/sysbus.c
@@ -0,0 +1,6 @@
+#include "hw/qdev-core.h"
+
+BusState *sysbus_get_default(void)
+{
+    return NULL;
+}
diff --git a/stubs/vm-stop.c b/stubs/vm-stop.c
new file mode 100644
index 0000000..4568935
--- /dev/null
+++ b/stubs/vm-stop.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "sysemu/sysemu.h"
+
+void vm_stop(RunState state)
+{
+    abort();
+}
diff --git a/stubs/vmstate.c b/stubs/vmstate.c
new file mode 100644
index 0000000..3682af5
--- /dev/null
+++ b/stubs/vmstate.c
@@ -0,0 +1,17 @@
+#include "qemu-common.h"
+#include "migration/vmstate.h"
+
+int vmstate_register_with_alias_id(DeviceState *dev,
+                                   int instance_id,
+                                   const VMStateDescription *vmsd,
+                                   void *base, int alias_id,
+                                   int required_for_version)
+{
+    return 0;
+}
+
+void vmstate_unregister(DeviceState *dev,
+                        const VMStateDescription *vmsd,
+                        void *opaque)
+{
+}
diff --git a/target-alpha/cpu-qom.h b/target-alpha/cpu-qom.h
index 6b4ca6d..16367d2 100644
--- a/target-alpha/cpu-qom.h
+++ b/target-alpha/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_ALPHA_CPU_QOM_H
 #define QEMU_ALPHA_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_ALPHA_CPU "alpha-cpu"
@@ -58,6 +58,9 @@ typedef struct AlphaCPU {
     /*< public >*/
 
     CPUAlphaState env;
+
+    /* This alarm doesn't exist in real hardware; we wish it did.  */
+    struct QEMUTimer *alarm_timer;
 } AlphaCPU;
 
 static inline AlphaCPU *alpha_env_get_cpu(CPUAlphaState *env)
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index 11a19eb..0ad69f0 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -21,8 +21,213 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
+#include "qapi/error.h"
 
 
+static void alpha_cpu_realize(Object *obj, Error **errp)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+
+    qemu_init_vcpu(&cpu->env);
+}
+
+/* Sort alphabetically by type name. */
+static gint alpha_cpu_list_compare(gconstpointer a, gconstpointer b)
+{
+    ObjectClass *class_a = (ObjectClass *)a;
+    ObjectClass *class_b = (ObjectClass *)b;
+    const char *name_a, *name_b;
+
+    name_a = object_class_get_name(class_a);
+    name_b = object_class_get_name(class_b);
+    return strcmp(name_a, name_b);
+}
+
+static void alpha_cpu_list_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CPUListState *s = user_data;
+
+    (*s->cpu_fprintf)(s->file, "  %s\n",
+                      object_class_get_name(oc));
+}
+
+void alpha_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+    CPUListState s = {
+        .file = f,
+        .cpu_fprintf = cpu_fprintf,
+    };
+    GSList *list;
+
+    list = object_class_get_list(TYPE_ALPHA_CPU, false);
+    list = g_slist_sort(list, alpha_cpu_list_compare);
+    (*cpu_fprintf)(f, "Available CPUs:\n");
+    g_slist_foreach(list, alpha_cpu_list_entry, &s);
+    g_slist_free(list);
+}
+
+/* Models */
+
+#define TYPE(model) model "-" TYPE_ALPHA_CPU
+
+typedef struct AlphaCPUAlias {
+    const char *alias;
+    const char *typename;
+} AlphaCPUAlias;
+
+static const AlphaCPUAlias alpha_cpu_aliases[] = {
+    { "21064",   TYPE("ev4") },
+    { "21164",   TYPE("ev5") },
+    { "21164a",  TYPE("ev56") },
+    { "21164pc", TYPE("pca56") },
+    { "21264",   TYPE("ev6") },
+    { "21264a",  TYPE("ev67") },
+};
+
+static ObjectClass *alpha_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc = NULL;
+    char *typename;
+    int i;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (oc != NULL && object_class_dynamic_cast(oc, TYPE_ALPHA_CPU) != NULL &&
+        !object_class_is_abstract(oc)) {
+        return oc;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(alpha_cpu_aliases); i++) {
+        if (strcmp(cpu_model, alpha_cpu_aliases[i].alias) == 0) {
+            oc = object_class_by_name(alpha_cpu_aliases[i].typename);
+            assert(oc != NULL && !object_class_is_abstract(oc));
+            return oc;
+        }
+    }
+
+    typename = g_strdup_printf("%s-" TYPE_ALPHA_CPU, cpu_model);
+    oc = object_class_by_name(typename);
+    g_free(typename);
+    if (oc != NULL && object_class_is_abstract(oc)) {
+        oc = NULL;
+    }
+    return oc;
+}
+
+AlphaCPU *cpu_alpha_init(const char *cpu_model)
+{
+    AlphaCPU *cpu;
+    CPUAlphaState *env;
+    ObjectClass *cpu_class;
+
+    cpu_class = alpha_cpu_class_by_name(cpu_model);
+    if (cpu_class == NULL) {
+        /* Default to ev67; no reason not to emulate insns by default.  */
+        cpu_class = object_class_by_name(TYPE("ev67"));
+    }
+    cpu = ALPHA_CPU(object_new(object_class_get_name(cpu_class)));
+    env = &cpu->env;
+
+    env->cpu_model_str = cpu_model;
+
+    alpha_cpu_realize(OBJECT(cpu), NULL);
+    return cpu;
+}
+
+static void ev4_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->implver = IMPLVER_2106x;
+}
+
+static const TypeInfo ev4_cpu_type_info = {
+    .name = TYPE("ev4"),
+    .parent = TYPE_ALPHA_CPU,
+    .instance_init = ev4_cpu_initfn,
+};
+
+static void ev5_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->implver = IMPLVER_21164;
+}
+
+static const TypeInfo ev5_cpu_type_info = {
+    .name = TYPE("ev5"),
+    .parent = TYPE_ALPHA_CPU,
+    .instance_init = ev5_cpu_initfn,
+};
+
+static void ev56_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->amask |= AMASK_BWX;
+}
+
+static const TypeInfo ev56_cpu_type_info = {
+    .name = TYPE("ev56"),
+    .parent = TYPE("ev5"),
+    .instance_init = ev56_cpu_initfn,
+};
+
+static void pca56_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->amask |= AMASK_MVI;
+}
+
+static const TypeInfo pca56_cpu_type_info = {
+    .name = TYPE("pca56"),
+    .parent = TYPE("ev56"),
+    .instance_init = pca56_cpu_initfn,
+};
+
+static void ev6_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->implver = IMPLVER_21264;
+    env->amask = AMASK_BWX | AMASK_FIX | AMASK_MVI | AMASK_TRAP;
+}
+
+static const TypeInfo ev6_cpu_type_info = {
+    .name = TYPE("ev6"),
+    .parent = TYPE_ALPHA_CPU,
+    .instance_init = ev6_cpu_initfn,
+};
+
+static void ev67_cpu_initfn(Object *obj)
+{
+    AlphaCPU *cpu = ALPHA_CPU(obj);
+    CPUAlphaState *env = &cpu->env;
+
+    env->amask |= AMASK_CIX | AMASK_PREFETCH;
+}
+
+static const TypeInfo ev67_cpu_type_info = {
+    .name = TYPE("ev67"),
+    .parent = TYPE("ev6"),
+    .instance_init = ev67_cpu_initfn,
+};
+
+static const TypeInfo ev68_cpu_type_info = {
+    .name = TYPE("ev68"),
+    .parent = TYPE("ev67"),
+};
+
 static void alpha_cpu_initfn(Object *obj)
 {
     AlphaCPU *cpu = ALPHA_CPU(obj);
@@ -31,6 +236,8 @@ static void alpha_cpu_initfn(Object *obj)
     cpu_exec_init(env);
     tlb_flush(env, 1);
 
+    alpha_translate_init();
+
 #if defined(CONFIG_USER_ONLY)
     env->ps = PS_USER_MODE;
     cpu_alpha_store_fpcr(env, (FPCR_INVD | FPCR_DZED | FPCR_OVFD
@@ -41,18 +248,33 @@ static void alpha_cpu_initfn(Object *obj)
     env->fen = 1;
 }
 
+static void alpha_cpu_class_init(ObjectClass *oc, void *data)
+{
+    CPUClass *cc = CPU_CLASS(oc);
+
+    cc->class_by_name = alpha_cpu_class_by_name;
+}
+
 static const TypeInfo alpha_cpu_type_info = {
     .name = TYPE_ALPHA_CPU,
     .parent = TYPE_CPU,
     .instance_size = sizeof(AlphaCPU),
     .instance_init = alpha_cpu_initfn,
-    .abstract = false,
+    .abstract = true,
     .class_size = sizeof(AlphaCPUClass),
+    .class_init = alpha_cpu_class_init,
 };
 
 static void alpha_cpu_register_types(void)
 {
     type_register_static(&alpha_cpu_type_info);
+    type_register_static(&ev4_cpu_type_info);
+    type_register_static(&ev5_cpu_type_info);
+    type_register_static(&ev56_cpu_type_info);
+    type_register_static(&pca56_cpu_type_info);
+    type_register_static(&ev6_cpu_type_info);
+    type_register_static(&ev67_cpu_type_info);
+    type_register_static(&ev68_cpu_type_info);
 }
 
 type_init(alpha_cpu_register_types)
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index 34221fb..f1db651 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -27,9 +27,9 @@
 
 #define CPUArchState struct CPUAlphaState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -277,16 +277,8 @@ struct CPUAlphaState {
 #endif
 
     /* This alarm doesn't exist in real hardware; we wish it did.  */
-    struct QEMUTimer *alarm_timer;
     uint64_t alarm_expire;
 
-#if TARGET_LONG_BITS > HOST_LONG_BITS
-    /* temporary fixed-point registers
-     * used to emulate 64 bits target on 32 bits hosts
-     */
-    target_ulong t0, t1;
-#endif
-
     /* Those resources are used only in QEMU core */
     CPU_COMMON
 
@@ -297,12 +289,12 @@ struct CPUAlphaState {
     int implver;
 };
 
-#define cpu_init cpu_alpha_init
+#define cpu_list alpha_cpu_list
 #define cpu_exec cpu_alpha_exec
 #define cpu_gen_code cpu_alpha_gen_code
 #define cpu_signal_handler cpu_alpha_signal_handler
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "cpu-qom.h"
 
 enum {
@@ -434,7 +426,20 @@ enum {
     IR_ZERO = 31,
 };
 
-CPUAlphaState * cpu_alpha_init (const char *cpu_model);
+void alpha_translate_init(void);
+
+AlphaCPU *cpu_alpha_init(const char *cpu_model);
+
+static inline CPUAlphaState *cpu_init(const char *cpu_model)
+{
+    AlphaCPU *cpu = cpu_alpha_init(cpu_model);
+    if (cpu == NULL) {
+        return NULL;
+    }
+    return &cpu->env;
+}
+
+void alpha_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 int cpu_alpha_exec(CPUAlphaState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -527,7 +532,7 @@ static inline bool cpu_has_work(CPUState *cpu)
                                      | CPU_INTERRUPT_MCHK);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUAlphaState *env, TranslationBlock *tb)
 {
diff --git a/target-alpha/fpu_helper.c b/target-alpha/fpu_helper.c
index fe988ec..fad3575 100644
--- a/target-alpha/fpu_helper.c
+++ b/target-alpha/fpu_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define FP_STATUS (env->fp_status)
 
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index d9d7f75..22c9c6e 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -22,7 +22,7 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 #include "helper.h"
 
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env)
@@ -494,16 +494,6 @@ void cpu_dump_state (CPUAlphaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "\n");
 }
 
-void do_restore_state(CPUAlphaState *env, uintptr_t retaddr)
-{
-    if (retaddr) {
-        TranslationBlock *tb = tb_find_pc(retaddr);
-        if (tb) {
-            cpu_restore_state(tb, env, retaddr);
-        }
-    }
-}
-
 /* This should only be called from translate, via gen_excp.
    We expect that ENV->PC has already been updated.  */
 void QEMU_NORETURN helper_excp(CPUAlphaState *env, int excp, int error)
@@ -519,7 +509,9 @@ void QEMU_NORETURN dynamic_excp(CPUAlphaState *env, uintptr_t retaddr,
 {
     env->exception_index = excp;
     env->error_code = error;
-    do_restore_state(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
     cpu_loop_exit(env);
 }
 
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index dd55f89..eac3041 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_3(excp, noreturn, env, int, int)
 DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_NO_RWG_SE, i64, env)
@@ -119,4 +119,4 @@ DEF_HELPER_FLAGS_0(get_time, TCG_CALL_NO_RWG, i64)
 DEF_HELPER_FLAGS_2(set_alarm, TCG_CALL_NO_RWG, void, env, i64)
 #endif
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-alpha/int_helper.c b/target-alpha/int_helper.c
index 1d832f0..c9b42b6 100644
--- a/target-alpha/int_helper.c
+++ b/target-alpha/int_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 
 uint64_t helper_umulh(uint64_t op1, uint64_t op2)
diff --git a/target-alpha/mem_helper.c b/target-alpha/mem_helper.c
index 617836c..3d2cd61 100644
--- a/target-alpha/mem_helper.c
+++ b/target-alpha/mem_helper.c
@@ -94,7 +94,9 @@ static void do_unaligned_access(CPUAlphaState *env, target_ulong addr,
     uint64_t pc;
     uint32_t insn;
 
-    do_restore_state(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
 
     pc = env->pc;
     insn = cpu_ldl_code(env, pc);
@@ -115,22 +117,22 @@ void cpu_unassigned_access(CPUAlphaState *env, hwaddr addr,
     dynamic_excp(env, 0, EXCP_MCHK, 0);
 }
 
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 #define ALIGNED_ONLY
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -143,7 +145,9 @@ void tlb_fill(CPUAlphaState *env, target_ulong addr, int is_write,
 
     ret = cpu_alpha_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret != 0)) {
-        do_restore_state(env, retaddr);
+        if (retaddr) {
+            cpu_restore_state(env, retaddr);
+        }
         /* Exception index and error code are already set */
         cpu_loop_exit(env);
     }
diff --git a/target-alpha/sys_helper.c b/target-alpha/sys_helper.c
index 40ca49c..339501a 100644
--- a/target-alpha/sys_helper.c
+++ b/target-alpha/sys_helper.c
@@ -19,8 +19,8 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "sysemu.h"
-#include "qemu-timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
 
 
 uint64_t helper_load_pcc(CPUAlphaState *env)
@@ -77,11 +77,13 @@ uint64_t helper_get_time(void)
 
 void helper_set_alarm(CPUAlphaState *env, uint64_t expire)
 {
+    AlphaCPU *cpu = alpha_env_get_cpu(env);
+
     if (expire) {
         env->alarm_expire = expire;
-        qemu_mod_timer(env->alarm_timer, expire);
+        qemu_mod_timer(cpu->alarm_timer, expire);
     } else {
-        qemu_del_timer(env->alarm_timer);
+        qemu_del_timer(cpu->alarm_timer);
     }
 }
 #endif /* CONFIG_USER_ONLY */
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 4045f78..f687b95 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -18,8 +18,8 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
-#include "host-utils.h"
+#include "disas/disas.h"
+#include "qemu/host-utils.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -88,9 +88,9 @@ static TCGv cpu_usp;
 /* register names */
 static char cpu_reg_names[10*4+21*5 + 10*5+21*6];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
-static void alpha_translate_init(void)
+void alpha_translate_init(void)
 {
     int i;
     char *p;
@@ -611,7 +611,7 @@ static void gen_qual_roundmode(DisasContext *ctx, int fn11)
     }
 
 #if defined(CONFIG_SOFTFLOAT_INLINE)
-    /* ??? The "softfloat.h" interface is to call set_float_rounding_mode.
+    /* ??? The "fpu/softfloat.h" interface is to call set_float_rounding_mode.
        With CONFIG_SOFTFLOAT that expands to an out-of-line call that just
        sets the one field.  */
     tcg_gen_st8_i32(tmp, cpu_env,
@@ -1579,7 +1579,7 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
         case 0x3C:
             /* WHAMI */
             tcg_gen_ld32s_i64(cpu_ir[IR_V0], cpu_env,
-                              offsetof(CPUAlphaState, cpu_index));
+                -offsetof(AlphaCPU, env) + offsetof(CPUState, cpu_index));
             break;
 
         default:
@@ -3410,11 +3410,11 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = ctx.pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
@@ -3468,7 +3468,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = ctx.pc - pc_start;
         tb->icount = num_insns;
@@ -3493,63 +3493,7 @@ void gen_intermediate_code_pc (CPUAlphaState *env, struct TranslationBlock *tb)
     gen_intermediate_code_internal(env, tb, 1);
 }
 
-struct cpu_def_t {
-    const char *name;
-    int implver, amask;
-};
-
-static const struct cpu_def_t cpu_defs[] = {
-    { "ev4",   IMPLVER_2106x, 0 },
-    { "ev5",   IMPLVER_21164, 0 },
-    { "ev56",  IMPLVER_21164, AMASK_BWX },
-    { "pca56", IMPLVER_21164, AMASK_BWX | AMASK_MVI },
-    { "ev6",   IMPLVER_21264, AMASK_BWX | AMASK_FIX | AMASK_MVI | AMASK_TRAP },
-    { "ev67",  IMPLVER_21264, (AMASK_BWX | AMASK_FIX | AMASK_CIX
-			       | AMASK_MVI | AMASK_TRAP | AMASK_PREFETCH), },
-    { "ev68",  IMPLVER_21264, (AMASK_BWX | AMASK_FIX | AMASK_CIX
-			       | AMASK_MVI | AMASK_TRAP | AMASK_PREFETCH), },
-    { "21064", IMPLVER_2106x, 0 },
-    { "21164", IMPLVER_21164, 0 },
-    { "21164a", IMPLVER_21164, AMASK_BWX },
-    { "21164pc", IMPLVER_21164, AMASK_BWX | AMASK_MVI },
-    { "21264", IMPLVER_21264, AMASK_BWX | AMASK_FIX | AMASK_MVI | AMASK_TRAP },
-    { "21264a", IMPLVER_21264, (AMASK_BWX | AMASK_FIX | AMASK_CIX
-				| AMASK_MVI | AMASK_TRAP | AMASK_PREFETCH), }
-};
-
-CPUAlphaState * cpu_alpha_init (const char *cpu_model)
-{
-    AlphaCPU *cpu;
-    CPUAlphaState *env;
-    int implver, amask, i, max;
-
-    cpu = ALPHA_CPU(object_new(TYPE_ALPHA_CPU));
-    env = &cpu->env;
-
-    alpha_translate_init();
-
-    /* Default to ev67; no reason not to emulate insns by default.  */
-    implver = IMPLVER_21264;
-    amask = (AMASK_BWX | AMASK_FIX | AMASK_CIX | AMASK_MVI
-	     | AMASK_TRAP | AMASK_PREFETCH);
-
-    max = ARRAY_SIZE(cpu_defs);
-    for (i = 0; i < max; i++) {
-        if (strcmp (cpu_model, cpu_defs[i].name) == 0) {
-            implver = cpu_defs[i].implver;
-            amask = cpu_defs[i].amask;
-            break;
-        }
-    }
-    env->implver = implver;
-    env->amask = amask;
-    env->cpu_model_str = cpu_model;
-
-    qemu_init_vcpu(env);
-    return env;
-}
-
 void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-arm/arm-semi.c b/target-arm/arm-semi.c
index 7743d67..847318d 100644
--- a/target-arm/arm-semi.c
+++ b/target-arm/arm-semi.c
@@ -33,7 +33,7 @@
 #define ARM_ANGEL_HEAP_SIZE (128 * 1024 * 1024)
 #else
 #include "qemu-common.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 #include "hw/arm-misc.h"
 #endif
 
@@ -113,7 +113,7 @@ static inline uint32_t set_swi_errno(CPUARMState *env, uint32_t code)
     return code;
 }
 
-#include "softmmu-semi.h"
+#include "exec/softmmu-semi.h"
 #endif
 
 static target_ulong arm_semi_syscall_len;
diff --git a/target-arm/cpu-qom.h b/target-arm/cpu-qom.h
index beabf9a..0f455c4 100644
--- a/target-arm/cpu-qom.h
+++ b/target-arm/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_ARM_CPU_QOM_H
 #define QEMU_ARM_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_ARM_CPU "arm-cpu"
 
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index 252c903..af0e134 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -23,7 +23,7 @@
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/loader.h"
 #endif
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
 {
@@ -64,7 +64,7 @@ static void arm_cpu_reset(CPUState *s)
     CPUARMState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -201,6 +201,25 @@ void arm_cpu_realize(ARMCPU *cpu)
 
 /* CPU models */
 
+static ObjectClass *arm_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+    char *typename;
+
+    if (!cpu_model) {
+        return NULL;
+    }
+
+    typename = g_strdup_printf("%s-" TYPE_ARM_CPU, cpu_model);
+    oc = object_class_by_name(typename);
+    g_free(typename);
+    if (!oc || !object_class_dynamic_cast(oc, TYPE_ARM_CPU) ||
+        object_class_is_abstract(oc)) {
+        return NULL;
+    }
+    return oc;
+}
+
 static void arm926_initfn(Object *obj)
 {
     ARMCPU *cpu = ARM_CPU(obj);
@@ -811,19 +830,22 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
 
     acc->parent_reset = cc->reset;
     cc->reset = arm_cpu_reset;
+
+    cc->class_by_name = arm_cpu_class_by_name;
 }
 
 static void cpu_register(const ARMCPUInfo *info)
 {
     TypeInfo type_info = {
-        .name = info->name,
         .parent = TYPE_ARM_CPU,
         .instance_size = sizeof(ARMCPU),
         .instance_init = info->initfn,
         .class_size = sizeof(ARMCPUClass),
     };
 
-    type_register_static(&type_info);
+    type_info.name = g_strdup_printf("%s-" TYPE_ARM_CPU, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
 }
 
 static const TypeInfo arm_cpu_type_info = {
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 474ee05..417a566 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -27,9 +27,9 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -667,7 +667,7 @@ static inline void cpu_clone_regs(CPUARMState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /* Bit usage in the TB flags field: */
 #define ARM_TBFLAG_THUMB_SHIFT      0
@@ -734,7 +734,7 @@ static inline bool cpu_has_work(CPUState *cpu)
         (CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD | CPU_INTERRUPT_EXITTB);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUARMState *env, TranslationBlock *tb)
 {
diff --git a/target-arm/helper.c b/target-arm/helper.c
index 6dd6904..34bdb64 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -1,9 +1,9 @@
 #include "cpu.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 #include "helper.h"
-#include "host-utils.h"
-#include "sysemu.h"
-#include "bitops.h"
+#include "qemu/host-utils.h"
+#include "sysemu/sysemu.h"
+#include "qemu/bitops.h"
 
 #ifndef CONFIG_USER_ONLY
 static inline int get_phys_addr(CPUARMState *env, uint32_t address,
@@ -899,7 +899,8 @@ static const ARMCPRegInfo strongarm_cp_reginfo[] = {
 static int mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri,
                       uint64_t *value)
 {
-    uint32_t mpidr = env->cpu_index;
+    CPUState *cs = CPU(arm_env_get_cpu(env));
+    uint32_t mpidr = cs->cpu_index;
     /* We don't support setting cluster ID ([8..11])
      * so these bits always RAZ.
      */
@@ -1290,12 +1291,14 @@ ARMCPU *cpu_arm_init(const char *cpu_model)
 {
     ARMCPU *cpu;
     CPUARMState *env;
+    ObjectClass *oc;
     static int inited = 0;
 
-    if (!object_class_by_name(cpu_model)) {
+    oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
+    if (!oc) {
         return NULL;
     }
-    cpu = ARM_CPU(object_new(cpu_model));
+    cpu = ARM_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
     env->cpu_model_str = cpu_model;
     arm_cpu_realize(cpu);
@@ -1320,11 +1323,6 @@ ARMCPU *cpu_arm_init(const char *cpu_model)
     return cpu;
 }
 
-typedef struct ARMCPUListState {
-    fprintf_function cpu_fprintf;
-    FILE *file;
-} ARMCPUListState;
-
 /* Sort alphabetically by type name, except for "any". */
 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
 {
@@ -1334,9 +1332,9 @@ static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
 
     name_a = object_class_get_name(class_a);
     name_b = object_class_get_name(class_b);
-    if (strcmp(name_a, "any") == 0) {
+    if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
         return 1;
-    } else if (strcmp(name_b, "any") == 0) {
+    } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
         return -1;
     } else {
         return strcmp(name_a, name_b);
@@ -1346,15 +1344,20 @@ static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
 {
     ObjectClass *oc = data;
-    ARMCPUListState *s = user_data;
+    CPUListState *s = user_data;
+    const char *typename;
+    char *name;
 
+    typename = object_class_get_name(oc);
+    name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
     (*s->cpu_fprintf)(s->file, "  %s\n",
-                      object_class_get_name(oc));
+                      name);
+    g_free(name);
 }
 
 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    ARMCPUListState s = {
+    CPUListState s = {
         .file = f,
         .cpu_fprintf = cpu_fprintf,
     };
@@ -1772,7 +1775,7 @@ static void do_interrupt_v7m(CPUARMState *env)
         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
         return;
     case EXCP_SWI:
-        env->regs[15] += 2;
+        /* The PC already points to the next instruction.  */
         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
         return;
     case EXCP_PREFETCH_ABORT:
diff --git a/target-arm/helper.h b/target-arm/helper.h
index 3d23ceb..8544f82 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
 DEF_HELPER_FLAGS_1(sxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
@@ -463,4 +463,4 @@ DEF_HELPER_3(neon_qzip8, void, env, i32, i32)
 DEF_HELPER_3(neon_qzip16, void, env, i32, i32)
 DEF_HELPER_3(neon_qzip32, void, env, i32, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-arm/iwmmxt_helper.c b/target-arm/iwmmxt_helper.c
index 1dd8d1a..7953b53 100644
--- a/target-arm/iwmmxt_helper.c
+++ b/target-arm/iwmmxt_helper.c
@@ -23,7 +23,7 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 #include "helper.h"
 
 /* iwMMXt macros extracted from GNU gdb.  */
diff --git a/target-arm/kvm.c b/target-arm/kvm.c
index ff3007b..c0f2d86 100644
--- a/target-arm/kvm.c
+++ b/target-arm/kvm.c
@@ -16,9 +16,9 @@
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "cpu.h"
 #include "hw/arm-misc.h"
@@ -36,7 +36,12 @@ int kvm_arch_init(KVMState *s)
     return 0;
 }
 
-int kvm_arch_init_vcpu(CPUARMState *env)
+unsigned long kvm_arch_vcpu_id(CPUState *cpu)
+{
+    return cpu->cpu_index;
+}
+
+int kvm_arch_init_vcpu(CPUState *cs)
 {
     struct kvm_vcpu_init init;
     int ret;
@@ -45,7 +50,7 @@ int kvm_arch_init_vcpu(CPUARMState *env)
 
     init.target = KVM_ARM_TARGET_CORTEX_A15;
     memset(init.features, 0, sizeof(init.features));
-    ret = kvm_vcpu_ioctl(env, KVM_ARM_VCPU_INIT, &init);
+    ret = kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
     if (ret) {
         return ret;
     }
@@ -56,7 +61,7 @@ int kvm_arch_init_vcpu(CPUARMState *env)
      */
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP | 31;
     r.addr = (uintptr_t)(&v);
-    ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
     if (ret == ENOENT) {
         return EINVAL;
     }
@@ -72,7 +77,7 @@ int kvm_arch_init_vcpu(CPUARMState *env)
  * need to do anything special for the KVM case.
  */
 typedef struct KVMDevice {
-    struct kvm_device_address kda;
+    struct kvm_arm_device_addr kda;
     MemoryRegion *mr;
     QSLIST_ENTRY(KVMDevice) entries;
 } KVMDevice;
@@ -112,8 +117,9 @@ static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
     memory_listener_unregister(&devlistener);
     QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
         if (kd->kda.addr != -1) {
-            if (kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ADDRESS, &kd->kda) < 0) {
-                fprintf(stderr, "KVM_SET_DEVICE_ADDRESS failed: %s\n",
+            if (kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR,
+                             &kd->kda) < 0) {
+                fprintf(stderr, "KVM_ARM_SET_DEVICE_ADDRESS failed: %s\n",
                         strerror(errno));
                 abort();
             }
@@ -231,8 +237,10 @@ static const Reg regs[] = {
     VFPSYSREG(FPINST2),
 };
 
-int kvm_arch_put_registers(CPUARMState *env, int level)
+int kvm_arch_put_registers(CPUState *cs, int level)
 {
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     struct kvm_one_reg r;
     int mode, bn;
     int ret, i;
@@ -255,7 +263,7 @@ int kvm_arch_put_registers(CPUARMState *env, int level)
     for (i = 0; i < ARRAY_SIZE(regs); i++) {
         r.id = regs[i].id;
         r.addr = (uintptr_t)(env) + regs[i].offset;
-        ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &r);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
         if (ret) {
             return ret;
         }
@@ -266,7 +274,7 @@ int kvm_arch_put_registers(CPUARMState *env, int level)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U32 |
         KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(usr_regs.ARM_cpsr);
     r.addr = (uintptr_t)(&cpsr);
-    ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -276,7 +284,7 @@ int kvm_arch_put_registers(CPUARMState *env, int level)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
         (2 << KVM_REG_ARM_CRM_SHIFT) | (0 << KVM_REG_ARM_OPC1_SHIFT);
     r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -286,7 +294,7 @@ int kvm_arch_put_registers(CPUARMState *env, int level)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
         (2 << KVM_REG_ARM_CRM_SHIFT) | (1 << KVM_REG_ARM_OPC1_SHIFT);
     r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -295,7 +303,7 @@ int kvm_arch_put_registers(CPUARMState *env, int level)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
     for (i = 0; i < 32; i++) {
         r.addr = (uintptr_t)(&env->vfp.regs[i]);
-        ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &r);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
         if (ret) {
             return ret;
         }
@@ -306,13 +314,15 @@ int kvm_arch_put_registers(CPUARMState *env, int level)
         KVM_REG_ARM_VFP_FPSCR;
     fpscr = vfp_get_fpscr(env);
     r.addr = (uintptr_t)&fpscr;
-    ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
 
     return ret;
 }
 
-int kvm_arch_get_registers(CPUARMState *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     struct kvm_one_reg r;
     int mode, bn;
     int ret, i;
@@ -322,7 +332,7 @@ int kvm_arch_get_registers(CPUARMState *env)
     for (i = 0; i < ARRAY_SIZE(regs); i++) {
         r.id = regs[i].id;
         r.addr = (uintptr_t)(env) + regs[i].offset;
-        ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
         if (ret) {
             return ret;
         }
@@ -332,7 +342,7 @@ int kvm_arch_get_registers(CPUARMState *env)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U32 |
         KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(usr_regs.ARM_cpsr);
     r.addr = (uintptr_t)(&cpsr);
-    ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -342,7 +352,7 @@ int kvm_arch_get_registers(CPUARMState *env)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
         (2 << KVM_REG_ARM_CRM_SHIFT) | (0 << KVM_REG_ARM_OPC1_SHIFT);
     r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -353,7 +363,7 @@ int kvm_arch_get_registers(CPUARMState *env)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | (15 << KVM_REG_ARM_COPROC_SHIFT) |
         (2 << KVM_REG_ARM_CRM_SHIFT) | (1 << KVM_REG_ARM_OPC1_SHIFT);
     r.addr = (uintptr_t)(&ttbr);
-    ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -385,7 +395,7 @@ int kvm_arch_get_registers(CPUARMState *env)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
     for (i = 0; i < 32; i++) {
         r.addr = (uintptr_t)(&env->vfp.regs[i]);
-        ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
         if (ret) {
             return ret;
         }
@@ -395,7 +405,7 @@ int kvm_arch_get_registers(CPUARMState *env)
     r.id = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_VFP |
         KVM_REG_ARM_VFP_FPSCR;
     r.addr = (uintptr_t)&fpscr;
-    ret = kvm_vcpu_ioctl(env, KVM_GET_ONE_REG, &r);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
     if (ret) {
         return ret;
     }
@@ -404,36 +414,36 @@ int kvm_arch_get_registers(CPUARMState *env)
     return 0;
 }
 
-void kvm_arch_pre_run(CPUARMState *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 {
 }
 
-void kvm_arch_post_run(CPUARMState *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
 }
 
-int kvm_arch_handle_exit(CPUARMState *env, struct kvm_run *run)
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
     int ret = 0;
 
     return ret;
 }
 
-void kvm_arch_reset_vcpu(CPUARMState *env)
+void kvm_arch_reset_vcpu(CPUState *cs)
 {
 }
 
-bool kvm_arch_stop_on_emulation_error(CPUARMState *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cs)
 {
     return true;
 }
 
-int kvm_arch_process_async_events(CPUARMState *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
     return 0;
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUARMState *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *cs, int code, void *addr)
 {
     return 1;
 }
@@ -443,12 +453,12 @@ int kvm_arch_on_sigbus(int code, void *addr)
     return 1;
 }
 
-void kvm_arch_update_guest_debug(CPUARMState *env, struct kvm_guest_debug *dbg)
+void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
 {
     qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
 }
 
-int kvm_arch_insert_sw_breakpoint(CPUARMState *env,
+int kvm_arch_insert_sw_breakpoint(CPUState *cs,
                                   struct kvm_sw_breakpoint *bp)
 {
     qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
@@ -469,7 +479,7 @@ int kvm_arch_remove_hw_breakpoint(target_ulong addr,
     return -EINVAL;
 }
 
-int kvm_arch_remove_sw_breakpoint(CPUARMState *env,
+int kvm_arch_remove_sw_breakpoint(CPUState *cs,
                                   struct kvm_sw_breakpoint *bp)
 {
     qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
diff --git a/target-arm/kvm_arm.h b/target-arm/kvm_arm.h
index 2f61d00..b1c54ff 100644
--- a/target-arm/kvm_arm.h
+++ b/target-arm/kvm_arm.h
@@ -11,8 +11,8 @@
 #ifndef QEMU_KVM_ARM_H
 #define QEMU_KVM_ARM_H
 
-#include "kvm.h"
-#include "memory.h"
+#include "sysemu/kvm.h"
+#include "exec/memory.h"
 
 /**
  * kvm_arm_register_device:
diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index 89280b6..b028cc2 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -10,7 +10,7 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 #include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index 6e3ab90..99610d7 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -52,21 +52,21 @@ uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
 
 #if !defined(CONFIG_USER_ONLY)
 
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -74,19 +74,13 @@ uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
 void tlb_fill(CPUARMState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_arm_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         raise_exception(env, env->exception_index);
     }
diff --git a/target-arm/translate.c b/target-arm/translate.c
index c1e3e0d..fb1048d 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -25,9 +25,9 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -99,7 +99,7 @@ static TCGv_i32 cpu_exclusive_info;
 static TCGv cpu_F0s, cpu_F1s;
 static TCGv_i64 cpu_F0d, cpu_F1d;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 static const char *regnames[] =
     { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
@@ -2744,7 +2744,6 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                     }
                 } else {
                     /* arm->vfp */
-                    tmp = load_reg(s, rd);
                     if (insn & (1 << 21)) {
                         rn >>= 1;
                         /* system register */
@@ -2755,6 +2754,7 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                             /* Writes are ignored.  */
                             break;
                         case ARM_VFP_FPSCR:
+                            tmp = load_reg(s, rd);
                             gen_helper_vfp_set_fpscr(cpu_env, tmp);
                             tcg_temp_free_i32(tmp);
                             gen_lookup_tb(s);
@@ -2764,18 +2764,21 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                                 return 1;
                             /* TODO: VFP subarchitecture support.
                              * For now, keep the EN bit only */
+                            tmp = load_reg(s, rd);
                             tcg_gen_andi_i32(tmp, tmp, 1 << 30);
                             store_cpu_field(tmp, vfp.xregs[rn]);
                             gen_lookup_tb(s);
                             break;
                         case ARM_VFP_FPINST:
                         case ARM_VFP_FPINST2:
+                            tmp = load_reg(s, rd);
                             store_cpu_field(tmp, vfp.xregs[rn]);
                             break;
                         default:
                             return 1;
                         }
                     } else {
+                        tmp = load_reg(s, rd);
                         gen_vfp_msr(tmp);
                         gen_mov_vreg_F0(0, rn);
                     }
@@ -9858,12 +9861,12 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
             gen_opc_condexec_bits[lj] = (dc->condexec_cond << 4) | (dc->condexec_mask >> 1);
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
@@ -10004,7 +10007,7 @@ done_generating:
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = dc->pc - pc_start;
         tb->icount = num_insns;
@@ -10070,6 +10073,6 @@ void cpu_dump_state(CPUARMState *env, FILE *f, fprintf_function cpu_fprintf,
 
 void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->regs[15] = gen_opc_pc[pc_pos];
+    env->regs[15] = tcg_ctx.gen_opc_pc[pc_pos];
     env->condexec_bits = gen_opc_condexec_bits[pc_pos];
 }
diff --git a/target-cris/cpu-qom.h b/target-cris/cpu-qom.h
index d0e5f04..41ab9b2 100644
--- a/target-cris/cpu-qom.h
+++ b/target-cris/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_CRIS_CPU_QOM_H
 #define QEMU_CRIS_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_CRIS_CPU "cris-cpu"
 
diff --git a/target-cris/cpu.c b/target-cris/cpu.c
index c596609..3f64a57 100644
--- a/target-cris/cpu.c
+++ b/target-cris/cpu.c
@@ -35,7 +35,7 @@ static void cris_cpu_reset(CPUState *s)
     uint32_t vr;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
diff --git a/target-cris/cpu.h b/target-cris/cpu.h
index 2c27506..257cb52 100644
--- a/target-cris/cpu.h
+++ b/target-cris/cpu.h
@@ -27,7 +27,7 @@
 
 #define CPUArchState struct CPUCRISState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -175,7 +175,6 @@ typedef struct CPUCRISState {
 
 CRISCPU *cpu_cris_init(const char *cpu_model);
 int cpu_cris_exec(CPUCRISState *s);
-void cpu_cris_close(CPUCRISState *s);
 void do_interrupt(CPUCRISState *env);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -270,7 +269,7 @@ static inline void cpu_set_tls(CPUCRISState *env, target_ulong newtls)
 #define SFR_RW_MM_TLB_LO   env->pregs[PR_SRS]][5
 #define SFR_RW_MM_TLB_HI   env->pregs[PR_SRS]][6
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
@@ -292,7 +291,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUCRISState *env, TranslationBlock *tb)
 {
diff --git a/target-cris/crisv32-decode.h b/target-cris/crisv32-decode.h
index ed141de..cdba377 100644
--- a/target-cris/crisv32-decode.h
+++ b/target-cris/crisv32-decode.h
@@ -17,6 +17,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef CRISV32_DECODE_H
+#define CRISV32_DECODE_H 1
 
 /* Convenient binary macros.  */
 #define HEX__(n) 0x##n##LU
@@ -126,3 +128,5 @@
 
 #define DEC_FTAG_FIDX_D_M {B8(10101011), B8(11111111)}
 #define DEC_FTAG_FIDX_I_M {B8(11010011), B8(11111111)}
+
+#endif
diff --git a/target-cris/helper.c b/target-cris/helper.c
index 324fe05..6e75e98 100644
--- a/target-cris/helper.c
+++ b/target-cris/helper.c
@@ -20,7 +20,7 @@
 
 #include "cpu.h"
 #include "mmu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 
 //#define CRIS_HELPER_DEBUG
@@ -28,7 +28,7 @@
 
 #ifdef CRIS_HELPER_DEBUG
 #define D(x) x
-#define D_LOG(...) qemu_log(__VA__ARGS__)
+#define D_LOG(...) qemu_log(__VA_ARGS__)
 #else
 #define D(x)
 #define D_LOG(...) do { } while (0)
diff --git a/target-cris/helper.h b/target-cris/helper.h
index fe12083..8e8365c 100644
--- a/target-cris/helper.h
+++ b/target-cris/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_2(tlb_flush_pid, void, env, i32)
@@ -26,4 +26,4 @@ DEF_HELPER_FLAGS_3(evaluate_flags_move_2, TCG_CALL_NO_SE, i32, env, i32, i32)
 DEF_HELPER_1(evaluate_flags, void, env)
 DEF_HELPER_1(top_evaluate_flags, void, env)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-cris/op_helper.c b/target-cris/op_helper.c
index a7468d4..b580513 100644
--- a/target-cris/op_helper.c
+++ b/target-cris/op_helper.c
@@ -21,35 +21,35 @@
 #include "cpu.h"
 #include "mmu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 //#define CRIS_OP_HELPER_DEBUG
 
 
 #ifdef CRIS_OP_HELPER_DEBUG
 #define D(x) x
-#define D_LOG(...) qemu_log(__VA__ARGS__)
+#define D_LOG(...) qemu_log(__VA_ARGS__)
 #else
 #define D(x)
 #define D_LOG(...) do { } while (0)
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -57,21 +57,15 @@
 void tlb_fill(CPUCRISState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     D_LOG("%s pc=%x tpc=%x ra=%p\n", __func__,
-          env->pc, env->debug1, (void *)retaddr);
+          env->pc, env->pregs[PR_EDA], (void *)retaddr);
     ret = cpu_cris_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-
+            if (cpu_restore_state(env, retaddr)) {
 		/* Evaluate flags after retranslation.  */
                 helper_top_evaluate_flags(env);
             }
diff --git a/target-cris/translate.c b/target-cris/translate.c
index 0b0e86d..09e6011 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -24,7 +24,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 #include "helper.h"
 #include "mmu.h"
@@ -70,7 +70,7 @@ static TCGv env_btaken;
 static TCGv env_btarget;
 static TCGv env_pc;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
@@ -3301,16 +3301,16 @@ gen_intermediate_code_internal(CPUCRISState *env, TranslationBlock *tb,
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
             if (dc->delayed_branch == 1) {
-                gen_opc_pc[lj] = dc->ppc | 1;
+                tcg_ctx.gen_opc_pc[lj] = dc->ppc | 1;
             } else {
-                gen_opc_pc[lj] = dc->pc;
+                tcg_ctx.gen_opc_pc[lj] = dc->pc;
             }
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         /* Pretty disas.  */
@@ -3439,7 +3439,7 @@ gen_intermediate_code_internal(CPUCRISState *env, TranslationBlock *tb,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -3621,5 +3621,5 @@ CRISCPU *cpu_cris_init(const char *cpu_model)
 
 void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-i386/arch_dump.c b/target-i386/arch_dump.c
index 4240278..2cd2f7f 100644
--- a/target-i386/arch_dump.c
+++ b/target-i386/arch_dump.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "dump.h"
+#include "exec/cpu-all.h"
+#include "sysemu/dump.h"
 #include "elf.h"
 
 #ifdef TARGET_X86_64
@@ -403,7 +403,7 @@ int cpu_get_dump_info(ArchDumpInfo *info)
     } else {
         info->d_class = ELFCLASS32;
 
-        QLIST_FOREACH(block, &ram_list.blocks, next) {
+        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
             if (block->offset + block->length > UINT_MAX) {
                 /* The memory size is greater than 4G */
                 info->d_class = ELFCLASS64;
diff --git a/target-i386/arch_memory_mapping.c b/target-i386/arch_memory_mapping.c
index 41f9d1c..844893f 100644
--- a/target-i386/arch_memory_mapping.c
+++ b/target-i386/arch_memory_mapping.c
@@ -12,8 +12,8 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
-#include "memory_mapping.h"
+#include "exec/cpu-all.h"
+#include "sysemu/memory_mapping.h"
 
 /* PAE Paging or IA-32e Paging */
 static void walk_pte(MemoryMappingList *list, hwaddr pte_start_addr,
@@ -115,7 +115,7 @@ static void walk_pde2(MemoryMappingList *list,
                       hwaddr pde_start_addr, int32_t a20_mask,
                       bool pse)
 {
-    hwaddr pde_addr, pte_start_addr, start_paddr;
+    hwaddr pde_addr, pte_start_addr, start_paddr, high_paddr;
     uint32_t pde;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -130,8 +130,13 @@ static void walk_pde2(MemoryMappingList *list,
 
         line_addr = (((unsigned int)i & 0x3ff) << 22);
         if ((pde & PG_PSE_MASK) && pse) {
-            /* 4 MB page */
-            start_paddr = (pde & ~0x3fffff) | ((pde & 0x1fe000) << 19);
+            /*
+             * 4 MB page:
+             * bits 39:32 are bits 20:13 of the PDE
+             * bit3 31:22 are bits 31:22 of the PDE
+             */
+            high_paddr = ((hwaddr)(pde & 0x1fe000) << 19);
+            start_paddr = (pde & ~0x3fffff) | high_paddr;
             if (cpu_physical_memory_is_io(start_paddr)) {
                 /* I/O region */
                 continue;
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index 5901140..332916a 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -20,9 +20,9 @@
 #ifndef QEMU_I386_CPU_QOM_H
 #define QEMU_I386_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
-#include "error.h"
+#include "qapi/error.h"
 
 #ifdef TARGET_X86_64
 #define TYPE_X86_CPU "x86_64-cpu"
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index c6c2ca0..aab35c7 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -22,13 +22,16 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
+#include "sysemu/cpus.h"
+#include "topology.h"
 
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "qapi/qmp/qerror.h"
 
-#include "qapi/qapi-visit-core.h"
-#include "arch_init.h"
+#include "qapi/visitor.h"
+#include "sysemu/arch_init.h"
 
 #include "hyperv.h"
 
@@ -37,13 +40,25 @@
 #include <linux/kvm_para.h>
 #endif
 
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #ifndef CONFIG_USER_ONLY
 #include "hw/xen.h"
 #include "hw/sysbus.h"
 #include "hw/apic_internal.h"
 #endif
 
+static void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1,
+                                     uint32_t vendor2, uint32_t vendor3)
+{
+    int i;
+    for (i = 0; i < 4; i++) {
+        dst[i] = vendor1 >> (8 * i);
+        dst[i + 4] = vendor2 >> (8 * i);
+        dst[i + 8] = vendor3 >> (8 * i);
+    }
+    dst[CPUID_VENDOR_SZ] = '\0';
+}
+
 /* feature flags taken from "Intel Processor Identification and the CPUID
  * Instruction" and AMD's "CPUID Specification".  In cases of disagreement
  * between feature naming conventions, aliases may be added.
@@ -94,6 +109,17 @@ static const char *ext3_feature_name[] = {
     NULL, NULL, NULL, NULL,
 };
 
+static const char *ext4_feature_name[] = {
+    NULL, NULL, "xstore", "xstore-en",
+    NULL, NULL, "xcrypt", "xcrypt-en",
+    "ace2", "ace2-en", "phe", "phe-en",
+    "pmm", "pmm-en", NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
 static const char *kvm_feature_name[] = {
     "kvmclock", "kvm_nopiodelay", "kvm_mmu", "kvmclock",
     "kvm_asyncpf", "kvm_steal_time", "kvm_pv_eoi", NULL,
@@ -123,36 +149,88 @@ static const char *cpuid_7_0_ebx_feature_name[] = {
     NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
 };
 
+typedef struct FeatureWordInfo {
+    const char **feat_names;
+    uint32_t cpuid_eax; /* Input EAX for CPUID */
+    int cpuid_reg;      /* R_* register constant */
+} FeatureWordInfo;
+
+static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
+    [FEAT_1_EDX] = {
+        .feat_names = feature_name,
+        .cpuid_eax = 1, .cpuid_reg = R_EDX,
+    },
+    [FEAT_1_ECX] = {
+        .feat_names = ext_feature_name,
+        .cpuid_eax = 1, .cpuid_reg = R_ECX,
+    },
+    [FEAT_8000_0001_EDX] = {
+        .feat_names = ext2_feature_name,
+        .cpuid_eax = 0x80000001, .cpuid_reg = R_EDX,
+    },
+    [FEAT_8000_0001_ECX] = {
+        .feat_names = ext3_feature_name,
+        .cpuid_eax = 0x80000001, .cpuid_reg = R_ECX,
+    },
+    [FEAT_C000_0001_EDX] = {
+        .feat_names = ext4_feature_name,
+        .cpuid_eax = 0xC0000001, .cpuid_reg = R_EDX,
+    },
+    [FEAT_KVM] = {
+        .feat_names = kvm_feature_name,
+        .cpuid_eax = KVM_CPUID_FEATURES, .cpuid_reg = R_EAX,
+    },
+    [FEAT_SVM] = {
+        .feat_names = svm_feature_name,
+        .cpuid_eax = 0x8000000A, .cpuid_reg = R_EDX,
+    },
+    [FEAT_7_0_EBX] = {
+        .feat_names = cpuid_7_0_ebx_feature_name,
+        .cpuid_eax = 7, .cpuid_reg = R_EBX,
+    },
+};
+
+const char *get_register_name_32(unsigned int reg)
+{
+    static const char *reg_names[CPU_NB_REGS32] = {
+        [R_EAX] = "EAX",
+        [R_ECX] = "ECX",
+        [R_EDX] = "EDX",
+        [R_EBX] = "EBX",
+        [R_ESP] = "ESP",
+        [R_EBP] = "EBP",
+        [R_ESI] = "ESI",
+        [R_EDI] = "EDI",
+    };
+
+    if (reg > CPU_NB_REGS32) {
+        return NULL;
+    }
+    return reg_names[reg];
+}
+
 /* collects per-function cpuid data
  */
 typedef struct model_features_t {
     uint32_t *guest_feat;
     uint32_t *host_feat;
-    uint32_t check_feat;
-    const char **flag_names;
-    uint32_t cpuid;
-    } model_features_t;
+    FeatureWord feat_word;
+} model_features_t;
 
 int check_cpuid = 0;
 int enforce_cpuid = 0;
 
-#if defined(CONFIG_KVM)
 static uint32_t kvm_default_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
         (1 << KVM_FEATURE_NOP_IO_DELAY) |
-        (1 << KVM_FEATURE_MMU_OP) |
         (1 << KVM_FEATURE_CLOCKSOURCE2) |
         (1 << KVM_FEATURE_ASYNC_PF) |
         (1 << KVM_FEATURE_STEAL_TIME) |
+        (1 << KVM_FEATURE_PV_EOI) |
         (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
-static const uint32_t kvm_pv_eoi_features = (0x1 << KVM_FEATURE_PV_EOI);
-#else
-static uint32_t kvm_default_features = 0;
-static const uint32_t kvm_pv_eoi_features = 0;
-#endif
 
-void enable_kvm_pv_eoi(void)
+void disable_kvm_pv_eoi(void)
 {
-    kvm_default_features |= kvm_pv_eoi_features;
+    kvm_default_features &= ~(1UL << KVM_FEATURE_PV_EOI);
 }
 
 void host_cpuid(uint32_t function, uint32_t count,
@@ -252,39 +330,34 @@ static bool lookup_feature(uint32_t *pval, const char *s, const char *e,
     return found;
 }
 
-static void add_flagname_to_bitmaps(const char *flagname, uint32_t *features,
-                                    uint32_t *ext_features,
-                                    uint32_t *ext2_features,
-                                    uint32_t *ext3_features,
-                                    uint32_t *kvm_features,
-                                    uint32_t *svm_features,
-                                    uint32_t *cpuid_7_0_ebx_features)
+static void add_flagname_to_bitmaps(const char *flagname,
+                                    FeatureWordArray words)
 {
-    if (!lookup_feature(features, flagname, NULL, feature_name) &&
-        !lookup_feature(ext_features, flagname, NULL, ext_feature_name) &&
-        !lookup_feature(ext2_features, flagname, NULL, ext2_feature_name) &&
-        !lookup_feature(ext3_features, flagname, NULL, ext3_feature_name) &&
-        !lookup_feature(kvm_features, flagname, NULL, kvm_feature_name) &&
-        !lookup_feature(svm_features, flagname, NULL, svm_feature_name) &&
-        !lookup_feature(cpuid_7_0_ebx_features, flagname, NULL,
-                        cpuid_7_0_ebx_feature_name))
-            fprintf(stderr, "CPU feature %s not found\n", flagname);
+    FeatureWord w;
+    for (w = 0; w < FEATURE_WORDS; w++) {
+        FeatureWordInfo *wi = &feature_word_info[w];
+        if (wi->feat_names &&
+            lookup_feature(&words[w], flagname, NULL, wi->feat_names)) {
+            break;
+        }
+    }
+    if (w == FEATURE_WORDS) {
+        fprintf(stderr, "CPU feature %s not found\n", flagname);
+    }
 }
 
 typedef struct x86_def_t {
-    struct x86_def_t *next;
     const char *name;
     uint32_t level;
-    uint32_t vendor1, vendor2, vendor3;
+    /* vendor is zero-terminated, 12 character ASCII string */
+    char vendor[CPUID_VENDOR_SZ + 1];
     int family;
     int model;
     int stepping;
-    int tsc_khz;
     uint32_t features, ext_features, ext2_features, ext3_features;
     uint32_t kvm_features, svm_features;
     uint32_t xlevel;
     char model_id[48];
-    int vendor_override;
     /* Store the results of Centaur's CPUID instructions */
     uint32_t ext4_features;
     uint32_t xlevel2;
@@ -330,19 +403,13 @@ typedef struct x86_def_t {
 #define TCG_SVM_FEATURES 0
 #define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP)
 
-/* maintains list of cpu model definitions
- */
-static x86_def_t *x86_defs = {NULL};
-
-/* built-in cpu model definitions (deprecated)
+/* built-in CPU model definitions
  */
 static x86_def_t builtin_x86_defs[] = {
     {
         .name = "qemu64",
         .level = 4,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 6,
         .model = 2,
         .stepping = 3,
@@ -359,9 +426,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "phenom",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 16,
         .model = 2,
         .stepping = 3,
@@ -387,6 +452,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "core2duo",
         .level = 10,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 15,
         .stepping = 11,
@@ -405,9 +471,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "kvm64",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 15,
         .model = 6,
         .stepping = 1,
@@ -431,6 +495,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "qemu32",
         .level = 4,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 3,
         .stepping = 3,
@@ -441,6 +506,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "kvm32",
         .level = 5,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 15,
         .model = 6,
         .stepping = 1,
@@ -455,6 +521,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "coreduo",
         .level = 10,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 14,
         .stepping = 8,
@@ -470,6 +537,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "486",
         .level = 1,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 4,
         .model = 0,
         .stepping = 0,
@@ -479,6 +547,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "pentium",
         .level = 1,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 5,
         .model = 4,
         .stepping = 3,
@@ -488,6 +557,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "pentium2",
         .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 5,
         .stepping = 2,
@@ -497,6 +567,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "pentium3",
         .level = 2,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 7,
         .stepping = 3,
@@ -506,9 +577,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "athlon",
         .level = 2,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 6,
         .model = 2,
         .stepping = 3,
@@ -522,6 +591,7 @@ static x86_def_t builtin_x86_defs[] = {
         .name = "n270",
         /* original is on level 10 */
         .level = 5,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 28,
         .stepping = 2,
@@ -540,9 +610,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Conroe",
         .level = 2,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 2,
         .stepping = 3,
@@ -560,9 +628,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Penryn",
         .level = 2,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 2,
         .stepping = 3,
@@ -581,9 +647,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Nehalem",
         .level = 2,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 2,
         .stepping = 3,
@@ -602,9 +666,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Westmere",
         .level = 11,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 44,
         .stepping = 1,
@@ -624,9 +686,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "SandyBridge",
         .level = 0xd,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 42,
         .stepping = 1,
@@ -649,9 +709,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Haswell",
         .level = 0xd,
-        .vendor1 = CPUID_VENDOR_INTEL_1,
-        .vendor2 = CPUID_VENDOR_INTEL_2,
-        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 60,
         .stepping = 1,
@@ -679,9 +737,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Opteron_G1",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 15,
         .model = 6,
         .stepping = 1,
@@ -703,9 +759,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Opteron_G2",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 15,
         .model = 6,
         .stepping = 1,
@@ -729,9 +783,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Opteron_G3",
         .level = 5,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 15,
         .model = 6,
         .stepping = 1,
@@ -757,9 +809,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Opteron_G4",
         .level = 0xd,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 21,
         .model = 1,
         .stepping = 2,
@@ -789,9 +839,7 @@ static x86_def_t builtin_x86_defs[] = {
     {
         .name = "Opteron_G5",
         .level = 0xd,
-        .vendor1 = CPUID_VENDOR_AMD_1,
-        .vendor2 = CPUID_VENDOR_AMD_2,
-        .vendor3 = CPUID_VENDOR_AMD_3,
+        .vendor = CPUID_VENDOR_AMD,
         .family = 21,
         .model = 2,
         .stepping = 0,
@@ -852,9 +900,7 @@ static void kvm_cpu_fill_host(x86_def_t *x86_cpu_def)
 
     x86_cpu_def->name = "host";
     host_cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->vendor1 = ebx;
-    x86_cpu_def->vendor2 = edx;
-    x86_cpu_def->vendor3 = ecx;
+    x86_cpu_vendor_words2str(x86_cpu_def->vendor, ebx, edx, ecx);
 
     host_cpuid(0x1, 0, &eax, &ebx, &ecx, &edx);
     x86_cpu_def->family = ((eax >> 8) & 0x0F) + ((eax >> 20) & 0xFF);
@@ -879,12 +925,9 @@ static void kvm_cpu_fill_host(x86_def_t *x86_cpu_def)
                 kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_ECX);
 
     cpu_x86_fill_model_id(x86_cpu_def->model_id);
-    x86_cpu_def->vendor_override = 0;
 
     /* Call Centaur's CPUID instruction. */
-    if (x86_cpu_def->vendor1 == CPUID_VENDOR_VIA_1 &&
-        x86_cpu_def->vendor2 == CPUID_VENDOR_VIA_2 &&
-        x86_cpu_def->vendor3 == CPUID_VENDOR_VIA_3) {
+    if (!strcmp(x86_cpu_def->vendor, CPUID_VENDOR_VIA)) {
         host_cpuid(0xC0000000, 0, &eax, &ebx, &ecx, &edx);
         eax = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0, R_EAX);
         if (eax >= 0xC0000001) {
@@ -896,62 +939,78 @@ static void kvm_cpu_fill_host(x86_def_t *x86_cpu_def)
         }
     }
 
-    /*
-     * Every SVM feature requires emulation support in KVM - so we can't just
-     * read the host features here. KVM might even support SVM features not
-     * available on the host hardware. Just set all bits and mask out the
-     * unsupported ones later.
-     */
-    x86_cpu_def->svm_features = -1;
+    /* Other KVM-specific feature fields: */
+    x86_cpu_def->svm_features =
+        kvm_arch_get_supported_cpuid(s, 0x8000000A, 0, R_EDX);
+    x86_cpu_def->kvm_features =
+        kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
+
 #endif /* CONFIG_KVM */
 }
 
-static int unavailable_host_feature(struct model_features_t *f, uint32_t mask)
+static int unavailable_host_feature(FeatureWordInfo *f, uint32_t mask)
 {
     int i;
 
     for (i = 0; i < 32; ++i)
         if (1 << i & mask) {
-            fprintf(stderr, "warning: host cpuid %04x_%04x lacks requested"
-                " flag '%s' [0x%08x]\n",
-                f->cpuid >> 16, f->cpuid & 0xffff,
-                f->flag_names[i] ? f->flag_names[i] : "[reserved]", mask);
+            const char *reg = get_register_name_32(f->cpuid_reg);
+            assert(reg);
+            fprintf(stderr, "warning: host doesn't support requested feature: "
+                "CPUID.%02XH:%s%s%s [bit %d]\n",
+                f->cpuid_eax, reg,
+                f->feat_names[i] ? "." : "",
+                f->feat_names[i] ? f->feat_names[i] : "", i);
             break;
         }
     return 0;
 }
 
-/* best effort attempt to inform user requested cpu flags aren't making
- * their way to the guest.  Note: ft[].check_feat ideally should be
- * specified via a guest_def field to suppress report of extraneous flags.
+/* Check if all requested cpu flags are making their way to the guest
+ *
+ * Returns 0 if all flags are supported by the host, non-zero otherwise.
  *
  * This function may be called only if KVM is enabled.
  */
-static int kvm_check_features_against_host(x86_def_t *guest_def)
+static int kvm_check_features_against_host(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     x86_def_t host_def;
     uint32_t mask;
     int rv, i;
     struct model_features_t ft[] = {
-        {&guest_def->features, &host_def.features,
-            ~0, feature_name, 0x00000000},
-        {&guest_def->ext_features, &host_def.ext_features,
-            ~CPUID_EXT_HYPERVISOR, ext_feature_name, 0x00000001},
-        {&guest_def->ext2_features, &host_def.ext2_features,
-            ~PPRO_FEATURES, ext2_feature_name, 0x80000000},
-        {&guest_def->ext3_features, &host_def.ext3_features,
-            ~CPUID_EXT3_SVM, ext3_feature_name, 0x80000001}};
+        {&env->cpuid_features, &host_def.features,
+            FEAT_1_EDX },
+        {&env->cpuid_ext_features, &host_def.ext_features,
+            FEAT_1_ECX },
+        {&env->cpuid_ext2_features, &host_def.ext2_features,
+            FEAT_8000_0001_EDX },
+        {&env->cpuid_ext3_features, &host_def.ext3_features,
+            FEAT_8000_0001_ECX },
+        {&env->cpuid_ext4_features, &host_def.ext4_features,
+            FEAT_C000_0001_EDX },
+        {&env->cpuid_7_0_ebx_features, &host_def.cpuid_7_0_ebx_features,
+            FEAT_7_0_EBX },
+        {&env->cpuid_svm_features, &host_def.svm_features,
+            FEAT_SVM },
+        {&env->cpuid_kvm_features, &host_def.kvm_features,
+            FEAT_KVM },
+    };
 
     assert(kvm_enabled());
 
     kvm_cpu_fill_host(&host_def);
-    for (rv = 0, i = 0; i < ARRAY_SIZE(ft); ++i)
-        for (mask = 1; mask; mask <<= 1)
-            if (ft[i].check_feat & mask && *ft[i].guest_feat & mask &&
+    for (rv = 0, i = 0; i < ARRAY_SIZE(ft); ++i) {
+        FeatureWord w = ft[i].feat_word;
+        FeatureWordInfo *wi = &feature_word_info[w];
+        for (mask = 1; mask; mask <<= 1) {
+            if (*ft[i].guest_feat & mask &&
                 !(*ft[i].host_feat & mask)) {
-                    unavailable_host_feature(&ft[i], mask);
-                    rv = 1;
-                }
+                unavailable_host_feature(wi, mask);
+                rv = 1;
+            }
+        }
+    }
     return rv;
 }
 
@@ -1104,15 +1163,10 @@ static char *x86_cpuid_get_vendor(Object *obj, Error **errp)
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
     char *value;
-    int i;
 
-    value = (char *)g_malloc(12 + 1);
-    for (i = 0; i < 4; i++) {
-        value[i    ] = env->cpuid_vendor1 >> (8 * i);
-        value[i + 4] = env->cpuid_vendor2 >> (8 * i);
-        value[i + 8] = env->cpuid_vendor3 >> (8 * i);
-    }
-    value[12] = '\0';
+    value = (char *)g_malloc(CPUID_VENDOR_SZ + 1);
+    x86_cpu_vendor_words2str(value, env->cpuid_vendor1, env->cpuid_vendor2,
+                             env->cpuid_vendor3);
     return value;
 }
 
@@ -1123,7 +1177,7 @@ static void x86_cpuid_set_vendor(Object *obj, const char *value,
     CPUX86State *env = &cpu->env;
     int i;
 
-    if (strlen(value) != 12) {
+    if (strlen(value) != CPUID_VENDOR_SZ) {
         error_set(errp, QERR_PROPERTY_VALUE_BAD, "",
                   "vendor", value);
         return;
@@ -1137,7 +1191,6 @@ static void x86_cpuid_set_vendor(Object *obj, const char *value,
         env->cpuid_vendor2 |= ((uint8_t)value[i + 4]) << (8 * i);
         env->cpuid_vendor3 |= ((uint8_t)value[i + 8]) << (8 * i);
     }
-    env->cpuid_vendor_override = 1;
 }
 
 static char *x86_cpuid_get_model_id(Object *obj, Error **errp)
@@ -1208,138 +1261,116 @@ static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
     cpu->env.tsc_khz = value / 1000;
 }
 
-static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
+static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *name)
 {
-    unsigned int i;
     x86_def_t *def;
+    int i;
 
-    char *s = g_strdup(cpu_model);
-    char *featurestr, *name = strtok(s, ",");
-    /* Features to be added*/
-    uint32_t plus_features = 0, plus_ext_features = 0;
-    uint32_t plus_ext2_features = 0, plus_ext3_features = 0;
-    uint32_t plus_kvm_features = kvm_default_features, plus_svm_features = 0;
-    uint32_t plus_7_0_ebx_features = 0;
-    /* Features to be removed */
-    uint32_t minus_features = 0, minus_ext_features = 0;
-    uint32_t minus_ext2_features = 0, minus_ext3_features = 0;
-    uint32_t minus_kvm_features = 0, minus_svm_features = 0;
-    uint32_t minus_7_0_ebx_features = 0;
-    uint32_t numvalue;
-
-    for (def = x86_defs; def; def = def->next)
-        if (name && !strcmp(name, def->name))
-            break;
-    if (kvm_enabled() && name && strcmp(name, "host") == 0) {
+    if (name == NULL) {
+        return -1;
+    }
+    if (kvm_enabled() && strcmp(name, "host") == 0) {
         kvm_cpu_fill_host(x86_cpu_def);
-    } else if (!def) {
-        goto error;
-    } else {
-        memcpy(x86_cpu_def, def, sizeof(*def));
+        return 0;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
+        def = &builtin_x86_defs[i];
+        if (strcmp(name, def->name) == 0) {
+            memcpy(x86_cpu_def, def, sizeof(*def));
+            /* sysenter isn't supported in compatibility mode on AMD,
+             * syscall isn't supported in compatibility mode on Intel.
+             * Normally we advertise the actual CPU vendor, but you can
+             * override this using the 'vendor' property if you want to use
+             * KVM's sysenter/syscall emulation in compatibility mode and
+             * when doing cross vendor migration
+             */
+            if (kvm_enabled()) {
+                uint32_t  ebx = 0, ecx = 0, edx = 0;
+                host_cpuid(0, 0, NULL, &ebx, &ecx, &edx);
+                x86_cpu_vendor_words2str(x86_cpu_def->vendor, ebx, edx, ecx);
+            }
+            return 0;
+        }
     }
 
-    add_flagname_to_bitmaps("hypervisor", &plus_features,
-            &plus_ext_features, &plus_ext2_features, &plus_ext3_features,
-            &plus_kvm_features, &plus_svm_features,  &plus_7_0_ebx_features);
+    return -1;
+}
+
+/* Parse "+feature,-feature,feature=foo" CPU feature string
+ */
+static void cpu_x86_parse_featurestr(X86CPU *cpu, char *features, Error **errp)
+{
+    char *featurestr; /* Single 'key=value" string being parsed */
+    /* Features to be added */
+    FeatureWordArray plus_features = { 0 };
+    /* Features to be removed */
+    FeatureWordArray minus_features = { 0 };
+    uint32_t numvalue;
+    CPUX86State *env = &cpu->env;
 
-    featurestr = strtok(NULL, ",");
+    featurestr = features ? strtok(features, ",") : NULL;
 
     while (featurestr) {
         char *val;
         if (featurestr[0] == '+') {
-            add_flagname_to_bitmaps(featurestr + 1, &plus_features,
-                            &plus_ext_features, &plus_ext2_features,
-                            &plus_ext3_features, &plus_kvm_features,
-                            &plus_svm_features, &plus_7_0_ebx_features);
+            add_flagname_to_bitmaps(featurestr + 1, plus_features);
         } else if (featurestr[0] == '-') {
-            add_flagname_to_bitmaps(featurestr + 1, &minus_features,
-                            &minus_ext_features, &minus_ext2_features,
-                            &minus_ext3_features, &minus_kvm_features,
-                            &minus_svm_features, &minus_7_0_ebx_features);
+            add_flagname_to_bitmaps(featurestr + 1, minus_features);
         } else if ((val = strchr(featurestr, '='))) {
             *val = 0; val++;
             if (!strcmp(featurestr, "family")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xff + 0xf) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->family = numvalue;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "model")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xff) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->model = numvalue;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "stepping")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err || numvalue > 0xf) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->stepping = numvalue ;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "level")) {
-                char *err;
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
-                }
-                x86_cpu_def->level = numvalue;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "xlevel")) {
                 char *err;
+                char num[32];
+
                 numvalue = strtoul(val, &err, 0);
                 if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
+                    error_setg(errp, "bad numerical value %s", val);
+                    goto out;
                 }
                 if (numvalue < 0x80000000) {
+                    fprintf(stderr, "xlevel value shall always be >= 0x80000000"
+                            ", fixup will be removed in future versions\n");
                     numvalue += 0x80000000;
                 }
-                x86_cpu_def->xlevel = numvalue;
+                snprintf(num, sizeof(num), "%" PRIu32, numvalue);
+                object_property_parse(OBJECT(cpu), num, featurestr, errp);
             } else if (!strcmp(featurestr, "vendor")) {
-                if (strlen(val) != 12) {
-                    fprintf(stderr, "vendor string must be 12 chars long\n");
-                    goto error;
-                }
-                x86_cpu_def->vendor1 = 0;
-                x86_cpu_def->vendor2 = 0;
-                x86_cpu_def->vendor3 = 0;
-                for(i = 0; i < 4; i++) {
-                    x86_cpu_def->vendor1 |= ((uint8_t)val[i    ]) << (8 * i);
-                    x86_cpu_def->vendor2 |= ((uint8_t)val[i + 4]) << (8 * i);
-                    x86_cpu_def->vendor3 |= ((uint8_t)val[i + 8]) << (8 * i);
-                }
-                x86_cpu_def->vendor_override = 1;
+                object_property_parse(OBJECT(cpu), val, featurestr, errp);
             } else if (!strcmp(featurestr, "model_id")) {
-                pstrcpy(x86_cpu_def->model_id, sizeof(x86_cpu_def->model_id),
-                        val);
+                object_property_parse(OBJECT(cpu), val, "model-id", errp);
             } else if (!strcmp(featurestr, "tsc_freq")) {
                 int64_t tsc_freq;
                 char *err;
+                char num[32];
 
                 tsc_freq = strtosz_suffix_unit(val, &err,
                                                STRTOSZ_DEFSUFFIX_B, 1000);
                 if (tsc_freq < 0 || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
+                    error_setg(errp, "bad numerical value %s", val);
+                    goto out;
                 }
-                x86_cpu_def->tsc_khz = tsc_freq / 1000;
+                snprintf(num, sizeof(num), "%" PRId64, tsc_freq);
+                object_property_parse(OBJECT(cpu), num, "tsc-frequency", errp);
             } else if (!strcmp(featurestr, "hv_spinlocks")) {
                 char *err;
                 numvalue = strtoul(val, &err, 0);
                 if (!*val || *err) {
-                    fprintf(stderr, "bad numerical value %s\n", val);
-                    goto error;
+                    error_setg(errp, "bad numerical value %s", val);
+                    goto out;
                 }
                 hyperv_set_spinlock_retries(numvalue);
             } else {
-                fprintf(stderr, "unrecognized feature %s\n", featurestr);
-                goto error;
+                error_setg(errp, "unrecognized feature %s", featurestr);
+                goto out;
             }
         } else if (!strcmp(featurestr, "check")) {
             check_cpuid = 1;
@@ -1350,38 +1381,34 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
         } else if (!strcmp(featurestr, "hv_vapic")) {
             hyperv_enable_vapic_recommended(true);
         } else {
-            fprintf(stderr, "feature string `%s' not in format (+feature|-feature|feature=xyz)\n", featurestr);
-            goto error;
+            error_setg(errp, "feature string `%s' not in format (+feature|"
+                       "-feature|feature=xyz)", featurestr);
+            goto out;
+        }
+        if (error_is_set(errp)) {
+            goto out;
         }
         featurestr = strtok(NULL, ",");
     }
-    x86_cpu_def->features |= plus_features;
-    x86_cpu_def->ext_features |= plus_ext_features;
-    x86_cpu_def->ext2_features |= plus_ext2_features;
-    x86_cpu_def->ext3_features |= plus_ext3_features;
-    x86_cpu_def->kvm_features |= plus_kvm_features;
-    x86_cpu_def->svm_features |= plus_svm_features;
-    x86_cpu_def->cpuid_7_0_ebx_features |= plus_7_0_ebx_features;
-    x86_cpu_def->features &= ~minus_features;
-    x86_cpu_def->ext_features &= ~minus_ext_features;
-    x86_cpu_def->ext2_features &= ~minus_ext2_features;
-    x86_cpu_def->ext3_features &= ~minus_ext3_features;
-    x86_cpu_def->kvm_features &= ~minus_kvm_features;
-    x86_cpu_def->svm_features &= ~minus_svm_features;
-    x86_cpu_def->cpuid_7_0_ebx_features &= ~minus_7_0_ebx_features;
-    if (check_cpuid && kvm_enabled()) {
-        if (kvm_check_features_against_host(x86_cpu_def) && enforce_cpuid)
-            goto error;
-    }
-    if (x86_cpu_def->cpuid_7_0_ebx_features && x86_cpu_def->level < 7) {
-        x86_cpu_def->level = 7;
-    }
-    g_free(s);
-    return 0;
-
-error:
-    g_free(s);
-    return -1;
+    env->cpuid_features |= plus_features[FEAT_1_EDX];
+    env->cpuid_ext_features |= plus_features[FEAT_1_ECX];
+    env->cpuid_ext2_features |= plus_features[FEAT_8000_0001_EDX];
+    env->cpuid_ext3_features |= plus_features[FEAT_8000_0001_ECX];
+    env->cpuid_ext4_features |= plus_features[FEAT_C000_0001_EDX];
+    env->cpuid_kvm_features |= plus_features[FEAT_KVM];
+    env->cpuid_svm_features |= plus_features[FEAT_SVM];
+    env->cpuid_7_0_ebx_features |= plus_features[FEAT_7_0_EBX];
+    env->cpuid_features &= ~minus_features[FEAT_1_EDX];
+    env->cpuid_ext_features &= ~minus_features[FEAT_1_ECX];
+    env->cpuid_ext2_features &= ~minus_features[FEAT_8000_0001_EDX];
+    env->cpuid_ext3_features &= ~minus_features[FEAT_8000_0001_ECX];
+    env->cpuid_ext4_features &= ~minus_features[FEAT_C000_0001_EDX];
+    env->cpuid_kvm_features &= ~minus_features[FEAT_KVM];
+    env->cpuid_svm_features &= ~minus_features[FEAT_SVM];
+    env->cpuid_7_0_ebx_features &= ~minus_features[FEAT_7_0_EBX];
+
+out:
+    return;
 }
 
 /* generate a composite string into buf of all cpuid names in featureset
@@ -1419,8 +1446,10 @@ void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
     x86_def_t *def;
     char buf[256];
+    int i;
 
-    for (def = x86_defs; def; def = def->next) {
+    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
+        def = &builtin_x86_defs[i];
         snprintf(buf, sizeof(buf), "%s", def->name);
         (*cpu_fprintf)(f, "x86 %16s  %-48s\n", buf, def->model_id);
     }
@@ -1442,11 +1471,13 @@ CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
 {
     CpuDefinitionInfoList *cpu_list = NULL;
     x86_def_t *def;
+    int i;
 
-    for (def = x86_defs; def; def = def->next) {
+    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
         CpuDefinitionInfoList *entry;
         CpuDefinitionInfo *info;
 
+        def = &builtin_x86_defs[i];
         info = g_malloc0(sizeof(*info));
         info->name = g_strdup(def->name);
 
@@ -1490,21 +1521,30 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
     CPUX86State *env = &cpu->env;
     x86_def_t def1, *def = &def1;
     Error *error = NULL;
+    char *name, *features;
+    gchar **model_pieces;
 
     memset(def, 0, sizeof(*def));
 
-    if (cpu_x86_find_by_name(def, cpu_model) < 0)
-        return -1;
-    if (def->vendor1) {
-        env->cpuid_vendor1 = def->vendor1;
-        env->cpuid_vendor2 = def->vendor2;
-        env->cpuid_vendor3 = def->vendor3;
-    } else {
-        env->cpuid_vendor1 = CPUID_VENDOR_INTEL_1;
-        env->cpuid_vendor2 = CPUID_VENDOR_INTEL_2;
-        env->cpuid_vendor3 = CPUID_VENDOR_INTEL_3;
+    model_pieces = g_strsplit(cpu_model, ",", 2);
+    if (!model_pieces[0]) {
+        error_setg(&error, "Invalid/empty CPU model name");
+        goto out;
+    }
+    name = model_pieces[0];
+    features = model_pieces[1];
+
+    if (cpu_x86_find_by_name(def, name) < 0) {
+        error_setg(&error, "Unable to find CPU definition: %s", name);
+        goto out;
+    }
+
+    if (kvm_enabled()) {
+        def->kvm_features |= kvm_default_features;
     }
-    env->cpuid_vendor_override = def->vendor_override;
+    def->ext_features |= CPUID_EXT_HYPERVISOR;
+
+    object_property_set_str(OBJECT(cpu), def->vendor, "vendor", &error);
     object_property_set_int(OBJECT(cpu), def->level, "level", &error);
     object_property_set_int(OBJECT(cpu), def->family, "family", &error);
     object_property_set_int(OBJECT(cpu), def->model, "model", &error);
@@ -1519,35 +1559,15 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
     env->cpuid_ext4_features = def->ext4_features;
     env->cpuid_7_0_ebx_features = def->cpuid_7_0_ebx_features;
     env->cpuid_xlevel2 = def->xlevel2;
-    object_property_set_int(OBJECT(cpu), (int64_t)def->tsc_khz * 1000,
-                            "tsc-frequency", &error);
 
-    /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
-     * CPUID[1].EDX.
-     */
-    if (env->cpuid_vendor1 == CPUID_VENDOR_AMD_1 &&
-            env->cpuid_vendor2 == CPUID_VENDOR_AMD_2 &&
-            env->cpuid_vendor3 == CPUID_VENDOR_AMD_3) {
-        env->cpuid_ext2_features &= ~CPUID_EXT2_AMD_ALIASES;
-        env->cpuid_ext2_features |= (def->features & CPUID_EXT2_AMD_ALIASES);
+    object_property_set_str(OBJECT(cpu), def->model_id, "model-id", &error);
+    if (error) {
+        goto out;
     }
 
-    if (!kvm_enabled()) {
-        env->cpuid_features &= TCG_FEATURES;
-        env->cpuid_ext_features &= TCG_EXT_FEATURES;
-        env->cpuid_ext2_features &= (TCG_EXT2_FEATURES
-#ifdef TARGET_X86_64
-            | CPUID_EXT2_SYSCALL | CPUID_EXT2_LM
-#endif
-            );
-        env->cpuid_ext3_features &= TCG_EXT3_FEATURES;
-        env->cpuid_svm_features &= TCG_SVM_FEATURES;
-    } else {
-#ifdef CONFIG_KVM
-        filter_features_for_kvm(cpu);
-#endif
-    }
-    object_property_set_str(OBJECT(cpu), def->model_id, "model-id", &error);
+    cpu_x86_parse_featurestr(cpu, features, &error);
+out:
+    g_strfreev(model_pieces);
     if (error) {
         fprintf(stderr, "%s\n", error_get_pretty(error));
         error_free(error);
@@ -1574,7 +1594,6 @@ void x86_cpudef_setup(void)
 
     for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); ++i) {
         x86_def_t *def = &builtin_x86_defs[i];
-        def->next = x86_defs;
 
         /* Look for specific "cpudef" models that */
         /* have the QEMU version in .model_id */
@@ -1587,8 +1606,6 @@ void x86_cpudef_setup(void)
                 break;
             }
         }
-
-        x86_defs = def;
     }
 }
 
@@ -1598,22 +1615,15 @@ static void get_cpuid_vendor(CPUX86State *env, uint32_t *ebx,
     *ebx = env->cpuid_vendor1;
     *edx = env->cpuid_vendor2;
     *ecx = env->cpuid_vendor3;
-
-    /* sysenter isn't supported on compatibility mode on AMD, syscall
-     * isn't supported in compatibility mode on Intel.
-     * Normally we advertise the actual cpu vendor, but you can override
-     * this if you want to use KVM's sysenter/syscall emulation
-     * in compatibility mode and when doing cross vendor migration
-     */
-    if (kvm_enabled() && ! env->cpuid_vendor_override) {
-        host_cpuid(0, 0, NULL, ebx, ecx, edx);
-    }
 }
 
 void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                    uint32_t *eax, uint32_t *ebx,
                    uint32_t *ecx, uint32_t *edx)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
     /* test if maximum index reached */
     if (index & 0x80000000) {
         if (index > env->cpuid_xlevel) {
@@ -1625,7 +1635,11 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                     index = env->cpuid_xlevel;
                 }
             } else {
-                index =  env->cpuid_xlevel;
+                /* Intel documentation states that invalid EAX input will
+                 * return the same information as EAX=cpuid_level
+                 * (Intel SDM Vol. 2A - Instruction Set Reference - CPUID)
+                 */
+                index =  env->cpuid_level;
             }
         }
     } else {
@@ -1643,8 +1657,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         *ebx = (env->cpuid_apic_id << 24) | 8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
         *ecx = env->cpuid_ext_features;
         *edx = env->cpuid_features;
-        if (env->nr_cores * env->nr_threads > 1) {
-            *ebx |= (env->nr_cores * env->nr_threads) << 16;
+        if (cs->nr_cores * cs->nr_threads > 1) {
+            *ebx |= (cs->nr_cores * cs->nr_threads) << 16;
             *edx |= 1 << 28;    /* HTT bit */
         }
         break;
@@ -1657,8 +1671,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 4:
         /* cache info: needed for Core compatibility */
-        if (env->nr_cores > 1) {
-            *eax = (env->nr_cores - 1) << 26;
+        if (cs->nr_cores > 1) {
+            *eax = (cs->nr_cores - 1) << 26;
         } else {
             *eax = 0;
         }
@@ -1677,8 +1691,8 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                 break;
             case 2: /* L2 cache info */
                 *eax |= 0x0000143;
-                if (env->nr_threads > 1) {
-                    *eax |= (env->nr_threads - 1) << 14;
+                if (cs->nr_threads > 1) {
+                    *eax |= (cs->nr_threads - 1) << 14;
                 }
                 *ebx = 0x3c0003f;
                 *ecx = 0x0000fff;
@@ -1730,7 +1744,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
     case 0xA:
         /* Architectural Performance Monitoring Leaf */
         if (kvm_enabled()) {
-            KVMState *s = env->kvm_state;
+            KVMState *s = cs->kvm_state;
 
             *eax = kvm_arch_get_supported_cpuid(s, 0xA, count, R_EAX);
             *ebx = kvm_arch_get_supported_cpuid(s, 0xA, count, R_EBX);
@@ -1753,7 +1767,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
             break;
         }
         if (kvm_enabled()) {
-            KVMState *s = env->kvm_state;
+            KVMState *s = cs->kvm_state;
 
             *eax = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EAX);
             *ebx = kvm_arch_get_supported_cpuid(s, 0xd, count, R_EBX);
@@ -1782,7 +1796,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
          * discards multiple thread information if it is set.
          * So dont set it here for Intel to make Linux guests happy.
          */
-        if (env->nr_cores * env->nr_threads > 1) {
+        if (cs->nr_cores * cs->nr_threads > 1) {
             uint32_t tebx, tecx, tedx;
             get_cpuid_vendor(env, &tebx, &tecx, &tedx);
             if (tebx != CPUID_VENDOR_INTEL_1 ||
@@ -1830,22 +1844,22 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         *ebx = 0;
         *ecx = 0;
         *edx = 0;
-        if (env->nr_cores * env->nr_threads > 1) {
-            *ecx |= (env->nr_cores * env->nr_threads) - 1;
+        if (cs->nr_cores * cs->nr_threads > 1) {
+            *ecx |= (cs->nr_cores * cs->nr_threads) - 1;
         }
         break;
     case 0x8000000A:
-	if (env->cpuid_ext3_features & CPUID_EXT3_SVM) {
-		*eax = 0x00000001; /* SVM Revision */
-		*ebx = 0x00000010; /* nr of ASIDs */
-		*ecx = 0;
-		*edx = env->cpuid_svm_features; /* optional features */
-	} else {
-		*eax = 0;
-		*ebx = 0;
-		*ecx = 0;
-		*edx = 0;
-	}
+        if (env->cpuid_ext3_features & CPUID_EXT3_SVM) {
+            *eax = 0x00000001; /* SVM Revision */
+            *ebx = 0x00000010; /* nr of ASIDs */
+            *ecx = 0;
+            *edx = env->cpuid_svm_features; /* optional features */
+        } else {
+            *eax = 0;
+            *ebx = 0;
+            *ecx = 0;
+            *edx = 0;
+        }
         break;
     case 0xC0000000:
         *eax = env->cpuid_xlevel2;
@@ -1888,7 +1902,7 @@ static void x86_cpu_reset(CPUState *s)
     int i;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
     }
 
@@ -1962,7 +1976,7 @@ static void x86_cpu_reset(CPUState *s)
 
 #if !defined(CONFIG_USER_ONLY)
     /* We hard-wire the BSP to the first CPU. */
-    if (env->cpu_index == 0) {
+    if (s->cpu_index == 0) {
         apic_designate_bsp(env->apic_state);
     }
 
@@ -2040,7 +2054,7 @@ static void x86_cpu_apic_init(X86CPU *cpu, Error **errp)
         /* NOTE: the APIC is directly connected to the CPU - it is not
            on the global memory bus. */
         /* XXX: what if the base changes? */
-        sysbus_mmio_map(sysbus_from_qdev(env->apic_state), 0, MSI_ADDR_BASE);
+        sysbus_mmio_map(SYS_BUS_DEVICE(env->apic_state), 0, MSI_ADDR_BASE);
         apic_mapped = 1;
     }
 }
@@ -2049,6 +2063,43 @@ static void x86_cpu_apic_init(X86CPU *cpu, Error **errp)
 void x86_cpu_realize(Object *obj, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
+    CPUX86State *env = &cpu->env;
+
+    if (env->cpuid_7_0_ebx_features && env->cpuid_level < 7) {
+        env->cpuid_level = 7;
+    }
+
+    /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
+     * CPUID[1].EDX.
+     */
+    if (env->cpuid_vendor1 == CPUID_VENDOR_AMD_1 &&
+        env->cpuid_vendor2 == CPUID_VENDOR_AMD_2 &&
+        env->cpuid_vendor3 == CPUID_VENDOR_AMD_3) {
+        env->cpuid_ext2_features &= ~CPUID_EXT2_AMD_ALIASES;
+        env->cpuid_ext2_features |= (env->cpuid_features
+           & CPUID_EXT2_AMD_ALIASES);
+    }
+
+    if (!kvm_enabled()) {
+        env->cpuid_features &= TCG_FEATURES;
+        env->cpuid_ext_features &= TCG_EXT_FEATURES;
+        env->cpuid_ext2_features &= (TCG_EXT2_FEATURES
+#ifdef TARGET_X86_64
+            | CPUID_EXT2_SYSCALL | CPUID_EXT2_LM
+#endif
+            );
+        env->cpuid_ext3_features &= TCG_EXT3_FEATURES;
+        env->cpuid_svm_features &= TCG_SVM_FEATURES;
+    } else {
+#ifdef CONFIG_KVM
+        filter_features_for_kvm(cpu);
+#endif
+        if (check_cpuid && kvm_check_features_against_host(cpu)
+            && enforce_cpuid) {
+            error_setg(errp, "Host's CPU doesn't support requested features");
+            return;
+        }
+    }
 
 #ifndef CONFIG_USER_ONLY
     qemu_register_reset(x86_cpu_machine_reset_cb, cpu);
@@ -2066,8 +2117,42 @@ void x86_cpu_realize(Object *obj, Error **errp)
     cpu_reset(CPU(cpu));
 }
 
+/* Enables contiguous-apic-ID mode, for compatibility */
+static bool compat_apic_id_mode;
+
+void enable_compat_apic_id_mode(void)
+{
+    compat_apic_id_mode = true;
+}
+
+/* Calculates initial APIC ID for a specific CPU index
+ *
+ * Currently we need to be able to calculate the APIC ID from the CPU index
+ * alone (without requiring a CPU object), as the QEMU<->Seabios interfaces have
+ * no concept of "CPU index", and the NUMA tables on fw_cfg need the APIC ID of
+ * all CPUs up to max_cpus.
+ */
+uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index)
+{
+    uint32_t correct_id;
+    static bool warned;
+
+    correct_id = x86_apicid_from_cpu_idx(smp_cores, smp_threads, cpu_index);
+    if (compat_apic_id_mode) {
+        if (cpu_index != correct_id && !warned) {
+            error_report("APIC IDs set in compatibility mode, "
+                         "CPU topology won't match the configuration");
+            warned = true;
+        }
+        return cpu_index;
+    } else {
+        return correct_id;
+    }
+}
+
 static void x86_cpu_initfn(Object *obj)
 {
+    CPUState *cs = CPU(obj);
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
     static int inited;
@@ -2099,7 +2184,7 @@ static void x86_cpu_initfn(Object *obj)
                         x86_cpuid_get_tsc_freq,
                         x86_cpuid_set_tsc_freq, NULL, NULL, NULL);
 
-    env->cpuid_apic_id = env->cpu_index;
+    env->cpuid_apic_id = x86_cpu_apic_id_from_index(cs->cpu_index);
 
     /* init various static tables used in TCG mode */
     if (tcg_enabled() && !inited) {
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 90ef1ff..9e6e1a6 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -44,9 +44,9 @@
 
 #define CPUArchState struct CPUX86State
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define R_EAX 0
 #define R_ECX 1
@@ -231,6 +231,12 @@
 #define DR7_TYPE_SHIFT  16
 #define DR7_LEN_SHIFT   18
 #define DR7_FIXED_1     0x00000400
+#define DR7_LOCAL_BP_MASK    0x55
+#define DR7_MAX_BP           4
+#define DR7_TYPE_BP_INST     0x0
+#define DR7_TYPE_DATA_WR     0x1
+#define DR7_TYPE_IO_RW       0x2
+#define DR7_TYPE_DATA_RW     0x3
 
 #define PG_PRESENT_BIT	0
 #define PG_RW_BIT	1
@@ -295,6 +301,7 @@
 #define MSR_IA32_APICBASE_BSP           (1<<8)
 #define MSR_IA32_APICBASE_ENABLE        (1<<11)
 #define MSR_IA32_APICBASE_BASE          (0xfffff<<12)
+#define MSR_TSC_ADJUST                  0x0000003b
 #define MSR_IA32_TSCDEADLINE            0x6e0
 
 #define MSR_MTRRcap			0xfe
@@ -360,6 +367,21 @@
 
 #define MSR_VM_HSAVE_PA                 0xc0010117
 
+/* CPUID feature words */
+typedef enum FeatureWord {
+    FEAT_1_EDX,         /* CPUID[1].EDX */
+    FEAT_1_ECX,         /* CPUID[1].ECX */
+    FEAT_7_0_EBX,       /* CPUID[EAX=7,ECX=0].EBX */
+    FEAT_8000_0001_EDX, /* CPUID[8000_0001].EDX */
+    FEAT_8000_0001_ECX, /* CPUID[8000_0001].ECX */
+    FEAT_C000_0001_EDX, /* CPUID[C000_0001].EDX */
+    FEAT_KVM,           /* CPUID[4000_0001].EAX (KVM_CPUID_FEATURES) */
+    FEAT_SVM,           /* CPUID[8000_000A].EDX */
+    FEATURE_WORDS,
+} FeatureWord;
+
+typedef uint32_t FeatureWordArray[FEATURE_WORDS];
+
 /* cpuid_features bits */
 #define CPUID_FP87 (1 << 0)
 #define CPUID_VME  (1 << 1)
@@ -510,17 +532,19 @@
 #define CPUID_7_0_EBX_ADX      (1 << 19)
 #define CPUID_7_0_EBX_SMAP     (1 << 20)
 
+#define CPUID_VENDOR_SZ      12
+
 #define CPUID_VENDOR_INTEL_1 0x756e6547 /* "Genu" */
 #define CPUID_VENDOR_INTEL_2 0x49656e69 /* "ineI" */
 #define CPUID_VENDOR_INTEL_3 0x6c65746e /* "ntel" */
+#define CPUID_VENDOR_INTEL "GenuineIntel"
 
 #define CPUID_VENDOR_AMD_1   0x68747541 /* "Auth" */
 #define CPUID_VENDOR_AMD_2   0x69746e65 /* "enti" */
 #define CPUID_VENDOR_AMD_3   0x444d4163 /* "cAMD" */
+#define CPUID_VENDOR_AMD   "AuthenticAMD"
 
-#define CPUID_VENDOR_VIA_1   0x746e6543 /* "Cent" */
-#define CPUID_VENDOR_VIA_2   0x48727561 /* "aurH" */
-#define CPUID_VENDOR_VIA_3   0x736c7561 /* "auls" */
+#define CPUID_VENDOR_VIA   "CentaurHauls"
 
 #define CPUID_MWAIT_IBE     (1 << 1) /* Interrupts can exit capability */
 #define CPUID_MWAIT_EMX     (1 << 0) /* enumeration supported */
@@ -772,6 +796,7 @@ typedef struct CPUX86State {
     uint64_t pv_eoi_en_msr;
 
     uint64_t tsc;
+    uint64_t tsc_adjust;
     uint64_t tsc_deadline;
 
     uint64_t mcg_status;
@@ -810,7 +835,6 @@ typedef struct CPUX86State {
     uint32_t cpuid_ext2_features;
     uint32_t cpuid_ext3_features;
     uint32_t cpuid_apic_id;
-    int cpuid_vendor_override;
     /* Store the results of Centaur's CPUID instructions */
     uint32_t cpuid_xlevel2;
     uint32_t cpuid_ext4_features;
@@ -987,11 +1011,22 @@ void host_cpuid(uint32_t function, uint32_t count,
 int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
                              int is_write, int mmu_idx);
 #define cpu_handle_mmu_fault cpu_x86_handle_mmu_fault
-void cpu_x86_set_a20(CPUX86State *env, int a20_state);
+void x86_cpu_set_a20(X86CPU *cpu, int a20_state);
+
+static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 1;
+}
 
-static inline int hw_breakpoint_enabled(unsigned long dr7, int index)
+static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
 {
-    return (dr7 >> (index * 2)) & 3;
+    return (dr7 >> (index * 2)) & 2;
+
+}
+static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return hw_global_breakpoint_enabled(dr7, index) ||
+           hw_local_breakpoint_enabled(dr7, index);
 }
 
 static inline int hw_breakpoint_type(unsigned long dr7, int index)
@@ -1007,7 +1042,7 @@ static inline int hw_breakpoint_len(unsigned long dr7, int index)
 
 void hw_breakpoint_insert(CPUX86State *env, int index);
 void hw_breakpoint_remove(CPUX86State *env, int index);
-int check_hw_breakpoints(CPUX86State *env, int force_dr6_update);
+bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
 void breakpoint_handler(CPUX86State *env);
 
 /* will be suppressed */
@@ -1115,7 +1150,7 @@ static inline void cpu_clone_regs(CPUX86State *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "svm.h"
 
 #if !defined(CONFIG_USER_ONLY)
@@ -1135,7 +1170,7 @@ static inline bool cpu_has_work(CPUState *cpu)
                                       CPU_INTERRUPT_MCE));
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUX86State *env, TranslationBlock *tb)
 {
@@ -1214,6 +1249,12 @@ void do_smm_enter(CPUX86State *env1);
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
 
-void enable_kvm_pv_eoi(void);
+void disable_kvm_pv_eoi(void);
+
+/* Return name of 32-bit register, from a R_* constant */
+const char *get_register_name_32(unsigned int reg);
+
+uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index);
+void enable_compat_apic_id_mode(void);
 
 #endif /* CPU_I386_H */
diff --git a/target-i386/excp_helper.c b/target-i386/excp_helper.c
index aaa5ca2..179ea82 100644
--- a/target-i386/excp_helper.c
+++ b/target-i386/excp_helper.c
@@ -18,8 +18,8 @@
  */
 
 #include "cpu.h"
-#include "qemu-log.h"
-#include "sysemu.h"
+#include "qemu/log.h"
+#include "sysemu/sysemu.h"
 #include "helper.h"
 
 #if 0
diff --git a/target-i386/fpu_helper.c b/target-i386/fpu_helper.c
index dfc34a6..44f3d27 100644
--- a/target-i386/fpu_helper.c
+++ b/target-i386/fpu_helper.c
@@ -22,7 +22,7 @@
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 #define FPU_RC_MASK         0xc00
diff --git a/target-i386/helper.c b/target-i386/helper.c
index bf206cf..d1cb4e2 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -18,10 +18,10 @@
  */
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #ifndef CONFIG_USER_ONLY
-#include "sysemu.h"
-#include "monitor.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
 #endif
 
 //#define DEBUG_MMU
@@ -366,8 +366,10 @@ void cpu_dump_state(CPUX86State *env, FILE *f, fprintf_function cpu_fprintf,
 /* x86 mmu */
 /* XXX: add PGE support */
 
-void cpu_x86_set_a20(CPUX86State *env, int a20_state)
+void x86_cpu_set_a20(X86CPU *cpu, int a20_state)
 {
+    CPUX86State *env = &cpu->env;
+
     a20_state = (a20_state != 0);
     if (a20_state != ((env->a20_mask >> 20) & 1)) {
 #if defined(DEBUG_MMU)
@@ -966,30 +968,35 @@ hwaddr cpu_get_phys_page_debug(CPUX86State *env, target_ulong addr)
 
 void hw_breakpoint_insert(CPUX86State *env, int index)
 {
-    int type, err = 0;
+    int type = 0, err = 0;
 
     switch (hw_breakpoint_type(env->dr[7], index)) {
-    case 0:
-        if (hw_breakpoint_enabled(env->dr[7], index))
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(env->dr[7], index)) {
             err = cpu_breakpoint_insert(env, env->dr[index], BP_CPU,
                                         &env->cpu_breakpoint[index]);
+        }
         break;
-    case 1:
+    case DR7_TYPE_DATA_WR:
         type = BP_CPU | BP_MEM_WRITE;
-        goto insert_wp;
-    case 2:
-         /* No support for I/O watchpoints yet */
         break;
-    case 3:
+    case DR7_TYPE_IO_RW:
+        /* No support for I/O watchpoints yet */
+        break;
+    case DR7_TYPE_DATA_RW:
         type = BP_CPU | BP_MEM_ACCESS;
-    insert_wp:
+        break;
+    }
+
+    if (type != 0) {
         err = cpu_watchpoint_insert(env, env->dr[index],
                                     hw_breakpoint_len(env->dr[7], index),
                                     type, &env->cpu_watchpoint[index]);
-        break;
     }
-    if (err)
+
+    if (err) {
         env->cpu_breakpoint[index] = NULL;
+    }
 }
 
 void hw_breakpoint_remove(CPUX86State *env, int index)
@@ -997,39 +1004,60 @@ void hw_breakpoint_remove(CPUX86State *env, int index)
     if (!env->cpu_breakpoint[index])
         return;
     switch (hw_breakpoint_type(env->dr[7], index)) {
-    case 0:
-        if (hw_breakpoint_enabled(env->dr[7], index))
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(env->dr[7], index)) {
             cpu_breakpoint_remove_by_ref(env, env->cpu_breakpoint[index]);
+        }
         break;
-    case 1:
-    case 3:
+    case DR7_TYPE_DATA_WR:
+    case DR7_TYPE_DATA_RW:
         cpu_watchpoint_remove_by_ref(env, env->cpu_watchpoint[index]);
         break;
-    case 2:
+    case DR7_TYPE_IO_RW:
         /* No support for I/O watchpoints yet */
         break;
     }
 }
 
-int check_hw_breakpoints(CPUX86State *env, int force_dr6_update)
+bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
 {
     target_ulong dr6;
-    int reg, type;
-    int hit_enabled = 0;
+    int reg;
+    bool hit_enabled = false;
 
     dr6 = env->dr[6] & ~0xf;
-    for (reg = 0; reg < 4; reg++) {
-        type = hw_breakpoint_type(env->dr[7], reg);
-        if ((type == 0 && env->dr[reg] == env->eip) ||
-            ((type & 1) && env->cpu_watchpoint[reg] &&
-             (env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT))) {
+    for (reg = 0; reg < DR7_MAX_BP; reg++) {
+        bool bp_match = false;
+        bool wp_match = false;
+
+        switch (hw_breakpoint_type(env->dr[7], reg)) {
+        case DR7_TYPE_BP_INST:
+            if (env->dr[reg] == env->eip) {
+                bp_match = true;
+            }
+            break;
+        case DR7_TYPE_DATA_WR:
+        case DR7_TYPE_DATA_RW:
+            if (env->cpu_watchpoint[reg] &&
+                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
+                wp_match = true;
+            }
+            break;
+        case DR7_TYPE_IO_RW:
+            break;
+        }
+        if (bp_match || wp_match) {
             dr6 |= 1 << reg;
-            if (hw_breakpoint_enabled(env->dr[7], reg))
-                hit_enabled = 1;
+            if (hw_breakpoint_enabled(env->dr[7], reg)) {
+                hit_enabled = true;
+            }
         }
     }
-    if (hit_enabled || force_dr6_update)
+
+    if (hit_enabled || force_dr6_update) {
         env->dr[6] = dr6;
+    }
+
     return hit_enabled;
 }
 
@@ -1040,16 +1068,17 @@ void breakpoint_handler(CPUX86State *env)
     if (env->watchpoint_hit) {
         if (env->watchpoint_hit->flags & BP_CPU) {
             env->watchpoint_hit = NULL;
-            if (check_hw_breakpoints(env, 0))
+            if (check_hw_breakpoints(env, false)) {
                 raise_exception(env, EXCP01_DB);
-            else
+            } else {
                 cpu_resume_from_signal(env, NULL);
+            }
         }
     } else {
         QTAILQ_FOREACH(bp, &env->breakpoints, entry)
             if (bp->pc == env->eip) {
                 if (bp->flags & BP_CPU) {
-                    check_hw_breakpoints(env, 1);
+                    check_hw_breakpoints(env, true);
                     raise_exception(env, EXCP01_DB);
                 }
                 break;
@@ -1059,7 +1088,7 @@ void breakpoint_handler(CPUX86State *env)
 
 typedef struct MCEInjectionParams {
     Monitor *mon;
-    CPUX86State *env;
+    X86CPU *cpu;
     int bank;
     uint64_t status;
     uint64_t mcg_status;
@@ -1071,7 +1100,8 @@ typedef struct MCEInjectionParams {
 static void do_inject_x86_mce(void *data)
 {
     MCEInjectionParams *params = data;
-    CPUX86State *cenv = params->env;
+    CPUX86State *cenv = &params->cpu->env;
+    CPUState *cpu = CPU(params->cpu);
     uint64_t *banks = cenv->mce_banks + 4 * params->bank;
 
     cpu_synchronize_state(cenv);
@@ -1094,7 +1124,7 @@ static void do_inject_x86_mce(void *data)
         if ((cenv->mcg_cap & MCG_CTL_P) && cenv->mcg_ctl != ~(uint64_t)0) {
             monitor_printf(params->mon,
                            "CPU %d: Uncorrected error reporting disabled\n",
-                           cenv->cpu_index);
+                           cpu->cpu_index);
             return;
         }
 
@@ -1106,7 +1136,7 @@ static void do_inject_x86_mce(void *data)
             monitor_printf(params->mon,
                            "CPU %d: Uncorrected error reporting disabled for"
                            " bank %d\n",
-                           cenv->cpu_index, params->bank);
+                           cpu->cpu_index, params->bank);
             return;
         }
 
@@ -1115,7 +1145,7 @@ static void do_inject_x86_mce(void *data)
             monitor_printf(params->mon,
                            "CPU %d: Previous MCE still in progress, raising"
                            " triple fault\n",
-                           cenv->cpu_index);
+                           cpu->cpu_index);
             qemu_log_mask(CPU_LOG_RESET, "Triple fault\n");
             qemu_system_reset_request();
             return;
@@ -1148,7 +1178,7 @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
     CPUX86State *cenv = &cpu->env;
     MCEInjectionParams params = {
         .mon = mon,
-        .env = cenv,
+        .cpu = cpu,
         .bank = bank,
         .status = status,
         .mcg_status = mcg_status,
@@ -1188,7 +1218,7 @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
             if (cenv == env) {
                 continue;
             }
-            params.env = env;
+            params.cpu = x86_env_get_cpu(env);
             run_on_cpu(CPU(cpu), do_inject_x86_mce, &params);
         }
     }
@@ -1196,15 +1226,12 @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access)
 {
-    TranslationBlock *tb;
-
     if (kvm_enabled()) {
         env->tpr_access_type = access;
 
         cpu_interrupt(env, CPU_INTERRUPT_TPR);
     } else {
-        tb = tb_find_pc(env->mem_io_pc);
-        cpu_restore_state(tb, env, env->mem_io_pc);
+        cpu_restore_state(env, env->mem_io_pc);
 
         apic_handle_tpr_access_report(env->apic_state, env->eip, access);
     }
@@ -1251,14 +1278,14 @@ X86CPU *cpu_x86_init(const char *cpu_model)
     env->cpu_model_str = cpu_model;
 
     if (cpu_x86_register(cpu, cpu_model) < 0) {
-        object_delete(OBJECT(cpu));
+        object_unref(OBJECT(cpu));
         return NULL;
     }
 
     x86_cpu_realize(OBJECT(cpu), &error);
     if (error) {
         error_free(error);
-        object_delete(OBJECT(cpu));
+        object_unref(OBJECT(cpu));
         return NULL;
     }
     return cpu;
diff --git a/target-i386/helper.h b/target-i386/helper.h
index 970fcd9..9ed720d 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_FLAGS_2(cc_compute_all, TCG_CALL_NO_SE, i32, env, int)
 DEF_HELPER_FLAGS_2(cc_compute_c, TCG_CALL_NO_SE, i32, env, int)
@@ -220,4 +220,4 @@ DEF_HELPER_3(rclq, tl, env, tl, tl)
 DEF_HELPER_3(rcrq, tl, env, tl, tl)
 #endif
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-i386/int_helper.c b/target-i386/int_helper.c
index f39747e..84b812d 100644
--- a/target-i386/int_helper.c
+++ b/target-i386/int_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 //#define DEBUG_MULDIV
diff --git a/target-i386/ioport-user.c b/target-i386/ioport-user.c
index 03fac22..f7636e0 100644
--- a/target-i386/ioport-user.c
+++ b/target-i386/ioport-user.c
@@ -21,7 +21,7 @@
 
 #include "qemu.h"
 #include "qemu-common.h"
-#include "ioport.h"
+#include "exec/ioport.h"
 
 void cpu_outb(pio_addr_t addr, uint8_t val)
 {
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index f669281..9ebf181 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -21,17 +21,18 @@
 #include <linux/kvm_para.h>
 
 #include "qemu-common.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #include "cpu.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
+#include "qemu/config-file.h"
 #include "hw/pc.h"
 #include "hw/apic.h"
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "hyperv.h"
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 
 //#define DEBUG_KVM
 
@@ -62,6 +63,7 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 
 static bool has_msr_star;
 static bool has_msr_hsave_pa;
+static bool has_msr_tsc_adjust;
 static bool has_msr_tsc_deadline;
 static bool has_msr_async_pf_en;
 static bool has_msr_pv_eoi_en;
@@ -306,16 +308,17 @@ static void hardware_memory_error(void)
     exit(1);
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(c);
+    CPUX86State *env = &cpu->env;
     ram_addr_t ram_addr;
     hwaddr paddr;
 
     if ((env->mcg_cap & MCG_SER_P) && addr
         && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
-            !kvm_physical_memory_addr_from_host(env->kvm_state, addr, &paddr)) {
+            !kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
             fprintf(stderr, "Hardware memory error for memory used by "
                     "QEMU itself instead of guest system!\n");
             /* Hope we are lucky for AO MCE */
@@ -347,8 +350,8 @@ int kvm_arch_on_sigbus(int code, void *addr)
 
         /* Hope we are lucky for AO MCE */
         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
-            !kvm_physical_memory_addr_from_host(first_cpu->kvm_state, addr,
-                                                &paddr)) {
+            !kvm_physical_memory_addr_from_host(CPU(first_cpu)->kvm_state,
+                                                addr, &paddr)) {
             fprintf(stderr, "Hardware memory error for memory used by "
                     "QEMU itself instead of guest system!: %p\n", addr);
             return 0;
@@ -367,8 +370,10 @@ int kvm_arch_on_sigbus(int code, void *addr)
     return 0;
 }
 
-static int kvm_inject_mce_oldstyle(CPUX86State *env)
+static int kvm_inject_mce_oldstyle(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
+
     if (!kvm_has_vcpu_events() && env->exception_injected == EXCP12_MCHK) {
         unsigned int bank, bank_num = env->mcg_cap & 0xff;
         struct kvm_x86_mce mce;
@@ -392,7 +397,7 @@ static int kvm_inject_mce_oldstyle(CPUX86State *env)
         mce.addr = env->mce_banks[bank * 4 + 2];
         mce.misc = env->mce_banks[bank * 4 + 3];
 
-        return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, &mce);
+        return kvm_vcpu_ioctl(CPU(cpu), KVM_X86_SET_MCE, &mce);
     }
     return 0;
 }
@@ -406,12 +411,22 @@ static void cpu_update_state(void *opaque, int running, RunState state)
     }
 }
 
-int kvm_arch_init_vcpu(CPUX86State *env)
+unsigned long kvm_arch_vcpu_id(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    return cpu->env.cpuid_apic_id;
+}
+
+#define KVM_MAX_CPUID_ENTRIES  100
+
+int kvm_arch_init_vcpu(CPUState *cs)
 {
     struct {
         struct kvm_cpuid2 cpuid;
-        struct kvm_cpuid_entry2 entries[100];
+        struct kvm_cpuid_entry2 entries[KVM_MAX_CPUID_ENTRIES];
     } QEMU_PACKED cpuid_data;
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
     uint32_t limit, i, j, cpuid_i;
     uint32_t unused;
     struct kvm_cpuid_entry2 *c;
@@ -495,6 +510,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0; i <= limit; i++) {
+        if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+            fprintf(stderr, "unsupported level value: 0x%x\n", limit);
+            abort();
+        }
         c = &cpuid_data.entries[cpuid_i++];
 
         switch (i) {
@@ -509,6 +528,11 @@ int kvm_arch_init_vcpu(CPUX86State *env)
             times = c->eax & 0xff;
 
             for (j = 1; j < times; ++j) {
+                if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+                    fprintf(stderr, "cpuid_data is full, no space for "
+                            "cpuid(eax:2):eax & 0xf = 0x%x\n", times);
+                    abort();
+                }
                 c = &cpuid_data.entries[cpuid_i++];
                 c->function = i;
                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
@@ -537,6 +561,11 @@ int kvm_arch_init_vcpu(CPUX86State *env)
                 if (i == 0xd && c->eax == 0) {
                     continue;
                 }
+                if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+                    fprintf(stderr, "cpuid_data is full, no space for "
+                            "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
+                    abort();
+                }
                 c = &cpuid_data.entries[cpuid_i++];
             }
             break;
@@ -550,6 +579,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0x80000000; i <= limit; i++) {
+        if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+            fprintf(stderr, "unsupported xlevel value: 0x%x\n", limit);
+            abort();
+        }
         c = &cpuid_data.entries[cpuid_i++];
 
         c->function = i;
@@ -562,6 +595,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
         cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
 
         for (i = 0xC0000000; i <= limit; i++) {
+            if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
+                fprintf(stderr, "unsupported xlevel2 value: 0x%x\n", limit);
+                abort();
+            }
             c = &cpuid_data.entries[cpuid_i++];
 
             c->function = i;
@@ -574,12 +611,12 @@ int kvm_arch_init_vcpu(CPUX86State *env)
 
     if (((env->cpuid_version >> 8)&0xF) >= 6
         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
-        && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
+        && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
         uint64_t mcg_cap;
         int banks;
         int ret;
 
-        ret = kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks);
+        ret = kvm_get_mce_cap_supported(cs->kvm_state, &mcg_cap, &banks);
         if (ret < 0) {
             fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
             return ret;
@@ -590,7 +627,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
         }
         mcg_cap &= MCE_CAP_DEF;
         mcg_cap |= banks;
-        ret = kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, &mcg_cap);
+        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &mcg_cap);
         if (ret < 0) {
             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
             return ret;
@@ -602,14 +639,14 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     qemu_add_vm_change_state_handler(cpu_update_state, env);
 
     cpuid_data.cpuid.padding = 0;
-    r = kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
+    r = kvm_vcpu_ioctl(cs, KVM_SET_CPUID2, &cpuid_data);
     if (r) {
         return r;
     }
 
-    r = kvm_check_extension(env->kvm_state, KVM_CAP_TSC_CONTROL);
+    r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL);
     if (r && env->tsc_khz) {
-        r = kvm_vcpu_ioctl(env, KVM_SET_TSC_KHZ, env->tsc_khz);
+        r = kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz);
         if (r < 0) {
             fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
             return r;
@@ -623,9 +660,10 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     return 0;
 }
 
-void kvm_arch_reset_vcpu(CPUX86State *env)
+void kvm_arch_reset_vcpu(CPUState *cs)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
 
     env->exception_injected = -1;
     env->interrupt_injected = -1;
@@ -676,6 +714,10 @@ static int kvm_get_supported_msrs(KVMState *s)
                     has_msr_hsave_pa = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_TSC_ADJUST) {
+                    has_msr_tsc_adjust = true;
+                    continue;
+                }
                 if (kvm_msr_list->indices[i] == MSR_IA32_TSCDEADLINE) {
                     has_msr_tsc_deadline = true;
                     continue;
@@ -816,13 +858,14 @@ static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
     }
 }
 
-static int kvm_getput_regs(CPUX86State *env, int set)
+static int kvm_getput_regs(X86CPU *cpu, int set)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_regs regs;
     int ret = 0;
 
     if (!set) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+        ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_REGS, &regs);
         if (ret < 0) {
             return ret;
         }
@@ -851,14 +894,15 @@ static int kvm_getput_regs(CPUX86State *env, int set)
     kvm_getput_reg(&regs.rip, &env->eip, set);
 
     if (set) {
-        ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
+        ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_REGS, &regs);
     }
 
     return ret;
 }
 
-static int kvm_put_fpu(CPUX86State *env)
+static int kvm_put_fpu(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_fpu fpu;
     int i;
 
@@ -876,7 +920,7 @@ static int kvm_put_fpu(CPUX86State *env)
     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
     fpu.mxcsr = env->mxcsr;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_FPU, &fpu);
 }
 
 #define XSAVE_FCW_FSW     0
@@ -889,14 +933,15 @@ static int kvm_put_fpu(CPUX86State *env)
 #define XSAVE_XSTATE_BV   128
 #define XSAVE_YMMH_SPACE  144
 
-static int kvm_put_xsave(CPUX86State *env)
+static int kvm_put_xsave(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_xsave* xsave = env->kvm_xsave_buf;
     uint16_t cwd, swd, twd;
     int i, r;
 
     if (!kvm_has_xsave()) {
-        return kvm_put_fpu(env);
+        return kvm_put_fpu(cpu);
     }
 
     memset(xsave, 0, sizeof(struct kvm_xsave));
@@ -919,12 +964,13 @@ static int kvm_put_xsave(CPUX86State *env)
     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
             sizeof env->ymmh_regs);
-    r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
+    r = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
     return r;
 }
 
-static int kvm_put_xcrs(CPUX86State *env)
+static int kvm_put_xcrs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_xcrs xcrs;
 
     if (!kvm_has_xcrs()) {
@@ -935,11 +981,12 @@ static int kvm_put_xcrs(CPUX86State *env)
     xcrs.flags = 0;
     xcrs.xcrs[0].xcr = 0;
     xcrs.xcrs[0].value = env->xcr0;
-    return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XCRS, &xcrs);
 }
 
-static int kvm_put_sregs(CPUX86State *env)
+static int kvm_put_sregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_sregs sregs;
 
     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
@@ -984,7 +1031,7 @@ static int kvm_put_sregs(CPUX86State *env)
 
     sregs.efer = env->efer;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
 }
 
 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
@@ -994,8 +1041,9 @@ static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
     entry->data = value;
 }
 
-static int kvm_put_msrs(CPUX86State *env, int level)
+static int kvm_put_msrs(X86CPU *cpu, int level)
 {
+    CPUX86State *env = &cpu->env;
     struct {
         struct kvm_msrs info;
         struct kvm_msr_entry entries[100];
@@ -1013,6 +1061,9 @@ static int kvm_put_msrs(CPUX86State *env, int level)
     if (has_msr_hsave_pa) {
         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
     }
+    if (has_msr_tsc_adjust) {
+        kvm_msr_entry_set(&msrs[n++], MSR_TSC_ADJUST, env->tsc_adjust);
+    }
     if (has_msr_tsc_deadline) {
         kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSCDEADLINE, env->tsc_deadline);
     }
@@ -1076,17 +1127,18 @@ static int kvm_put_msrs(CPUX86State *env, int level)
 
     msr_data.info.nmsrs = n;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
 
 }
 
 
-static int kvm_get_fpu(CPUX86State *env)
+static int kvm_get_fpu(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_fpu fpu;
     int i, ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_FPU, &fpu);
     if (ret < 0) {
         return ret;
     }
@@ -1107,17 +1159,18 @@ static int kvm_get_fpu(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_xsave(CPUX86State *env)
+static int kvm_get_xsave(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_xsave* xsave = env->kvm_xsave_buf;
     int ret, i;
     uint16_t cwd, swd, twd;
 
     if (!kvm_has_xsave()) {
-        return kvm_get_fpu(env);
+        return kvm_get_fpu(cpu);
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XSAVE, xsave);
     if (ret < 0) {
         return ret;
     }
@@ -1145,8 +1198,9 @@ static int kvm_get_xsave(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_xcrs(CPUX86State *env)
+static int kvm_get_xcrs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     int i, ret;
     struct kvm_xcrs xcrs;
 
@@ -1154,7 +1208,7 @@ static int kvm_get_xcrs(CPUX86State *env)
         return 0;
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XCRS, &xcrs);
     if (ret < 0) {
         return ret;
     }
@@ -1169,13 +1223,14 @@ static int kvm_get_xcrs(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_sregs(CPUX86State *env)
+static int kvm_get_sregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_sregs sregs;
     uint32_t hflags;
     int bit, i, ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
     if (ret < 0) {
         return ret;
     }
@@ -1253,8 +1308,9 @@ static int kvm_get_sregs(CPUX86State *env)
     return 0;
 }
 
-static int kvm_get_msrs(CPUX86State *env)
+static int kvm_get_msrs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct {
         struct kvm_msrs info;
         struct kvm_msr_entry entries[100];
@@ -1273,6 +1329,9 @@ static int kvm_get_msrs(CPUX86State *env)
     if (has_msr_hsave_pa) {
         msrs[n++].index = MSR_VM_HSAVE_PA;
     }
+    if (has_msr_tsc_adjust) {
+        msrs[n++].index = MSR_TSC_ADJUST;
+    }
     if (has_msr_tsc_deadline) {
         msrs[n++].index = MSR_IA32_TSCDEADLINE;
     }
@@ -1311,7 +1370,7 @@ static int kvm_get_msrs(CPUX86State *env)
     }
 
     msr_data.info.nmsrs = n;
-    ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
     if (ret < 0) {
         return ret;
     }
@@ -1350,6 +1409,9 @@ static int kvm_get_msrs(CPUX86State *env)
         case MSR_IA32_TSC:
             env->tsc = msrs[i].data;
             break;
+        case MSR_TSC_ADJUST:
+            env->tsc_adjust = msrs[i].data;
+            break;
         case MSR_IA32_TSCDEADLINE:
             env->tsc_deadline = msrs[i].data;
             break;
@@ -1389,11 +1451,11 @@ static int kvm_get_msrs(CPUX86State *env)
     return 0;
 }
 
-static int kvm_put_mp_state(CPUX86State *env)
+static int kvm_put_mp_state(X86CPU *cpu)
 {
-    struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
+    struct kvm_mp_state mp_state = { .mp_state = cpu->env.mp_state };
 
-    return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
 }
 
 static int kvm_get_mp_state(X86CPU *cpu)
@@ -1402,7 +1464,7 @@ static int kvm_get_mp_state(X86CPU *cpu)
     struct kvm_mp_state mp_state;
     int ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
     if (ret < 0) {
         return ret;
     }
@@ -1413,14 +1475,15 @@ static int kvm_get_mp_state(X86CPU *cpu)
     return 0;
 }
 
-static int kvm_get_apic(CPUX86State *env)
+static int kvm_get_apic(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     DeviceState *apic = env->apic_state;
     struct kvm_lapic_state kapic;
     int ret;
 
     if (apic && kvm_irqchip_in_kernel()) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_LAPIC, &kapic);
+        ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_LAPIC, &kapic);
         if (ret < 0) {
             return ret;
         }
@@ -1430,21 +1493,23 @@ static int kvm_get_apic(CPUX86State *env)
     return 0;
 }
 
-static int kvm_put_apic(CPUX86State *env)
+static int kvm_put_apic(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     DeviceState *apic = env->apic_state;
     struct kvm_lapic_state kapic;
 
     if (apic && kvm_irqchip_in_kernel()) {
         kvm_put_apic_state(apic, &kapic);
 
-        return kvm_vcpu_ioctl(env, KVM_SET_LAPIC, &kapic);
+        return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_LAPIC, &kapic);
     }
     return 0;
 }
 
-static int kvm_put_vcpu_events(CPUX86State *env, int level)
+static int kvm_put_vcpu_events(X86CPU *cpu, int level)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_vcpu_events events;
 
     if (!kvm_has_vcpu_events()) {
@@ -1474,11 +1539,12 @@ static int kvm_put_vcpu_events(CPUX86State *env, int level)
             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
     }
 
-    return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
 }
 
-static int kvm_get_vcpu_events(CPUX86State *env)
+static int kvm_get_vcpu_events(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_vcpu_events events;
     int ret;
 
@@ -1486,7 +1552,7 @@ static int kvm_get_vcpu_events(CPUX86State *env)
         return 0;
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
     if (ret < 0) {
        return ret;
     }
@@ -1512,8 +1578,9 @@ static int kvm_get_vcpu_events(CPUX86State *env)
     return 0;
 }
 
-static int kvm_guest_debug_workarounds(CPUX86State *env)
+static int kvm_guest_debug_workarounds(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     int ret = 0;
     unsigned long reinject_trap = 0;
 
@@ -1541,8 +1608,9 @@ static int kvm_guest_debug_workarounds(CPUX86State *env)
     return ret;
 }
 
-static int kvm_put_debugregs(CPUX86State *env)
+static int kvm_put_debugregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_debugregs dbgregs;
     int i;
 
@@ -1557,11 +1625,12 @@ static int kvm_put_debugregs(CPUX86State *env)
     dbgregs.dr7 = env->dr[7];
     dbgregs.flags = 0;
 
-    return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEBUGREGS, &dbgregs);
 }
 
-static int kvm_get_debugregs(CPUX86State *env)
+static int kvm_get_debugregs(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_debugregs dbgregs;
     int i, ret;
 
@@ -1569,7 +1638,7 @@ static int kvm_get_debugregs(CPUX86State *env)
         return 0;
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_DEBUGREGS, &dbgregs);
     if (ret < 0) {
         return ret;
     }
@@ -1582,88 +1651,88 @@ static int kvm_get_debugregs(CPUX86State *env)
     return 0;
 }
 
-int kvm_arch_put_registers(CPUX86State *env, int level)
+int kvm_arch_put_registers(CPUState *cpu, int level)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
+    X86CPU *x86_cpu = X86_CPU(cpu);
     int ret;
 
     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
-    ret = kvm_getput_regs(env, 1);
+    ret = kvm_getput_regs(x86_cpu, 1);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_xsave(env);
+    ret = kvm_put_xsave(x86_cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_xcrs(env);
+    ret = kvm_put_xcrs(x86_cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_sregs(env);
+    ret = kvm_put_sregs(x86_cpu);
     if (ret < 0) {
         return ret;
     }
     /* must be before kvm_put_msrs */
-    ret = kvm_inject_mce_oldstyle(env);
+    ret = kvm_inject_mce_oldstyle(x86_cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_msrs(env, level);
+    ret = kvm_put_msrs(x86_cpu, level);
     if (ret < 0) {
         return ret;
     }
     if (level >= KVM_PUT_RESET_STATE) {
-        ret = kvm_put_mp_state(env);
+        ret = kvm_put_mp_state(x86_cpu);
         if (ret < 0) {
             return ret;
         }
-        ret = kvm_put_apic(env);
+        ret = kvm_put_apic(x86_cpu);
         if (ret < 0) {
             return ret;
         }
     }
-    ret = kvm_put_vcpu_events(env, level);
+    ret = kvm_put_vcpu_events(x86_cpu, level);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_put_debugregs(env);
+    ret = kvm_put_debugregs(x86_cpu);
     if (ret < 0) {
         return ret;
     }
     /* must be last */
-    ret = kvm_guest_debug_workarounds(env);
+    ret = kvm_guest_debug_workarounds(x86_cpu);
     if (ret < 0) {
         return ret;
     }
     return 0;
 }
 
-int kvm_arch_get_registers(CPUX86State *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(cs);
     int ret;
 
-    assert(cpu_is_stopped(CPU(cpu)) || qemu_cpu_is_self(CPU(cpu)));
+    assert(cpu_is_stopped(cs) || qemu_cpu_is_self(cs));
 
-    ret = kvm_getput_regs(env, 0);
+    ret = kvm_getput_regs(cpu, 0);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_xsave(env);
+    ret = kvm_get_xsave(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_xcrs(env);
+    ret = kvm_get_xcrs(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_sregs(env);
+    ret = kvm_get_sregs(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_msrs(env);
+    ret = kvm_get_msrs(cpu);
     if (ret < 0) {
         return ret;
     }
@@ -1671,30 +1740,32 @@ int kvm_arch_get_registers(CPUX86State *env)
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_apic(env);
+    ret = kvm_get_apic(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_vcpu_events(env);
+    ret = kvm_get_vcpu_events(cpu);
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_debugregs(env);
+    ret = kvm_get_debugregs(cpu);
     if (ret < 0) {
         return ret;
     }
     return 0;
 }
 
-void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
 {
+    X86CPU *x86_cpu = X86_CPU(cpu);
+    CPUX86State *env = &x86_cpu->env;
     int ret;
 
     /* Inject NMI */
     if (env->interrupt_request & CPU_INTERRUPT_NMI) {
         env->interrupt_request &= ~CPU_INTERRUPT_NMI;
         DPRINTF("injected NMI\n");
-        ret = kvm_vcpu_ioctl(env, KVM_NMI);
+        ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
         if (ret < 0) {
             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
                     strerror(-ret));
@@ -1722,7 +1793,7 @@ void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
 
                 intr.irq = irq;
                 DPRINTF("injected interrupt %d\n", irq);
-                ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
+                ret = kvm_vcpu_ioctl(cpu, KVM_INTERRUPT, &intr);
                 if (ret < 0) {
                     fprintf(stderr,
                             "KVM: injection failed, interrupt lost (%s)\n",
@@ -1746,8 +1817,11 @@ void kvm_arch_pre_run(CPUX86State *env, struct kvm_run *run)
     }
 }
 
-void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
+    X86CPU *x86_cpu = X86_CPU(cpu);
+    CPUX86State *env = &x86_cpu->env;
+
     if (run->if_flag) {
         env->eflags |= IF_MASK;
     } else {
@@ -1757,9 +1831,10 @@ void kvm_arch_post_run(CPUX86State *env, struct kvm_run *run)
     cpu_set_apic_base(env->apic_state, run->apic_base);
 }
 
-int kvm_arch_process_async_events(CPUX86State *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
 
     if (env->interrupt_request & CPU_INTERRUPT_MCE) {
         /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
@@ -1829,9 +1904,11 @@ static int kvm_handle_halt(X86CPU *cpu)
     return 0;
 }
 
-static int kvm_handle_tpr_access(CPUX86State *env)
+static int kvm_handle_tpr_access(X86CPU *cpu)
 {
-    struct kvm_run *run = env->kvm_run;
+    CPUX86State *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
 
     apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip,
                                   run->tpr_access.is_write ? TPR_ACCESS_WRITE
@@ -1839,8 +1916,9 @@ static int kvm_handle_tpr_access(CPUX86State *env)
     return 1;
 }
 
-int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_insert_sw_breakpoint(CPUState *cpu, struct kvm_sw_breakpoint *bp)
 {
+    CPUX86State *env = &X86_CPU(cpu)->env;
     static const uint8_t int3 = 0xcc;
 
     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
@@ -1850,8 +1928,9 @@ int kvm_arch_insert_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp
     return 0;
 }
 
-int kvm_arch_remove_sw_breakpoint(CPUX86State *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_remove_sw_breakpoint(CPUState *cpu, struct kvm_sw_breakpoint *bp)
 {
+    CPUX86State *env = &X86_CPU(cpu)->env;
     uint8_t int3;
 
     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
@@ -1945,9 +2024,10 @@ void kvm_arch_remove_all_hw_breakpoints(void)
 
 static CPUWatchpoint hw_watchpoint;
 
-static int kvm_handle_debug(CPUX86State *env,
+static int kvm_handle_debug(X86CPU *cpu,
                             struct kvm_debug_exit_arch *arch_info)
 {
+    CPUX86State *env = &cpu->env;
     int ret = 0;
     int n;
 
@@ -1979,7 +2059,7 @@ static int kvm_handle_debug(CPUX86State *env,
                 }
             }
         }
-    } else if (kvm_find_sw_breakpoint(env, arch_info->pc)) {
+    } else if (kvm_find_sw_breakpoint(CPU(cpu), arch_info->pc)) {
         ret = EXCP_DEBUG;
     }
     if (ret == 0) {
@@ -1994,7 +2074,7 @@ static int kvm_handle_debug(CPUX86State *env,
     return ret;
 }
 
-void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
+void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
 {
     const uint8_t type_code[] = {
         [GDB_BREAKPOINT_HW] = 0x0,
@@ -2006,7 +2086,7 @@ void kvm_arch_update_guest_debug(CPUX86State *env, struct kvm_guest_debug *dbg)
     };
     int n;
 
-    if (kvm_sw_breakpoints_active(env)) {
+    if (kvm_sw_breakpoints_active(cpu)) {
         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
     }
     if (nb_hw_breakpoint > 0) {
@@ -2031,9 +2111,9 @@ static bool host_supports_vmx(void)
 
 #define VMX_INVALID_GUEST_STATE 0x80000021
 
-int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
-    X86CPU *cpu = x86_env_get_cpu(env);
+    X86CPU *cpu = X86_CPU(cs);
     uint64_t code;
     int ret;
 
@@ -2046,7 +2126,7 @@ int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
         ret = 0;
         break;
     case KVM_EXIT_TPR_ACCESS:
-        ret = kvm_handle_tpr_access(env);
+        ret = kvm_handle_tpr_access(cpu);
         break;
     case KVM_EXIT_FAIL_ENTRY:
         code = run->fail_entry.hardware_entry_failure_reason;
@@ -2072,7 +2152,7 @@ int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
         break;
     case KVM_EXIT_DEBUG:
         DPRINTF("kvm_exit_debug\n");
-        ret = kvm_handle_debug(env, &run->debug.arch);
+        ret = kvm_handle_debug(cpu, &run->debug.arch);
         break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
@@ -2083,8 +2163,11 @@ int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
     return ret;
 }
 
-bool kvm_arch_stop_on_emulation_error(CPUX86State *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cs)
 {
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
     kvm_cpu_synchronize_state(env);
     return !(env->cr[0] & CR0_PE_MASK) ||
            ((env->segs[R_CS].selector  & 3) != 3);
diff --git a/target-i386/kvm_i386.h b/target-i386/kvm_i386.h
index f6ab82f..4392ab4 100644
--- a/target-i386/kvm_i386.h
+++ b/target-i386/kvm_i386.h
@@ -11,7 +11,7 @@
 #ifndef QEMU_KVM_I386_H
 #define QEMU_KVM_I386_H
 
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 bool kvm_allows_irq0_override(void);
 
diff --git a/target-i386/machine.c b/target-i386/machine.c
index 4771508..8df6a6b 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -4,7 +4,7 @@
 #include "hw/isa.h"
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 static const VMStateDescription vmstate_segment = {
     .name = "segment",
@@ -265,10 +265,11 @@ static int cpu_post_load(void *opaque, int version_id)
 
     cpu_breakpoint_remove_all(env, BP_CPU);
     cpu_watchpoint_remove_all(env, BP_CPU);
-    for (i = 0; i < 4; i++)
+    for (i = 0; i < DR7_MAX_BP; i++) {
         hw_breakpoint_insert(env, i);
-
+    }
     tlb_flush(env, 1);
+
     return 0;
 }
 
@@ -328,6 +329,24 @@ static const VMStateDescription vmstate_fpop_ip_dp = {
     }
 };
 
+static bool tsc_adjust_needed(void *opaque)
+{
+    CPUX86State *env = opaque;
+
+    return env->tsc_adjust != 0;
+}
+
+static const VMStateDescription vmstate_msr_tsc_adjust = {
+    .name = "cpu/msr_tsc_adjust",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT64(tsc_adjust, CPUX86State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool tscdeadline_needed(void *opaque)
 {
     CPUX86State *env = opaque;
@@ -478,6 +497,9 @@ static const VMStateDescription vmstate_cpu = {
             .vmsd = &vmstate_fpop_ip_dp,
             .needed = fpop_ip_dp_needed,
         }, {
+            .vmsd = &vmstate_msr_tsc_adjust,
+            .needed = tsc_adjust_needed,
+        }, {
             .vmsd = &vmstate_msr_tscdeadline,
             .needed = tscdeadline_needed,
         }, {
diff --git a/target-i386/mem_helper.c b/target-i386/mem_helper.c
index 7f99c7c..6cf9ba0 100644
--- a/target-i386/mem_helper.c
+++ b/target-i386/mem_helper.c
@@ -21,7 +21,7 @@
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 /* broken thread support */
@@ -114,16 +114,16 @@ void helper_boundl(CPUX86State *env, target_ulong a0, int v)
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #endif
 
@@ -135,19 +135,13 @@ void helper_boundl(CPUX86State *env, target_ulong a0, int v)
 void tlb_fill(CPUX86State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_x86_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         raise_exception_err(env, env->exception_index, env->error_code);
     }
diff --git a/target-i386/misc_helper.c b/target-i386/misc_helper.c
index a020379..b6d5740 100644
--- a/target-i386/misc_helper.c
+++ b/target-i386/misc_helper.c
@@ -18,11 +18,11 @@
  */
 
 #include "cpu.h"
-#include "ioport.h"
+#include "exec/ioport.h"
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 /* check if Port I/O is allowed in TSS */
@@ -110,7 +110,7 @@ void helper_into(CPUX86State *env, int next_eip_addend)
 void helper_single_step(CPUX86State *env)
 {
 #ifndef CONFIG_USER_ONLY
-    check_hw_breakpoints(env, 1);
+    check_hw_breakpoints(env, true);
     env->dr[6] |= DR6_BS;
 #endif
     raise_exception(env, EXCP01_DB);
@@ -197,11 +197,11 @@ void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
         env->dr[reg] = t0;
         hw_breakpoint_insert(env, reg);
     } else if (reg == 7) {
-        for (i = 0; i < 4; i++) {
+        for (i = 0; i < DR7_MAX_BP; i++) {
             hw_breakpoint_remove(env, i);
         }
         env->dr[7] = t0;
-        for (i = 0; i < 4; i++) {
+        for (i = 0; i < DR7_MAX_BP; i++) {
             hw_breakpoint_insert(env, i);
         }
     } else {
@@ -580,14 +580,17 @@ void helper_monitor(CPUX86State *env, target_ulong ptr)
 
 void helper_mwait(CPUX86State *env, int next_eip_addend)
 {
+    CPUState *cpu;
+
     if ((uint32_t)ECX != 0) {
         raise_exception(env, EXCP0D_GPF);
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_MWAIT, 0);
     EIP += next_eip_addend;
 
+    cpu = CPU(x86_env_get_cpu(env));
     /* XXX: not complete but not completely erroneous */
-    if (env->cpu_index != 0 || env->next_cpu != NULL) {
+    if (cpu->cpu_index != 0 || env->next_cpu != NULL) {
         /* more than one CPU: do not sleep because another CPU may
            wake this one */
     } else {
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index ff93374..3247dee 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -19,13 +19,13 @@
  */
 
 #include "cpu.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "helper.h"
 
 //#define DEBUG_PCALL
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 #ifdef DEBUG_PCALL
@@ -465,13 +465,14 @@ static void switch_tss(CPUX86State *env, int tss_selector,
 
 #ifndef CONFIG_USER_ONLY
     /* reset local breakpoints */
-    if (env->dr[7] & 0x55) {
-        for (i = 0; i < 4; i++) {
-            if (hw_breakpoint_enabled(env->dr[7], i) == 0x1) {
+    if (env->dr[7] & DR7_LOCAL_BP_MASK) {
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if (hw_local_breakpoint_enabled(env->dr[7], i) &&
+                !hw_global_breakpoint_enabled(env->dr[7], i)) {
                 hw_breakpoint_remove(env, i);
             }
         }
-        env->dr[7] &= ~0x55;
+        env->dr[7] &= ~DR7_LOCAL_BP_MASK;
     }
 #endif
 }
diff --git a/target-i386/svm_helper.c b/target-i386/svm_helper.c
index a238d95..3f246e9 100644
--- a/target-i386/svm_helper.c
+++ b/target-i386/svm_helper.c
@@ -18,11 +18,11 @@
  */
 
 #include "cpu.h"
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 /* Secure Virtual Machine helpers */
diff --git a/target-i386/topology.h b/target-i386/topology.h
new file mode 100644
index 0000000..24ed525
--- /dev/null
+++ b/target-i386/topology.h
@@ -0,0 +1,136 @@
+/*
+ *  x86 CPU topology data structures and functions
+ *
+ *  Copyright (c) 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef TARGET_I386_TOPOLOGY_H
+#define TARGET_I386_TOPOLOGY_H
+
+/* This file implements the APIC-ID-based CPU topology enumeration logic,
+ * documented at the following document:
+ *   Intel® 64 Architecture Processor Topology Enumeration
+ *   http://software.intel.com/en-us/articles/intel-64-architecture-processor-topology-enumeration/
+ *
+ * This code should be compatible with AMD's "Extended Method" described at:
+ *   AMD CPUID Specification (Publication #25481)
+ *   Section 3: Multiple Core Calcuation
+ * as long as:
+ *  nr_threads is set to 1;
+ *  OFFSET_IDX is assumed to be 0;
+ *  CPUID Fn8000_0008_ECX[ApicIdCoreIdSize[3:0]] is set to apicid_core_width().
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include "qemu/bitops.h"
+
+/* APIC IDs can be 32-bit, but beware: APIC IDs > 255 require x2APIC support
+ */
+typedef uint32_t apic_id_t;
+
+/* Return the bit width needed for 'count' IDs
+ */
+static unsigned apicid_bitwidth_for_count(unsigned count)
+{
+    g_assert(count >= 1);
+    if (count == 1) {
+        return 0;
+    }
+    return bitops_flsl(count - 1) + 1;
+}
+
+/* Bit width of the SMT_ID (thread ID) field on the APIC ID
+ */
+static inline unsigned apicid_smt_width(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_bitwidth_for_count(nr_threads);
+}
+
+/* Bit width of the Core_ID field
+ */
+static inline unsigned apicid_core_width(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_bitwidth_for_count(nr_cores);
+}
+
+/* Bit offset of the Core_ID field
+ */
+static inline unsigned apicid_core_offset(unsigned nr_cores,
+                                          unsigned nr_threads)
+{
+    return apicid_smt_width(nr_cores, nr_threads);
+}
+
+/* Bit offset of the Pkg_ID (socket ID) field
+ */
+static inline unsigned apicid_pkg_offset(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_core_offset(nr_cores, nr_threads) +
+           apicid_core_width(nr_cores, nr_threads);
+}
+
+/* Make APIC ID for the CPU based on Pkg_ID, Core_ID, SMT_ID
+ *
+ * The caller must make sure core_id < nr_cores and smt_id < nr_threads.
+ */
+static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
+                                             unsigned nr_threads,
+                                             unsigned pkg_id,
+                                             unsigned core_id,
+                                             unsigned smt_id)
+{
+    return (pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
+           (core_id << apicid_core_offset(nr_cores, nr_threads)) |
+           smt_id;
+}
+
+/* Calculate thread/core/package IDs for a specific topology,
+ * based on (contiguous) CPU index
+ */
+static inline void x86_topo_ids_from_idx(unsigned nr_cores,
+                                         unsigned nr_threads,
+                                         unsigned cpu_index,
+                                         unsigned *pkg_id,
+                                         unsigned *core_id,
+                                         unsigned *smt_id)
+{
+    unsigned core_index = cpu_index / nr_threads;
+    *smt_id = cpu_index % nr_threads;
+    *core_id = core_index % nr_cores;
+    *pkg_id = core_index / nr_cores;
+}
+
+/* Make APIC ID for the CPU 'cpu_index'
+ *
+ * 'cpu_index' is a sequential, contiguous ID for the CPU.
+ */
+static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_cores,
+                                                unsigned nr_threads,
+                                                unsigned cpu_index)
+{
+    unsigned pkg_id, core_id, smt_id;
+    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index,
+                          &pkg_id, &core_id, &smt_id);
+    return apicid_from_topo_ids(nr_cores, nr_threads, pkg_id, core_id, smt_id);
+}
+
+#endif /* TARGET_I386_TOPOLOGY_H */
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 8e676ba..32d21f5 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -24,7 +24,7 @@
 #include <signal.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -65,7 +65,7 @@ static TCGv cpu_tmp5;
 
 static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 #ifdef TARGET_X86_64
 static int x86_64_hregs;
@@ -7988,12 +7988,12 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = pc_ptr;
+            tcg_ctx.gen_opc_pc[lj] = pc_ptr;
             gen_opc_cc_op[lj] = dc->cc_op;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
@@ -8037,7 +8037,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     }
 
 #ifdef DEBUG_DISAS
@@ -8080,16 +8080,17 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, int pc_pos)
         int i;
         qemu_log("RESTORE:\n");
         for(i = 0;i <= pc_pos; i++) {
-            if (gen_opc_instr_start[i]) {
-                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i, gen_opc_pc[i]);
+            if (tcg_ctx.gen_opc_instr_start[i]) {
+                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i,
+                        tcg_ctx.gen_opc_pc[i]);
             }
         }
         qemu_log("pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
-                pc_pos, gen_opc_pc[pc_pos] - tb->cs_base,
+                pc_pos, tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base,
                 (uint32_t)tb->cs_base);
     }
 #endif
-    env->eip = gen_opc_pc[pc_pos] - tb->cs_base;
+    env->eip = tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base;
     cc_op = gen_opc_cc_op[pc_pos];
     if (cc_op != CC_OP_DYNAMIC)
         env->cc_op = cc_op;
diff --git a/target-lm32/cpu-qom.h b/target-lm32/cpu-qom.h
index 4ae2edd..400cdbd 100644
--- a/target-lm32/cpu-qom.h
+++ b/target-lm32/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_LM32_CPU_QOM_H
 #define QEMU_LM32_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_LM32_CPU "lm32-cpu"
diff --git a/target-lm32/cpu.c b/target-lm32/cpu.c
index caa4834..eca2dca 100644
--- a/target-lm32/cpu.c
+++ b/target-lm32/cpu.c
@@ -30,7 +30,7 @@ static void lm32_cpu_reset(CPUState *s)
     CPULM32State *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index 7243b4f..4e202db 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -26,7 +26,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 struct CPULM32State;
 typedef struct CPULM32State CPULM32State;
 
@@ -238,7 +238,7 @@ static inline int cpu_interrupts_enabled(CPULM32State *env)
     return env->ie & IE_IE;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline target_ulong cpu_get_pc(CPULM32State *env)
 {
@@ -260,7 +260,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPULM32State *env, TranslationBlock *tb)
 {
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index 0ed7cfd..d76ea3f 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 int cpu_lm32_handle_mmu_fault(CPULM32State *env, target_ulong address, int rw,
                               int mmu_idx)
diff --git a/target-lm32/helper.h b/target-lm32/helper.h
index 07f5670..3ea15a6 100644
--- a/target-lm32/helper.h
+++ b/target-lm32/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_1(hlt, void, env)
@@ -11,4 +11,4 @@ DEF_HELPER_1(rcsr_ip, i32, env)
 DEF_HELPER_1(rcsr_jtx, i32, env)
 DEF_HELPER_1(rcsr_jrx, i32, env)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-lm32/op_helper.c b/target-lm32/op_helper.c
index 7b91d8c..53410b1 100644
--- a/target-lm32/op_helper.c
+++ b/target-lm32/op_helper.c
@@ -1,7 +1,7 @@
 #include <assert.h>
 #include "cpu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "hw/lm32_pic.h"
 #include "hw/lm32_juart.h"
@@ -9,13 +9,13 @@
 #if !defined(CONFIG_USER_ONLY)
 #define MMUSUFFIX _mmu
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 void helper_raise_exception(CPULM32State *env, uint32_t index)
 {
@@ -76,19 +76,13 @@ uint32_t helper_rcsr_jrx(CPULM32State *env)
 void tlb_fill(CPULM32State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_lm32_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index af98649..6b87340 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "helper.h"
 #include "tcg-op.h"
 
@@ -53,7 +53,7 @@ static TCGv cpu_deba;
 static TCGv cpu_bp[4];
 static TCGv cpu_wp[4];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 enum {
     OP_FMT_RI,
@@ -1051,12 +1051,12 @@ static void gen_intermediate_code_internal(CPULM32State *env,
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         /* Pretty disas.  */
@@ -1110,7 +1110,7 @@ static void gen_intermediate_code_internal(CPULM32State *env,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -1172,7 +1172,7 @@ void cpu_dump_state(CPULM32State *env, FILE *f, fprintf_function cpu_fprintf,
 
 void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
 
 void lm32_translate_init(void)
diff --git a/target-m68k/Makefile.objs b/target-m68k/Makefile.objs
index 7eccfab..2e2b850 100644
--- a/target-m68k/Makefile.objs
+++ b/target-m68k/Makefile.objs
@@ -1,3 +1,2 @@
 obj-y += m68k-semi.o
 obj-y += translate.o op_helper.o helper.o cpu.o
-obj-$(CONFIG_SOFTMMU) += machine.o
diff --git a/target-m68k/cpu-qom.h b/target-m68k/cpu-qom.h
index 805786b..170daa7 100644
--- a/target-m68k/cpu-qom.h
+++ b/target-m68k/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_M68K_CPU_QOM_H
 #define QEMU_M68K_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_M68K_CPU "m68k-cpu"
 
diff --git a/target-m68k/cpu.c b/target-m68k/cpu.c
index 3e70bb0..c71f715 100644
--- a/target-m68k/cpu.c
+++ b/target-m68k/cpu.c
@@ -20,6 +20,7 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
+#include "migration/vmstate.h"
 
 
 static void m68k_set_feature(CPUM68KState *env, int feature)
@@ -35,7 +36,7 @@ static void m68k_cpu_reset(CPUState *s)
     CPUM68KState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -55,6 +56,25 @@ static void m68k_cpu_reset(CPUState *s)
 
 /* CPU models */
 
+static ObjectClass *m68k_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+    char *typename;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    typename = g_strdup_printf("%s-" TYPE_M68K_CPU, cpu_model);
+    oc = object_class_by_name(typename);
+    g_free(typename);
+    if (oc != NULL && (object_class_dynamic_cast(oc, TYPE_M68K_CPU) == NULL ||
+                       object_class_is_abstract(oc))) {
+        return NULL;
+    }
+    return oc;
+}
+
 static void m5206_cpu_initfn(Object *obj)
 {
     M68kCPU *cpu = M68K_CPU(obj);
@@ -127,24 +147,34 @@ static void m68k_cpu_initfn(Object *obj)
     cpu_exec_init(env);
 }
 
+static const VMStateDescription vmstate_m68k_cpu = {
+    .name = "cpu",
+    .unmigratable = 1,
+};
+
 static void m68k_cpu_class_init(ObjectClass *c, void *data)
 {
     M68kCPUClass *mcc = M68K_CPU_CLASS(c);
     CPUClass *cc = CPU_CLASS(c);
+    DeviceClass *dc = DEVICE_CLASS(c);
 
     mcc->parent_reset = cc->reset;
     cc->reset = m68k_cpu_reset;
+
+    cc->class_by_name = m68k_cpu_class_by_name;
+    dc->vmsd = &vmstate_m68k_cpu;
 }
 
 static void register_cpu_type(const M68kCPUInfo *info)
 {
     TypeInfo type_info = {
-        .name = info->name,
         .parent = TYPE_M68K_CPU,
         .instance_init = info->instance_init,
     };
 
-    type_register_static(&type_info);
+    type_info.name = g_strdup_printf("%s-" TYPE_M68K_CPU, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
 }
 
 static const TypeInfo m68k_cpu_type_info = {
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index 780e2c9..adaf56c 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -26,9 +26,9 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define MAX_QREGS 32
 
@@ -103,9 +103,6 @@ typedef struct CPUM68KState {
     uint32_t rambar0;
     uint32_t cacr;
 
-    /* ??? remove this.  */
-    uint32_t t1;
-
     int pending_vector;
     int pending_level;
 
@@ -245,7 +242,7 @@ static inline void cpu_clone_regs(CPUM68KState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
@@ -264,7 +261,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUM68KState *env, TranslationBlock *tb)
 {
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index a5d0100..5ddcd70 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -19,17 +19,12 @@
  */
 
 #include "cpu.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 
 #include "helpers.h"
 
 #define SIGNBIT (1u << 31)
 
-typedef struct M68kCPUListState {
-    fprintf_function cpu_fprintf;
-    FILE *file;
-} M68kCPUListState;
-
 /* Sort alphabetically, except for "any". */
 static gint m68k_cpu_list_compare(gconstpointer a, gconstpointer b)
 {
@@ -39,9 +34,9 @@ static gint m68k_cpu_list_compare(gconstpointer a, gconstpointer b)
 
     name_a = object_class_get_name(class_a);
     name_b = object_class_get_name(class_b);
-    if (strcmp(name_a, "any") == 0) {
+    if (strcmp(name_a, "any-" TYPE_M68K_CPU) == 0) {
         return 1;
-    } else if (strcmp(name_b, "any") == 0) {
+    } else if (strcmp(name_b, "any-" TYPE_M68K_CPU) == 0) {
         return -1;
     } else {
         return strcasecmp(name_a, name_b);
@@ -51,15 +46,20 @@ static gint m68k_cpu_list_compare(gconstpointer a, gconstpointer b)
 static void m68k_cpu_list_entry(gpointer data, gpointer user_data)
 {
     ObjectClass *c = data;
-    M68kCPUListState *s = user_data;
+    CPUListState *s = user_data;
+    const char *typename;
+    char *name;
 
+    typename = object_class_get_name(c);
+    name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_M68K_CPU));
     (*s->cpu_fprintf)(s->file, "%s\n",
-                      object_class_get_name(c));
+                      name);
+    g_free(name);
 }
 
 void m68k_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    M68kCPUListState s = {
+    CPUListState s = {
         .file = f,
         .cpu_fprintf = cpu_fprintf,
     };
@@ -102,12 +102,14 @@ CPUM68KState *cpu_m68k_init(const char *cpu_model)
 {
     M68kCPU *cpu;
     CPUM68KState *env;
+    ObjectClass *oc;
     static int inited;
 
-    if (object_class_by_name(cpu_model) == NULL) {
+    oc = cpu_class_by_name(TYPE_M68K_CPU, cpu_model);
+    if (oc == NULL) {
         return NULL;
     }
-    cpu = M68K_CPU(object_new(cpu_model));
+    cpu = M68K_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
 
     if (!inited) {
diff --git a/target-m68k/helpers.h b/target-m68k/helpers.h
index 8112b44..2b02450 100644
--- a/target-m68k/helpers.h
+++ b/target-m68k/helpers.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_1(bitrev, i32, i32)
 DEF_HELPER_1(ff1, i32, i32)
@@ -51,4 +51,4 @@ DEF_HELPER_3(set_mac_extu, void, env, i32, i32)
 DEF_HELPER_2(flush_flags, void, env, i32)
 DEF_HELPER_2(raise_exception, void, env, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-m68k/m68k-semi.c b/target-m68k/m68k-semi.c
index 9f7a24c..239fadb 100644
--- a/target-m68k/m68k-semi.c
+++ b/target-m68k/m68k-semi.c
@@ -33,10 +33,10 @@
 #define SEMIHOSTING_HEAP_SIZE (128 * 1024 * 1024)
 #else
 #include "qemu-common.h"
-#include "gdbstub.h"
-#include "softmmu-semi.h"
+#include "exec/gdbstub.h"
+#include "exec/softmmu-semi.h"
 #endif
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define HOSTED_EXIT  0
 #define HOSTED_INIT_SIM 1
diff --git a/target-m68k/machine.c b/target-m68k/machine.c
deleted file mode 100644
index e69de29..0000000
--- a/target-m68k/machine.c
+++ /dev/null
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index aa00504..16df24c 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -34,21 +34,21 @@ void do_interrupt_m68k_hardirq(CPUM68KState *env)
 
 extern int semihosting_enabled;
 
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -56,19 +56,13 @@ extern int semihosting_enabled;
 void tlb_fill(CPUM68KState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_m68k_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index b13be48..3f1478c 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -19,9 +19,9 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #include "helpers.h"
 #define GEN_HELPER 1
@@ -61,7 +61,7 @@ static TCGv NULL_QREG;
 /* Used to distinguish stores from bad addressing modes.  */
 static TCGv store_dummy;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 void m68k_tcg_init(void)
 {
@@ -574,7 +574,7 @@ static inline TCGv gen_ea_once(CPUM68KState *env, DisasContext *s,
     return gen_ldst(s, opsize, tmp, val, what);
 }
 
-/* Generate code to load/store a value ito/from an EA.  If VAL > 0 this is
+/* Generate code to load/store a value from/into an EA.  If VAL > 0 this is
    a write otherwise it is a read (0 == sign extend, -1 == zero extend).
    ADDRP is non-null for readwrite operands.  */
 static TCGv gen_ea(CPUM68KState *env, DisasContext *s, uint16_t insn,
@@ -3019,11 +3019,11 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
@@ -3078,7 +3078,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = dc->pc - pc_start;
         tb->icount = num_insns;
@@ -3121,5 +3121,5 @@ void cpu_dump_state(CPUM68KState *env, FILE *f, fprintf_function cpu_fprintf,
 
 void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-microblaze/Makefile.objs b/target-microblaze/Makefile.objs
index afb87bc..985330e 100644
--- a/target-microblaze/Makefile.objs
+++ b/target-microblaze/Makefile.objs
@@ -1,2 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
-obj-$(CONFIG_SOFTMMU) += mmu.o machine.o
+obj-$(CONFIG_SOFTMMU) += mmu.o
diff --git a/target-microblaze/cpu-qom.h b/target-microblaze/cpu-qom.h
index 4b23303..f75549d 100644
--- a/target-microblaze/cpu-qom.h
+++ b/target-microblaze/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_MICROBLAZE_CPU_QOM_H
 #define QEMU_MICROBLAZE_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_MICROBLAZE_CPU "microblaze-cpu"
 
diff --git a/target-microblaze/cpu.c b/target-microblaze/cpu.c
index 34b3a9b..39230fd 100644
--- a/target-microblaze/cpu.c
+++ b/target-microblaze/cpu.c
@@ -22,6 +22,7 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
+#include "migration/vmstate.h"
 
 
 /* CPUClass::reset() */
@@ -32,7 +33,7 @@ static void mb_cpu_reset(CPUState *s)
     CPUMBState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -94,13 +95,21 @@ static void mb_cpu_initfn(Object *obj)
     set_float_rounding_mode(float_round_nearest_even, &env->fp_status);
 }
 
+static const VMStateDescription vmstate_mb_cpu = {
+    .name = "cpu",
+    .unmigratable = 1,
+};
+
 static void mb_cpu_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     CPUClass *cc = CPU_CLASS(oc);
     MicroBlazeCPUClass *mcc = MICROBLAZE_CPU_CLASS(oc);
 
     mcc->parent_reset = cc->reset;
     cc->reset = mb_cpu_reset;
+
+    dc->vmsd = &vmstate_mb_cpu;
 }
 
 static const TypeInfo mb_cpu_type_info = {
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 585bbd6..41480e7 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -26,8 +26,8 @@
 
 #define CPUArchState struct CPUMBState
 
-#include "cpu-defs.h"
-#include "softfloat.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 struct CPUMBState;
 typedef struct CPUMBState CPUMBState;
 #if !defined(CONFIG_USER_ONLY)
@@ -274,7 +274,6 @@ struct CPUMBState {
 
 MicroBlazeCPU *cpu_mb_init(const char *cpu_model);
 int cpu_mb_exec(CPUMBState *s);
-void cpu_mb_close(CPUMBState *s);
 void do_interrupt(CPUMBState *env);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -308,8 +307,6 @@ static inline CPUMBState *cpu_init(const char *cpu_model)
 #define cpu_gen_code cpu_mb_gen_code
 #define cpu_signal_handler cpu_mb_signal_handler
 
-#define CPU_SAVE_VERSION 1
-
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _nommu
 #define MMU_MODE1_SUFFIX _kernel
@@ -353,7 +350,7 @@ static inline int cpu_interrupts_enabled(CPUMBState *env)
     return env->sregs[SR_MSR] & MSR_IE;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline target_ulong cpu_get_pc(CPUMBState *env)
 {
@@ -381,7 +378,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUMBState *env, TranslationBlock *tb)
 {
diff --git a/target-microblaze/helper.c b/target-microblaze/helper.c
index efaa123..97aedc5 100644
--- a/target-microblaze/helper.c
+++ b/target-microblaze/helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #define D(x)
 #define DMMU(x)
@@ -198,7 +198,7 @@ void do_interrupt(CPUMBState *env)
             t = (env->sregs[SR_MSR] & (MSR_VM | MSR_UM)) << 1;
 
 #if 0
-#include "disas.h"
+#include "disas/disas.h"
 
 /* Useful instrumentation when debugging interrupt issues in either
    the models or in sw.  */
diff --git a/target-microblaze/helper.h b/target-microblaze/helper.h
index a667122..4e51429 100644
--- a/target-microblaze/helper.h
+++ b/target-microblaze/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_1(debug, void, env)
@@ -38,4 +38,4 @@ DEF_HELPER_2(stackprot, void, env, i32)
 DEF_HELPER_2(get, i32, i32, i32)
 DEF_HELPER_3(put, void, i32, i32, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-microblaze/machine.c b/target-microblaze/machine.c
deleted file mode 100644
index 1be1c35..0000000
--- a/target-microblaze/machine.c
+++ /dev/null
@@ -1,11 +0,0 @@
-#include "hw/hw.h"
-#include "hw/boards.h"
-
-void cpu_save(QEMUFile *f, void *opaque)
-{
-}
-
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-    return 0;
-}
diff --git a/target-microblaze/op_helper.c b/target-microblaze/op_helper.c
index 210296b..1c62f3c 100644
--- a/target-microblaze/op_helper.c
+++ b/target-microblaze/op_helper.c
@@ -21,22 +21,22 @@
 #include <assert.h>
 #include "cpu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #define D(x)
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -44,19 +44,13 @@
 void tlb_fill(CPUMBState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_mb_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index cce4494..58ce712 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 #include "helper.h"
 #include "microblaze-decode.h"
@@ -50,7 +50,7 @@ static TCGv env_btaken;
 static TCGv env_btarget;
 static TCGv env_iflags;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
@@ -1788,11 +1788,11 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-                        gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+                        tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         /* Pretty disas.  */
@@ -1902,7 +1902,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = dc->pc - pc_start;
                 tb->icount = num_insns;
@@ -2014,5 +2014,5 @@ MicroBlazeCPU *cpu_mb_init(const char *cpu_model)
 
 void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->sregs[SR_PC] = gen_opc_pc[pc_pos];
+    env->sregs[SR_PC] = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-mips/cpu-qom.h b/target-mips/cpu-qom.h
index 6e22371..2a4b812 100644
--- a/target-mips/cpu-qom.h
+++ b/target-mips/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_MIPS_CPU_QOM_H
 #define QEMU_MIPS_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #ifdef TARGET_MIPS64
 #define TYPE_MIPS_CPU "mips64-cpu"
diff --git a/target-mips/cpu.c b/target-mips/cpu.c
index 0044062..10ff46d 100644
--- a/target-mips/cpu.c
+++ b/target-mips/cpu.c
@@ -29,8 +29,16 @@ static void mips_cpu_reset(CPUState *s)
     MIPSCPUClass *mcc = MIPS_CPU_GET_CLASS(cpu);
     CPUMIPSState *env = &cpu->env;
 
+    if (qemu_loglevel_mask(CPU_LOG_RESET)) {
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
+        log_cpu_state(env, 0);
+    }
+
     mcc->parent_reset(s);
 
+    memset(env, 0, offsetof(CPUMIPSState, breakpoints));
+    tlb_flush(env, 1);
+
     cpu_state_reset(env);
 }
 
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index aebb2d5..5963d62 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -12,8 +12,8 @@
 #include "config.h"
 #include "qemu-common.h"
 #include "mips-defs.h"
-#include "cpu-defs.h"
-#include "softfloat.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 
 struct CPUMIPSState;
 
@@ -560,7 +560,7 @@ static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
     return r;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /* Memory access type :
  * may be needed for precise access rights control and precise exceptions.
@@ -738,7 +738,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return has_work;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUMIPSState *env, TranslationBlock *tb)
 {
@@ -751,7 +751,7 @@ static inline void compute_hflags(CPUMIPSState *env)
 {
     env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
                      MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
-                     MIPS_HFLAG_UX);
+                     MIPS_HFLAG_UX | MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2);
     if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
         !(env->CP0_Status & (1 << CP0St_ERL)) &&
         !(env->hflags & MIPS_HFLAG_DM)) {
diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
index e7949c2..96cb044 100644
--- a/target-mips/dsp_helper.c
+++ b/target-mips/dsp_helper.c
@@ -2,7 +2,7 @@
  * MIPS ASE DSP Instruction emulation helpers for QEMU.
  *
  * Copyright (c) 2012  Jia Liu <proljc@gmail.com>
- *                     Dongxue Zhang <elat.era@gmail.com>
+ *                     Dongxue Zhang <elta.era@gmail.com>
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
@@ -20,6 +20,28 @@
 #include "cpu.h"
 #include "helper.h"
 
+/* As the byte ordering doesn't matter, i.e. all columns are treated
+   identically, these unions can be used directly.  */
+typedef union {
+    uint8_t  ub[4];
+    int8_t   sb[4];
+    uint16_t uh[2];
+    int16_t  sh[2];
+    uint32_t uw[1];
+    int32_t  sw[1];
+} DSP32Value;
+
+typedef union {
+    uint8_t  ub[8];
+    int8_t   sb[8];
+    uint16_t uh[4];
+    int16_t  sh[4];
+    uint32_t uw[2];
+    int32_t  sw[2];
+    uint64_t ul[1];
+    int64_t  sl[1];
+} DSP64Value;
+
 /*** MIPS DSP internal functions begin ***/
 #define MIPSDSP_ABS(x) (((x) >= 0) ? x : -x)
 #define MIPSDSP_OVERFLOW(a, b, c, d) (!(!((a ^ b ^ -1) & (a ^ c) & d)))
@@ -484,35 +506,6 @@ static inline uint8_t mipsdsp_rrshift1_sub_u8(uint8_t a, uint8_t b)
     return (temp >> 1) & 0x00FF;
 }
 
-static inline int64_t mipsdsp_rashift_short_acc(int32_t ac,
-                                                int32_t shift,
-                                                CPUMIPSState *env)
-{
-    int32_t sign, temp31;
-    int64_t temp, acc;
-
-    sign = (env->active_tc.HI[ac] >> 31) & 0x01;
-    acc = ((int64_t)env->active_tc.HI[ac] << 32) |
-          ((int64_t)env->active_tc.LO[ac] & 0xFFFFFFFF);
-    if (shift == 0) {
-        temp = acc;
-    } else {
-        if (sign == 0) {
-            temp = (((int64_t)0x01 << (32 - shift + 1)) - 1) & (acc >> shift);
-        } else {
-            temp = ((((int64_t)0x01 << (shift + 1)) - 1) << (32 - shift)) |
-                   (acc >> shift);
-        }
-    }
-
-    temp31 = (temp >> 31) & 0x01;
-    if (sign != temp31) {
-        set_DSPControl_overflow_flag(1, 23, env);
-    }
-
-    return temp;
-}
-
 /*  128 bits long. p[0] is LO, p[1] is HI. */
 static inline void mipsdsp_rndrashift_short_acc(int64_t *p,
                                                 int32_t ac,
@@ -1085,7 +1078,6 @@ static inline int32_t mipsdsp_cmpu_lt(uint32_t a, uint32_t b)
         b = num & MIPSDSP_LO;               \
     } while (0)
 
-#define MIPSDSP_RETURN32(a)             ((target_long)(int32_t)a)
 #define MIPSDSP_RETURN32_8(a, b, c, d)  ((target_long)(int32_t) \
                                          (((uint32_t)a << 24) | \
                                          (((uint32_t)b << 16) | \
@@ -1118,287 +1110,169 @@ static inline int32_t mipsdsp_cmpu_lt(uint32_t a, uint32_t b)
 #endif
 
 /** DSP Arithmetic Sub-class insns **/
-#define ARITH_PH(name, func)                                      \
-target_ulong helper_##name##_ph(target_ulong rs, target_ulong rt) \
-{                                                                 \
-    uint16_t  rsh, rsl, rth, rtl, temph, templ;                   \
-                                                                  \
-    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                             \
-    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
-                                                                  \
-    temph = mipsdsp_##func(rsh, rth);                             \
-    templ = mipsdsp_##func(rsl, rtl);                             \
-                                                                  \
-    return MIPSDSP_RETURN32_16(temph, templ);                     \
-}
-
-#define ARITH_PH_ENV(name, func)                                  \
-target_ulong helper_##name##_ph(target_ulong rs, target_ulong rt, \
-                                CPUMIPSState *env)                \
-{                                                                 \
-    uint16_t  rsh, rsl, rth, rtl, temph, templ;                   \
-                                                                  \
-    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                             \
-    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
-                                                                  \
-    temph = mipsdsp_##func(rsh, rth, env);                        \
-    templ = mipsdsp_##func(rsl, rtl, env);                        \
-                                                                  \
-    return MIPSDSP_RETURN32_16(temph, templ);                     \
-}
-
-
-ARITH_PH_ENV(addq, add_i16);
-ARITH_PH_ENV(addq_s, sat_add_i16);
-ARITH_PH_ENV(addu, add_u16);
-ARITH_PH_ENV(addu_s, sat_add_u16);
-
-ARITH_PH(addqh, rshift1_add_q16);
-ARITH_PH(addqh_r, rrshift1_add_q16);
-
-ARITH_PH_ENV(subq, sub_i16);
-ARITH_PH_ENV(subq_s, sat16_sub);
-ARITH_PH_ENV(subu, sub_u16_u16);
-ARITH_PH_ENV(subu_s, satu16_sub_u16_u16);
-
-ARITH_PH(subqh, rshift1_sub_q16);
-ARITH_PH(subqh_r, rrshift1_sub_q16);
-
-#undef ARITH_PH
-#undef ARITH_PH_ENV
-
-#ifdef TARGET_MIPS64
-#define ARITH_QH_ENV(name, func) \
-target_ulong helper_##name##_qh(target_ulong rs, target_ulong rt, \
-                                CPUMIPSState *env)           \
-{                                                            \
-    uint16_t rs3, rs2, rs1, rs0;                             \
-    uint16_t rt3, rt2, rt1, rt0;                             \
-    uint16_t tempD, tempC, tempB, tempA;                     \
-                                                             \
-    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);              \
-    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);              \
-                                                             \
-    tempD = mipsdsp_##func(rs3, rt3, env);                   \
-    tempC = mipsdsp_##func(rs2, rt2, env);                   \
-    tempB = mipsdsp_##func(rs1, rt1, env);                   \
-    tempA = mipsdsp_##func(rs0, rt0, env);                   \
-                                                             \
-    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
-}
-
-ARITH_QH_ENV(addq, add_i16);
-ARITH_QH_ENV(addq_s, sat_add_i16);
-ARITH_QH_ENV(addu, add_u16);
-ARITH_QH_ENV(addu_s, sat_add_u16);
-
-ARITH_QH_ENV(subq, sub_i16);
-ARITH_QH_ENV(subq_s, sat16_sub);
-ARITH_QH_ENV(subu, sub_u16_u16);
-ARITH_QH_ENV(subu_s, satu16_sub_u16_u16);
-
-#undef ARITH_QH_ENV
-
-#endif
-
-#define ARITH_W(name, func) \
-target_ulong helper_##name##_w(target_ulong rs, target_ulong rt) \
-{                                                                \
-    uint32_t rd;                                                 \
-    rd = mipsdsp_##func(rs, rt);                                 \
-    return MIPSDSP_RETURN32(rd);                                 \
-}
-
-#define ARITH_W_ENV(name, func) \
-target_ulong helper_##name##_w(target_ulong rs, target_ulong rt, \
-                               CPUMIPSState *env)                \
-{                                                                \
-    uint32_t rd;                                                 \
-    rd = mipsdsp_##func(rs, rt, env);                            \
-    return MIPSDSP_RETURN32(rd);                                 \
-}
-
-ARITH_W_ENV(addq_s, sat_add_i32);
-
-ARITH_W(addqh, rshift1_add_q32);
-ARITH_W(addqh_r, rrshift1_add_q32);
-
-ARITH_W_ENV(subq_s, sat32_sub);
-
-ARITH_W(subqh, rshift1_sub_q32);
-ARITH_W(subqh_r, rrshift1_sub_q32);
-
-#undef ARITH_W
-#undef ARITH_W_ENV
-
-target_ulong helper_absq_s_w(target_ulong rt, CPUMIPSState *env)
-{
-    uint32_t rd;
-
-    rd = mipsdsp_sat_abs32(rt, env);
-
-    return (target_ulong)rd;
-}
-
+#define MIPSDSP32_UNOP_ENV(name, func, element)                            \
+target_ulong helper_##name(target_ulong rt, CPUMIPSState *env)             \
+{                                                                          \
+    DSP32Value dt;                                                         \
+    unsigned int i, n;                                                     \
+                                                                           \
+    n = sizeof(DSP32Value) / sizeof(dt.element[0]);                        \
+    dt.sw[0] = rt;                                                         \
+                                                                           \
+    for (i = 0; i < n; i++) {                                              \
+        dt.element[i] = mipsdsp_##func(dt.element[i], env);                \
+    }                                                                      \
+                                                                           \
+    return (target_long)dt.sw[0];                                          \
+}
+MIPSDSP32_UNOP_ENV(absq_s_ph, sat_abs16, sh)
+MIPSDSP32_UNOP_ENV(absq_s_qb, sat_abs8, sb)
+MIPSDSP32_UNOP_ENV(absq_s_w, sat_abs32, sw)
+#undef MIPSDSP32_UNOP_ENV
 
 #if defined(TARGET_MIPS64)
-
-#define ARITH_PW_ENV(name, func) \
-target_ulong helper_##name##_pw(target_ulong rs, target_ulong rt, \
-                                CPUMIPSState *env)                \
-{                                                                 \
-    uint32_t rs1, rs0;                                            \
-    uint32_t rt1, rt0;                                            \
-    uint32_t tempB, tempA;                                        \
-                                                                  \
-    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                             \
-    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
-                                                                  \
-    tempB = mipsdsp_##func(rs1, rt1, env);                        \
-    tempA = mipsdsp_##func(rs0, rt0, env);                        \
-                                                                  \
-    return MIPSDSP_RETURN64_32(tempB, tempA);                     \
-}
-
-ARITH_PW_ENV(addq, add_i32);
-ARITH_PW_ENV(addq_s, sat_add_i32);
-ARITH_PW_ENV(subq, sub32);
-ARITH_PW_ENV(subq_s, sat32_sub);
-
-#undef ARITH_PW_ENV
-
+#define MIPSDSP64_UNOP_ENV(name, func, element)                            \
+target_ulong helper_##name(target_ulong rt, CPUMIPSState *env)             \
+{                                                                          \
+    DSP64Value dt;                                                         \
+    unsigned int i, n;                                                     \
+                                                                           \
+    n = sizeof(DSP64Value) / sizeof(dt.element[0]);                        \
+    dt.sl[0] = rt;                                                         \
+                                                                           \
+    for (i = 0; i < n; i++) {                                              \
+        dt.element[i] = mipsdsp_##func(dt.element[i], env);                \
+    }                                                                      \
+                                                                           \
+    return dt.sl[0];                                                       \
+}
+MIPSDSP64_UNOP_ENV(absq_s_ob, sat_abs8, sb)
+MIPSDSP64_UNOP_ENV(absq_s_qh, sat_abs16, sh)
+MIPSDSP64_UNOP_ENV(absq_s_pw, sat_abs32, sw)
+#undef MIPSDSP64_UNOP_ENV
 #endif
 
-#define ARITH_QB(name, func) \
-target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt) \
-{                                                                 \
-    uint8_t  rs0, rs1, rs2, rs3;                                  \
-    uint8_t  rt0, rt1, rt2, rt3;                                  \
-    uint8_t  temp0, temp1, temp2, temp3;                          \
-                                                                  \
-    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                    \
-    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                    \
-                                                                  \
-    temp0 = mipsdsp_##func(rs0, rt0);                             \
-    temp1 = mipsdsp_##func(rs1, rt1);                             \
-    temp2 = mipsdsp_##func(rs2, rt2);                             \
-    temp3 = mipsdsp_##func(rs3, rt3);                             \
-                                                                  \
-    return MIPSDSP_RETURN32_8(temp3, temp2, temp1, temp0);        \
-}
+#define MIPSDSP32_BINOP(name, func, element)                               \
+target_ulong helper_##name(target_ulong rs, target_ulong rt)               \
+{                                                                          \
+    DSP32Value ds, dt;                                                     \
+    unsigned int i, n;                                                     \
+                                                                           \
+    n = sizeof(DSP32Value) / sizeof(ds.element[0]);                        \
+    ds.sw[0] = rs;                                                         \
+    dt.sw[0] = rt;                                                         \
+                                                                           \
+    for (i = 0; i < n; i++) {                                              \
+        ds.element[i] = mipsdsp_##func(ds.element[i], dt.element[i]);      \
+    }                                                                      \
+                                                                           \
+    return (target_long)ds.sw[0];                                          \
+}
+MIPSDSP32_BINOP(addqh_ph, rshift1_add_q16, sh);
+MIPSDSP32_BINOP(addqh_r_ph, rrshift1_add_q16, sh);
+MIPSDSP32_BINOP(addqh_r_w, rrshift1_add_q32, sw);
+MIPSDSP32_BINOP(addqh_w, rshift1_add_q32, sw);
+MIPSDSP32_BINOP(adduh_qb, rshift1_add_u8, ub);
+MIPSDSP32_BINOP(adduh_r_qb, rrshift1_add_u8, ub);
+MIPSDSP32_BINOP(subqh_ph, rshift1_sub_q16, sh);
+MIPSDSP32_BINOP(subqh_r_ph, rrshift1_sub_q16, sh);
+MIPSDSP32_BINOP(subqh_r_w, rrshift1_sub_q32, sw);
+MIPSDSP32_BINOP(subqh_w, rshift1_sub_q32, sw);
+#undef MIPSDSP32_BINOP
+
+#define MIPSDSP32_BINOP_ENV(name, func, element)                           \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,               \
+                           CPUMIPSState *env)                              \
+{                                                                          \
+    DSP32Value ds, dt;                                                     \
+    unsigned int i, n;                                                     \
+                                                                           \
+    n = sizeof(DSP32Value) / sizeof(ds.element[0]);                        \
+    ds.sw[0] = rs;                                                         \
+    dt.sw[0] = rt;                                                         \
+                                                                           \
+    for (i = 0 ; i < n ; i++) {                                            \
+        ds.element[i] = mipsdsp_##func(ds.element[i], dt.element[i], env); \
+    }                                                                      \
+                                                                           \
+    return (target_long)ds.sw[0];                                          \
+}
+MIPSDSP32_BINOP_ENV(addq_ph, add_i16, sh)
+MIPSDSP32_BINOP_ENV(addq_s_ph, sat_add_i16, sh)
+MIPSDSP32_BINOP_ENV(addq_s_w, sat_add_i32, sw);
+MIPSDSP32_BINOP_ENV(addu_ph, add_u16, sh)
+MIPSDSP32_BINOP_ENV(addu_qb, add_u8, ub);
+MIPSDSP32_BINOP_ENV(addu_s_ph, sat_add_u16, sh)
+MIPSDSP32_BINOP_ENV(addu_s_qb, sat_add_u8, ub);
+MIPSDSP32_BINOP_ENV(subq_ph, sub_i16, sh);
+MIPSDSP32_BINOP_ENV(subq_s_ph, sat16_sub, sh);
+MIPSDSP32_BINOP_ENV(subq_s_w, sat32_sub, sw);
+MIPSDSP32_BINOP_ENV(subu_ph, sub_u16_u16, sh);
+MIPSDSP32_BINOP_ENV(subu_qb, sub_u8, ub);
+MIPSDSP32_BINOP_ENV(subu_s_ph, satu16_sub_u16_u16, sh);
+MIPSDSP32_BINOP_ENV(subu_s_qb, satu8_sub, ub);
+#undef MIPSDSP32_BINOP_ENV
 
-#define ARITH_QB_ENV(name, func) \
-target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt, \
-                                CPUMIPSState *env)          \
-{                                                           \
-    uint8_t  rs0, rs1, rs2, rs3;                            \
-    uint8_t  rt0, rt1, rt2, rt3;                            \
-    uint8_t  temp0, temp1, temp2, temp3;                    \
-                                                            \
-    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);              \
-    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);              \
-                                                            \
-    temp0 = mipsdsp_##func(rs0, rt0, env);                  \
-    temp1 = mipsdsp_##func(rs1, rt1, env);                  \
-    temp2 = mipsdsp_##func(rs2, rt2, env);                  \
-    temp3 = mipsdsp_##func(rs3, rt3, env);                  \
-                                                            \
-    return MIPSDSP_RETURN32_8(temp3, temp2, temp1, temp0);  \
-}
-
-ARITH_QB(adduh, rshift1_add_u8);
-ARITH_QB(adduh_r, rrshift1_add_u8);
-
-ARITH_QB_ENV(addu, add_u8);
-ARITH_QB_ENV(addu_s, sat_add_u8);
-
-#undef ADDU_QB
-#undef ADDU_QB_ENV
-
-#if defined(TARGET_MIPS64)
-#define ARITH_OB(name, func) \
-target_ulong helper_##name##_ob(target_ulong rs, target_ulong rt) \
-{                                                                 \
-    int i;                                                        \
-    uint8_t rs_t[8], rt_t[8];                                     \
-    uint8_t temp[8];                                              \
-    uint64_t result;                                              \
-                                                                  \
-    result = 0;                                                   \
-                                                                  \
-    for (i = 0; i < 8; i++) {                                     \
-        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;                   \
-        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                   \
-        temp[i] = mipsdsp_##func(rs_t[i], rt_t[i]);               \
-        result |= (uint64_t)temp[i] << (8 * i);                   \
-    }                                                             \
-                                                                  \
-    return result;                                                \
-}
-
-#define ARITH_OB_ENV(name, func) \
-target_ulong helper_##name##_ob(target_ulong rs, target_ulong rt, \
-                                CPUMIPSState *env)                \
-{                                                                 \
-    int i;                                                        \
-    uint8_t rs_t[8], rt_t[8];                                     \
-    uint8_t temp[8];                                              \
-    uint64_t result;                                              \
-                                                                  \
-    result = 0;                                                   \
-                                                                  \
-    for (i = 0; i < 8; i++) {                                     \
-        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;                   \
-        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                   \
-        temp[i] = mipsdsp_##func(rs_t[i], rt_t[i], env);          \
-        result |= (uint64_t)temp[i] << (8 * i);                   \
-    }                                                             \
-                                                                  \
-    return result;                                                \
-}
-
-ARITH_OB_ENV(addu, add_u8);
-ARITH_OB_ENV(addu_s, sat_add_u8);
-
-ARITH_OB(adduh, rshift1_add_u8);
-ARITH_OB(adduh_r, rrshift1_add_u8);
-
-ARITH_OB_ENV(subu, sub_u8);
-ARITH_OB_ENV(subu_s, satu8_sub);
-
-ARITH_OB(subuh, rshift1_sub_u8);
-ARITH_OB(subuh_r, rrshift1_sub_u8);
-
-#undef ARITH_OB
-#undef ARITH_OB_ENV
+#ifdef TARGET_MIPS64
+#define MIPSDSP64_BINOP(name, func, element)                               \
+target_ulong helper_##name(target_ulong rs, target_ulong rt)               \
+{                                                                          \
+    DSP64Value ds, dt;                                                     \
+    unsigned int i, n;                                                     \
+                                                                           \
+    n = sizeof(DSP64Value) / sizeof(ds.element[0]);                        \
+    ds.sl[0] = rs;                                                         \
+    dt.sl[0] = rt;                                                         \
+                                                                           \
+    for (i = 0 ; i < n ; i++) {                                            \
+        ds.element[i] = mipsdsp_##func(ds.element[i], dt.element[i]);      \
+    }                                                                      \
+                                                                           \
+    return ds.sl[0];                                                       \
+}
+MIPSDSP64_BINOP(adduh_ob, rshift1_add_u8, ub);
+MIPSDSP64_BINOP(adduh_r_ob, rrshift1_add_u8, ub);
+MIPSDSP64_BINOP(subuh_ob, rshift1_sub_u8, ub);
+MIPSDSP64_BINOP(subuh_r_ob, rrshift1_sub_u8, ub);
+#undef MIPSDSP64_BINOP
+
+#define MIPSDSP64_BINOP_ENV(name, func, element)                           \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,               \
+                           CPUMIPSState *env)                              \
+{                                                                          \
+    DSP64Value ds, dt;                                                     \
+    unsigned int i, n;                                                     \
+                                                                           \
+    n = sizeof(DSP64Value) / sizeof(ds.element[0]);                        \
+    ds.sl[0] = rs;                                                         \
+    dt.sl[0] = rt;                                                         \
+                                                                           \
+    for (i = 0 ; i < n ; i++) {                                            \
+        ds.element[i] = mipsdsp_##func(ds.element[i], dt.element[i], env); \
+    }                                                                      \
+                                                                           \
+    return ds.sl[0];                                                       \
+}
+MIPSDSP64_BINOP_ENV(addq_pw, add_i32, sw);
+MIPSDSP64_BINOP_ENV(addq_qh, add_i16, sh);
+MIPSDSP64_BINOP_ENV(addq_s_pw, sat_add_i32, sw);
+MIPSDSP64_BINOP_ENV(addq_s_qh, sat_add_i16, sh);
+MIPSDSP64_BINOP_ENV(addu_ob, add_u8, uh);
+MIPSDSP64_BINOP_ENV(addu_qh, add_u16, uh);
+MIPSDSP64_BINOP_ENV(addu_s_ob, sat_add_u8, uh);
+MIPSDSP64_BINOP_ENV(addu_s_qh, sat_add_u16, uh);
+MIPSDSP64_BINOP_ENV(subq_pw, sub32, sw);
+MIPSDSP64_BINOP_ENV(subq_qh, sub_i16, sh);
+MIPSDSP64_BINOP_ENV(subq_s_pw, sat32_sub, sw);
+MIPSDSP64_BINOP_ENV(subq_s_qh, sat16_sub, sh);
+MIPSDSP64_BINOP_ENV(subu_ob, sub_u8, uh);
+MIPSDSP64_BINOP_ENV(subu_qh, sub_u16_u16, uh);
+MIPSDSP64_BINOP_ENV(subu_s_ob, satu8_sub, uh);
+MIPSDSP64_BINOP_ENV(subu_s_qh, satu16_sub_u16_u16, uh);
+#undef MIPSDSP64_BINOP_ENV
 
 #endif
 
-#define SUBU_QB(name, func) \
-target_ulong helper_##name##_qb(target_ulong rs,               \
-                                target_ulong rt,               \
-                                CPUMIPSState *env)             \
-{                                                              \
-    uint8_t rs3, rs2, rs1, rs0;                                \
-    uint8_t rt3, rt2, rt1, rt0;                                \
-    uint8_t tempD, tempC, tempB, tempA;                        \
-                                                               \
-    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                 \
-    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                 \
-                                                               \
-    tempD = mipsdsp_##func(rs3, rt3, env);                     \
-    tempC = mipsdsp_##func(rs2, rt2, env);                     \
-    tempB = mipsdsp_##func(rs1, rt1, env);                     \
-    tempA = mipsdsp_##func(rs0, rt0, env);                     \
-                                                               \
-    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);     \
-}
-
-SUBU_QB(subu, sub_u8);
-SUBU_QB(subu_s, satu8_sub);
-
-#undef SUBU_QB
-
 #define SUBUH_QB(name, var) \
 target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt) \
 {                                                                 \
@@ -1478,103 +1352,29 @@ target_ulong helper_modsub(target_ulong rs, target_ulong rt)
 
 target_ulong helper_raddu_w_qb(target_ulong rs)
 {
-    uint8_t  rs3, rs2, rs1, rs0;
-    uint16_t temp;
-
-    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);
-
-    temp = (uint16_t)rs3 + (uint16_t)rs2 + (uint16_t)rs1 + (uint16_t)rs0;
-
-    return (target_ulong)temp;
-}
-
-#if defined(TARGET_MIPS64)
-target_ulong helper_raddu_l_ob(target_ulong rs)
-{
-    int i;
-    uint16_t rs_t[8];
-    uint64_t temp;
+    target_ulong ret = 0;
+    DSP32Value ds;
+    unsigned int i;
 
-    temp = 0;
-
-    for (i = 0; i < 8; i++) {
-        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;
-        temp += (uint64_t)rs_t[i];
+    ds.uw[0] = rs;
+    for (i = 0; i < 4; i++) {
+        ret += ds.ub[i];
     }
-
-    return temp;
-}
-#endif
-
-target_ulong helper_absq_s_qb(target_ulong rt, CPUMIPSState *env)
-{
-    uint8_t tempD, tempC, tempB, tempA;
-
-    MIPSDSP_SPLIT32_8(rt, tempD, tempC, tempB, tempA);
-
-    tempD = mipsdsp_sat_abs8(tempD, env);
-    tempC = mipsdsp_sat_abs8(tempC, env);
-    tempB = mipsdsp_sat_abs8(tempB, env);
-    tempA = mipsdsp_sat_abs8(tempA, env);
-
-    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);
-}
-
-target_ulong helper_absq_s_ph(target_ulong rt, CPUMIPSState *env)
-{
-    uint16_t tempB, tempA;
-
-    MIPSDSP_SPLIT32_16(rt, tempB, tempA);
-
-    tempB = mipsdsp_sat_abs16 (tempB, env);
-    tempA = mipsdsp_sat_abs16 (tempA, env);
-
-    return MIPSDSP_RETURN32_16(tempB, tempA);
+    return ret;
 }
 
 #if defined(TARGET_MIPS64)
-target_ulong helper_absq_s_ob(target_ulong rt, CPUMIPSState *env)
+target_ulong helper_raddu_l_ob(target_ulong rs)
 {
-    int i;
-    int8_t temp[8];
-    uint64_t result;
-
-    for (i = 0; i < 8; i++) {
-        temp[i] = (rt >> (8 * i)) & MIPSDSP_Q0;
-        temp[i] = mipsdsp_sat_abs8(temp[i], env);
-    }
+    target_ulong ret = 0;
+    DSP64Value ds;
+    unsigned int i;
 
+    ds.ul[0] = rs;
     for (i = 0; i < 8; i++) {
-        result = (uint64_t)(uint8_t)temp[i] << (8 * i);
+        ret += ds.ub[i];
     }
-
-    return result;
-}
-
-target_ulong helper_absq_s_qh(target_ulong rt, CPUMIPSState *env)
-{
-    int16_t tempD, tempC, tempB, tempA;
-
-    MIPSDSP_SPLIT64_16(rt, tempD, tempC, tempB, tempA);
-
-    tempD = mipsdsp_sat_abs16(tempD, env);
-    tempC = mipsdsp_sat_abs16(tempC, env);
-    tempB = mipsdsp_sat_abs16(tempB, env);
-    tempA = mipsdsp_sat_abs16(tempA, env);
-
-    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
-}
-
-target_ulong helper_absq_s_pw(target_ulong rt, CPUMIPSState *env)
-{
-    int32_t tempB, tempA;
-
-    MIPSDSP_SPLIT64_32(rt, tempB, tempA);
-
-    tempB = mipsdsp_sat_abs32(tempB, env);
-    tempA = mipsdsp_sat_abs32(tempA, env);
-
-    return MIPSDSP_RETURN64_32(tempB, tempA);
+    return ret;
 }
 #endif
 
@@ -2502,7 +2302,7 @@ DP_OB(dpsu_h_obr, 0, 24, 16, 8, 0, 24, 16, 8, 0);
 void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,              \
                    CPUMIPSState *env)                                          \
 {                                                                              \
-    uint16_t rsB, rsA, rtB, rtA;                                               \
+    int16_t rsB, rsA, rtB, rtA;                                                \
     int32_t  tempA, tempB;                                                     \
     int64_t  acc;                                                              \
                                                                                \
@@ -3152,7 +2952,7 @@ target_ulong helper_##name(CPUMIPSState *env, target_ulong rs,  \
                                                                 \
     filter = ((int32_t)0x01 << size) - 1;                       \
     filter = filter << pos;                                     \
-    temprs = rs & filter;                                       \
+    temprs = (rs << pos) & filter;                              \
     temprt = rt & ~filter;                                      \
     temp = temprs | temprt;                                     \
                                                                 \
@@ -3311,73 +3111,6 @@ PICK_INSN(pick_pw, 2, MIPSDSP_LLO, 32, 0);
 #endif
 #undef PICK_INSN
 
-#define APPEND_INSN(name, ret_32) \
-target_ulong helper_##name(target_ulong rt, target_ulong rs, uint32_t sa) \
-{                                                                         \
-    target_ulong temp;                                                    \
-                                                                          \
-    if (ret_32) {                                                         \
-        temp = ((rt & MIPSDSP_LLO) << sa) |                               \
-               ((rs & MIPSDSP_LLO) & ((0x01 << sa) - 1));                 \
-        temp = (target_long)(int32_t)(temp & MIPSDSP_LLO);                \
-    } else {                                                              \
-        temp = (rt << sa) | (rs & ((0x01 << sa) - 1));                    \
-    }                                                                     \
-                                                                          \
-    return temp;                                                          \
-}
-
-APPEND_INSN(append, 1);
-#ifdef TARGET_MIPS64
-APPEND_INSN(dappend, 0);
-#endif
-#undef APPEND_INSN
-
-#define PREPEND_INSN(name, or_val, ret_32)                    \
-target_ulong helper_##name(target_ulong rs, target_ulong rt,  \
-                           uint32_t sa)                       \
-{                                                             \
-    sa |= or_val;                                             \
-                                                              \
-    if (1) {                                                  \
-        return (target_long)(int32_t)(uint32_t)               \
-            (((rs & MIPSDSP_LLO) << (32 - sa)) |              \
-             ((rt & MIPSDSP_LLO) >> sa));                     \
-    } else {                                                  \
-        return (rs << (64 - sa)) | (rt >> sa);                \
-    }                                                         \
-}
-
-PREPEND_INSN(prepend, 0, 1);
-#ifdef TARGET_MIPS64
-PREPEND_INSN(prependw, 0, 0);
-PREPEND_INSN(prependd, 0x20, 0);
-#endif
-#undef PREPEND_INSN
-
-#define BALIGN_INSN(name, filter, ret32) \
-target_ulong helper_##name(target_ulong rs, target_ulong rt, uint32_t bp) \
-{                                                                         \
-    bp = bp & 0x03;                                                       \
-                                                                          \
-    if ((bp & 1) == 0) {                                                  \
-        return rt;                                                        \
-    } else {                                                              \
-        if (ret32) {                                                      \
-            return (target_long)(int32_t)((rt << (8 * bp)) |              \
-                                          (rs >> (8 * (4 - bp))));        \
-        } else {                                                          \
-            return (rt << (8 * bp)) | (rs >> (8 * (8 - bp)));             \
-        }                                                                 \
-    }                                                                     \
-}
-
-BALIGN_INSN(balign, 0x03, 1);
-#if defined(TARGET_MIPS64)
-BALIGN_INSN(dbalign, 0x07, 0);
-#endif
-#undef BALIGN_INSN
-
 target_ulong helper_packrl_ph(target_ulong rs, target_ulong rt)
 {
     uint32_t rsl, rth;
@@ -3407,7 +3140,7 @@ target_ulong helper_extr_w(target_ulong ac, target_ulong shift,
     int32_t tempI;
     int64_t tempDL[2];
 
-    shift = shift & 0x0F;
+    shift = shift & 0x1F;
 
     mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
     if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
@@ -3435,7 +3168,7 @@ target_ulong helper_extr_r_w(target_ulong ac, target_ulong shift,
 {
     int64_t tempDL[2];
 
-    shift = shift & 0x0F;
+    shift = shift & 0x1F;
 
     mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
     if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
@@ -3462,7 +3195,7 @@ target_ulong helper_extr_rs_w(target_ulong ac, target_ulong shift,
     int32_t tempI, temp64;
     int64_t tempDL[2];
 
-    shift = shift & 0x0F;
+    shift = shift & 0x1F;
 
     mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
     if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
@@ -3645,11 +3378,15 @@ target_ulong helper_dextr_rs_l(target_ulong ac, target_ulong shift,
 target_ulong helper_extr_s_h(target_ulong ac, target_ulong shift,
                              CPUMIPSState *env)
 {
-    int64_t temp;
+    int64_t temp, acc;
+
+    shift = shift & 0x1F;
+
+    acc = ((int64_t)env->active_tc.HI[ac] << 32) |
+          ((int64_t)env->active_tc.LO[ac] & 0xFFFFFFFF);
 
-    shift = shift & 0x0F;
+    temp = acc >> shift;
 
-    temp = mipsdsp_rashift_short_acc(ac, shift, env);
     if (temp > (int64_t)0x7FFF) {
         temp = 0x00007FFF;
         set_DSPControl_overflow_flag(1, 23, env);
@@ -3814,17 +3551,18 @@ void helper_shilo(target_ulong ac, target_ulong rs, CPUMIPSState *env)
 
     rs5_0 = rs & 0x3F;
     rs5_0 = (int8_t)(rs5_0 << 2) >> 2;
-    rs5_0 = MIPSDSP_ABS(rs5_0);
+
+    if (unlikely(rs5_0 == 0)) {
+        return;
+    }
+
     acc   = (((uint64_t)env->active_tc.HI[ac] << 32) & MIPSDSP_LHI) |
             ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
-    if (rs5_0 == 0) {
-        temp = acc;
+
+    if (rs5_0 > 0) {
+        temp = acc >> rs5_0;
     } else {
-        if (rs5_0 > 0) {
-            temp = acc >> rs5_0;
-        } else {
-            temp = acc << rs5_0;
-        }
+        temp = acc << -rs5_0;
     }
 
     env->active_tc.HI[ac] = (target_ulong)(int32_t)((temp & MIPSDSP_LHI) >> 32);
@@ -3947,7 +3685,11 @@ void helper_wrdsp(target_ulong rs, target_ulong mask_num, CPUMIPSState *env)
     if (mask[4] == 1) {
         overwrite &= 0x00FFFFFF;
         newbits   &= 0x00FFFFFF;
+#if defined(TARGET_MIPS64)
         newbits   |= 0xFF000000 & rs;
+#else
+        newbits   |= 0x0F000000 & rs;
+#endif
     }
 
     if (mask[5] == 1) {
@@ -3998,7 +3740,11 @@ target_ulong helper_rddsp(target_ulong masknum, CPUMIPSState *env)
     }
 
     if (mask[4] == 1) {
+#if defined(TARGET_MIPS64)
         temp |= dsp & 0xFF000000;
+#else
+        temp |= dsp & 0x0F000000;
+#endif
     }
 
     if (mask[5] == 1) {
@@ -4021,7 +3767,6 @@ target_ulong helper_rddsp(target_ulong masknum, CPUMIPSState *env)
 #undef MIPSDSP_SPLIT32_8
 #undef MIPSDSP_SPLIT32_16
 
-#undef MIPSDSP_RETURN32
 #undef MIPSDSP_RETURN32_8
 #undef MIPSDSP_RETURN32_16
 
diff --git a/target-mips/helper.h b/target-mips/helper.h
index acf9ebd..cd48738 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
 DEF_HELPER_2(raise_exception, noreturn, env, i32)
@@ -654,19 +654,6 @@ DEF_HELPER_FLAGS_3(pick_ob, 0, tl, tl, tl, env)
 DEF_HELPER_FLAGS_3(pick_qh, 0, tl, tl, tl, env)
 DEF_HELPER_FLAGS_3(pick_pw, 0, tl, tl, tl, env)
 #endif
-DEF_HELPER_FLAGS_3(append, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-#if defined(TARGET_MIPS64)
-DEF_HELPER_FLAGS_3(dappend, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-#endif
-DEF_HELPER_FLAGS_3(prepend, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-#if defined(TARGET_MIPS64)
-DEF_HELPER_FLAGS_3(prependd, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-DEF_HELPER_FLAGS_3(prependw, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-#endif
-DEF_HELPER_FLAGS_3(balign, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-#if defined(TARGET_MIPS64)
-DEF_HELPER_FLAGS_3(dbalign, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
-#endif
 DEF_HELPER_FLAGS_2(packrl_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
 #if defined(TARGET_MIPS64)
 DEF_HELPER_FLAGS_2(packrl_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
@@ -707,4 +694,4 @@ DEF_HELPER_FLAGS_2(rddsp, 0, tl, tl, env)
 
 
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index f45d494..526f84f 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -18,12 +18,12 @@
  */
 #include <stdlib.h>
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 #ifndef CONFIG_USER_ONLY
@@ -38,22 +38,15 @@ static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
                                                         int error_code,
                                                         uintptr_t pc)
 {
-    TranslationBlock *tb;
-#if 1
-    if (exception < 0x100)
+    if (exception < EXCP_SC) {
         qemu_log("%s: %d %d\n", __func__, exception, error_code);
-#endif
+    }
     env->exception_index = exception;
     env->error_code = error_code;
 
     if (pc) {
         /* now we have a real cpu fault */
-        tb = tb_find_pc(pc);
-        if (tb) {
-            /* the PC is inside the translated code. It means that we have
-               a virtual CPU fault */
-            cpu_restore_state(tb, env, pc);
-        }
+        cpu_restore_state(env, pc);
     }
 
     cpu_loop_exit(env);
@@ -579,17 +572,23 @@ static inline void mips_tc_sleep(MIPSCPU *cpu, int tc)
     }
 }
 
-/* tc should point to an int with the value of the global TC index.
-   This function will transform it into a local index within the
-   returned CPUMIPSState.
-
-   FIXME: This code assumes that all VPEs have the same number of TCs,
+/**
+ * mips_cpu_map_tc:
+ * @env: CPU from which mapping is performed.
+ * @tc: Should point to an int with the value of the global TC index.
+ *
+ * This function will transform @tc into a local index within the
+ * returned #CPUMIPSState.
+ */
+/* FIXME: This code assumes that all VPEs have the same number of TCs,
           which depends on runtime setup. Can probably be fixed by
           walking the list of CPUMIPSStates.  */
 static CPUMIPSState *mips_cpu_map_tc(CPUMIPSState *env, int *tc)
 {
-    CPUMIPSState *other;
-    int vpe_idx, nr_threads = env->nr_threads;
+    MIPSCPU *cpu;
+    CPUState *cs;
+    CPUState *other_cs;
+    int vpe_idx;
     int tc_idx = *tc;
 
     if (!(env->CP0_VPEConf0 & (1 << CP0VPEC0_MVP))) {
@@ -598,10 +597,15 @@ static CPUMIPSState *mips_cpu_map_tc(CPUMIPSState *env, int *tc)
         return env;
     }
 
-    vpe_idx = tc_idx / nr_threads;
-    *tc = tc_idx % nr_threads;
-    other = qemu_get_cpu(vpe_idx);
-    return other ? other : env;
+    cs = CPU(mips_env_get_cpu(env));
+    vpe_idx = tc_idx / cs->nr_threads;
+    *tc = tc_idx % cs->nr_threads;
+    other_cs = qemu_get_cpu(vpe_idx);
+    if (other_cs == NULL) {
+        return env;
+    }
+    cpu = MIPS_CPU(other_cs);
+    return &cpu->env;
 }
 
 /* The per VPE CP0_Status register shares some fields with the per TC
@@ -2122,16 +2126,16 @@ static void QEMU_NORETURN do_unaligned_access(CPUMIPSState *env,
 #define ALIGNED_ONLY
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 static void do_unaligned_access(CPUMIPSState *env, target_ulong addr,
                                 int is_write, int is_user, uintptr_t retaddr)
@@ -2177,11 +2181,17 @@ static unsigned int ieee_rm[] = {
     float_round_down
 };
 
-#define RESTORE_ROUNDING_MODE \
-    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3], &env->active_fpu.fp_status)
+static inline void restore_rounding_mode(CPUMIPSState *env)
+{
+    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3],
+                            &env->active_fpu.fp_status);
+}
 
-#define RESTORE_FLUSH_MODE \
-    set_flush_to_zero((env->active_fpu.fcr31 & (1 << 24)) != 0, &env->active_fpu.fp_status);
+static inline void restore_flush_mode(CPUMIPSState *env)
+{
+    set_flush_to_zero((env->active_fpu.fcr31 & (1 << 24)) != 0,
+                      &env->active_fpu.fp_status);
+}
 
 target_ulong helper_cfc1(CPUMIPSState *env, uint32_t reg)
 {
@@ -2237,9 +2247,9 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t reg)
         return;
     }
     /* set rounding mode */
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     /* set flush-to-zero mode */
-    RESTORE_FLUSH_MODE;
+    restore_flush_mode(env);
     set_float_exception_flags(0, &env->active_fpu.fp_status);
     if ((GET_FP_ENABLE(env->active_fpu.fcr31) | 0x20) & GET_FP_CAUSE(env->active_fpu.fcr31))
         do_raise_exception(env, EXCP_FPE, GETPC());
@@ -2471,7 +2481,7 @@ uint64_t helper_float_roundl_d(CPUMIPSState *env, uint64_t fdt0)
 
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         dt2 = FP_TO_INT64_OVERFLOW;
@@ -2486,7 +2496,7 @@ uint64_t helper_float_roundl_s(CPUMIPSState *env, uint32_t fst0)
 
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         dt2 = FP_TO_INT64_OVERFLOW;
@@ -2501,7 +2511,7 @@ uint32_t helper_float_roundw_d(CPUMIPSState *env, uint64_t fdt0)
 
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
@@ -2516,7 +2526,7 @@ uint32_t helper_float_roundw_s(CPUMIPSState *env, uint32_t fst0)
 
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
@@ -2583,7 +2593,7 @@ uint64_t helper_float_ceill_d(CPUMIPSState *env, uint64_t fdt0)
 
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         dt2 = FP_TO_INT64_OVERFLOW;
@@ -2598,7 +2608,7 @@ uint64_t helper_float_ceill_s(CPUMIPSState *env, uint32_t fst0)
 
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         dt2 = FP_TO_INT64_OVERFLOW;
@@ -2613,7 +2623,7 @@ uint32_t helper_float_ceilw_d(CPUMIPSState *env, uint64_t fdt0)
 
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
@@ -2628,7 +2638,7 @@ uint32_t helper_float_ceilw_s(CPUMIPSState *env, uint32_t fst0)
 
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
@@ -2643,7 +2653,7 @@ uint64_t helper_float_floorl_d(CPUMIPSState *env, uint64_t fdt0)
 
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         dt2 = FP_TO_INT64_OVERFLOW;
@@ -2658,7 +2668,7 @@ uint64_t helper_float_floorl_s(CPUMIPSState *env, uint32_t fst0)
 
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         dt2 = FP_TO_INT64_OVERFLOW;
@@ -2673,7 +2683,7 @@ uint32_t helper_float_floorw_d(CPUMIPSState *env, uint64_t fdt0)
 
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
@@ -2688,7 +2698,7 @@ uint32_t helper_float_floorw_s(CPUMIPSState *env, uint32_t fst0)
 
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    RESTORE_ROUNDING_MODE;
+    restore_rounding_mode(env);
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
@@ -2868,14 +2878,26 @@ FLOAT_BINOP(mul)
 FLOAT_BINOP(div)
 #undef FLOAT_BINOP
 
+#define UNFUSED_FMA(prefix, a, b, c, flags)                          \
+{                                                                    \
+    a = prefix##_mul(a, b, &env->active_fpu.fp_status);              \
+    if ((flags) & float_muladd_negate_c) {                           \
+        a = prefix##_sub(a, c, &env->active_fpu.fp_status);          \
+    } else {                                                         \
+        a = prefix##_add(a, c, &env->active_fpu.fp_status);          \
+    }                                                                \
+    if ((flags) & float_muladd_negate_result) {                      \
+        a = prefix##_chs(a);                                         \
+    }                                                                \
+}
+
 /* FMA based operations */
 #define FLOAT_FMA(name, type)                                        \
 uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,              \
                                      uint64_t fdt0, uint64_t fdt1,   \
                                      uint64_t fdt2)                  \
 {                                                                    \
-    fdt0 = float64_muladd(fdt0, fdt1, fdt2, type,                    \
-                         &env->active_fpu.fp_status);                \
+    UNFUSED_FMA(float64, fdt0, fdt1, fdt2, type);                    \
     update_fcr31(env, GETPC());                                      \
     return fdt0;                                                     \
 }                                                                    \
@@ -2884,8 +2906,7 @@ uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,              \
                                      uint32_t fst0, uint32_t fst1,   \
                                      uint32_t fst2)                  \
 {                                                                    \
-    fst0 = float32_muladd(fst0, fst1, fst2, type,                    \
-                         &env->active_fpu.fp_status);                \
+    UNFUSED_FMA(float32, fst0, fst1, fst2, type);                    \
     update_fcr31(env, GETPC());                                      \
     return fst0;                                                     \
 }                                                                    \
@@ -2901,10 +2922,8 @@ uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,             \
     uint32_t fst2 = fdt2 & 0XFFFFFFFF;                               \
     uint32_t fsth2 = fdt2 >> 32;                                     \
                                                                      \
-    fst0 = float32_muladd(fst0, fst1, fst2, type,                    \
-                          &env->active_fpu.fp_status);               \
-    fsth0 = float32_muladd(fsth0, fsth1, fsth2, type,                \
-                           &env->active_fpu.fp_status);              \
+    UNFUSED_FMA(float32, fst0, fst1, fst2, type);                    \
+    UNFUSED_FMA(float32, fsth0, fsth1, fsth2, type);                 \
     update_fcr31(env, GETPC());                                      \
     return ((uint64_t)fsth0 << 32) | fst0;                           \
 }
diff --git a/target-mips/translate.c b/target-mips/translate.c
index 71c55bc..3b77b53 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -22,7 +22,7 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -336,7 +336,7 @@ enum {
     /* DSP Bit/Manipulation Sub-class */
     OPC_INSV_DSP       = 0x0C | OPC_SPECIAL3,
     OPC_DINSV_DSP      = 0x0D | OPC_SPECIAL3,
-    /* MIPS DSP Compare-Pick Sub-class */
+    /* MIPS DSP Append Sub-class */
     OPC_APPEND_DSP     = 0x31 | OPC_SPECIAL3,
     OPC_DAPPEND_DSP    = 0x35 | OPC_SPECIAL3,
     /* MIPS DSP Accumulator and DSPControl Access Sub-class */
@@ -543,7 +543,7 @@ enum {
 
 #define MASK_APPEND(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
 enum {
-    /* MIPS DSP Compare-Pick Sub-class */
+    /* MIPS DSP Append Sub-class */
     OPC_APPEND  = (0x00 << 6) | OPC_APPEND_DSP,
     OPC_PREPEND = (0x01 << 6) | OPC_APPEND_DSP,
     OPC_BALIGN  = (0x10 << 6) | OPC_APPEND_DSP,
@@ -667,7 +667,7 @@ enum {
 
 #define MASK_DAPPEND(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
 enum {
-    /* DSP Compare-Pick Sub-class */
+    /* DSP Append Sub-class */
     OPC_DAPPEND  = (0x00 << 6) | OPC_DAPPEND_DSP,
     OPC_PREPENDD = (0x03 << 6) | OPC_DAPPEND_DSP,
     OPC_PREPENDW = (0x01 << 6) | OPC_DAPPEND_DSP,
@@ -1017,7 +1017,7 @@ static TCGv_i64 fpu_f64[32];
 static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
 static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 #define gen_helper_0e0i(name, arg) do {                           \
     TCGv_i32 helper_tmp = tcg_const_i32(arg);                     \
@@ -1066,6 +1066,7 @@ typedef struct DisasContext {
     target_ulong pc, saved_pc;
     uint32_t opcode;
     int singlestep_enabled;
+    int insn_flags;
     /* Routine used to access memory */
     int mem_idx;
     uint32_t hflags, saved_hflags;
@@ -1393,23 +1394,32 @@ static inline void check_cp1_registers(DisasContext *ctx, int regs)
 static inline void check_dsp(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSP))) {
-        generate_exception(ctx, EXCP_DSPDIS);
+        if (ctx->insn_flags & ASE_DSP) {
+            generate_exception(ctx, EXCP_DSPDIS);
+        } else {
+            generate_exception(ctx, EXCP_RI);
+        }
     }
 }
 
 static inline void check_dspr2(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSPR2))) {
-        generate_exception(ctx, EXCP_DSPDIS);
+        if (ctx->insn_flags & ASE_DSP) {
+            generate_exception(ctx, EXCP_DSPDIS);
+        } else {
+            generate_exception(ctx, EXCP_RI);
+        }
     }
 }
 
 /* This code generates a "reserved instruction" exception if the
    CPU does not support the instruction set corresponding to flags. */
-static inline void check_insn(CPUMIPSState *env, DisasContext *ctx, int flags)
+static inline void check_insn(DisasContext *ctx, int flags)
 {
-    if (unlikely(!(env->insn_flags & flags)))
+    if (unlikely(!(ctx->insn_flags & flags))) {
         generate_exception(ctx, EXCP_RI);
+    }
 }
 
 /* This code generates a "reserved instruction" exception if 64-bit
@@ -1576,13 +1586,13 @@ static target_ulong pc_relative_pc (DisasContext *ctx)
 }
 
 /* Load */
-static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
-                    int rt, int base, int16_t offset)
+static void gen_ld(DisasContext *ctx, uint32_t opc,
+                   int rt, int base, int16_t offset)
 {
     const char *opn = "ld";
     TCGv t0, t1, t2;
 
-    if (rt == 0 && env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
+    if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
         /* Loongson CPU uses a load to zero register for prefetch.
            We emulate it as a NOP. On other CPU we must perform the
            actual memory access. */
@@ -1735,6 +1745,7 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
+        tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
         opn = "lwr";
         break;
@@ -1921,8 +1932,8 @@ static void gen_cop1_ldst(CPUMIPSState *env, DisasContext *ctx,
 }
 
 /* Arithmetic with immediate operand */
-static void gen_arith_imm (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
-                           int rt, int rs, int16_t imm)
+static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
+                          int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
     const char *opn = "imm arith";
@@ -2009,7 +2020,7 @@ static void gen_arith_imm (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Logic with immediate operand */
-static void gen_logic_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_logic_imm(DisasContext *ctx, uint32_t opc,
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm;
@@ -2057,7 +2068,7 @@ static void gen_logic_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Set on less than with immediate operand */
-static void gen_slt_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_slt_imm(DisasContext *ctx, uint32_t opc,
                         int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
@@ -2087,7 +2098,7 @@ static void gen_slt_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Shifts with immediate operand */
-static void gen_shift_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = ((uint16_t)imm) & 0x1f;
@@ -2179,8 +2190,8 @@ static void gen_shift_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Arithmetic */
-static void gen_arith (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
-                       int rd, int rs, int rt)
+static void gen_arith(DisasContext *ctx, uint32_t opc,
+                      int rd, int rs, int rt)
 {
     const char *opn = "arith";
 
@@ -2359,7 +2370,7 @@ static void gen_arith (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Conditional move */
-static void gen_cond_move(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_cond_move(DisasContext *ctx, uint32_t opc,
                           int rd, int rs, int rt)
 {
     const char *opn = "cond move";
@@ -2395,7 +2406,7 @@ static void gen_cond_move(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Logic */
-static void gen_logic(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_logic(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
     const char *opn = "logic";
@@ -2457,7 +2468,7 @@ static void gen_logic(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Set on lower than */
-static void gen_slt(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_slt(DisasContext *ctx, uint32_t opc,
                     int rd, int rs, int rt)
 {
     const char *opn = "slt";
@@ -2490,8 +2501,8 @@ static void gen_slt(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Shifts */
-static void gen_shift (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
-                       int rd, int rs, int rt)
+static void gen_shift(DisasContext *ctx, uint32_t opc,
+                      int rd, int rs, int rt)
 {
     const char *opn = "shifts";
     TCGv t0, t1;
@@ -4097,12 +4108,12 @@ static inline void gen_mtc0_store64 (TCGv arg, target_ulong off)
     tcg_gen_st_tl(arg, cpu_env, off);
 }
 
-static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, int sel)
+static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
 
     if (sel != 0)
-        check_insn(env, ctx, ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS32);
 
     switch (reg) {
     case 0:
@@ -4112,17 +4123,17 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Index";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
@@ -4137,37 +4148,37 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Random";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEControl));
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf0));
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf1));
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load64(arg, offsetof(CPUMIPSState, CP0_YQMask));
             rn = "YQMask";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEOpt));
             rn = "VPEOpt";
             break;
@@ -4183,37 +4194,37 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
@@ -4254,7 +4265,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "PageMask";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_PageGrain));
             rn = "PageGrain";
             break;
@@ -4269,27 +4280,27 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Wired";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf0));
             rn = "SRSConf0";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf1));
             rn = "SRSConf1";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf2));
             rn = "SRSConf2";
             break;
         case 4:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf3));
             rn = "SRSConf3";
             break;
         case 5:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf4));
             rn = "SRSConf4";
             break;
@@ -4300,7 +4311,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 7:
         switch (sel) {
         case 0:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_HWREna));
             rn = "HWREna";
             break;
@@ -4368,17 +4379,17 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Status";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_IntCtl));
             rn = "IntCtl";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSCtl));
             rn = "SRSCtl";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSMap));
             rn = "SRSMap";
             break;
@@ -4414,7 +4425,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "PRid";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_EBase));
             rn = "EBase";
             break;
@@ -4488,7 +4499,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
 #if defined(TARGET_MIPS64)
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_XContext));
             tcg_gen_ext32s_tl(arg, arg);
             rn = "XContext";
@@ -4677,12 +4688,12 @@ die:
     generate_exception(ctx, EXCP_RI);
 }
 
-static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, int sel)
+static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
 
     if (sel != 0)
-        check_insn(env, ctx, ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS32);
 
     if (use_icount)
         gen_io_start();
@@ -4695,17 +4706,17 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Index";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             /* ignored */
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             /* ignored */
             rn = "MVPConf1";
             break;
@@ -4720,37 +4731,37 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Random";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
@@ -4765,37 +4776,37 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
@@ -4834,7 +4845,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "PageMask";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
             break;
@@ -4849,27 +4860,27 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Wired";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf0(cpu_env, arg);
             rn = "SRSConf0";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf1(cpu_env, arg);
             rn = "SRSConf1";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf2(cpu_env, arg);
             rn = "SRSConf2";
             break;
         case 4:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf3(cpu_env, arg);
             rn = "SRSConf3";
             break;
         case 5:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf4(cpu_env, arg);
             rn = "SRSConf4";
             break;
@@ -4880,7 +4891,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 7:
         switch (sel) {
         case 0:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_hwrena(cpu_env, arg);
             rn = "HWREna";
             break;
@@ -4935,21 +4946,21 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Status";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_intctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "IntCtl";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "SRSCtl";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mtc0_store32(arg, offsetof(CPUMIPSState, CP0_SRSMap));
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4987,7 +4998,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "PRid";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_ebase(cpu_env, arg);
             rn = "EBase";
             break;
@@ -5066,7 +5077,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
 #if defined(TARGET_MIPS64)
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             gen_helper_mtc0_xcontext(cpu_env, arg);
             rn = "XContext";
             break;
@@ -5274,12 +5285,12 @@ die:
 }
 
 #if defined(TARGET_MIPS64)
-static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, int sel)
+static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
 
     if (sel != 0)
-        check_insn(env, ctx, ISA_MIPS64);
+        check_insn(ctx, ISA_MIPS64);
 
     switch (reg) {
     case 0:
@@ -5289,17 +5300,17 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Index";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
@@ -5314,37 +5325,37 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Random";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEControl));
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf0));
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEConf1));
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_YQMask));
             rn = "YQMask";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPEOpt));
             rn = "VPEOpt";
             break;
@@ -5359,37 +5370,37 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_dmfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_dmfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_dmfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_dmfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_dmfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
@@ -5428,7 +5439,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "PageMask";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_PageGrain));
             rn = "PageGrain";
             break;
@@ -5443,27 +5454,27 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Wired";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf0));
             rn = "SRSConf0";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf1));
             rn = "SRSConf1";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf2));
             rn = "SRSConf2";
             break;
         case 4:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf3));
             rn = "SRSConf3";
             break;
         case 5:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSConf4));
             rn = "SRSConf4";
             break;
@@ -5474,7 +5485,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 7:
         switch (sel) {
         case 0:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_HWREna));
             rn = "HWREna";
             break;
@@ -5540,17 +5551,17 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Status";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_IntCtl));
             rn = "IntCtl";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSCtl));
             rn = "SRSCtl";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_SRSMap));
             rn = "SRSMap";
             break;
@@ -5585,7 +5596,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "PRid";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_EBase));
             rn = "EBase";
             break;
@@ -5657,7 +5668,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 20:
         switch (sel) {
         case 0:
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_XContext));
             rn = "XContext";
             break;
@@ -5843,12 +5854,12 @@ die:
     generate_exception(ctx, EXCP_RI);
 }
 
-static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, int sel)
+static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
 
     if (sel != 0)
-        check_insn(env, ctx, ISA_MIPS64);
+        check_insn(ctx, ISA_MIPS64);
 
     if (use_icount)
         gen_io_start();
@@ -5861,17 +5872,17 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Index";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             /* ignored */
             rn = "MVPConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             /* ignored */
             rn = "MVPConf1";
             break;
@@ -5886,37 +5897,37 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Random";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
@@ -5931,37 +5942,37 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "EntryLo0";
             break;
         case 1:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
@@ -6000,7 +6011,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "PageMask";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
             break;
@@ -6015,27 +6026,27 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Wired";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf0(cpu_env, arg);
             rn = "SRSConf0";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf1(cpu_env, arg);
             rn = "SRSConf1";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf2(cpu_env, arg);
             rn = "SRSConf2";
             break;
         case 4:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf3(cpu_env, arg);
             rn = "SRSConf3";
             break;
         case 5:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsconf4(cpu_env, arg);
             rn = "SRSConf4";
             break;
@@ -6046,7 +6057,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 7:
         switch (sel) {
         case 0:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_hwrena(cpu_env, arg);
             rn = "HWREna";
             break;
@@ -6105,21 +6116,21 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Status";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_intctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "IntCtl";
             break;
         case 2:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_srsctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "SRSCtl";
             break;
         case 3:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_mtc0_store32(arg, offsetof(CPUMIPSState, CP0_SRSMap));
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -6167,7 +6178,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "PRid";
             break;
         case 1:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_ebase(cpu_env, arg);
             rn = "EBase";
             break;
@@ -6236,7 +6247,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 20:
         switch (sel) {
         case 0:
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             gen_helper_mtc0_xcontext(cpu_env, arg);
             rn = "XContext";
             break;
@@ -6493,7 +6504,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
                 gen_helper_mftc0_tcschefback(t0, cpu_env);
                 break;
             default:
-                gen_mfc0(env, ctx, t0, rt, sel);
+                gen_mfc0(ctx, t0, rt, sel);
                 break;
             }
             break;
@@ -6503,7 +6514,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
                 gen_helper_mftc0_entryhi(t0, cpu_env);
                 break;
             default:
-                gen_mfc0(env, ctx, t0, rt, sel);
+                gen_mfc0(ctx, t0, rt, sel);
                 break;
             }
         case 12:
@@ -6512,7 +6523,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
                 gen_helper_mftc0_status(t0, cpu_env);
                 break;
             default:
-                gen_mfc0(env, ctx, t0, rt, sel);
+                gen_mfc0(ctx, t0, rt, sel);
                 break;
             }
         case 13:
@@ -6561,12 +6572,12 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
                 gen_helper_mftc0_debug(t0, cpu_env);
                 break;
             default:
-                gen_mfc0(env, ctx, t0, rt, sel);
+                gen_mfc0(ctx, t0, rt, sel);
                 break;
             }
             break;
         default:
-            gen_mfc0(env, ctx, t0, rt, sel);
+            gen_mfc0(ctx, t0, rt, sel);
         }
     } else switch (sel) {
     /* GPR registers. */
@@ -6711,7 +6722,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
                 gen_helper_mttc0_tcschefback(cpu_env, t0);
                 break;
             default:
-                gen_mtc0(env, ctx, t0, rd, sel);
+                gen_mtc0(ctx, t0, rd, sel);
                 break;
             }
             break;
@@ -6721,7 +6732,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
                 gen_helper_mttc0_entryhi(cpu_env, t0);
                 break;
             default:
-                gen_mtc0(env, ctx, t0, rd, sel);
+                gen_mtc0(ctx, t0, rd, sel);
                 break;
             }
         case 12:
@@ -6730,7 +6741,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
                 gen_helper_mttc0_status(cpu_env, t0);
                 break;
             default:
-                gen_mtc0(env, ctx, t0, rd, sel);
+                gen_mtc0(ctx, t0, rd, sel);
                 break;
             }
         case 13:
@@ -6759,12 +6770,12 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
                 gen_helper_mttc0_debug(cpu_env, t0);
                 break;
             default:
-                gen_mtc0(env, ctx, t0, rd, sel);
+                gen_mtc0(ctx, t0, rd, sel);
                 break;
             }
             break;
         default:
-            gen_mtc0(env, ctx, t0, rd, sel);
+            gen_mtc0(ctx, t0, rd, sel);
         }
     } else switch (sel) {
     /* GPR registers. */
@@ -6866,7 +6877,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
             /* Treat as NOP. */
             return;
         }
-        gen_mfc0(env, ctx, cpu_gpr[rt], rd, ctx->opcode & 0x7);
+        gen_mfc0(ctx, cpu_gpr[rt], rd, ctx->opcode & 0x7);
         opn = "mfc0";
         break;
     case OPC_MTC0:
@@ -6874,35 +6885,35 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
             TCGv t0 = tcg_temp_new();
 
             gen_load_gpr(t0, rt);
-            gen_mtc0(env, ctx, t0, rd, ctx->opcode & 0x7);
+            gen_mtc0(ctx, t0, rd, ctx->opcode & 0x7);
             tcg_temp_free(t0);
         }
         opn = "mtc0";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMFC0:
-        check_insn(env, ctx, ISA_MIPS3);
+        check_insn(ctx, ISA_MIPS3);
         if (rt == 0) {
             /* Treat as NOP. */
             return;
         }
-        gen_dmfc0(env, ctx, cpu_gpr[rt], rd, ctx->opcode & 0x7);
+        gen_dmfc0(ctx, cpu_gpr[rt], rd, ctx->opcode & 0x7);
         opn = "dmfc0";
         break;
     case OPC_DMTC0:
-        check_insn(env, ctx, ISA_MIPS3);
+        check_insn(ctx, ISA_MIPS3);
         {
             TCGv t0 = tcg_temp_new();
 
             gen_load_gpr(t0, rt);
-            gen_dmtc0(env, ctx, t0, rd, ctx->opcode & 0x7);
+            gen_dmtc0(ctx, t0, rd, ctx->opcode & 0x7);
             tcg_temp_free(t0);
         }
         opn = "dmtc0";
         break;
 #endif
     case OPC_MFTR:
-        check_insn(env, ctx, ASE_MT);
+        check_insn(ctx, ASE_MT);
         if (rd == 0) {
             /* Treat as NOP. */
             return;
@@ -6912,7 +6923,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         opn = "mftr";
         break;
     case OPC_MTTR:
-        check_insn(env, ctx, ASE_MT);
+        check_insn(ctx, ASE_MT);
         gen_mttr(env, ctx, rd, rt, (ctx->opcode >> 5) & 1,
                  ctx->opcode & 0x7, (ctx->opcode >> 4) & 1);
         opn = "mttr";
@@ -6943,13 +6954,13 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         break;
     case OPC_ERET:
         opn = "eret";
-        check_insn(env, ctx, ISA_MIPS2);
+        check_insn(ctx, ISA_MIPS2);
         gen_helper_eret(cpu_env);
         ctx->bstate = BS_EXCP;
         break;
     case OPC_DERET:
         opn = "deret";
-        check_insn(env, ctx, ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS32);
         if (!(ctx->hflags & MIPS_HFLAG_DM)) {
             MIPS_INVAL(opn);
             generate_exception(ctx, EXCP_RI);
@@ -6960,7 +6971,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         break;
     case OPC_WAIT:
         opn = "wait";
-        check_insn(env, ctx, ISA_MIPS3 | ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
         /* If we get an exception, we want to restart at next instruction */
         ctx->pc += 4;
         save_cpu_state(ctx, 1);
@@ -6980,15 +6991,15 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
 #endif /* !CONFIG_USER_ONLY */
 
 /* CP1 Branches (before delay slot) */
-static void gen_compute_branch1 (CPUMIPSState *env, DisasContext *ctx, uint32_t op,
-                                 int32_t cc, int32_t offset)
+static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
+                                int32_t cc, int32_t offset)
 {
     target_ulong btarget;
     const char *opn = "cp1 cond branch";
     TCGv_i32 t0 = tcg_temp_new_i32();
 
     if (cc != 0)
-        check_insn(env, ctx, ISA_MIPS4 | ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
 
     btarget = ctx->pc + 4 + offset;
 
@@ -9032,15 +9043,14 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
                fregnames[fs], fregnames[ft]);
 }
 
-static void
-gen_rdhwr (CPUMIPSState *env, DisasContext *ctx, int rt, int rd)
+static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
 {
     TCGv t0;
 
 #if !defined(CONFIG_USER_ONLY)
     /* The Linux kernel will emulate rdhwr if it's not supported natively.
        Therefore only check the ISA in system mode.  */
-    check_insn(env, ctx, ISA_MIPS32R2);
+    check_insn(ctx, ISA_MIPS32R2);
 #endif
     t0 = tcg_temp_new();
 
@@ -9082,8 +9092,7 @@ gen_rdhwr (CPUMIPSState *env, DisasContext *ctx, int rt, int rd)
     tcg_temp_free(t0);
 }
 
-static void handle_delay_slot (CPUMIPSState *env, DisasContext *ctx,
-                               int insn_bytes)
+static void handle_delay_slot(DisasContext *ctx, int insn_bytes)
 {
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
         int proc_hflags = ctx->hflags & MIPS_HFLAG_BMASK;
@@ -9121,7 +9130,7 @@ static void handle_delay_slot (CPUMIPSState *env, DisasContext *ctx,
         case MIPS_HFLAG_BR:
             /* unconditional branch to register */
             MIPS_DEBUG("branch to register");
-            if (env->insn_flags & (ASE_MIPS16 | ASE_MICROMIPS)) {
+            if (ctx->insn_flags & (ASE_MIPS16 | ASE_MICROMIPS)) {
                 TCGv t0 = tcg_temp_new();
                 TCGv_i32 t1 = tcg_temp_new_i32();
 
@@ -9438,7 +9447,7 @@ static void gen_mips16_restore (DisasContext *ctx,
 
 #define DECR_AND_LOAD(reg) do {                   \
         tcg_gen_subi_tl(t0, t0, 4);               \
-        tcg_gen_qemu_ld32u(t1, t0, ctx->mem_idx); \
+        tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx); \
         gen_store_gpr(t1, reg);                   \
     } while (0)
 
@@ -9548,7 +9557,7 @@ static void gen_addiupc (DisasContext *ctx, int rx, int imm,
 }
 
 #if defined(TARGET_MIPS64)
-static void decode_i64_mips16 (CPUMIPSState *env, DisasContext *ctx,
+static void decode_i64_mips16 (DisasContext *ctx,
                                int ry, int funct, int16_t offset,
                                int extended)
 {
@@ -9556,7 +9565,7 @@ static void decode_i64_mips16 (CPUMIPSState *env, DisasContext *ctx,
     case I64_LDSP:
         check_mips_64(ctx);
         offset = extended ? offset : offset << 3;
-        gen_ld(env, ctx, OPC_LD, ry, 29, offset);
+        gen_ld(ctx, OPC_LD, ry, 29, offset);
         break;
     case I64_SDSP:
         check_mips_64(ctx);
@@ -9571,20 +9580,20 @@ static void decode_i64_mips16 (CPUMIPSState *env, DisasContext *ctx,
     case I64_DADJSP:
         check_mips_64(ctx);
         offset = extended ? offset : ((int8_t)ctx->opcode) << 3;
-        gen_arith_imm(env, ctx, OPC_DADDIU, 29, 29, offset);
+        gen_arith_imm(ctx, OPC_DADDIU, 29, 29, offset);
         break;
     case I64_LDPC:
         if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
             generate_exception(ctx, EXCP_RI);
         } else {
             offset = extended ? offset : offset << 3;
-            gen_ld(env, ctx, OPC_LDPC, ry, 0, offset);
+            gen_ld(ctx, OPC_LDPC, ry, 0, offset);
         }
         break;
     case I64_DADDIU5:
         check_mips_64(ctx);
         offset = extended ? offset : ((int8_t)(offset << 3)) >> 3;
-        gen_arith_imm(env, ctx, OPC_DADDIU, ry, ry, offset);
+        gen_arith_imm(ctx, OPC_DADDIU, ry, ry, offset);
         break;
     case I64_DADDIUPC:
         check_mips_64(ctx);
@@ -9594,7 +9603,7 @@ static void decode_i64_mips16 (CPUMIPSState *env, DisasContext *ctx,
     case I64_DADDIUSP:
         check_mips_64(ctx);
         offset = extended ? offset : offset << 2;
-        gen_arith_imm(env, ctx, OPC_DADDIU, ry, 29, offset);
+        gen_arith_imm(ctx, OPC_DADDIU, ry, 29, offset);
         break;
     }
 }
@@ -9621,7 +9630,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
        counterparts.  */
     switch (op) {
     case M16_OPC_ADDIUSP:
-        gen_arith_imm(env, ctx, OPC_ADDIU, rx, 29, imm);
+        gen_arith_imm(ctx, OPC_ADDIU, rx, 29, imm);
         break;
     case M16_OPC_ADDIUPC:
         gen_addiupc(ctx, rx, imm, 0, 1);
@@ -9641,28 +9650,28 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     case M16_OPC_SHIFT:
         switch (ctx->opcode & 0x3) {
         case 0x0:
-            gen_shift_imm(env, ctx, OPC_SLL, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_SLL, rx, ry, sa);
             break;
         case 0x1:
 #if defined(TARGET_MIPS64)
             check_mips_64(ctx);
-            gen_shift_imm(env, ctx, OPC_DSLL, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
             generate_exception(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
-            gen_shift_imm(env, ctx, OPC_SRL, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_SRL, rx, ry, sa);
             break;
         case 0x3:
-            gen_shift_imm(env, ctx, OPC_SRA, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_SRA, rx, ry, sa);
             break;
         }
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_LD:
             check_mips_64(ctx);
-        gen_ld(env, ctx, OPC_LD, ry, rx, offset);
+        gen_ld(ctx, OPC_LD, ry, rx, offset);
         break;
 #endif
     case M16_OPC_RRIA:
@@ -9673,22 +9682,22 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         if ((ctx->opcode >> 4) & 0x1) {
 #if defined(TARGET_MIPS64)
             check_mips_64(ctx);
-            gen_arith_imm(env, ctx, OPC_DADDIU, ry, rx, imm);
+            gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
             generate_exception(ctx, EXCP_RI);
 #endif
         } else {
-            gen_arith_imm(env, ctx, OPC_ADDIU, ry, rx, imm);
+            gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
         }
         break;
     case M16_OPC_ADDIU8:
-        gen_arith_imm(env, ctx, OPC_ADDIU, rx, rx, imm);
+        gen_arith_imm(ctx, OPC_ADDIU, rx, rx, imm);
         break;
     case M16_OPC_SLTI:
-        gen_slt_imm(env, ctx, OPC_SLTI, 24, rx, imm);
+        gen_slt_imm(ctx, OPC_SLTI, 24, rx, imm);
         break;
     case M16_OPC_SLTIU:
-        gen_slt_imm(env, ctx, OPC_SLTIU, 24, rx, imm);
+        gen_slt_imm(ctx, OPC_SLTIU, 24, rx, imm);
         break;
     case M16_OPC_I8:
         switch (funct) {
@@ -9702,7 +9711,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             gen_st(ctx, OPC_SW, 31, 29, imm);
             break;
         case I8_ADJSP:
-            gen_arith_imm(env, ctx, OPC_ADDIU, 29, 29, imm);
+            gen_arith_imm(ctx, OPC_ADDIU, 29, 29, imm);
             break;
         case I8_SVRS:
             {
@@ -9742,29 +9751,29 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         break;
 #endif
     case M16_OPC_LB:
-        gen_ld(env, ctx, OPC_LB, ry, rx, offset);
+        gen_ld(ctx, OPC_LB, ry, rx, offset);
         break;
     case M16_OPC_LH:
-        gen_ld(env, ctx, OPC_LH, ry, rx, offset);
+        gen_ld(ctx, OPC_LH, ry, rx, offset);
         break;
     case M16_OPC_LWSP:
-        gen_ld(env, ctx, OPC_LW, rx, 29, offset);
+        gen_ld(ctx, OPC_LW, rx, 29, offset);
         break;
     case M16_OPC_LW:
-        gen_ld(env, ctx, OPC_LW, ry, rx, offset);
+        gen_ld(ctx, OPC_LW, ry, rx, offset);
         break;
     case M16_OPC_LBU:
-        gen_ld(env, ctx, OPC_LBU, ry, rx, offset);
+        gen_ld(ctx, OPC_LBU, ry, rx, offset);
         break;
     case M16_OPC_LHU:
-        gen_ld(env, ctx, OPC_LHU, ry, rx, offset);
+        gen_ld(ctx, OPC_LHU, ry, rx, offset);
         break;
     case M16_OPC_LWPC:
-        gen_ld(env, ctx, OPC_LWPC, rx, 0, offset);
+        gen_ld(ctx, OPC_LWPC, rx, 0, offset);
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_LWU:
-        gen_ld(env, ctx, OPC_LWU, ry, rx, offset);
+        gen_ld(ctx, OPC_LWU, ry, rx, offset);
         break;
 #endif
     case M16_OPC_SB:
@@ -9781,7 +9790,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_I64:
-        decode_i64_mips16(env, ctx, ry, funct, offset, 1);
+        decode_i64_mips16(ctx, ry, funct, offset, 1);
         break;
 #endif
     default:
@@ -9816,7 +9825,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         {
             int16_t imm = ((uint8_t) ctx->opcode) << 2;
 
-            gen_arith_imm(env, ctx, OPC_ADDIU, rx, 29, imm);
+            gen_arith_imm(ctx, OPC_ADDIU, rx, 29, imm);
         }
         break;
     case M16_OPC_ADDIUPC:
@@ -9849,28 +9858,28 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     case M16_OPC_SHIFT:
         switch (ctx->opcode & 0x3) {
         case 0x0:
-            gen_shift_imm(env, ctx, OPC_SLL, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_SLL, rx, ry, sa);
             break;
         case 0x1:
 #if defined(TARGET_MIPS64)
             check_mips_64(ctx);
-            gen_shift_imm(env, ctx, OPC_DSLL, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
             generate_exception(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
-            gen_shift_imm(env, ctx, OPC_SRL, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_SRL, rx, ry, sa);
             break;
         case 0x3:
-            gen_shift_imm(env, ctx, OPC_SRA, rx, ry, sa);
+            gen_shift_imm(ctx, OPC_SRA, rx, ry, sa);
             break;
         }
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_LD:
         check_mips_64(ctx);
-        gen_ld(env, ctx, OPC_LD, ry, rx, offset << 3);
+        gen_ld(ctx, OPC_LD, ry, rx, offset << 3);
         break;
 #endif
     case M16_OPC_RRIA:
@@ -9880,12 +9889,12 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             if ((ctx->opcode >> 4) & 1) {
 #if defined(TARGET_MIPS64)
                 check_mips_64(ctx);
-                gen_arith_imm(env, ctx, OPC_DADDIU, ry, rx, imm);
+                gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
                 generate_exception(ctx, EXCP_RI);
 #endif
             } else {
-                gen_arith_imm(env, ctx, OPC_ADDIU, ry, rx, imm);
+                gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
             }
         }
         break;
@@ -9893,19 +9902,19 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         {
             int16_t imm = (int8_t) ctx->opcode;
 
-            gen_arith_imm(env, ctx, OPC_ADDIU, rx, rx, imm);
+            gen_arith_imm(ctx, OPC_ADDIU, rx, rx, imm);
         }
         break;
     case M16_OPC_SLTI:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-            gen_slt_imm(env, ctx, OPC_SLTI, 24, rx, imm);
+            gen_slt_imm(ctx, OPC_SLTI, 24, rx, imm);
         }
         break;
     case M16_OPC_SLTIU:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-            gen_slt_imm(env, ctx, OPC_SLTIU, 24, rx, imm);
+            gen_slt_imm(ctx, OPC_SLTIU, 24, rx, imm);
         }
         break;
     case M16_OPC_I8:
@@ -9926,7 +9935,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
                 gen_st(ctx, OPC_SW, 31, 29, (ctx->opcode & 0xff) << 2);
                 break;
             case I8_ADJSP:
-                gen_arith_imm(env, ctx, OPC_ADDIU, 29, 29,
+                gen_arith_imm(ctx, OPC_ADDIU, 29, 29,
                               ((int8_t)ctx->opcode) << 3);
                 break;
             case I8_SVRS:
@@ -9957,12 +9966,12 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
 
                     reg32 = (((ctx->opcode >> 3) & 0x3) << 3) |
                         ((ctx->opcode >> 5) & 0x7);
-                    gen_arith(env, ctx, OPC_ADDU, reg32, rz, 0);
+                    gen_arith(ctx, OPC_ADDU, reg32, rz, 0);
                 }
                 break;
             case I8_MOVR32:
                 reg32 = ctx->opcode & 0x1f;
-                gen_arith(env, ctx, OPC_ADDU, ry, reg32, 0);
+                gen_arith(ctx, OPC_ADDU, ry, reg32, 0);
                 break;
             default:
                 generate_exception(ctx, EXCP_RI);
@@ -9974,13 +9983,13 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         {
             int16_t imm = (uint8_t) ctx->opcode;
 
-            gen_arith_imm(env, ctx, OPC_ADDIU, rx, 0, imm);
+            gen_arith_imm(ctx, OPC_ADDIU, rx, 0, imm);
         }
         break;
     case M16_OPC_CMPI:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-            gen_logic_imm(env, ctx, OPC_XORI, 24, rx, imm);
+            gen_logic_imm(ctx, OPC_XORI, 24, rx, imm);
         }
         break;
 #if defined(TARGET_MIPS64)
@@ -9990,30 +9999,30 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         break;
 #endif
     case M16_OPC_LB:
-        gen_ld(env, ctx, OPC_LB, ry, rx, offset);
+        gen_ld(ctx, OPC_LB, ry, rx, offset);
         break;
     case M16_OPC_LH:
-        gen_ld(env, ctx, OPC_LH, ry, rx, offset << 1);
+        gen_ld(ctx, OPC_LH, ry, rx, offset << 1);
         break;
     case M16_OPC_LWSP:
-        gen_ld(env, ctx, OPC_LW, rx, 29, ((uint8_t)ctx->opcode) << 2);
+        gen_ld(ctx, OPC_LW, rx, 29, ((uint8_t)ctx->opcode) << 2);
         break;
     case M16_OPC_LW:
-        gen_ld(env, ctx, OPC_LW, ry, rx, offset << 2);
+        gen_ld(ctx, OPC_LW, ry, rx, offset << 2);
         break;
     case M16_OPC_LBU:
-        gen_ld(env, ctx, OPC_LBU, ry, rx, offset);
+        gen_ld(ctx, OPC_LBU, ry, rx, offset);
         break;
     case M16_OPC_LHU:
-        gen_ld(env, ctx, OPC_LHU, ry, rx, offset << 1);
+        gen_ld(ctx, OPC_LHU, ry, rx, offset << 1);
         break;
     case M16_OPC_LWPC:
-        gen_ld(env, ctx, OPC_LWPC, rx, 0, ((uint8_t)ctx->opcode) << 2);
+        gen_ld(ctx, OPC_LWPC, rx, 0, ((uint8_t)ctx->opcode) << 2);
         break;
 #if defined (TARGET_MIPS64)
     case M16_OPC_LWU:
         check_mips_64(ctx);
-        gen_ld(env, ctx, OPC_LWU, ry, rx, offset << 2);
+        gen_ld(ctx, OPC_LWU, ry, rx, offset << 2);
         break;
 #endif
     case M16_OPC_SB:
@@ -10055,7 +10064,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
                 goto done;
             }
 
-            gen_arith(env, ctx, mips32_op, rz, rx, ry);
+            gen_arith(ctx, mips32_op, rz, rx, ry);
         done:
             ;
         }
@@ -10084,7 +10093,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             /* XXX: not clear which exception should be raised
              *      when in debug mode...
              */
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             if (!(ctx->hflags & MIPS_HFLAG_DM)) {
                 generate_exception(ctx, EXCP_DBp);
             } else {
@@ -10092,46 +10101,46 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             }
             break;
         case RR_SLT:
-            gen_slt(env, ctx, OPC_SLT, 24, rx, ry);
+            gen_slt(ctx, OPC_SLT, 24, rx, ry);
             break;
         case RR_SLTU:
-            gen_slt(env, ctx, OPC_SLTU, 24, rx, ry);
+            gen_slt(ctx, OPC_SLTU, 24, rx, ry);
             break;
         case RR_BREAK:
             generate_exception(ctx, EXCP_BREAK);
             break;
         case RR_SLLV:
-            gen_shift(env, ctx, OPC_SLLV, ry, rx, ry);
+            gen_shift(ctx, OPC_SLLV, ry, rx, ry);
             break;
         case RR_SRLV:
-            gen_shift(env, ctx, OPC_SRLV, ry, rx, ry);
+            gen_shift(ctx, OPC_SRLV, ry, rx, ry);
             break;
         case RR_SRAV:
-            gen_shift(env, ctx, OPC_SRAV, ry, rx, ry);
+            gen_shift(ctx, OPC_SRAV, ry, rx, ry);
             break;
 #if defined (TARGET_MIPS64)
         case RR_DSRL:
             check_mips_64(ctx);
-            gen_shift_imm(env, ctx, OPC_DSRL, ry, ry, sa);
+            gen_shift_imm(ctx, OPC_DSRL, ry, ry, sa);
             break;
 #endif
         case RR_CMP:
-            gen_logic(env, ctx, OPC_XOR, 24, rx, ry);
+            gen_logic(ctx, OPC_XOR, 24, rx, ry);
             break;
         case RR_NEG:
-            gen_arith(env, ctx, OPC_SUBU, rx, 0, ry);
+            gen_arith(ctx, OPC_SUBU, rx, 0, ry);
             break;
         case RR_AND:
-            gen_logic(env, ctx, OPC_AND, rx, rx, ry);
+            gen_logic(ctx, OPC_AND, rx, rx, ry);
             break;
         case RR_OR:
-            gen_logic(env, ctx, OPC_OR, rx, rx, ry);
+            gen_logic(ctx, OPC_OR, rx, rx, ry);
             break;
         case RR_XOR:
-            gen_logic(env, ctx, OPC_XOR, rx, rx, ry);
+            gen_logic(ctx, OPC_XOR, rx, rx, ry);
             break;
         case RR_NOT:
-            gen_logic(env, ctx, OPC_NOR, rx, ry, 0);
+            gen_logic(ctx, OPC_NOR, rx, ry, 0);
             break;
         case RR_MFHI:
             gen_HILO(ctx, OPC_MFHI, rx);
@@ -10171,19 +10180,19 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
 #if defined (TARGET_MIPS64)
         case RR_DSRA:
             check_mips_64(ctx);
-            gen_shift_imm(env, ctx, OPC_DSRA, ry, ry, sa);
+            gen_shift_imm(ctx, OPC_DSRA, ry, ry, sa);
             break;
         case RR_DSLLV:
             check_mips_64(ctx);
-            gen_shift(env, ctx, OPC_DSLLV, ry, rx, ry);
+            gen_shift(ctx, OPC_DSLLV, ry, rx, ry);
             break;
         case RR_DSRLV:
             check_mips_64(ctx);
-            gen_shift(env, ctx, OPC_DSRLV, ry, rx, ry);
+            gen_shift(ctx, OPC_DSRLV, ry, rx, ry);
             break;
         case RR_DSRAV:
             check_mips_64(ctx);
-            gen_shift(env, ctx, OPC_DSRAV, ry, rx, ry);
+            gen_shift(ctx, OPC_DSRAV, ry, rx, ry);
             break;
 #endif
         case RR_MULT:
@@ -10228,7 +10237,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
 #if defined(TARGET_MIPS64)
     case M16_OPC_I64:
         funct = (ctx->opcode >> 8) & 0x7;
-        decode_i64_mips16(env, ctx, ry, funct, offset, 0);
+        decode_i64_mips16(ctx, ry, funct, offset, 0);
         break;
 #endif
     default:
@@ -10730,23 +10739,23 @@ static int mmreg2 (int r)
 /* Zero-extended immediate */
 #define ZIMM(op, start, width) ((op >> start) & ((~0U) >> (32-width)))
 
-static void gen_addiur1sp (CPUMIPSState *env, DisasContext *ctx)
+static void gen_addiur1sp(DisasContext *ctx)
 {
     int rd = mmreg(uMIPS_RD(ctx->opcode));
 
-    gen_arith_imm(env, ctx, OPC_ADDIU, rd, 29, ((ctx->opcode >> 1) & 0x3f) << 2);
+    gen_arith_imm(ctx, OPC_ADDIU, rd, 29, ((ctx->opcode >> 1) & 0x3f) << 2);
 }
 
-static void gen_addiur2 (CPUMIPSState *env, DisasContext *ctx)
+static void gen_addiur2(DisasContext *ctx)
 {
     static const int decoded_imm[] = { 1, 4, 8, 12, 16, 20, 24, -1 };
     int rd = mmreg(uMIPS_RD(ctx->opcode));
     int rs = mmreg(uMIPS_RS(ctx->opcode));
 
-    gen_arith_imm(env, ctx, OPC_ADDIU, rd, rs, decoded_imm[ZIMM(ctx->opcode, 1, 3)]);
+    gen_arith_imm(ctx, OPC_ADDIU, rd, rs, decoded_imm[ZIMM(ctx->opcode, 1, 3)]);
 }
 
-static void gen_addiusp (CPUMIPSState *env, DisasContext *ctx)
+static void gen_addiusp(DisasContext *ctx)
 {
     int encoded = ZIMM(ctx->opcode, 1, 9);
     int decoded;
@@ -10761,18 +10770,18 @@ static void gen_addiusp (CPUMIPSState *env, DisasContext *ctx)
         decoded = encoded - 768;
     }
 
-    gen_arith_imm(env, ctx, OPC_ADDIU, 29, 29, decoded << 2);
+    gen_arith_imm(ctx, OPC_ADDIU, 29, 29, decoded << 2);
 }
 
-static void gen_addius5 (CPUMIPSState *env, DisasContext *ctx)
+static void gen_addius5(DisasContext *ctx)
 {
     int imm = SIMM(ctx->opcode, 1, 4);
     int rd = (ctx->opcode >> 5) & 0x1f;
 
-    gen_arith_imm(env, ctx, OPC_ADDIU, rd, rd, imm);
+    gen_arith_imm(ctx, OPC_ADDIU, rd, rd, imm);
 }
 
-static void gen_andi16 (CPUMIPSState *env, DisasContext *ctx)
+static void gen_andi16(DisasContext *ctx)
 {
     static const int decoded_imm[] = { 128, 1, 2, 3, 4, 7, 8, 15, 16,
                                  31, 32, 63, 64, 255, 32768, 65535 };
@@ -10780,7 +10789,7 @@ static void gen_andi16 (CPUMIPSState *env, DisasContext *ctx)
     int rs = mmreg(uMIPS_RS(ctx->opcode));
     int encoded = ZIMM(ctx->opcode, 0, 4);
 
-    gen_logic_imm(env, ctx, OPC_ANDI, rd, rs, decoded_imm[encoded]);
+    gen_logic_imm(ctx, OPC_ANDI, rd, rs, decoded_imm[encoded]);
 }
 
 static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
@@ -10831,7 +10840,7 @@ static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
 }
 
 
-static void gen_pool16c_insn (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
+static void gen_pool16c_insn(DisasContext *ctx, int *is_branch)
 {
     int rd = mmreg((ctx->opcode >> 3) & 0x7);
     int rs = mmreg(ctx->opcode & 0x7);
@@ -10842,25 +10851,25 @@ static void gen_pool16c_insn (CPUMIPSState *env, DisasContext *ctx, int *is_bran
     case NOT16 + 1:
     case NOT16 + 2:
     case NOT16 + 3:
-        gen_logic(env, ctx, OPC_NOR, rd, rs, 0);
+        gen_logic(ctx, OPC_NOR, rd, rs, 0);
         break;
     case XOR16 + 0:
     case XOR16 + 1:
     case XOR16 + 2:
     case XOR16 + 3:
-        gen_logic(env, ctx, OPC_XOR, rd, rd, rs);
+        gen_logic(ctx, OPC_XOR, rd, rd, rs);
         break;
     case AND16 + 0:
     case AND16 + 1:
     case AND16 + 2:
     case AND16 + 3:
-        gen_logic(env, ctx, OPC_AND, rd, rd, rs);
+        gen_logic(ctx, OPC_AND, rd, rd, rs);
         break;
     case OR16 + 0:
     case OR16 + 1:
     case OR16 + 2:
     case OR16 + 3:
-        gen_logic(env, ctx, OPC_OR, rd, rd, rs);
+        gen_logic(ctx, OPC_OR, rd, rd, rs);
         break;
     case LWM16 + 0:
     case LWM16 + 1:
@@ -10935,7 +10944,7 @@ static void gen_pool16c_insn (CPUMIPSState *env, DisasContext *ctx, int *is_bran
         /* XXX: not clear which exception should be raised
          *      when in debug mode...
          */
-        check_insn(env, ctx, ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS32);
         if (!(ctx->hflags & MIPS_HFLAG_DM)) {
             generate_exception(ctx, EXCP_DBp);
         } else {
@@ -10948,7 +10957,7 @@ static void gen_pool16c_insn (CPUMIPSState *env, DisasContext *ctx, int *is_bran
             int imm = ZIMM(ctx->opcode, 0, 5);
 
             gen_compute_branch(ctx, OPC_JR, 2, 31, 0, 0);
-            gen_arith_imm(env, ctx, OPC_ADDIU, 29, 29, imm << 2);
+            gen_arith_imm(ctx, OPC_ADDIU, 29, 29, imm << 2);
             /* Let normal delay slot handling in our caller take us
                to the branch target.  */
         }
@@ -11085,7 +11094,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
             /* Treat as NOP. */
             break;
         }
-        gen_mfc0(env, ctx, cpu_gpr[rt], rs, (ctx->opcode >> 11) & 0x7);
+        gen_mfc0(ctx, cpu_gpr[rt], rs, (ctx->opcode >> 11) & 0x7);
         break;
     case MTC0:
     case MTC0 + 32:
@@ -11094,7 +11103,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
             TCGv t0 = tcg_temp_new();
 
             gen_load_gpr(t0, rt);
-            gen_mtc0(env, ctx, t0, rs, (ctx->opcode >> 11) & 0x7);
+            gen_mtc0(ctx, t0, rs, (ctx->opcode >> 11) & 0x7);
             tcg_temp_free(t0);
         }
         break;
@@ -11113,11 +11122,11 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
         case CLZ:
             mips32_op = OPC_CLZ;
         do_cl:
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             gen_cl(ctx, mips32_op, rt, rs);
             break;
         case RDHWR:
-            gen_rdhwr(env, ctx, rt, rs);
+            gen_rdhwr(ctx, rt, rs);
             break;
         case WSBH:
             gen_bshfl(ctx, OPC_WSBH, rs, rt);
@@ -11146,7 +11155,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
         case MSUBU:
             mips32_op = OPC_MSUBU;
         do_muldiv:
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             gen_muldiv(ctx, mips32_op, rs, rt);
             break;
         default:
@@ -11187,12 +11196,12 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
         switch (minor) {
         case RDPGPR:
             check_cp0_enabled(ctx);
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_load_srsgpr(rt, rs);
             break;
         case WRPGPR:
             check_cp0_enabled(ctx);
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_store_srsgpr(rt, rs);
             break;
         default:
@@ -11272,7 +11281,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
             ctx->bstate = BS_STOP;
             break;
         case SDBBP:
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             if (!(ctx->hflags & MIPS_HFLAG_DM)) {
                 generate_exception(ctx, EXCP_DBp);
             } else {
@@ -11329,7 +11338,7 @@ enum {
     FMT_DWL_L = 2
 };
 
-static void gen_pool32fxf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
+static void gen_pool32fxf(DisasContext *ctx, int rt, int rs)
 {
     int extension = (ctx->opcode >> 6) & 0x3ff;
     uint32_t mips32_op;
@@ -11614,7 +11623,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case ROTR:
                 mips32_op = OPC_ROTR;
             do_shifti:
-                gen_shift_imm(env, ctx, mips32_op, rt, rs, rd);
+                gen_shift_imm(ctx, mips32_op, rt, rs, rd);
                 break;
             default:
                 goto pool32a_invalid;
@@ -11639,7 +11648,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case MUL:
                 mips32_op = OPC_MUL;
             do_arith:
-                gen_arith(env, ctx, mips32_op, rd, rs, rt);
+                gen_arith(ctx, mips32_op, rd, rs, rt);
                 break;
                 /* Shifts */
             case SLLV:
@@ -11654,7 +11663,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case ROTRV:
                 mips32_op = OPC_ROTRV;
             do_shift:
-                gen_shift(env, ctx, mips32_op, rd, rs, rt);
+                gen_shift(ctx, mips32_op, rd, rs, rt);
                 break;
                 /* Logical operations */
             case AND:
@@ -11669,7 +11678,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case XOR32:
                 mips32_op = OPC_XOR;
             do_logic:
-                gen_logic(env, ctx, mips32_op, rd, rs, rt);
+                gen_logic(ctx, mips32_op, rd, rs, rt);
                 break;
                 /* Set less than */
             case SLT:
@@ -11678,7 +11687,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case SLTU:
                 mips32_op = OPC_SLTU;
             do_slt:
-                gen_slt(env, ctx, mips32_op, rd, rs, rt);
+                gen_slt(ctx, mips32_op, rd, rs, rt);
                 break;
             default:
                 goto pool32a_invalid;
@@ -11694,7 +11703,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case MOVZ:
                 mips32_op = OPC_MOVZ;
             do_cmov:
-                gen_cond_move(env, ctx, mips32_op, rd, rs, rt);
+                gen_cond_move(ctx, mips32_op, rd, rs, rt);
                 break;
             case LWXS:
                 gen_ldxs(ctx, rs, rt, rd);
@@ -11839,7 +11848,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
                 }
                 break;
             case POOL32FXF:
-                gen_pool32fxf(env, ctx, rt, rs);
+                gen_pool32fxf(ctx, rt, rs);
                 break;
             case 0x00:
                 /* PLL foo */
@@ -12107,7 +12116,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
                target. */
             break;
         case LUI:
-            gen_logic_imm(env, ctx, OPC_LUI, rs, -1, imm);
+            gen_logic_imm(ctx, OPC_LUI, rs, -1, imm);
             break;
         case SYNCI:
             break;
@@ -12129,10 +12138,10 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             mips32_op = OPC_BC1TANY4;
         do_cp1mips3d:
             check_cop1x(ctx);
-            check_insn(env, ctx, ASE_MIPS3D);
+            check_insn(ctx, ASE_MIPS3D);
             /* Fall through */
         do_cp1branch:
-            gen_compute_branch1(env, ctx, mips32_op,
+            gen_compute_branch1(ctx, mips32_op,
                                 (ctx->opcode >> 18) & 0x7, imm << 1);
             *is_branch = 1;
             break;
@@ -12185,7 +12194,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             mips32_op = OPC_LL;
             goto do_ld_lr;
         do_ld_lr:
-            gen_ld(env, ctx, mips32_op, rt, rs, SIMM(ctx->opcode, 0, 12));
+            gen_ld(ctx, mips32_op, rt, rs, SIMM(ctx->opcode, 0, 12));
             break;
         do_st_lr:
             gen_st(ctx, mips32_op, rt, rs, SIMM(ctx->opcode, 0, 12));
@@ -12213,7 +12222,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case ADDIU32:
         mips32_op = OPC_ADDIU;
     do_addi:
-        gen_arith_imm(env, ctx, mips32_op, rt, rs, imm);
+        gen_arith_imm(ctx, mips32_op, rt, rs, imm);
         break;
 
         /* Logical operations */
@@ -12226,7 +12235,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case ANDI32:
         mips32_op = OPC_ANDI;
     do_logici:
-        gen_logic_imm(env, ctx, mips32_op, rt, rs, imm);
+        gen_logic_imm(ctx, mips32_op, rt, rs, imm);
         break;
 
         /* Set less than immediate */
@@ -12236,7 +12245,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case SLTIU32:
         mips32_op = OPC_SLTIU;
     do_slti:
-        gen_slt_imm(env, ctx, mips32_op, rt, rs, imm);
+        gen_slt_imm(ctx, mips32_op, rt, rs, imm);
         break;
     case JALX32:
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
@@ -12323,7 +12332,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
         mips32_op = OPC_SW;
         goto do_st;
     do_ld:
-        gen_ld(env, ctx, mips32_op, rt, rs, imm);
+        gen_ld(ctx, mips32_op, rt, rs, imm);
         break;
     do_st:
         gen_st(ctx, mips32_op, rt, rs, imm);
@@ -12443,7 +12452,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
                 break;
             }
 
-            gen_arith(env, ctx, opc, rd, rs1, rs2);
+            gen_arith(ctx, opc, rd, rs1, rs2);
         }
         break;
     case POOL16B:
@@ -12463,11 +12472,11 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
                 break;
             }
 
-            gen_shift_imm(env, ctx, opc, rd, rs, amount);
+            gen_shift_imm(ctx, opc, rd, rs, amount);
         }
         break;
     case POOL16C:
-        gen_pool16c_insn(env, ctx, is_branch);
+        gen_pool16c_insn(ctx, is_branch);
         break;
     case LWGP16:
         {
@@ -12475,7 +12484,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             int rb = 28;            /* GP */
             int16_t offset = SIMM(ctx->opcode, 0, 7) << 2;
 
-            gen_ld(env, ctx, OPC_LW, rd, rb, offset);
+            gen_ld(ctx, OPC_LW, rd, rb, offset);
         }
         break;
     case POOL16F:
@@ -12496,8 +12505,8 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             rs = rs_rt_enc[enc_rs];
             rt = rs_rt_enc[enc_rt];
 
-            gen_arith_imm(env, ctx, OPC_ADDIU, rd, rs, 0);
-            gen_arith_imm(env, ctx, OPC_ADDIU, re, rt, 0);
+            gen_arith_imm(ctx, OPC_ADDIU, rd, rs, 0);
+            gen_arith_imm(ctx, OPC_ADDIU, re, rt, 0);
         }
         break;
     case LBU16:
@@ -12507,7 +12516,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             int16_t offset = ZIMM(ctx->opcode, 0, 4);
             offset = (offset == 0xf ? -1 : offset);
 
-            gen_ld(env, ctx, OPC_LBU, rd, rb, offset);
+            gen_ld(ctx, OPC_LBU, rd, rb, offset);
         }
         break;
     case LHU16:
@@ -12516,7 +12525,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             int rb = mmreg(uMIPS_RS(ctx->opcode));
             int16_t offset = ZIMM(ctx->opcode, 0, 4) << 1;
 
-            gen_ld(env, ctx, OPC_LHU, rd, rb, offset);
+            gen_ld(ctx, OPC_LHU, rd, rb, offset);
         }
         break;
     case LWSP16:
@@ -12525,7 +12534,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             int rb = 29;            /* SP */
             int16_t offset = ZIMM(ctx->opcode, 0, 5) << 2;
 
-            gen_ld(env, ctx, OPC_LW, rd, rb, offset);
+            gen_ld(ctx, OPC_LW, rd, rb, offset);
         }
         break;
     case LW16:
@@ -12534,7 +12543,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             int rb = mmreg(uMIPS_RS(ctx->opcode));
             int16_t offset = ZIMM(ctx->opcode, 0, 4) << 2;
 
-            gen_ld(env, ctx, OPC_LW, rd, rb, offset);
+            gen_ld(ctx, OPC_LW, rd, rb, offset);
         }
         break;
     case SB16:
@@ -12578,29 +12587,29 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
             int rd = uMIPS_RD5(ctx->opcode);
             int rs = uMIPS_RS5(ctx->opcode);
 
-            gen_arith_imm(env, ctx, OPC_ADDIU, rd, rs, 0);
+            gen_arith_imm(ctx, OPC_ADDIU, rd, rs, 0);
         }
         break;
     case ANDI16:
-        gen_andi16(env, ctx);
+        gen_andi16(ctx);
         break;
     case POOL16D:
         switch (ctx->opcode & 0x1) {
         case ADDIUS5:
-            gen_addius5(env, ctx);
+            gen_addius5(ctx);
             break;
         case ADDIUSP:
-            gen_addiusp(env, ctx);
+            gen_addiusp(ctx);
             break;
         }
         break;
     case POOL16E:
         switch (ctx->opcode & 0x1) {
         case ADDIUR2:
-            gen_addiur2(env, ctx);
+            gen_addiur2(ctx);
             break;
         case ADDIUR1SP:
-            gen_addiur1sp(env, ctx);
+            gen_addiur1sp(ctx);
             break;
         }
         break;
@@ -12651,17 +12660,12 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
 #endif
 
 /* MIPSDSP functions. */
-static void gen_mipsdsp_ld(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+static void gen_mipsdsp_ld(DisasContext *ctx, uint32_t opc,
                            int rd, int base, int offset)
 {
     const char *opn = "ldx";
     TCGv t0;
 
-    if (rd == 0) {
-        MIPS_DEBUG("NOP");
-        return;
-    }
-
     check_dsp(ctx);
     t0 = tcg_temp_new();
 
@@ -13717,8 +13721,7 @@ static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
 
 }
 
-static void gen_mipsdsp_bitinsn(CPUMIPSState *env, DisasContext *ctx,
-                                uint32_t op1, uint32_t op2,
+static void gen_mipsdsp_bitinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int val)
 {
     const char *opn = "mipsdsp Bit/ Manipulation";
@@ -13769,9 +13772,10 @@ static void gen_mipsdsp_bitinsn(CPUMIPSState *env, DisasContext *ctx,
             check_dsp(ctx);
             {
                 imm = (ctx->opcode >> 16) & 0x03FF;
+                imm = (int16_t)(imm << 6) >> 6;
                 tcg_gen_movi_tl(cpu_gpr[ret], \
                                 (target_long)((int32_t)imm << 16 | \
-                                (uint32_t)(uint16_t)imm));
+                                (uint16_t)imm));
             }
             break;
         case OPC_REPLV_PH:
@@ -13865,7 +13869,6 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
                                      int ret, int v1, int v2, int check_ret)
 {
     const char *opn = "mipsdsp add compare pick";
-    TCGv_i32 t0;
     TCGv t1;
     TCGv v1_t;
     TCGv v2_t;
@@ -13876,7 +13879,6 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
         return;
     }
 
-    t0 = tcg_temp_new_i32();
     t1 = tcg_temp_new();
     v1_t = tcg_temp_new();
     v2_t = tcg_temp_new();
@@ -13885,26 +13887,6 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
     gen_load_gpr(v2_t, v2);
 
     switch (op1) {
-    case OPC_APPEND_DSP:
-        switch (op2) {
-        case OPC_APPEND:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_append(cpu_gpr[ret], cpu_gpr[ret], v1_t, t0);
-            break;
-        case OPC_PREPEND:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_prepend(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
-            break;
-        case OPC_BALIGN:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_balign(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
-            break;
-        default:            /* Invid */
-            MIPS_INVAL("MASK APPEND");
-            generate_exception(ctx, EXCP_RI);
-            break;
-        }
-        break;
     case OPC_CMPU_EQ_QB_DSP:
         switch (op2) {
         case OPC_CMPU_EQ_QB:
@@ -14062,23 +14044,95 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
             break;
         }
         break;
+#endif
+    }
+
+    tcg_temp_free(t1);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
+                               uint32_t op1, int rt, int rs, int sa)
+{
+    const char *opn = "mipsdsp append/dappend";
+    TCGv t0;
+
+    check_dspr2(ctx);
+
+    if (rt == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    gen_load_gpr(t0, rs);
+
+    switch (op1) {
+    case OPC_APPEND_DSP:
+        switch (MASK_APPEND(ctx->opcode)) {
+        case OPC_APPEND:
+            if (sa != 0) {
+                tcg_gen_deposit_tl(cpu_gpr[rt], t0, cpu_gpr[rt], sa, 32 - sa);
+            }
+            tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
+            break;
+        case OPC_PREPEND:
+            if (sa != 0) {
+                tcg_gen_ext32u_tl(cpu_gpr[rt], cpu_gpr[rt]);
+                tcg_gen_shri_tl(cpu_gpr[rt], cpu_gpr[rt], sa);
+                tcg_gen_shli_tl(t0, t0, 32 - sa);
+                tcg_gen_or_tl(cpu_gpr[rt], cpu_gpr[rt], t0);
+            }
+            tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
+            break;
+        case OPC_BALIGN:
+            sa &= 3;
+            if (sa != 0 && sa != 2) {
+                tcg_gen_shli_tl(cpu_gpr[rt], cpu_gpr[rt], 8 * sa);
+                tcg_gen_ext32u_tl(t0, t0);
+                tcg_gen_shri_tl(t0, t0, 8 * (4 - sa));
+                tcg_gen_or_tl(cpu_gpr[rt], cpu_gpr[rt], t0);
+            }
+            tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK APPEND");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
     case OPC_DAPPEND_DSP:
-        switch (op2) {
+        switch (MASK_DAPPEND(ctx->opcode)) {
         case OPC_DAPPEND:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_dappend(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            if (sa != 0) {
+                tcg_gen_deposit_tl(cpu_gpr[rt], t0, cpu_gpr[rt], sa, 64 - sa);
+            }
             break;
         case OPC_PREPENDD:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_prependd(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            tcg_gen_shri_tl(cpu_gpr[rt], cpu_gpr[rt], 0x20 | sa);
+            tcg_gen_shli_tl(t0, t0, 64 - (0x20 | sa));
+            tcg_gen_or_tl(cpu_gpr[rt], t0, t0);
             break;
         case OPC_PREPENDW:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_prependw(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            if (sa != 0) {
+                tcg_gen_shri_tl(cpu_gpr[rt], cpu_gpr[rt], sa);
+                tcg_gen_shli_tl(t0, t0, 64 - sa);
+                tcg_gen_or_tl(cpu_gpr[rt], cpu_gpr[rt], t0);
+            }
             break;
         case OPC_DBALIGN:
-            tcg_gen_movi_i32(t0, v2);
-            gen_helper_dbalign(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            sa &= 7;
+            if (sa != 0 && sa != 2 && sa != 4) {
+                tcg_gen_shli_tl(cpu_gpr[rt], cpu_gpr[rt], 8 * sa);
+                tcg_gen_shri_tl(t0, t0, 8 * (8 - sa));
+                tcg_gen_or_tl(cpu_gpr[rt], cpu_gpr[rt], t0);
+            }
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DAPPEND");
@@ -14088,12 +14142,7 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
         break;
 #endif
     }
-
-    tcg_temp_free_i32(t0);
-    tcg_temp_free(t1);
-    tcg_temp_free(v1_t);
-    tcg_temp_free(v2_t);
-
+    tcg_temp_free(t0);
     (void)opn; /* avoid a compiler warning */
     MIPS_DEBUG("%s", opn);
 }
@@ -14371,18 +14420,18 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         switch (op1) {
         case OPC_SLL:          /* Shift with immediate */
         case OPC_SRA:
-            gen_shift_imm(env, ctx, op1, rd, rt, sa);
+            gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         case OPC_SRL:
             switch ((ctx->opcode >> 21) & 0x1f) {
             case 1:
                 /* rotr is decoded as srl on non-R2 CPUs */
-                if (env->insn_flags & ISA_MIPS32R2) {
+                if (ctx->insn_flags & ISA_MIPS32R2) {
                     op1 = OPC_ROTR;
                 }
                 /* Fallthrough */
             case 0:
-                gen_shift_imm(env, ctx, op1, rd, rt, sa);
+                gen_shift_imm(ctx, op1, rd, rt, sa);
                 break;
             default:
                 generate_exception(ctx, EXCP_RI);
@@ -14391,27 +14440,27 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             break;
         case OPC_MOVN:         /* Conditional move */
         case OPC_MOVZ:
-            check_insn(env, ctx, ISA_MIPS4 | ISA_MIPS32 |
+            check_insn(ctx, ISA_MIPS4 | ISA_MIPS32 |
                                  INSN_LOONGSON2E | INSN_LOONGSON2F);
-            gen_cond_move(env, ctx, op1, rd, rs, rt);
+            gen_cond_move(ctx, op1, rd, rs, rt);
             break;
         case OPC_ADD ... OPC_SUBU:
-            gen_arith(env, ctx, op1, rd, rs, rt);
+            gen_arith(ctx, op1, rd, rs, rt);
             break;
         case OPC_SLLV:         /* Shifts */
         case OPC_SRAV:
-            gen_shift(env, ctx, op1, rd, rs, rt);
+            gen_shift(ctx, op1, rd, rs, rt);
             break;
         case OPC_SRLV:
             switch ((ctx->opcode >> 6) & 0x1f) {
             case 1:
                 /* rotrv is decoded as srlv on non-R2 CPUs */
-                if (env->insn_flags & ISA_MIPS32R2) {
+                if (ctx->insn_flags & ISA_MIPS32R2) {
                     op1 = OPC_ROTRV;
                 }
                 /* Fallthrough */
             case 0:
-                gen_shift(env, ctx, op1, rd, rs, rt);
+                gen_shift(ctx, op1, rd, rs, rt);
                 break;
             default:
                 generate_exception(ctx, EXCP_RI);
@@ -14420,17 +14469,17 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             break;
         case OPC_SLT:          /* Set on less than */
         case OPC_SLTU:
-            gen_slt(env, ctx, op1, rd, rs, rt);
+            gen_slt(ctx, op1, rd, rs, rt);
             break;
         case OPC_AND:          /* Logic*/
         case OPC_OR:
         case OPC_NOR:
         case OPC_XOR:
-            gen_logic(env, ctx, op1, rd, rs, rt);
+            gen_logic(ctx, op1, rd, rs, rt);
             break;
         case OPC_MULT ... OPC_DIVU:
             if (sa) {
-                check_insn(env, ctx, INSN_VR54XX);
+                check_insn(ctx, INSN_VR54XX);
                 op1 = MASK_MUL_VR54XX(ctx->opcode);
                 gen_mul_vr54xx(ctx, op1, rd, rs, rt);
             } else
@@ -14482,7 +14531,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             break;
 
         case OPC_MOVCI:
-            check_insn(env, ctx, ISA_MIPS4 | ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
             if (env->CP0_Config1 & (1 << CP0C1_FP)) {
                 check_cp1_enabled(ctx);
                 gen_movci(ctx, rd, rs, (ctx->opcode >> 18) & 0x7,
@@ -14498,22 +14547,22 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_DSRA:
         case OPC_DSLL32:
         case OPC_DSRA32:
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
-            gen_shift_imm(env, ctx, op1, rd, rt, sa);
+            gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         case OPC_DSRL:
             switch ((ctx->opcode >> 21) & 0x1f) {
             case 1:
                 /* drotr is decoded as dsrl on non-R2 CPUs */
-                if (env->insn_flags & ISA_MIPS32R2) {
+                if (ctx->insn_flags & ISA_MIPS32R2) {
                     op1 = OPC_DROTR;
                 }
                 /* Fallthrough */
             case 0:
-                check_insn(env, ctx, ISA_MIPS3);
+                check_insn(ctx, ISA_MIPS3);
                 check_mips_64(ctx);
-                gen_shift_imm(env, ctx, op1, rd, rt, sa);
+                gen_shift_imm(ctx, op1, rd, rt, sa);
                 break;
             default:
                 generate_exception(ctx, EXCP_RI);
@@ -14524,14 +14573,14 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             switch ((ctx->opcode >> 21) & 0x1f) {
             case 1:
                 /* drotr32 is decoded as dsrl32 on non-R2 CPUs */
-                if (env->insn_flags & ISA_MIPS32R2) {
+                if (ctx->insn_flags & ISA_MIPS32R2) {
                     op1 = OPC_DROTR32;
                 }
                 /* Fallthrough */
             case 0:
-                check_insn(env, ctx, ISA_MIPS3);
+                check_insn(ctx, ISA_MIPS3);
                 check_mips_64(ctx);
-                gen_shift_imm(env, ctx, op1, rd, rt, sa);
+                gen_shift_imm(ctx, op1, rd, rt, sa);
                 break;
             default:
                 generate_exception(ctx, EXCP_RI);
@@ -14539,28 +14588,28 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             }
             break;
         case OPC_DADD ... OPC_DSUBU:
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
-            gen_arith(env, ctx, op1, rd, rs, rt);
+            gen_arith(ctx, op1, rd, rs, rt);
             break;
         case OPC_DSLLV:
         case OPC_DSRAV:
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
-            gen_shift(env, ctx, op1, rd, rs, rt);
+            gen_shift(ctx, op1, rd, rs, rt);
             break;
         case OPC_DSRLV:
             switch ((ctx->opcode >> 6) & 0x1f) {
             case 1:
                 /* drotrv is decoded as dsrlv on non-R2 CPUs */
-                if (env->insn_flags & ISA_MIPS32R2) {
+                if (ctx->insn_flags & ISA_MIPS32R2) {
                     op1 = OPC_DROTRV;
                 }
                 /* Fallthrough */
             case 0:
-                check_insn(env, ctx, ISA_MIPS3);
+                check_insn(ctx, ISA_MIPS3);
                 check_mips_64(ctx);
-                gen_shift(env, ctx, op1, rd, rs, rt);
+                gen_shift(ctx, op1, rd, rs, rt);
                 break;
             default:
                 generate_exception(ctx, EXCP_RI);
@@ -14568,7 +14617,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             }
             break;
         case OPC_DMULT ... OPC_DDIVU:
-            check_insn(env, ctx, ISA_MIPS3);
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_muldiv(ctx, op1, rs, rt);
             break;
@@ -14584,22 +14633,22 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         switch (op1) {
         case OPC_MADD ... OPC_MADDU: /* Multiply and add/sub */
         case OPC_MSUB ... OPC_MSUBU:
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             gen_muldiv(ctx, op1, rs, rt);
             break;
         case OPC_MUL:
-            gen_arith(env, ctx, op1, rd, rs, rt);
+            gen_arith(ctx, op1, rd, rs, rt);
             break;
         case OPC_CLO:
         case OPC_CLZ:
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             gen_cl(ctx, op1, rd, rs);
             break;
         case OPC_SDBBP:
             /* XXX: not clear which exception should be raised
              *      when in debug mode...
              */
-            check_insn(env, ctx, ISA_MIPS32);
+            check_insn(ctx, ISA_MIPS32);
             if (!(ctx->hflags & MIPS_HFLAG_DM)) {
                 generate_exception(ctx, EXCP_DBp);
             } else {
@@ -14613,13 +14662,13 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_MULTU_G_2F:
         case OPC_MOD_G_2F:
         case OPC_MODU_G_2F:
-            check_insn(env, ctx, INSN_LOONGSON2F);
+            check_insn(ctx, INSN_LOONGSON2F);
             gen_loongson_integer(ctx, op1, rd, rs, rt);
             break;
 #if defined(TARGET_MIPS64)
         case OPC_DCLO:
         case OPC_DCLZ:
-            check_insn(env, ctx, ISA_MIPS64);
+            check_insn(ctx, ISA_MIPS64);
             check_mips_64(ctx);
             gen_cl(ctx, op1, rd, rs);
             break;
@@ -14629,7 +14678,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_DDIVU_G_2F:
         case OPC_DMOD_G_2F:
         case OPC_DMODU_G_2F:
-            check_insn(env, ctx, INSN_LOONGSON2F);
+            check_insn(ctx, INSN_LOONGSON2F);
             gen_loongson_integer(ctx, op1, rd, rs, rt);
             break;
 #endif
@@ -14644,19 +14693,19 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         switch (op1) {
         case OPC_EXT:
         case OPC_INS:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_bitops(ctx, op1, rt, rs, sa, rd);
             break;
         case OPC_BSHFL:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             op2 = MASK_BSHFL(ctx->opcode);
             gen_bshfl(ctx, op2, rt, rd);
             break;
         case OPC_RDHWR:
-            gen_rdhwr(env, ctx, rt, rd);
+            gen_rdhwr(ctx, rt, rd);
             break;
         case OPC_FORK:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             {
                 TCGv t0 = tcg_temp_new();
                 TCGv t1 = tcg_temp_new();
@@ -14669,7 +14718,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             }
             break;
         case OPC_YIELD:
-            check_insn(env, ctx, ASE_MT);
+            check_insn(ctx, ASE_MT);
             {
                 TCGv t0 = tcg_temp_new();
 
@@ -14685,7 +14734,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
         /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
          * the same mask and op1. */
-            if ((env->insn_flags & ASE_DSPR2) && (op1 == OPC_MULT_G_2E)) {
+            if ((ctx->insn_flags & ASE_DSPR2) && (op1 == OPC_MULT_G_2E)) {
                 op2 = MASK_ADDUH_QB(ctx->opcode);
                 switch (op2) {
                 case OPC_ADDUH_QB:
@@ -14713,7 +14762,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                     generate_exception(ctx, EXCP_RI);
                     break;
                 }
-            } else if (env->insn_flags & INSN_LOONGSON2E) {
+            } else if (ctx->insn_flags & INSN_LOONGSON2E) {
                 gen_loongson_integer(ctx, op1, rd, rs, rt);
             } else {
                 generate_exception(ctx, EXCP_RI);
@@ -14728,7 +14777,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             case OPC_LBUX:
             case OPC_LHX:
             case OPC_LWX:
-                gen_mipsdsp_ld(env, ctx, op2, rd, rs, rt);
+                gen_mipsdsp_ld(ctx, op2, rd, rs, rt);
                 break;
             default:            /* Invalid */
                 MIPS_INVAL("MASK LX");
@@ -14759,7 +14808,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             case OPC_REPLV_QB:
             case OPC_REPL_PH:
             case OPC_REPLV_PH:
-                gen_mipsdsp_bitinsn(env, ctx, op1, op2, rd, rt);
+                gen_mipsdsp_bitinsn(ctx, op1, op2, rd, rt);
                 break;
             default:
                 MIPS_INVAL("MASK ABSQ_S.PH");
@@ -14912,9 +14961,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             }
             break;
         case OPC_APPEND_DSP:
-            check_dspr2(ctx);
-            op2 = MASK_APPEND(ctx->opcode);
-            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rt, rs, rd, 1);
+            gen_mipsdsp_append(env, ctx, op1, rt, rs, rd);
             break;
         case OPC_EXTR_W_DSP:
             op2 = MASK_EXTR_W(ctx->opcode);
@@ -14951,12 +14998,12 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 #if defined(TARGET_MIPS64)
         case OPC_DEXTM ... OPC_DEXT:
         case OPC_DINSM ... OPC_DINS:
-            check_insn(env, ctx, ISA_MIPS64R2);
+            check_insn(ctx, ISA_MIPS64R2);
             check_mips_64(ctx);
             gen_bitops(ctx, op1, rt, rs, sa, rd);
             break;
         case OPC_DBSHFL:
-            check_insn(env, ctx, ISA_MIPS64R2);
+            check_insn(ctx, ISA_MIPS64R2);
             check_mips_64(ctx);
             op2 = MASK_DBSHFL(ctx->opcode);
             gen_bshfl(ctx, op2, rt, rd);
@@ -14964,7 +15011,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_DDIV_G_2E ... OPC_DDIVU_G_2E:
         case OPC_DMULT_G_2E ... OPC_DMULTU_G_2E:
         case OPC_DMOD_G_2E ... OPC_DMODU_G_2E:
-            check_insn(env, ctx, INSN_LOONGSON2E);
+            check_insn(ctx, INSN_LOONGSON2E);
             gen_loongson_integer(ctx, op1, rd, rs, rt);
             break;
         case OPC_ABSQ_S_QH_DSP:
@@ -14995,7 +15042,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             case OPC_REPLV_OB:
             case OPC_REPLV_PW:
             case OPC_REPLV_QH:
-                gen_mipsdsp_bitinsn(env, ctx, op1, op2, rd, rt);
+                gen_mipsdsp_bitinsn(ctx, op1, op2, rd, rt);
                 break;
             default:            /* Invalid */
                 MIPS_INVAL("MASK ABSQ_S.QH");
@@ -15088,9 +15135,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             }
             break;
         case OPC_DAPPEND_DSP:
-            check_dspr2(ctx);
-            op2 = MASK_DAPPEND(ctx->opcode);
-            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rt, rs, rd, 1);
+            gen_mipsdsp_append(env, ctx, op1, rt, rs, rd);
             break;
         case OPC_DEXTR_W_DSP:
             op2 = MASK_DEXTR_W(ctx->opcode);
@@ -15216,7 +15261,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             gen_trap(ctx, op1, rs, -1, imm);
             break;
         case OPC_SYNCI:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             /* Treat as NOP. */
             break;
         case OPC_BPOSGE32:    /* MIPS DSP branch */
@@ -15262,27 +15307,27 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                 op2 = MASK_MFMC0(ctx->opcode);
                 switch (op2) {
                 case OPC_DMT:
-                    check_insn(env, ctx, ASE_MT);
+                    check_insn(ctx, ASE_MT);
                     gen_helper_dmt(t0);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_EMT:
-                    check_insn(env, ctx, ASE_MT);
+                    check_insn(ctx, ASE_MT);
                     gen_helper_emt(t0);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_DVPE:
-                    check_insn(env, ctx, ASE_MT);
+                    check_insn(ctx, ASE_MT);
                     gen_helper_dvpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_EVPE:
-                    check_insn(env, ctx, ASE_MT);
+                    check_insn(ctx, ASE_MT);
                     gen_helper_evpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_DI:
-                    check_insn(env, ctx, ISA_MIPS32R2);
+                    check_insn(ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
                     gen_helper_di(t0, cpu_env);
                     gen_store_gpr(t0, rt);
@@ -15290,7 +15335,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                     ctx->bstate = BS_STOP;
                     break;
                 case OPC_EI:
-                    check_insn(env, ctx, ISA_MIPS32R2);
+                    check_insn(ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
                     gen_helper_ei(t0, cpu_env);
                     gen_store_gpr(t0, rt);
@@ -15307,11 +15352,11 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 #endif /* !CONFIG_USER_ONLY */
             break;
         case OPC_RDPGPR:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_load_srsgpr(rt, rd);
             break;
         case OPC_WRPGPR:
-            check_insn(env, ctx, ISA_MIPS32R2);
+            check_insn(ctx, ISA_MIPS32R2);
             gen_store_srsgpr(rt, rd);
             break;
         default:
@@ -15322,17 +15367,17 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         break;
     case OPC_ADDI: /* Arithmetic with immediate opcode */
     case OPC_ADDIU:
-         gen_arith_imm(env, ctx, op, rt, rs, imm);
+         gen_arith_imm(ctx, op, rt, rs, imm);
          break;
     case OPC_SLTI: /* Set on less than with immediate opcode */
     case OPC_SLTIU:
-         gen_slt_imm(env, ctx, op, rt, rs, imm);
+         gen_slt_imm(ctx, op, rt, rs, imm);
          break;
     case OPC_ANDI: /* Arithmetic with immediate opcode */
     case OPC_LUI:
     case OPC_ORI:
     case OPC_XORI:
-         gen_logic_imm(env, ctx, op, rt, rs, imm);
+         gen_logic_imm(ctx, op, rt, rs, imm);
          break;
     case OPC_J ... OPC_JAL: /* Jump */
          offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
@@ -15346,7 +15391,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
          break;
     case OPC_LB ... OPC_LWR: /* Load and stores */
     case OPC_LL:
-         gen_ld(env, ctx, op, rt, rs, imm);
+         gen_ld(ctx, op, rt, rs, imm);
          break;
     case OPC_SB ... OPC_SW:
     case OPC_SWR:
@@ -15357,11 +15402,11 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
          break;
     case OPC_CACHE:
         check_cp0_enabled(ctx);
-        check_insn(env, ctx, ISA_MIPS3 | ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
         /* Treat as NOP. */
         break;
     case OPC_PREF:
-        check_insn(env, ctx, ISA_MIPS4 | ISA_MIPS32);
+        check_insn(ctx, ISA_MIPS4 | ISA_MIPS32);
         /* Treat as NOP. */
         break;
 
@@ -15380,7 +15425,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             switch (op1) {
             case OPC_MFHC1:
             case OPC_MTHC1:
-                check_insn(env, ctx, ISA_MIPS32R2);
+                check_insn(ctx, ISA_MIPS32R2);
             case OPC_MFC1:
             case OPC_CFC1:
             case OPC_MTC1:
@@ -15390,17 +15435,17 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 #if defined(TARGET_MIPS64)
             case OPC_DMFC1:
             case OPC_DMTC1:
-                check_insn(env, ctx, ISA_MIPS3);
+                check_insn(ctx, ISA_MIPS3);
                 gen_cp1(ctx, op1, rt, rd);
                 break;
 #endif
             case OPC_BC1ANY2:
             case OPC_BC1ANY4:
                 check_cop1x(ctx);
-                check_insn(env, ctx, ASE_MIPS3D);
+                check_insn(ctx, ASE_MIPS3D);
                 /* fall through */
             case OPC_BC1:
-                gen_compute_branch1(env, ctx, MASK_BC1(ctx->opcode),
+                gen_compute_branch1(ctx, MASK_BC1(ctx->opcode),
                                     (rt >> 2) & 0x7, imm << 2);
                 *is_branch = 1;
                 break;
@@ -15431,7 +15476,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         generate_exception_err(ctx, EXCP_CpU, 2);
         break;
     case OPC_CP2:
-        check_insn(env, ctx, INSN_LOONGSON2F);
+        check_insn(ctx, INSN_LOONGSON2F);
         /* Note that these instructions use different fields.  */
         gen_loongson_multimedia(ctx, sa, rd, rt);
         break;
@@ -15483,36 +15528,36 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
     case OPC_LDL ... OPC_LDR:
     case OPC_LLD:
     case OPC_LD:
-        check_insn(env, ctx, ISA_MIPS3);
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
-        gen_ld(env, ctx, op, rt, rs, imm);
+        gen_ld(ctx, op, rt, rs, imm);
         break;
     case OPC_SDL ... OPC_SDR:
     case OPC_SD:
-        check_insn(env, ctx, ISA_MIPS3);
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_st(ctx, op, rt, rs, imm);
         break;
     case OPC_SCD:
-        check_insn(env, ctx, ISA_MIPS3);
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_st_cond(ctx, op, rt, rs, imm);
         break;
     case OPC_DADDI:
     case OPC_DADDIU:
-        check_insn(env, ctx, ISA_MIPS3);
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
-        gen_arith_imm(env, ctx, op, rt, rs, imm);
+        gen_arith_imm(ctx, op, rt, rs, imm);
         break;
 #endif
     case OPC_JALX:
-        check_insn(env, ctx, ASE_MIPS16 | ASE_MICROMIPS);
+        check_insn(ctx, ASE_MIPS16 | ASE_MICROMIPS);
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
         gen_compute_branch(ctx, op, 4, rs, rt, offset);
         *is_branch = 1;
         break;
     case OPC_MDMX:
-        check_insn(env, ctx, ASE_MDMX);
+        check_insn(ctx, ASE_MDMX);
         /* MDMX: Not implemented. */
     default:            /* Invalid */
         MIPS_INVAL("major opcode");
@@ -15543,6 +15588,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.singlestep_enabled = env->singlestep_enabled;
+    ctx.insn_flags = env->insn_flags;
     ctx.tb = tb;
     ctx.bstate = BS_NONE;
     /* Restore delay slot state from the tb context.  */
@@ -15579,13 +15625,13 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = ctx.pc;
+            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
             gen_opc_hflags[lj] = ctx.hflags & MIPS_HFLAG_BMASK;
             gen_opc_btarget[lj] = ctx.btarget;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
@@ -15595,10 +15641,10 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
             ctx.opcode = cpu_ldl_code(env, ctx.pc);
             insn_bytes = 4;
             decode_opc(env, &ctx, &is_branch);
-        } else if (env->insn_flags & ASE_MICROMIPS) {
+        } else if (ctx.insn_flags & ASE_MICROMIPS) {
             ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_micromips_opc(env, &ctx, &is_branch);
-        } else if (env->insn_flags & ASE_MIPS16) {
+        } else if (ctx.insn_flags & ASE_MIPS16) {
             ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_mips16_opc(env, &ctx, &is_branch);
         } else {
@@ -15607,7 +15653,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
             break;
         }
         if (!is_branch) {
-            handle_delay_slot(env, &ctx, insn_bytes);
+            handle_delay_slot(&ctx, insn_bytes);
         }
         ctx.pc += insn_bytes;
 
@@ -15662,7 +15708,7 @@ done_generating:
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = ctx.pc - pc_start;
         tb->icount = num_insns;
@@ -15877,13 +15923,10 @@ MIPSCPU *cpu_mips_init(const char *cpu_model)
 
 void cpu_state_reset(CPUMIPSState *env)
 {
-    if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
-        log_cpu_state(env, 0);
-    }
-
-    memset(env, 0, offsetof(CPUMIPSState, breakpoints));
-    tlb_flush(env, 1);
+#ifndef CONFIG_USER_ONLY
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+#endif
 
     /* Reset registers to their default values */
     env->CP0_PRid = env->cpu_model->CP0_PRid;
@@ -15935,10 +15978,8 @@ void cpu_state_reset(CPUMIPSState *env)
     if (env->CP0_Config1 & (1 << CP0C1_FP)) {
         env->CP0_Status |= (1 << CP0St_CU1);
     }
-    if (env->cpu_model->insn_flags & ASE_DSPR2) {
-        env->hflags |= MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2;
-    } else if (env->cpu_model->insn_flags & ASE_DSP) {
-        env->hflags |= MIPS_HFLAG_DSP;
+    if (env->CP0_Config3 & (1 << CP0C3_DSPP)) {
+        env->CP0_Status |= (1 << CP0St_MX);
     }
 #else
     if (env->hflags & MIPS_HFLAG_BMASK) {
@@ -15952,7 +15993,7 @@ void cpu_state_reset(CPUMIPSState *env)
     env->CP0_Random = env->tlb->nb_tlb - 1;
     env->tlb->tlb_in_use = env->tlb->nb_tlb;
     env->CP0_Wired = 0;
-    env->CP0_EBase = 0x80000000 | (env->cpu_index & 0x3FF);
+    env->CP0_EBase = 0x80000000 | (cs->cpu_index & 0x3FF);
     env->CP0_Status = (1 << CP0St_BEV) | (1 << CP0St_ERL);
     /* vectored interrupts not implemented, timer on int 7,
        no performance counters. */
@@ -15975,13 +16016,13 @@ void cpu_state_reset(CPUMIPSState *env)
 
         /* Only TC0 on VPE 0 starts as active.  */
         for (i = 0; i < ARRAY_SIZE(env->tcs); i++) {
-            env->tcs[i].CP0_TCBind = env->cpu_index << CP0TCBd_CurVPE;
+            env->tcs[i].CP0_TCBind = cs->cpu_index << CP0TCBd_CurVPE;
             env->tcs[i].CP0_TCHalt = 1;
         }
         env->active_tc.CP0_TCHalt = 1;
         env->halted = 1;
 
-        if (!env->cpu_index) {
+        if (cs->cpu_index == 0) {
             /* VPE0 starts up enabled.  */
             env->mvp->CP0_MVPControl |= (1 << CP0MVPCo_EVP);
             env->CP0_VPEConf0 |= (1 << CP0VPEC0_MVP) | (1 << CP0VPEC0_VPA);
@@ -16002,7 +16043,7 @@ void cpu_state_reset(CPUMIPSState *env)
 
 void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->active_tc.PC = gen_opc_pc[pc_pos];
+    env->active_tc.PC = tcg_ctx.gen_opc_pc[pc_pos];
     env->hflags &= ~MIPS_HFLAG_BMASK;
     env->hflags |= gen_opc_hflags[pc_pos];
     switch (env->hflags & MIPS_HFLAG_BMASK_BASE) {
diff --git a/target-openrisc/cpu.c b/target-openrisc/cpu.c
index ba35b17..a7a8de8 100644
--- a/target-openrisc/cpu.c
+++ b/target-openrisc/cpu.c
@@ -27,7 +27,7 @@ static void openrisc_cpu_reset(CPUState *s)
     OpenRISCCPUClass *occ = OPENRISC_CPU_GET_CLASS(cpu);
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", cpu->env.cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(&cpu->env, 0);
     }
 
@@ -88,6 +88,23 @@ static void openrisc_cpu_initfn(Object *obj)
 }
 
 /* CPU models */
+
+static ObjectClass *openrisc_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    oc = object_class_by_name(cpu_model);
+    if (oc != NULL && (!object_class_dynamic_cast(oc, TYPE_OPENRISC_CPU) ||
+                       object_class_is_abstract(oc))) {
+        return NULL;
+    }
+    return oc;
+}
+
 static void or1200_initfn(Object *obj)
 {
     OpenRISCCPU *cpu = OPENRISC_CPU(obj);
@@ -120,19 +137,22 @@ static void openrisc_cpu_class_init(ObjectClass *oc, void *data)
 
     occ->parent_reset = cc->reset;
     cc->reset = openrisc_cpu_reset;
+
+    cc->class_by_name = openrisc_cpu_class_by_name;
 }
 
 static void cpu_register(const OpenRISCCPUInfo *info)
 {
     TypeInfo type_info = {
-        .name = info->name,
         .parent = TYPE_OPENRISC_CPU,
         .instance_size = sizeof(OpenRISCCPU),
         .instance_init = info->initfn,
         .class_size = sizeof(OpenRISCCPUClass),
     };
 
-    type_register_static(&type_info);
+    type_info.name = g_strdup_printf("%s-" TYPE_OPENRISC_CPU, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
 }
 
 static const TypeInfo openrisc_cpu_type_info = {
@@ -140,7 +160,7 @@ static const TypeInfo openrisc_cpu_type_info = {
     .parent = TYPE_CPU,
     .instance_size = sizeof(OpenRISCCPU),
     .instance_init = openrisc_cpu_initfn,
-    .abstract = false,
+    .abstract = true,
     .class_size = sizeof(OpenRISCCPUClass),
     .class_init = openrisc_cpu_class_init,
 };
@@ -158,11 +178,13 @@ static void openrisc_cpu_register_types(void)
 OpenRISCCPU *cpu_openrisc_init(const char *cpu_model)
 {
     OpenRISCCPU *cpu;
+    ObjectClass *oc;
 
-    if (!object_class_by_name(cpu_model)) {
+    oc = openrisc_cpu_class_by_name(cpu_model);
+    if (oc == NULL) {
         return NULL;
     }
-    cpu = OPENRISC_CPU(object_new(cpu_model));
+    cpu = OPENRISC_CPU(object_new(object_class_get_name(oc)));
     cpu->env.cpu_model_str = cpu_model;
 
     openrisc_cpu_realize(OBJECT(cpu), NULL);
@@ -170,11 +192,6 @@ OpenRISCCPU *cpu_openrisc_init(const char *cpu_model)
     return cpu;
 }
 
-typedef struct OpenRISCCPUList {
-    fprintf_function cpu_fprintf;
-    FILE *file;
-} OpenRISCCPUList;
-
 /* Sort alphabetically by type name, except for "any". */
 static gint openrisc_cpu_list_compare(gconstpointer a, gconstpointer b)
 {
@@ -184,9 +201,9 @@ static gint openrisc_cpu_list_compare(gconstpointer a, gconstpointer b)
 
     name_a = object_class_get_name(class_a);
     name_b = object_class_get_name(class_b);
-    if (strcmp(name_a, "any") == 0) {
+    if (strcmp(name_a, "any-" TYPE_OPENRISC_CPU) == 0) {
         return 1;
-    } else if (strcmp(name_b, "any") == 0) {
+    } else if (strcmp(name_b, "any-" TYPE_OPENRISC_CPU) == 0) {
         return -1;
     } else {
         return strcmp(name_a, name_b);
@@ -196,15 +213,21 @@ static gint openrisc_cpu_list_compare(gconstpointer a, gconstpointer b)
 static void openrisc_cpu_list_entry(gpointer data, gpointer user_data)
 {
     ObjectClass *oc = data;
-    OpenRISCCPUList *s = user_data;
+    CPUListState *s = user_data;
+    const char *typename;
+    char *name;
 
+    typename = object_class_get_name(oc);
+    name = g_strndup(typename,
+                     strlen(typename) - strlen("-" TYPE_OPENRISC_CPU));
     (*s->cpu_fprintf)(s->file, "  %s\n",
-                      object_class_get_name(oc));
+                      name);
+    g_free(name);
 }
 
 void cpu_openrisc_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    OpenRISCCPUList s = {
+    CPUListState s = {
         .file = f,
         .cpu_fprintf = cpu_fprintf,
     };
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index ebb5ad3..3beab45 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -30,10 +30,10 @@ struct OpenRISCCPU;
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
-#include "softfloat.h"
-#include "qemu/cpu.h"
-#include "error.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
+#include "qom/cpu.h"
+#include "qapi/error.h"
 
 #define TYPE_OPENRISC_CPU "or32-cpu"
 
@@ -398,7 +398,7 @@ static inline void cpu_clone_regs(CPUOpenRISCState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env,
                                         target_ulong *pc,
@@ -427,7 +427,7 @@ static inline bool cpu_has_work(CPUState *cpu)
                                      CPU_INTERRUPT_TIMER);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline target_ulong cpu_get_pc(CPUOpenRISCState *env)
 {
diff --git a/target-openrisc/exception_helper.c b/target-openrisc/exception_helper.c
index dab4148..0c53b77 100644
--- a/target-openrisc/exception_helper.c
+++ b/target-openrisc/exception_helper.c
@@ -23,7 +23,7 @@
 
 void HELPER(exception)(CPUOpenRISCState *env, uint32_t excp)
 {
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     raise_exception(cpu, excp);
 }
diff --git a/target-openrisc/fpu_helper.c b/target-openrisc/fpu_helper.c
index b184d5e..4615a36 100644
--- a/target-openrisc/fpu_helper.c
+++ b/target-openrisc/fpu_helper.c
@@ -68,7 +68,7 @@ static inline void update_fpcsr(OpenRISCCPU *cpu)
 uint64_t HELPER(itofd)(CPUOpenRISCState *env, uint64_t val)
 {
     uint64_t itofd;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     itofd = int32_to_float64(val, &cpu->env.fp_status);
@@ -80,7 +80,7 @@ uint64_t HELPER(itofd)(CPUOpenRISCState *env, uint64_t val)
 uint32_t HELPER(itofs)(CPUOpenRISCState *env, uint32_t val)
 {
     uint32_t itofs;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     itofs = int32_to_float32(val, &cpu->env.fp_status);
@@ -92,7 +92,7 @@ uint32_t HELPER(itofs)(CPUOpenRISCState *env, uint32_t val)
 uint64_t HELPER(ftoid)(CPUOpenRISCState *env, uint64_t val)
 {
     uint64_t ftoid;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     ftoid = float32_to_int64(val, &cpu->env.fp_status);
@@ -104,7 +104,7 @@ uint64_t HELPER(ftoid)(CPUOpenRISCState *env, uint64_t val)
 uint32_t HELPER(ftois)(CPUOpenRISCState *env, uint32_t val)
 {
     uint32_t ftois;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     set_float_exception_flags(0, &cpu->env.fp_status);
     ftois = float32_to_int32(val, &cpu->env.fp_status);
@@ -120,7 +120,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     uint64_t result;                                                      \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     result = float64_ ## name(fdt0, fdt1, &cpu->env.fp_status);           \
     update_fpcsr(cpu);                                                    \
@@ -131,7 +131,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     uint32_t result;                                                      \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     result = float32_ ## name(fdt0, fdt1, &cpu->env.fp_status);           \
     update_fpcsr(cpu);                                                    \
@@ -152,7 +152,7 @@ uint64_t helper_float_ ## name1 ## name2 ## _d(CPUOpenRISCState *env,     \
 {                                                                         \
     uint64_t result, temp, hi, lo;                                        \
     uint32_t val1, val2;                                                  \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     hi = env->fpmaddhi;                                                   \
     lo = env->fpmaddlo;                                                   \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
@@ -174,7 +174,7 @@ uint32_t helper_float_ ## name1 ## name2 ## _s(CPUOpenRISCState *env,     \
 {                                                                         \
     uint64_t result, temp, hi, lo;                                        \
     uint32_t val1, val2;                                                  \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     hi = cpu->env.fpmaddhi;                                               \
     lo = cpu->env.fpmaddlo;                                               \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
@@ -198,7 +198,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = float64_ ## name(fdt0, fdt1, &cpu->env.fp_status);              \
     update_fpcsr(cpu);                                                    \
@@ -209,7 +209,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                              uint32_t fdt0, uint32_t fdt1)\
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = float32_ ## name(fdt0, fdt1, &cpu->env.fp_status);              \
     update_fpcsr(cpu);                                                    \
@@ -227,7 +227,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float64_eq_quiet(fdt0, fdt1, &cpu->env.fp_status);             \
     update_fpcsr(cpu);                                                    \
@@ -238,7 +238,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float32_eq_quiet(fdt0, fdt1, &cpu->env.fp_status);             \
     update_fpcsr(cpu);                                                    \
@@ -253,7 +253,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float64_le(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
@@ -264,7 +264,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float32_le(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
@@ -278,7 +278,7 @@ uint64_t helper_float_ ## name ## _d(CPUOpenRISCState *env,               \
                                      uint64_t fdt0, uint64_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float64_lt(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
@@ -289,7 +289,7 @@ uint32_t helper_float_ ## name ## _s(CPUOpenRISCState *env,               \
                                      uint32_t fdt0, uint32_t fdt1)        \
 {                                                                         \
     int res;                                                              \
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));                    \
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);                         \
     set_float_exception_flags(0, &cpu->env.fp_status);                    \
     res = !float32_lt(fdt0, fdt1, &cpu->env.fp_status);                   \
     update_fpcsr(cpu);                                                    \
diff --git a/target-openrisc/helper.h b/target-openrisc/helper.h
index 404d464..2af9790 100644
--- a/target-openrisc/helper.h
+++ b/target-openrisc/helper.h
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 /* exception */
 DEF_HELPER_FLAGS_2(exception, 0, void, env, i32)
@@ -67,4 +67,4 @@ DEF_HELPER_FLAGS_1(rfe, 0, void, env)
 DEF_HELPER_FLAGS_4(mtspr, 0, void, env, tl, tl, tl)
 DEF_HELPER_FLAGS_4(mfspr, 0, tl, env, tl, tl, tl)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-openrisc/int_helper.c b/target-openrisc/int_helper.c
index 2fdfd27..16cb5ab 100644
--- a/target-openrisc/int_helper.c
+++ b/target-openrisc/int_helper.c
@@ -21,7 +21,7 @@
 #include "cpu.h"
 #include "helper.h"
 #include "exception.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 target_ulong HELPER(ff1)(target_ulong x)
 {
@@ -48,7 +48,7 @@ uint32_t HELPER(mul32)(CPUOpenRISCState *env,
     uint64_t result;
     uint32_t high, cy;
 
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     result = (uint64_t)ra * rb;
     /* regisiers in or32 is 32bit, so 32 is NOT a magic number.
diff --git a/target-openrisc/interrupt.c b/target-openrisc/interrupt.c
index 642da7d..7f2c025 100644
--- a/target-openrisc/interrupt.c
+++ b/target-openrisc/interrupt.c
@@ -19,8 +19,8 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 #ifndef CONFIG_USER_ONLY
 #include "hw/loader.h"
 #endif
diff --git a/target-openrisc/interrupt_helper.c b/target-openrisc/interrupt_helper.c
index 79f5afe..a176441 100644
--- a/target-openrisc/interrupt_helper.c
+++ b/target-openrisc/interrupt_helper.c
@@ -23,7 +23,7 @@
 
 void HELPER(rfe)(CPUOpenRISCState *env)
 {
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 #ifndef CONFIG_USER_ONLY
     int need_flush_tlb = (cpu->env.sr & (SR_SM | SR_IME | SR_DME)) ^
                          (cpu->env.esr & (SR_SM | SR_IME | SR_DME));
diff --git a/target-openrisc/mmu.c b/target-openrisc/mmu.c
index f2a6523..d354e1f 100644
--- a/target-openrisc/mmu.c
+++ b/target-openrisc/mmu.c
@@ -20,8 +20,8 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 #ifndef CONFIG_USER_ONLY
 #include "hw/loader.h"
 #endif
@@ -187,7 +187,7 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
     int ret = 0;
     hwaddr physical = 0;
     int prot = 0;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     ret = cpu_openrisc_get_phys_addr(cpu, &physical, &prot,
                                      address, rw);
@@ -209,7 +209,7 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
                                   target_ulong address, int rw, int mmu_idx)
 {
     int ret = 0;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     cpu_openrisc_raise_mmu_exception(cpu, address, rw, ret);
     ret = 1;
@@ -224,7 +224,7 @@ hwaddr cpu_get_phys_page_debug(CPUOpenRISCState *env,
 {
     hwaddr phys_addr;
     int prot;
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     if (cpu_openrisc_get_phys_addr(cpu, &phys_addr, &prot, addr, 0)) {
         return -1;
diff --git a/target-openrisc/mmu_helper.c b/target-openrisc/mmu_helper.c
index 59ed371..e46b092 100644
--- a/target-openrisc/mmu_helper.c
+++ b/target-openrisc/mmu_helper.c
@@ -21,26 +21,24 @@
 #include "cpu.h"
 
 #ifndef CONFIG_USER_ONLY
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 void tlb_fill(CPUOpenRISCState *env, target_ulong addr, int is_write,
               int mmu_idx, uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    unsigned long pc;
     int ret;
 
     ret = cpu_openrisc_handle_mmu_fault(env, addr, is_write, mmu_idx);
@@ -48,13 +46,7 @@ void tlb_fill(CPUOpenRISCState *env, target_ulong addr, int is_write,
     if (ret) {
         if (retaddr) {
             /* now we have a real cpu fault.  */
-            pc = (unsigned long)retaddr;
-            tb = tb_find_pc(pc);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we
-                   have a virtual CPU fault.  */
-                cpu_restore_state(tb, env, pc);
-            }
+            cpu_restore_state(env, retaddr);
         }
         /* Raise Exception.  */
         cpu_loop_exit(env);
diff --git a/target-openrisc/sys_helper.c b/target-openrisc/sys_helper.c
index f160dc3..3c5f45a 100644
--- a/target-openrisc/sys_helper.c
+++ b/target-openrisc/sys_helper.c
@@ -30,7 +30,7 @@ void HELPER(mtspr)(CPUOpenRISCState *env,
     int spr = (ra | offset);
     int idx;
 
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     switch (spr) {
     case TO_SPR(0, 0): /* VR */
@@ -177,7 +177,7 @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env,
     int spr = (ra | offset);
     int idx;
 
-    OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
 
     switch (spr) {
     case TO_SPR(0, 0): /* VR */
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index f14da7b..1e1b30c 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -19,13 +19,13 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "disas.h"
+#include "exec/exec-all.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu-common.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 #include "config.h"
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -61,7 +61,7 @@ static TCGv_i32 fpcsr;
 static TCGv machi, maclo;
 static TCGv fpmaddhi, fpmaddlo;
 static TCGv_i32 env_flags;
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 void openrisc_translate_init(void)
 {
@@ -1707,12 +1707,12 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
             if (k < j) {
                 k++;
                 while (k < j) {
-                    gen_opc_instr_start[k++] = 0;
+                    tcg_ctx.gen_opc_instr_start[k++] = 0;
                 }
             }
-            gen_opc_pc[k] = dc->pc;
-            gen_opc_instr_start[k] = 1;
-            gen_opc_icount[k] = num_insns;
+            tcg_ctx.gen_opc_pc[k] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[k] = 1;
+            tcg_ctx.gen_opc_icount[k] = num_insns;
         }
 
         if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
@@ -1787,7 +1787,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         k++;
         while (k <= j) {
-            gen_opc_instr_start[k++] = 0;
+            tcg_ctx.gen_opc_instr_start[k++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -1832,5 +1832,5 @@ void cpu_dump_state(CPUOpenRISCState *env, FILE *f,
 void restore_state_to_opc(CPUOpenRISCState *env, TranslationBlock *tb,
                           int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-ppc/Makefile.objs b/target-ppc/Makefile.objs
index 237a0ed..a028dcd 100644
--- a/target-ppc/Makefile.objs
+++ b/target-ppc/Makefile.objs
@@ -1,7 +1,6 @@
-obj-y += translate.o helper.o
+obj-y += translate.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o kvm_ppc.o
-obj-y += helper.o
 obj-y += excp_helper.o
 obj-y += fpu_helper.o
 obj-y += int_helper.o
@@ -9,4 +8,3 @@ obj-y += mmu_helper.o
 obj-y += timebase_helper.o
 obj-y += misc_helper.o
 obj-y += mem_helper.o
-obj-y += mpic_helper.o
diff --git a/target-ppc/cpu-qom.h b/target-ppc/cpu-qom.h
index fef6f95..b338f8f 100644
--- a/target-ppc/cpu-qom.h
+++ b/target-ppc/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_PPC_CPU_QOM_H
 #define QEMU_PPC_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #ifdef TARGET_PPC64
@@ -50,6 +50,9 @@ typedef struct PowerPCCPUClass {
     /*< public >*/
 
     void (*parent_reset)(CPUState *cpu);
+
+    /* TODO inline fields here */
+    ppc_def_t *info;
 } PowerPCCPUClass;
 
 /**
@@ -73,5 +76,7 @@ static inline PowerPCCPU *ppc_env_get_cpu(CPUPPCState *env)
 
 #define ENV_GET_CPU(e) CPU(ppc_env_get_cpu(e))
 
+PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr);
+
 
 #endif
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index 5f1dc8b..8c081db 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -73,9 +73,9 @@
 
 #define CPUArchState struct CPUPPCState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -355,7 +355,7 @@ struct ppc6xx_tlb_t {
 
 typedef struct ppcemb_tlb_t ppcemb_tlb_t;
 struct ppcemb_tlb_t {
-    hwaddr RPN;
+    uint64_t RPN;
     target_ulong EPN;
     target_ulong PID;
     target_ulong size;
@@ -1067,7 +1067,9 @@ struct CPUPPCState {
     target_ulong ivor_mask;
     target_ulong ivpr_mask;
     target_ulong hreset_vector;
-    hwaddr mpic_cpu_base;
+    hwaddr mpic_iack;
+    /* true when the external proxy facility mode is enabled */
+    bool mpic_proxy;
 #endif
 
     /* Those resources are used only during code translation */
@@ -1156,10 +1158,6 @@ void ppc_store_msr (CPUPPCState *env, target_ulong value);
 
 void ppc_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 
-const ppc_def_t *ppc_find_by_pvr(uint32_t pvr);
-const ppc_def_t *cpu_ppc_find_by_name (const char *name);
-int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def);
-
 /* Time-base and decrementer management */
 #ifndef NO_CPU_IO_DEFS
 uint64_t cpu_ppc_load_tbl (CPUPPCState *env);
@@ -1251,7 +1249,7 @@ static inline void cpu_clone_regs(CPUPPCState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /*****************************************************************************/
 /* CRF definitions */
@@ -1859,10 +1857,8 @@ enum {
     PPC_CACHE          = 0x0000000200000000ULL,
     /*   icbi instruction                                                    */
     PPC_CACHE_ICBI     = 0x0000000400000000ULL,
-    /*   dcbz instruction with fixed cache line size                         */
+    /*   dcbz instruction                                                    */
     PPC_CACHE_DCBZ     = 0x0000000800000000ULL,
-    /*   dcbz instruction with tunable cache line size                       */
-    PPC_CACHE_DCBZT    = 0x0000001000000000ULL,
     /*   dcba instruction                                                    */
     PPC_CACHE_DCBA     = 0x0000002000000000ULL,
     /*   Freescale cache locking instructions                                */
@@ -1930,7 +1926,7 @@ enum {
                         | PPC_MEM_TLBIE | PPC_MEM_TLBSYNC \
                         | PPC_MEM_SYNC | PPC_MEM_EIEIO \
                         | PPC_CACHE | PPC_CACHE_ICBI \
-                        | PPC_CACHE_DCBZ | PPC_CACHE_DCBZT \
+                        | PPC_CACHE_DCBZ \
                         | PPC_CACHE_DCBA | PPC_CACHE_LOCK \
                         | PPC_EXTERN | PPC_SEGMENT | PPC_6xx_TLB \
                         | PPC_74xx_TLB | PPC_40x_TLB | PPC_SEGMENT_64B \
@@ -2224,7 +2220,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return msr_ee && (env->interrupt_request & CPU_INTERRUPT_HARD);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUPPCState *env, TranslationBlock *tb)
 {
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index 5e34ad0..0a1ac86 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -84,7 +84,11 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
                   " => %08x (%02x)\n", env->nip, excp, env->error_code);
 
     /* new srr1 value excluding must-be-zero bits */
-    msr = env->msr & ~0x783f0000ULL;
+    if (excp_model == POWERPC_EXCP_BOOKE) {
+        msr = env->msr;
+    } else {
+        msr = env->msr & ~0x783f0000ULL;
+    }
 
     /* new interrupt handler msr */
     new_msr = env->msr & ((target_ulong)1 << MSR_ME);
@@ -145,6 +149,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
             srr1 = SPR_40x_SRR3;
             break;
         case POWERPC_EXCP_BOOKE:
+            /* FIXME: choose one or the other based on CPU type */
             srr0 = SPR_BOOKE_MCSRR0;
             srr1 = SPR_BOOKE_MCSRR1;
             asrr0 = SPR_BOOKE_CSRR0;
@@ -173,6 +178,10 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
         if (lpes0 == 1) {
             new_msr |= (target_ulong)MSR_HVB;
         }
+        if (env->mpic_proxy) {
+            /* IACK the IRQ on delivery */
+            env->spr[SPR_BOOKE_EPR] = ldl_phys(env->mpic_iack);
+        }
         goto store_next;
     case POWERPC_EXCP_ALIGN:     /* Alignment exception                      */
         if (lpes1 == 0) {
@@ -275,6 +284,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
     case POWERPC_EXCP_DEBUG:     /* Debug interrupt                          */
         switch (excp_model) {
         case POWERPC_EXCP_BOOKE:
+            /* FIXME: choose one or the other based on CPU type */
             srr0 = SPR_BOOKE_DSRR0;
             srr1 = SPR_BOOKE_DSRR1;
             asrr0 = SPR_BOOKE_CSRR0;
@@ -836,8 +846,13 @@ static inline void do_rfi(CPUPPCState *env, target_ulong nip, target_ulong msr,
 
 void helper_rfi(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
-           ~((target_ulong)0x783F0000), 1);
+    if (env->excp_model == POWERPC_EXCP_BOOKE) {
+        do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
+               ~((target_ulong)0), 0);
+    } else {
+        do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
+               ~((target_ulong)0x783F0000), 1);
+    }
 }
 
 #if defined(TARGET_PPC64)
@@ -864,20 +879,22 @@ void helper_40x_rfci(CPUPPCState *env)
 
 void helper_rfci(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_CSRR0], SPR_BOOKE_CSRR1,
-           ~((target_ulong)0x3FFF0000), 0);
+    do_rfi(env, env->spr[SPR_BOOKE_CSRR0], env->spr[SPR_BOOKE_CSRR1],
+           ~((target_ulong)0), 0);
 }
 
 void helper_rfdi(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_DSRR0], SPR_BOOKE_DSRR1,
-           ~((target_ulong)0x3FFF0000), 0);
+    /* FIXME: choose CSRR1 or DSRR1 based on cpu type */
+    do_rfi(env, env->spr[SPR_BOOKE_DSRR0], env->spr[SPR_BOOKE_DSRR1],
+           ~((target_ulong)0), 0);
 }
 
 void helper_rfmci(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_MCSRR0], SPR_BOOKE_MCSRR1,
-           ~((target_ulong)0x3FFF0000), 0);
+    /* FIXME: choose CSRR1 or MCSRR1 based on cpu type */
+    do_rfi(env, env->spr[SPR_BOOKE_MCSRR0], env->spr[SPR_BOOKE_MCSRR1],
+           ~((target_ulong)0), 0);
 }
 #endif
 
diff --git a/target-ppc/helper.c b/target-ppc/helper.c
deleted file mode 100644
index 48b19a7..0000000
--- a/target-ppc/helper.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- *  PowerPC emulation helpers for QEMU.
- *
- *  Copyright (c) 2003-2007 Jocelyn Mayer
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cpu.h"
-#include "helper_regs.h"
-#include "kvm.h"
-#include "kvm_ppc.h"
-#include "cpus.h"
-
-PowerPCCPU *cpu_ppc_init(const char *cpu_model)
-{
-    PowerPCCPU *cpu;
-    CPUPPCState *env;
-    const ppc_def_t *def;
-
-    def = cpu_ppc_find_by_name(cpu_model);
-    if (!def) {
-        return NULL;
-    }
-
-    cpu = POWERPC_CPU(object_new(TYPE_POWERPC_CPU));
-    env = &cpu->env;
-
-    if (tcg_enabled()) {
-        ppc_translate_init();
-    }
-
-    env->cpu_model_str = cpu_model;
-    cpu_ppc_register_internal(env, def);
-
-    qemu_init_vcpu(env);
-
-    return cpu;
-}
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index e588370..18e0394 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_3(raise_exception_err, void, env, i32, i32)
 DEF_HELPER_2(raise_exception, void, env, i32)
@@ -25,8 +25,7 @@ DEF_HELPER_3(stmw, void, env, tl, i32)
 DEF_HELPER_4(lsw, void, env, tl, i32, i32)
 DEF_HELPER_5(lswx, void, env, tl, i32, i32, i32)
 DEF_HELPER_4(stsw, void, env, tl, i32, i32)
-DEF_HELPER_2(dcbz, void, env, tl)
-DEF_HELPER_2(dcbz_970, void, env, tl)
+DEF_HELPER_3(dcbz, void, env, tl, i32)
 DEF_HELPER_2(icbi, void, env, tl)
 DEF_HELPER_5(lscbx, tl, env, tl, i32, i32, i32)
 
@@ -405,7 +404,6 @@ DEF_HELPER_2(store_40x_dbcr0, void, env, tl)
 DEF_HELPER_2(store_40x_sler, void, env, tl)
 DEF_HELPER_2(store_booke_tcr, void, env, tl)
 DEF_HELPER_2(store_booke_tsr, void, env, tl)
-DEF_HELPER_1(load_epr, tl, env)
 DEF_HELPER_3(store_ibatl, void, env, i32, tl)
 DEF_HELPER_3(store_ibatu, void, env, i32, tl)
 DEF_HELPER_3(store_dbatl, void, env, i32, tl)
@@ -414,4 +412,4 @@ DEF_HELPER_3(store_601_batl, void, env, i32, tl)
 DEF_HELPER_3(store_601_batu, void, env, i32, tl)
 #endif
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-ppc/int_helper.c b/target-ppc/int_helper.c
index f39b4f6..783079d 100644
--- a/target-ppc/int_helper.c
+++ b/target-ppc/int_helper.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 #include "helper_regs.h"
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index 3f5df57..2c64c63 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -23,13 +23,13 @@
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "cpu.h"
-#include "cpus.h"
-#include "device_tree.h"
+#include "sysemu/cpus.h"
+#include "sysemu/device_tree.h"
 #include "hw/sysbus.h"
 #include "hw/spapr.h"
 
@@ -99,8 +99,10 @@ int kvm_arch_init(KVMState *s)
     return 0;
 }
 
-static int kvm_arch_sync_sregs(CPUPPCState *cenv)
+static int kvm_arch_sync_sregs(PowerPCCPU *cpu)
 {
+    CPUPPCState *cenv = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_sregs sregs;
     int ret;
 
@@ -117,18 +119,20 @@ static int kvm_arch_sync_sregs(CPUPPCState *cenv)
         }
     }
 
-    ret = kvm_vcpu_ioctl(cenv, KVM_GET_SREGS, &sregs);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
     if (ret) {
         return ret;
     }
 
     sregs.pvr = cenv->spr[SPR_PVR];
-    return kvm_vcpu_ioctl(cenv, KVM_SET_SREGS, &sregs);
+    return kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
 }
 
 /* Set up a shared TLB array with KVM */
-static int kvm_booke206_tlb_init(CPUPPCState *env)
+static int kvm_booke206_tlb_init(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_book3e_206_tlb_params params = {};
     struct kvm_config_tlb cfg = {};
     struct kvm_enable_cap encap = {};
@@ -136,7 +140,7 @@ static int kvm_booke206_tlb_init(CPUPPCState *env)
     int ret, i;
 
     if (!kvm_enabled() ||
-        !kvm_check_extension(env->kvm_state, KVM_CAP_SW_TLB)) {
+        !kvm_check_extension(cs->kvm_state, KVM_CAP_SW_TLB)) {
         return 0;
     }
 
@@ -161,7 +165,7 @@ static int kvm_booke206_tlb_init(CPUPPCState *env)
     encap.cap = KVM_CAP_SW_TLB;
     encap.args[0] = (uintptr_t)&cfg;
 
-    ret = kvm_vcpu_ioctl(env, KVM_ENABLE_CAP, &encap);
+    ret = kvm_vcpu_ioctl(cs, KVM_ENABLE_CAP, &encap);
     if (ret < 0) {
         fprintf(stderr, "%s: couldn't enable KVM_CAP_SW_TLB: %s\n",
                 __func__, strerror(-ret));
@@ -174,9 +178,12 @@ static int kvm_booke206_tlb_init(CPUPPCState *env)
 
 
 #if defined(TARGET_PPC64)
-static void kvm_get_fallback_smmu_info(CPUPPCState *env,
+static void kvm_get_fallback_smmu_info(PowerPCCPU *cpu,
                                        struct kvm_ppc_smmu_info *info)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+
     memset(info, 0, sizeof(*info));
 
     /* We don't have the new KVM_PPC_GET_SMMU_INFO ioctl, so
@@ -202,7 +209,7 @@ static void kvm_get_fallback_smmu_info(CPUPPCState *env,
      *   implements KVM_CAP_PPC_GET_SMMU_INFO and thus doesn't hit
      *   this fallback.
      */
-    if (kvm_check_extension(env->kvm_state, KVM_CAP_PPC_GET_PVINFO)) {
+    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_PVINFO)) {
         /* No flags */
         info->flags = 0;
         info->slb_size = 64;
@@ -258,18 +265,19 @@ static void kvm_get_fallback_smmu_info(CPUPPCState *env,
     }
 }
 
-static void kvm_get_smmu_info(CPUPPCState *env, struct kvm_ppc_smmu_info *info)
+static void kvm_get_smmu_info(PowerPCCPU *cpu, struct kvm_ppc_smmu_info *info)
 {
+    CPUState *cs = CPU(cpu);
     int ret;
 
-    if (kvm_check_extension(env->kvm_state, KVM_CAP_PPC_GET_SMMU_INFO)) {
-        ret = kvm_vm_ioctl(env->kvm_state, KVM_PPC_GET_SMMU_INFO, info);
+    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_SMMU_INFO)) {
+        ret = kvm_vm_ioctl(cs->kvm_state, KVM_PPC_GET_SMMU_INFO, info);
         if (ret == 0) {
             return;
         }
     }
 
-    kvm_get_fallback_smmu_info(env, info);
+    kvm_get_fallback_smmu_info(cpu, info);
 }
 
 static long getrampagesize(void)
@@ -312,10 +320,11 @@ static bool kvm_valid_page_size(uint32_t flags, long rampgsize, uint32_t shift)
     return (1ul << shift) <= rampgsize;
 }
 
-static void kvm_fixup_page_sizes(CPUPPCState *env)
+static void kvm_fixup_page_sizes(PowerPCCPU *cpu)
 {
     static struct kvm_ppc_smmu_info smmu_info;
     static bool has_smmu_info;
+    CPUPPCState *env = &cpu->env;
     long rampagesize;
     int iq, ik, jq, jk;
 
@@ -326,7 +335,7 @@ static void kvm_fixup_page_sizes(CPUPPCState *env)
 
     /* Collect MMU info from kernel if not already */
     if (!has_smmu_info) {
-        kvm_get_smmu_info(env, &smmu_info);
+        kvm_get_smmu_info(cpu, &smmu_info);
         has_smmu_info = true;
     }
 
@@ -369,22 +378,28 @@ static void kvm_fixup_page_sizes(CPUPPCState *env)
 }
 #else /* defined (TARGET_PPC64) */
 
-static inline void kvm_fixup_page_sizes(CPUPPCState *env)
+static inline void kvm_fixup_page_sizes(PowerPCCPU *cpu)
 {
 }
 
 #endif /* !defined (TARGET_PPC64) */
 
-int kvm_arch_init_vcpu(CPUPPCState *cenv)
+unsigned long kvm_arch_vcpu_id(CPUState *cpu)
+{
+    return cpu->cpu_index;
+}
+
+int kvm_arch_init_vcpu(CPUState *cs)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(cenv);
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *cenv = &cpu->env;
     int ret;
 
     /* Gather server mmu info from KVM and update the CPU state */
-    kvm_fixup_page_sizes(cenv);
+    kvm_fixup_page_sizes(cpu);
 
     /* Synchronize sregs with kvm */
-    ret = kvm_arch_sync_sregs(cenv);
+    ret = kvm_arch_sync_sregs(cpu);
     if (ret) {
         return ret;
     }
@@ -394,7 +409,7 @@ int kvm_arch_init_vcpu(CPUPPCState *cenv)
     /* Some targets support access to KVM's guest TLB. */
     switch (cenv->mmu_model) {
     case POWERPC_MMU_BOOKE206:
-        ret = kvm_booke206_tlb_init(cenv);
+        ret = kvm_booke206_tlb_init(cpu);
         break;
     default:
         break;
@@ -403,12 +418,14 @@ int kvm_arch_init_vcpu(CPUPPCState *cenv)
     return ret;
 }
 
-void kvm_arch_reset_vcpu(CPUPPCState *env)
+void kvm_arch_reset_vcpu(CPUState *cpu)
 {
 }
 
-static void kvm_sw_tlb_put(CPUPPCState *env)
+static void kvm_sw_tlb_put(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_dirty_tlb dirty_tlb;
     unsigned char *bitmap;
     int ret;
@@ -423,7 +440,7 @@ static void kvm_sw_tlb_put(CPUPPCState *env)
     dirty_tlb.bitmap = (uintptr_t)bitmap;
     dirty_tlb.num_dirty = env->nb_tlb;
 
-    ret = kvm_vcpu_ioctl(env, KVM_DIRTY_TLB, &dirty_tlb);
+    ret = kvm_vcpu_ioctl(cs, KVM_DIRTY_TLB, &dirty_tlb);
     if (ret) {
         fprintf(stderr, "%s: KVM_DIRTY_TLB: %s\n",
                 __func__, strerror(-ret));
@@ -432,15 +449,18 @@ static void kvm_sw_tlb_put(CPUPPCState *env)
     g_free(bitmap);
 }
 
-int kvm_arch_put_registers(CPUPPCState *env, int level)
+int kvm_arch_put_registers(CPUState *cs, int level)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     struct kvm_regs regs;
     int ret;
     int i;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
-    if (ret < 0)
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REGS, &regs);
+    if (ret < 0) {
         return ret;
+    }
 
     regs.ctr = env->ctr;
     regs.lr  = env->lr;
@@ -465,12 +485,12 @@ int kvm_arch_put_registers(CPUPPCState *env, int level)
     for (i = 0;i < 32; i++)
         regs.gpr[i] = env->gpr[i];
 
-    ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_REGS, &regs);
     if (ret < 0)
         return ret;
 
     if (env->tlb_dirty) {
-        kvm_sw_tlb_put(env);
+        kvm_sw_tlb_put(cpu);
         env->tlb_dirty = false;
     }
 
@@ -503,7 +523,7 @@ int kvm_arch_put_registers(CPUPPCState *env, int level)
                 | env->IBAT[1][i];
         }
 
-        ret = kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
         if (ret) {
             return ret;
         }
@@ -516,7 +536,7 @@ int kvm_arch_put_registers(CPUPPCState *env, int level)
             .addr = (uintptr_t) &hior,
         };
 
-        ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &reg);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
         if (ret) {
             return ret;
         }
@@ -525,14 +545,16 @@ int kvm_arch_put_registers(CPUPPCState *env, int level)
     return ret;
 }
 
-int kvm_arch_get_registers(CPUPPCState *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     struct kvm_regs regs;
     struct kvm_sregs sregs;
     uint32_t cr;
     int i, ret;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REGS, &regs);
     if (ret < 0)
         return ret;
 
@@ -566,7 +588,7 @@ int kvm_arch_get_registers(CPUPPCState *env)
         env->gpr[i] = regs.gpr[i];
 
     if (cap_booke_sregs) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
         if (ret < 0) {
             return ret;
         }
@@ -670,7 +692,7 @@ int kvm_arch_get_registers(CPUPPCState *env)
     }
 
     if (cap_segstate) {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
         if (ret < 0) {
             return ret;
         }
@@ -702,7 +724,7 @@ int kvm_arch_get_registers(CPUPPCState *env)
     return 0;
 }
 
-int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
+int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
 {
     unsigned virq = level ? KVM_INTERRUPT_SET_LEVEL : KVM_INTERRUPT_UNSET;
 
@@ -714,7 +736,7 @@ int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
         return 0;
     }
 
-    kvm_vcpu_ioctl(env, KVM_INTERRUPT, &virq);
+    kvm_vcpu_ioctl(CPU(cpu), KVM_INTERRUPT, &virq);
 
     return 0;
 }
@@ -727,8 +749,10 @@ int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
 #define PPC_INPUT_INT PPC6xx_INPUT_INT
 #endif
 
-void kvm_arch_pre_run(CPUPPCState *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     int r;
     unsigned irq;
 
@@ -746,9 +770,10 @@ void kvm_arch_pre_run(CPUPPCState *env, struct kvm_run *run)
         irq = KVM_INTERRUPT_SET;
 
         dprintf("injected interrupt %d\n", irq);
-        r = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &irq);
-        if (r < 0)
-            printf("cpu %d fail inject %x\n", env->cpu_index, irq);
+        r = kvm_vcpu_ioctl(cs, KVM_INTERRUPT, &irq);
+        if (r < 0) {
+            printf("cpu %d fail inject %x\n", cs->cpu_index, irq);
+        }
 
         /* Always wake up soon in case the interrupt was level based */
         qemu_mod_timer(idle_timer, qemu_get_clock_ns(vm_clock) +
@@ -760,13 +785,14 @@ void kvm_arch_pre_run(CPUPPCState *env, struct kvm_run *run)
      * anyways, so we will get a chance to deliver the rest. */
 }
 
-void kvm_arch_post_run(CPUPPCState *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
 }
 
-int kvm_arch_process_async_events(CPUPPCState *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
-    return env->halted;
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    return cpu->env.halted;
 }
 
 static int kvmppc_handle_halt(CPUPPCState *env)
@@ -796,8 +822,10 @@ static int kvmppc_handle_dcr_write(CPUPPCState *env, uint32_t dcrn, uint32_t dat
     return 0;
 }
 
-int kvm_arch_handle_exit(CPUPPCState *env, struct kvm_run *run)
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
     int ret;
 
     switch (run->exit_reason) {
@@ -817,12 +845,17 @@ int kvm_arch_handle_exit(CPUPPCState *env, struct kvm_run *run)
 #ifdef CONFIG_PSERIES
     case KVM_EXIT_PAPR_HCALL:
         dprintf("handle PAPR hypercall\n");
-        run->papr_hcall.ret = spapr_hypercall(ppc_env_get_cpu(env),
+        run->papr_hcall.ret = spapr_hypercall(cpu,
                                               run->papr_hcall.nr,
                                               run->papr_hcall.args);
         ret = 0;
         break;
 #endif
+    case KVM_EXIT_EPR:
+        dprintf("handle epr\n");
+        run->epr.epr = ldl_phys(env->mpic_iack);
+        ret = 0;
+        break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
         ret = -1;
@@ -967,16 +1000,38 @@ uint32_t kvmppc_get_dfp(void)
     return kvmppc_read_int_cpu_dt("ibm,dfp");
 }
 
+static int kvmppc_get_pvinfo(CPUPPCState *env, struct kvm_ppc_pvinfo *pvinfo)
+ {
+     PowerPCCPU *cpu = ppc_env_get_cpu(env);
+     CPUState *cs = CPU(cpu);
+
+    if (kvm_check_extension(cs->kvm_state, KVM_CAP_PPC_GET_PVINFO) &&
+        !kvm_vm_ioctl(cs->kvm_state, KVM_PPC_GET_PVINFO, pvinfo)) {
+        return 0;
+    }
+
+    return 1;
+}
+
+int kvmppc_get_hasidle(CPUPPCState *env)
+{
+    struct kvm_ppc_pvinfo pvinfo;
+
+    if (!kvmppc_get_pvinfo(env, &pvinfo) &&
+        (pvinfo.flags & KVM_PPC_PVINFO_FLAGS_EV_IDLE)) {
+        return 1;
+    }
+
+    return 0;
+}
+
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
 {
     uint32_t *hc = (uint32_t*)buf;
-
     struct kvm_ppc_pvinfo pvinfo;
 
-    if (kvm_check_extension(env->kvm_state, KVM_CAP_PPC_GET_PVINFO) &&
-        !kvm_vm_ioctl(env->kvm_state, KVM_PPC_GET_PVINFO, &pvinfo)) {
+    if (!kvmppc_get_pvinfo(env, &pvinfo)) {
         memcpy(buf, pvinfo.hcall, buf_len);
-
         return 0;
     }
 
@@ -997,19 +1052,37 @@ int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
     return 0;
 }
 
-void kvmppc_set_papr(CPUPPCState *env)
+void kvmppc_set_papr(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
     struct kvm_enable_cap cap = {};
     int ret;
 
     cap.cap = KVM_CAP_PPC_PAPR;
-    ret = kvm_vcpu_ioctl(env, KVM_ENABLE_CAP, &cap);
+    ret = kvm_vcpu_ioctl(cs, KVM_ENABLE_CAP, &cap);
 
     if (ret) {
         cpu_abort(env, "This KVM version does not support PAPR\n");
     }
 }
 
+void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy)
+{
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    struct kvm_enable_cap cap = {};
+    int ret;
+
+    cap.cap = KVM_CAP_PPC_EPR;
+    cap.args[0] = mpic_proxy;
+    ret = kvm_vcpu_ioctl(cs, KVM_ENABLE_CAP, &cap);
+
+    if (ret && mpic_proxy) {
+        cpu_abort(env, "This KVM version does not support EPR\n");
+    }
+}
+
 int kvmppc_smt_threads(void)
 {
     return cap_ppc_smt ? cap_ppc_smt : 1;
@@ -1184,18 +1257,37 @@ static void alter_insns(uint64_t *word, uint64_t flags, bool on)
     }
 }
 
-const ppc_def_t *kvmppc_host_cpu_def(void)
+static void kvmppc_host_cpu_initfn(Object *obj)
+{
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(obj);
+
+    assert(kvm_enabled());
+
+    if (pcc->info->pvr != mfpvr()) {
+        fprintf(stderr, "Your host CPU is unsupported.\n"
+                "Please choose a supported model instead, see -cpu ?.\n");
+        exit(1);
+    }
+}
+
+static void kvmppc_host_cpu_class_init(ObjectClass *oc, void *data)
 {
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
     uint32_t host_pvr = mfpvr();
-    const ppc_def_t *base_spec;
+    PowerPCCPUClass *pvr_pcc;
     ppc_def_t *spec;
     uint32_t vmx = kvmppc_get_vmx();
     uint32_t dfp = kvmppc_get_dfp();
 
-    base_spec = ppc_find_by_pvr(host_pvr);
-
     spec = g_malloc0(sizeof(*spec));
-    memcpy(spec, base_spec, sizeof(*spec));
+
+    pvr_pcc = ppc_cpu_class_by_pvr(host_pvr);
+    if (pvr_pcc != NULL) {
+        memcpy(spec, pvr_pcc->info, sizeof(*spec));
+    }
+    pcc->info = spec;
+    /* Override the display name for -cpu ? and QMP */
+    pcc->info->name = "host";
 
     /* Now fix up the spec with information we can query from the host */
 
@@ -1208,29 +1300,28 @@ const ppc_def_t *kvmppc_host_cpu_def(void)
         /* Only override when we know what the host supports */
         alter_insns(&spec->insns_flags2, PPC2_DFP, dfp);
     }
-
-    return spec;
 }
 
-int kvmppc_fixup_cpu(CPUPPCState *env)
+int kvmppc_fixup_cpu(PowerPCCPU *cpu)
 {
+    CPUState *cs = CPU(cpu);
     int smt;
 
     /* Adjust cpu index for SMT */
     smt = kvmppc_smt_threads();
-    env->cpu_index = (env->cpu_index / smp_threads) * smt
-        + (env->cpu_index % smp_threads);
+    cs->cpu_index = (cs->cpu_index / smp_threads) * smt
+        + (cs->cpu_index % smp_threads);
 
     return 0;
 }
 
 
-bool kvm_arch_stop_on_emulation_error(CPUPPCState *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cpu)
 {
     return true;
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUPPCState *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
     return 1;
 }
@@ -1239,3 +1330,17 @@ int kvm_arch_on_sigbus(int code, void *addr)
 {
     return 1;
 }
+
+static const TypeInfo kvm_host_cpu_type_info = {
+    .name = TYPE_HOST_POWERPC_CPU,
+    .parent = TYPE_POWERPC_CPU,
+    .instance_init = kvmppc_host_cpu_initfn,
+    .class_init = kvmppc_host_cpu_class_init,
+};
+
+static void kvm_ppc_register_types(void)
+{
+    type_register_static(&kvm_host_cpu_type_info);
+}
+
+type_init(kvm_ppc_register_types)
diff --git a/target-ppc/kvm_ppc.c b/target-ppc/kvm_ppc.c
index a2e49cd..1b192a8 100644
--- a/target-ppc/kvm_ppc.c
+++ b/target-ppc/kvm_ppc.c
@@ -12,9 +12,9 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "kvm_ppc.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 #define PROC_DEVTREE_PATH "/proc/device-tree"
 
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index baad6eb..c30b006 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -9,7 +9,9 @@
 #ifndef __KVM_PPC_H__
 #define __KVM_PPC_H__
 
-#include "memory.h"
+#include "exec/memory.h"
+
+#define TYPE_HOST_POWERPC_CPU "host-" TYPE_POWERPC_CPU
 
 void kvmppc_init(void);
 
@@ -19,9 +21,11 @@ uint32_t kvmppc_get_tbfreq(void);
 uint64_t kvmppc_get_clockfreq(void);
 uint32_t kvmppc_get_vmx(void);
 uint32_t kvmppc_get_dfp(void);
+int kvmppc_get_hasidle(CPUPPCState *env);
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len);
-int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level);
-void kvmppc_set_papr(CPUPPCState *env);
+int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level);
+void kvmppc_set_papr(PowerPCCPU *cpu);
+void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
 int kvmppc_smt_threads(void);
 #ifndef CONFIG_USER_ONLY
 off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem);
@@ -30,8 +34,7 @@ int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
 int kvmppc_reset_htab(int shift_hint);
 uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
 #endif /* !CONFIG_USER_ONLY */
-const ppc_def_t *kvmppc_host_cpu_def(void);
-int kvmppc_fixup_cpu(CPUPPCState *env);
+int kvmppc_fixup_cpu(PowerPCCPU *cpu);
 
 #else
 
@@ -55,6 +58,11 @@ static inline uint32_t kvmppc_get_dfp(void)
     return 0;
 }
 
+static inline int kvmppc_get_hasidle(CPUPPCState *env)
+{
+    return 0;
+}
+
 static inline int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
 {
     return -1;
@@ -65,12 +73,16 @@ static inline int kvmppc_read_segment_page_sizes(uint32_t *prop, int maxcells)
     return -1;
 }
 
-static inline int kvmppc_set_interrupt(CPUPPCState *env, int irq, int level)
+static inline int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
 {
     return -1;
 }
 
-static inline void kvmppc_set_papr(CPUPPCState *env)
+static inline void kvmppc_set_papr(PowerPCCPU *cpu)
+{
+}
+
+static inline void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy)
 {
 }
 
@@ -115,12 +127,7 @@ static inline int kvmppc_update_sdr1(CPUPPCState *env)
 
 #endif /* !CONFIG_USER_ONLY */
 
-static inline const ppc_def_t *kvmppc_host_cpu_def(void)
-{
-    return NULL;
-}
-
-static inline int kvmppc_fixup_cpu(CPUPPCState *env)
+static inline int kvmppc_fixup_cpu(PowerPCCPU *cpu)
 {
     return -1;
 }
diff --git a/target-ppc/machine.c b/target-ppc/machine.c
index 5e7bc00..e014c0c 100644
--- a/target-ppc/machine.c
+++ b/target-ppc/machine.c
@@ -1,6 +1,6 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 void cpu_save(QEMUFile *f, void *opaque)
 {
diff --git a/target-ppc/mem_helper.c b/target-ppc/mem_helper.c
index 5b5f1bd..ba383c8 100644
--- a/target-ppc/mem_helper.c
+++ b/target-ppc/mem_helper.c
@@ -17,13 +17,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 #include "helper_regs.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif /* !defined(CONFIG_USER_ONLY) */
 
 //#define DEBUG_OP
@@ -136,18 +136,21 @@ static void do_dcbz(CPUPPCState *env, target_ulong addr, int dcache_line_size)
     }
 }
 
-void helper_dcbz(CPUPPCState *env, target_ulong addr)
+void helper_dcbz(CPUPPCState *env, target_ulong addr, uint32_t is_dcbzl)
 {
-    do_dcbz(env, addr, env->dcache_line_size);
-}
+    int dcbz_size = env->dcache_line_size;
 
-void helper_dcbz_970(CPUPPCState *env, target_ulong addr)
-{
-    if (((env->spr[SPR_970_HID5] >> 7) & 0x3) == 1) {
-        do_dcbz(env, addr, 32);
-    } else {
-        do_dcbz(env, addr, env->dcache_line_size);
+#if !defined(CONFIG_USER_ONLY) && defined(TARGET_PPC64)
+    if (!is_dcbzl &&
+        (env->excp_model == POWERPC_EXCP_970) &&
+        ((env->spr[SPR_970_HID5] >> 7) & 0x3) == 1) {
+        dcbz_size = 32;
     }
+#endif
+
+    /* XXX add e500mc support */
+
+    do_dcbz(env, addr, dcbz_size);
 }
 
 void helper_icbi(CPUPPCState *env, target_ulong addr)
@@ -257,16 +260,16 @@ STVE(stvewx, cpu_stl_data, bswap32, u32)
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -275,19 +278,13 @@ STVE(stvewx, cpu_stl_data, bswap32, u32)
 void tlb_fill(CPUPPCState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_ppc_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret != 0)) {
         if (likely(retaddr)) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (likely(tb)) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         helper_raise_exception_err(env, env->exception_index, env->error_code);
     }
diff --git a/target-ppc/mmu_helper.c b/target-ppc/mmu_helper.c
index 318ce92..1cc1c16 100644
--- a/target-ppc/mmu_helper.c
+++ b/target-ppc/mmu_helper.c
@@ -18,7 +18,7 @@
  */
 #include "cpu.h"
 #include "helper.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 //#define DEBUG_MMU
@@ -587,7 +587,7 @@ static inline int find_pte2(CPUPPCState *env, mmu_ctx_t *ctx, int is_64b, int h,
             }
 
             r = pte64_check(ctx, pte0, pte1, h, rw, type);
-            LOG_MMU("Load pte from " TARGET_FMT_lx " => " TARGET_FMT_lx " "
+            LOG_MMU("Load pte from %016" HWADDR_PRIx " => " TARGET_FMT_lx " "
                     TARGET_FMT_lx " %d %d %d " TARGET_FMT_lx "\n",
                     pteg_off + (i * 16), pte0, pte1, (int)(pte0 & 1), h,
                     (int)((pte0 >> 1) & 1), ctx->ptem);
@@ -602,7 +602,7 @@ static inline int find_pte2(CPUPPCState *env, mmu_ctx_t *ctx, int is_64b, int h,
                 pte1 = ldl_phys(env->htab_base + pteg_off + (i * 8) + 4);
             }
             r = pte32_check(ctx, pte0, pte1, h, rw, type);
-            LOG_MMU("Load pte from " TARGET_FMT_lx " => " TARGET_FMT_lx " "
+            LOG_MMU("Load pte from %08" HWADDR_PRIx " => " TARGET_FMT_lx " "
                     TARGET_FMT_lx " %d %d %d " TARGET_FMT_lx "\n",
                     pteg_off + (i * 8), pte0, pte1, (int)(pte0 >> 31), h,
                     (int)((pte0 >> 6) & 1), ctx->ptem);
@@ -633,7 +633,7 @@ static inline int find_pte2(CPUPPCState *env, mmu_ctx_t *ctx, int is_64b, int h,
     }
     if (good != -1) {
     done:
-        LOG_MMU("found PTE at addr " TARGET_FMT_lx " prot=%01x ret=%d\n",
+        LOG_MMU("found PTE at addr %08" HWADDR_PRIx " prot=%01x ret=%d\n",
                 ctx->raddr, ctx->prot, ret);
         /* Update page flags */
         pte1 = ctx->raddr;
@@ -2260,8 +2260,9 @@ void helper_store_601_batu(CPUPPCState *env, uint32_t nr, target_ulong value)
 
 void helper_store_601_batl(CPUPPCState *env, uint32_t nr, target_ulong value)
 {
+#if !defined(FLUSH_ALL_TLBS)
     target_ulong mask;
-#if defined(FLUSH_ALL_TLBS)
+#else
     int do_inval;
 #endif
 
diff --git a/target-ppc/mpic_helper.c b/target-ppc/mpic_helper.c
deleted file mode 100644
index 2c6a4d3..0000000
--- a/target-ppc/mpic_helper.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- *  PowerPC emulation helpers for QEMU.
- *
- *  Copyright (c) 2003-2007 Jocelyn Mayer
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-#include "cpu.h"
-#include "helper.h"
-
-/*****************************************************************************/
-/* SPR accesses */
-
-#if !defined(CONFIG_USER_ONLY)
-/*
- * This is an ugly helper for EPR, which is basically the same as accessing
- * the IACK (PIAC) register on the MPIC. Because we model the MPIC as a device
- * that can only talk to the CPU through MMIO, let's access it that way!
- */
-target_ulong helper_load_epr(CPUPPCState *env)
-{
-    return ldl_phys(env->mpic_cpu_base + 0xA0);
-}
-#endif
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index 987b04e..2ac5794 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -19,9 +19,9 @@
  */
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -71,7 +71,7 @@ static TCGv cpu_reserve;
 static TCGv cpu_fpscr;
 static TCGv_i32 cpu_access_type;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 void ppc_translate_init(void)
 {
@@ -4118,29 +4118,21 @@ static void gen_dcbtst(DisasContext *ctx)
 /* dcbz */
 static void gen_dcbz(DisasContext *ctx)
 {
-    TCGv t0;
-    gen_set_access_type(ctx, ACCESS_CACHE);
-    /* NIP cannot be restored if the memory exception comes from an helper */
-    gen_update_nip(ctx, ctx->nip - 4);
-    t0 = tcg_temp_new();
-    gen_addr_reg_index(ctx, t0);
-    gen_helper_dcbz(cpu_env, t0);
-    tcg_temp_free(t0);
-}
+    TCGv tcgv_addr;
+    TCGv_i32 tcgv_is_dcbzl;
+    int is_dcbzl = ctx->opcode & 0x00200000 ? 1 : 0;
 
-static void gen_dcbz_970(DisasContext *ctx)
-{
-    TCGv t0;
     gen_set_access_type(ctx, ACCESS_CACHE);
     /* NIP cannot be restored if the memory exception comes from an helper */
     gen_update_nip(ctx, ctx->nip - 4);
-    t0 = tcg_temp_new();
-    gen_addr_reg_index(ctx, t0);
-    if (ctx->opcode & 0x00200000)
-        gen_helper_dcbz(cpu_env, t0);
-    else
-        gen_helper_dcbz_970(cpu_env, t0);
-    tcg_temp_free(t0);
+    tcgv_addr = tcg_temp_new();
+    tcgv_is_dcbzl = tcg_const_i32(is_dcbzl);
+
+    gen_addr_reg_index(ctx, tcgv_addr);
+    gen_helper_dcbz(cpu_env, tcgv_addr, tcgv_is_dcbzl);
+
+    tcg_temp_free(tcgv_addr);
+    tcg_temp_free_i32(tcgv_is_dcbzl);
 }
 
 /* dst / dstt */
@@ -8648,8 +8640,7 @@ GEN_HANDLER(dcbi, 0x1F, 0x16, 0x0E, 0x03E00001, PPC_CACHE),
 GEN_HANDLER(dcbst, 0x1F, 0x16, 0x01, 0x03E00001, PPC_CACHE),
 GEN_HANDLER(dcbt, 0x1F, 0x16, 0x08, 0x02000001, PPC_CACHE),
 GEN_HANDLER(dcbtst, 0x1F, 0x16, 0x07, 0x02000001, PPC_CACHE),
-GEN_HANDLER(dcbz, 0x1F, 0x16, 0x1F, 0x03E00001, PPC_CACHE_DCBZ),
-GEN_HANDLER2(dcbz_970, "dcbz", 0x1F, 0x16, 0x1F, 0x03C00001, PPC_CACHE_DCBZT),
+GEN_HANDLER(dcbz, 0x1F, 0x16, 0x1F, 0x03C00001, PPC_CACHE_DCBZ),
 GEN_HANDLER(dst, 0x1F, 0x16, 0x0A, 0x01800001, PPC_ALTIVEC),
 GEN_HANDLER(dstst, 0x1F, 0x16, 0x0B, 0x02000001, PPC_ALTIVEC),
 GEN_HANDLER(dss, 0x1F, 0x16, 0x19, 0x019FF801, PPC_ALTIVEC),
@@ -9680,11 +9671,11 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
             }
-            gen_opc_pc[lj] = ctx.nip;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = ctx.nip;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
         LOG_DISAS("----------------\n");
         LOG_DISAS("nip=" TARGET_FMT_lx " super=%d ir=%d\n",
@@ -9698,7 +9689,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         }
         LOG_DISAS("translate opcode %08x (%02x %02x %02x) (%s)\n",
                     ctx.opcode, opc1(ctx.opcode), opc2(ctx.opcode),
-                    opc3(ctx.opcode), little_endian ? "little" : "big");
+                    opc3(ctx.opcode), ctx.le_mode ? "little" : "big");
         if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(ctx.nip);
         }
@@ -9781,7 +9772,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
     } else {
         tb->size = ctx.nip - pc_start;
         tb->icount = num_insns;
@@ -9810,5 +9801,5 @@ void gen_intermediate_code_pc (CPUPPCState *env, struct TranslationBlock *tb)
 
 void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->nip = gen_opc_pc[pc_pos];
+    env->nip = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index e63627c..6cebaa1 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -23,11 +23,12 @@
  * inside "#if defined(TODO) ... #endif" statements to make tests easier.
  */
 
-#include "dis-asm.h"
-#include "gdbstub.h"
-#include <kvm.h>
+#include "disas/bfd.h"
+#include "exec/gdbstub.h"
+#include <sysemu/kvm.h>
 #include "kvm_ppc.h"
-#include "arch_init.h"
+#include "sysemu/arch_init.h"
+#include "sysemu/cpus.h"
 
 //#define PPC_DUMP_CPU
 //#define PPC_DEBUG_SPR
@@ -4493,11 +4494,6 @@ static void spr_read_mas73(void *opaque, int gprn, int sprn)
     tcg_temp_free(mas7);
 }
 
-static void spr_load_epr(void *opaque, int gprn, int sprn)
-{
-    gen_helper_load_epr(cpu_gpr[gprn], cpu_env);
-}
-
 #endif
 
 enum fsl_e500_version {
@@ -4656,7 +4652,7 @@ static void init_proc_e500 (CPUPPCState *env, int version)
                  0x00000000);
     spr_register(env, SPR_BOOKE_EPR, "EPR",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_load_epr, SPR_NOACCESS,
+                 &spr_read_generic, SPR_NOACCESS,
                  0x00000000);
     /* XXX better abstract into Emb.xxx features */
     if (version == fsl_e5500) {
@@ -6302,7 +6298,7 @@ static void init_proc_7457 (CPUPPCState *env)
                               PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |   \
                               PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |           \
                               PPC_FLOAT_STFIWX |                              \
-                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZT |  \
+                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |  \
                               PPC_MEM_SYNC | PPC_MEM_EIEIO |                  \
                               PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |               \
                               PPC_64B | PPC_ALTIVEC |                         \
@@ -6398,7 +6394,7 @@ static void init_proc_970 (CPUPPCState *env)
                               PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |   \
                               PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |           \
                               PPC_FLOAT_STFIWX |                              \
-                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZT |  \
+                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |  \
                               PPC_MEM_SYNC | PPC_MEM_EIEIO |                  \
                               PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |               \
                               PPC_64B | PPC_ALTIVEC |                         \
@@ -6500,7 +6496,7 @@ static void init_proc_970FX (CPUPPCState *env)
                               PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |   \
                               PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |           \
                               PPC_FLOAT_STFIWX |                              \
-                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZT |  \
+                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |  \
                               PPC_MEM_SYNC | PPC_MEM_EIEIO |                  \
                               PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |               \
                               PPC_64B | PPC_ALTIVEC |                         \
@@ -6590,7 +6586,7 @@ static void init_proc_970GX (CPUPPCState *env)
                               PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |   \
                               PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |           \
                               PPC_FLOAT_STFIWX |                              \
-                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZT |  \
+                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |  \
                               PPC_MEM_SYNC | PPC_MEM_EIEIO |                  \
                               PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |               \
                               PPC_64B | PPC_ALTIVEC |                         \
@@ -6681,7 +6677,7 @@ static void init_proc_970MP (CPUPPCState *env)
                               PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |   \
                               PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |           \
                               PPC_FLOAT_STFIWX |                              \
-                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZT |  \
+                              PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |  \
                               PPC_MEM_SYNC | PPC_MEM_EIEIO |                  \
                               PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |               \
                               PPC_64B | PPC_ALTIVEC |                         \
@@ -8638,9 +8634,9 @@ static const ppc_def_t ppc_defs[] = {
     POWERPC_DEF("e500v2_v22",    CPU_POWERPC_e500v2_v22,             e500v2),
     /* PowerPC e500v2 v3.0 core                                              */
     POWERPC_DEF("e500v2_v30",    CPU_POWERPC_e500v2_v30,             e500v2),
-    POWERPC_DEF("e500mc",        CPU_POWERPC_e500mc,                 e500mc),
+    POWERPC_DEF_SVR("e500mc", CPU_POWERPC_e500mc, POWERPC_SVR_E500,  e500mc),
 #ifdef TARGET_PPC64
-    POWERPC_DEF("e5500",         CPU_POWERPC_e5500,                  e5500),
+    POWERPC_DEF_SVR("e5500",    CPU_POWERPC_e5500, POWERPC_SVR_E500, e5500),
 #endif
     /* PowerPC e500 microcontrollers                                         */
     /* MPC8533                                                               */
@@ -9797,8 +9793,11 @@ static void fix_opcode_tables (opc_handler_t **ppc_opcodes)
 }
 
 /*****************************************************************************/
-static int create_ppc_opcodes (CPUPPCState *env, const ppc_def_t *def)
+static void create_ppc_opcodes(PowerPCCPU *cpu, Error **errp)
 {
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
+    CPUPPCState *env = &cpu->env;
+    const ppc_def_t *def = pcc->info;
     opcode_t *opc;
 
     fill_new_table(env->opcodes, 0x40);
@@ -9806,18 +9805,16 @@ static int create_ppc_opcodes (CPUPPCState *env, const ppc_def_t *def)
         if (((opc->handler.type & def->insns_flags) != 0) ||
             ((opc->handler.type2 & def->insns_flags2) != 0)) {
             if (register_insn(env->opcodes, opc) < 0) {
-                printf("*** ERROR initializing PowerPC instruction "
-                       "0x%02x 0x%02x 0x%02x\n", opc->opc1, opc->opc2,
-                       opc->opc3);
-                return -1;
+                error_setg(errp, "ERROR initializing PowerPC instruction "
+                           "0x%02x 0x%02x 0x%02x", opc->opc1, opc->opc2,
+                           opc->opc3);
+                return;
             }
         }
     }
     fix_opcode_tables(env->opcodes);
     fflush(stdout);
     fflush(stderr);
-
-    return 0;
 }
 
 #if defined(PPC_DUMP_CPU)
@@ -10009,8 +10006,10 @@ static int gdb_set_spe_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
     return 0;
 }
 
-static int ppc_fixup_cpu(CPUPPCState *env)
+static int ppc_fixup_cpu(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+
     /* TCG doesn't (yet) emulate some groups of instructions that
      * are implemented on some otherwise supported CPUs (e.g. VSX
      * and decimal floating point instructions on POWER7).  We
@@ -10031,53 +10030,42 @@ static int ppc_fixup_cpu(CPUPPCState *env)
     return 0;
 }
 
-int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def)
+static void ppc_cpu_realize(Object *obj, Error **errp)
 {
-    env->msr_mask = def->msr_mask;
-    env->mmu_model = def->mmu_model;
-    env->excp_model = def->excp_model;
-    env->bus_model = def->bus_model;
-    env->insns_flags = def->insns_flags;
-    env->insns_flags2 = def->insns_flags2;
-    env->flags = def->flags;
-    env->bfd_mach = def->bfd_mach;
-    env->check_pow = def->check_pow;
+    PowerPCCPU *cpu = POWERPC_CPU(obj);
+    CPUPPCState *env = &cpu->env;
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
+    ppc_def_t *def = pcc->info;
+    Error *local_err = NULL;
+#if !defined(CONFIG_USER_ONLY)
+    int max_smt = kvm_enabled() ? kvmppc_smt_threads() : 1;
+#endif
 
-#if defined(TARGET_PPC64)
-    if (def->sps)
-        env->sps = *def->sps;
-    else if (env->mmu_model & POWERPC_MMU_64) {
-        /* Use default sets of page sizes */
-        static const struct ppc_segment_page_sizes defsps = {
-            .sps = {
-                { .page_shift = 12, /* 4K */
-                  .slb_enc = 0,
-                  .enc = { { .page_shift = 12, .pte_enc = 0 } }
-                },
-                { .page_shift = 24, /* 16M */
-                  .slb_enc = 0x100,
-                  .enc = { { .page_shift = 24, .pte_enc = 0 } }
-                },
-            },
-        };
-        env->sps = defsps;
+#if !defined(CONFIG_USER_ONLY)
+    if (smp_threads > max_smt) {
+        fprintf(stderr, "Cannot support more than %d threads on PPC with %s\n",
+                max_smt, kvm_enabled() ? "KVM" : "TCG");
+        exit(1);
     }
-#endif /* defined(TARGET_PPC64) */
+#endif
 
     if (kvm_enabled()) {
-        if (kvmppc_fixup_cpu(env) != 0) {
-            fprintf(stderr, "Unable to virtualize selected CPU with KVM\n");
-            exit(1);
+        if (kvmppc_fixup_cpu(cpu) != 0) {
+            error_setg(errp, "Unable to virtualize selected CPU with KVM");
+            return;
         }
     } else {
-        if (ppc_fixup_cpu(env) != 0) {
-            fprintf(stderr, "Unable to emulate selected CPU with TCG\n");
-            exit(1);
+        if (ppc_fixup_cpu(cpu) != 0) {
+            error_setg(errp, "Unable to emulate selected CPU with TCG");
+            return;
         }
     }
 
-    if (create_ppc_opcodes(env, def) < 0)
-        return -1;
+    create_ppc_opcodes(cpu, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
     init_ppc_proc(env, def);
 
     if (def->insns_flags & PPC_FLOAT) {
@@ -10093,6 +10081,8 @@ int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def)
                                  34, "power-spe.xml", 0);
     }
 
+    qemu_init_vcpu(env);
+
 #if defined(PPC_DUMP_CPU)
     {
         const char *mmu_model, *excp_model, *bus_model;
@@ -10254,50 +10244,65 @@ int cpu_ppc_register_internal (CPUPPCState *env, const ppc_def_t *def)
     dump_ppc_sprs(env);
     fflush(stdout);
 #endif
-
-    return 0;
 }
 
-static bool ppc_cpu_usable(const ppc_def_t *def)
+static gint ppc_cpu_compare_class_pvr(gconstpointer a, gconstpointer b)
 {
-#if defined(TARGET_PPCEMB)
-    /* When using the ppcemb target, we only support 440 style cores */
-    if (def->mmu_model != POWERPC_MMU_BOOKE) {
-        return false;
+    ObjectClass *oc = (ObjectClass *)a;
+    uint32_t pvr = *(uint32_t *)b;
+    PowerPCCPUClass *pcc = (PowerPCCPUClass *)a;
+
+    /* -cpu host does a PVR lookup during construction */
+    if (unlikely(strcmp(object_class_get_name(oc),
+                        TYPE_HOST_POWERPC_CPU) == 0)) {
+        return -1;
     }
-#endif
 
-    return true;
+    return pcc->info->pvr == pvr ? 0 : -1;
 }
 
-const ppc_def_t *ppc_find_by_pvr(uint32_t pvr)
+PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr)
 {
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(ppc_defs); i++) {
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
-        }
+    GSList *list, *item;
+    PowerPCCPUClass *pcc = NULL;
 
-        /* If we have an exact match, we're done */
-        if (pvr == ppc_defs[i].pvr) {
-            return &ppc_defs[i];
-        }
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    item = g_slist_find_custom(list, &pvr, ppc_cpu_compare_class_pvr);
+    if (item != NULL) {
+        pcc = POWERPC_CPU_CLASS(item->data);
     }
+    g_slist_free(list);
+
+    return pcc;
+}
+
+static gint ppc_cpu_compare_class_name(gconstpointer a, gconstpointer b)
+{
+    ObjectClass *oc = (ObjectClass *)a;
+    const char *name = b;
 
-    return NULL;
+    if (strncasecmp(name, object_class_get_name(oc), strlen(name)) == 0 &&
+        strcmp(object_class_get_name(oc) + strlen(name),
+               "-" TYPE_POWERPC_CPU) == 0) {
+        return 0;
+    }
+    return -1;
 }
 
 #include <ctype.h>
 
-const ppc_def_t *cpu_ppc_find_by_name (const char *name)
+static ObjectClass *ppc_cpu_class_by_name(const char *name)
 {
-    const ppc_def_t *ret;
+    GSList *list, *item;
+    ObjectClass *ret = NULL;
     const char *p;
-    int i, max, len;
+    int i, len;
 
-    if (kvm_enabled() && (strcasecmp(name, "host") == 0)) {
-        return kvmppc_host_cpu_def();
+    if (strcasecmp(name, "host") == 0) {
+        if (kvm_enabled()) {
+            ret = object_class_by_name(TYPE_HOST_POWERPC_CPU);
+        }
+        return ret;
     }
 
     /* Check if the given name is a PVR */
@@ -10312,63 +10317,152 @@ const ppc_def_t *cpu_ppc_find_by_name (const char *name)
             if (!qemu_isxdigit(*p++))
                 break;
         }
-        if (i == 8)
-            return ppc_find_by_pvr(strtoul(name, NULL, 16));
-    }
-    ret = NULL;
-    max = ARRAY_SIZE(ppc_defs);
-    for (i = 0; i < max; i++) {
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
+        if (i == 8) {
+            ret = OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
+            return ret;
         }
+    }
 
-        if (strcasecmp(name, ppc_defs[i].name) == 0) {
-            ret = &ppc_defs[i];
-            break;
-        }
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    item = g_slist_find_custom(list, name, ppc_cpu_compare_class_name);
+    if (item != NULL) {
+        ret = OBJECT_CLASS(item->data);
     }
+    g_slist_free(list);
 
     return ret;
 }
 
-void ppc_cpu_list (FILE *f, fprintf_function cpu_fprintf)
+PowerPCCPU *cpu_ppc_init(const char *cpu_model)
 {
-    int i, max;
+    PowerPCCPU *cpu;
+    CPUPPCState *env;
+    ObjectClass *oc;
+    Error *err = NULL;
 
-    max = ARRAY_SIZE(ppc_defs);
-    for (i = 0; i < max; i++) {
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
-        }
+    oc = ppc_cpu_class_by_name(cpu_model);
+    if (oc == NULL) {
+        return NULL;
+    }
 
-        (*cpu_fprintf)(f, "PowerPC %-16s PVR %08x\n",
-                       ppc_defs[i].name, ppc_defs[i].pvr);
+    cpu = POWERPC_CPU(object_new(object_class_get_name(oc)));
+    env = &cpu->env;
+
+    if (tcg_enabled()) {
+        ppc_translate_init();
     }
+
+    env->cpu_model_str = cpu_model;
+
+    ppc_cpu_realize(OBJECT(cpu), &err);
+    if (err != NULL) {
+        fprintf(stderr, "%s\n", error_get_pretty(err));
+        error_free(err);
+        object_unref(OBJECT(cpu));
+        return NULL;
+    }
+
+    return cpu;
+}
+
+/* Sort by PVR, ordering special case "host" last. */
+static gint ppc_cpu_list_compare(gconstpointer a, gconstpointer b)
+{
+    ObjectClass *oc_a = (ObjectClass *)a;
+    ObjectClass *oc_b = (ObjectClass *)b;
+    PowerPCCPUClass *pcc_a = POWERPC_CPU_CLASS(oc_a);
+    PowerPCCPUClass *pcc_b = POWERPC_CPU_CLASS(oc_b);
+    const char *name_a = object_class_get_name(oc_a);
+    const char *name_b = object_class_get_name(oc_b);
+
+    if (strcmp(name_a, TYPE_HOST_POWERPC_CPU) == 0) {
+        return 1;
+    } else if (strcmp(name_b, TYPE_HOST_POWERPC_CPU) == 0) {
+        return -1;
+    } else {
+        /* Avoid an integer overflow during subtraction */
+        if (pcc_a->info->pvr < pcc_b->info->pvr) {
+            return -1;
+        } else if (pcc_a->info->pvr > pcc_b->info->pvr) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static void ppc_cpu_list_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CPUListState *s = user_data;
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+
+    (*s->cpu_fprintf)(s->file, "PowerPC %-16s PVR %08x\n",
+                      pcc->info->name, pcc->info->pvr);
+}
+
+void ppc_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+    CPUListState s = {
+        .file = f,
+        .cpu_fprintf = cpu_fprintf,
+    };
+    GSList *list;
+
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    list = g_slist_sort(list, ppc_cpu_list_compare);
+    g_slist_foreach(list, ppc_cpu_list_entry, &s);
+    g_slist_free(list);
+}
+
+static void ppc_cpu_defs_entry(gpointer data, gpointer user_data)
+{
+    ObjectClass *oc = data;
+    CpuDefinitionInfoList **first = user_data;
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+    CpuDefinitionInfoList *entry;
+    CpuDefinitionInfo *info;
+
+    info = g_malloc0(sizeof(*info));
+    info->name = g_strdup(pcc->info->name);
+
+    entry = g_malloc0(sizeof(*entry));
+    entry->value = info;
+    entry->next = *first;
+    *first = entry;
 }
 
 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
 {
     CpuDefinitionInfoList *cpu_list = NULL;
-    int i;
+    GSList *list;
 
-    for (i = 0; i < ARRAY_SIZE(ppc_defs); i++) {
-        CpuDefinitionInfoList *entry;
-        CpuDefinitionInfo *info;
+    list = object_class_get_list(TYPE_POWERPC_CPU, false);
+    g_slist_foreach(list, ppc_cpu_defs_entry, &cpu_list);
+    g_slist_free(list);
 
-        if (!ppc_cpu_usable(&ppc_defs[i])) {
-            continue;
-        }
+    return cpu_list;
+}
 
-        info = g_malloc0(sizeof(*info));
-        info->name = g_strdup(ppc_defs[i].name);
+static void ppc_cpu_def_class_init(ObjectClass *oc, void *data)
+{
+    PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+    ppc_def_t *info = data;
 
-        entry = g_malloc0(sizeof(*entry));
-        entry->value = info;
-        entry->next = cpu_list;
-        cpu_list = entry;
-    }
+    pcc->info = info;
+}
 
-    return cpu_list;
+static void ppc_cpu_register_model(const ppc_def_t *def)
+{
+    TypeInfo type_info = {
+        .parent = TYPE_POWERPC_CPU,
+        .class_init = ppc_cpu_def_class_init,
+        .class_data = (void *)def,
+    };
+
+    type_info.name = g_strdup_printf("%s-" TYPE_POWERPC_CPU, def->name),
+    type_register(&type_info);
+    g_free((gpointer)type_info.name);
 }
 
 /* CPUClass::reset() */
@@ -10380,7 +10474,7 @@ static void ppc_cpu_reset(CPUState *s)
     target_ulong msr;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -10439,9 +10533,42 @@ static void ppc_cpu_reset(CPUState *s)
 static void ppc_cpu_initfn(Object *obj)
 {
     PowerPCCPU *cpu = POWERPC_CPU(obj);
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
     CPUPPCState *env = &cpu->env;
+    ppc_def_t *def = pcc->info;
 
     cpu_exec_init(env);
+
+    env->msr_mask = def->msr_mask;
+    env->mmu_model = def->mmu_model;
+    env->excp_model = def->excp_model;
+    env->bus_model = def->bus_model;
+    env->insns_flags = def->insns_flags;
+    env->insns_flags2 = def->insns_flags2;
+    env->flags = def->flags;
+    env->bfd_mach = def->bfd_mach;
+    env->check_pow = def->check_pow;
+
+#if defined(TARGET_PPC64)
+    if (def->sps) {
+        env->sps = *def->sps;
+    } else if (env->mmu_model & POWERPC_MMU_64) {
+        /* Use default sets of page sizes */
+        static const struct ppc_segment_page_sizes defsps = {
+            .sps = {
+                { .page_shift = 12, /* 4K */
+                  .slb_enc = 0,
+                  .enc = { { .page_shift = 12, .pte_enc = 0 } }
+                },
+                { .page_shift = 24, /* 16M */
+                  .slb_enc = 0x100,
+                  .enc = { { .page_shift = 24, .pte_enc = 0 } }
+                },
+            },
+        };
+        env->sps = defsps;
+    }
+#endif /* defined(TARGET_PPC64) */
 }
 
 static void ppc_cpu_class_init(ObjectClass *oc, void *data)
@@ -10451,6 +10578,8 @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data)
 
     pcc->parent_reset = cc->reset;
     cc->reset = ppc_cpu_reset;
+
+    cc->class_by_name = ppc_cpu_class_by_name;
 }
 
 static const TypeInfo ppc_cpu_type_info = {
@@ -10458,14 +10587,27 @@ static const TypeInfo ppc_cpu_type_info = {
     .parent = TYPE_CPU,
     .instance_size = sizeof(PowerPCCPU),
     .instance_init = ppc_cpu_initfn,
-    .abstract = false,
+    .abstract = true,
     .class_size = sizeof(PowerPCCPUClass),
     .class_init = ppc_cpu_class_init,
 };
 
 static void ppc_cpu_register_types(void)
 {
+    int i;
+
     type_register_static(&ppc_cpu_type_info);
+
+    for (i = 0; i < ARRAY_SIZE(ppc_defs); i++) {
+        const ppc_def_t *def = &ppc_defs[i];
+#if defined(TARGET_PPCEMB)
+        /* When using the ppcemb target, we only support 440 style cores */
+        if (def->mmu_model != POWERPC_MMU_BOOKE) {
+            continue;
+        }
+#endif
+        ppc_cpu_register_model(def);
+    }
 }
 
 type_init(ppc_cpu_register_types)
diff --git a/target-s390x/Makefile.objs b/target-s390x/Makefile.objs
index e728abf..4e63417 100644
--- a/target-s390x/Makefile.objs
+++ b/target-s390x/Makefile.objs
@@ -1,4 +1,4 @@
 obj-y += translate.o helper.o cpu.o interrupt.o
 obj-y += int_helper.o fpu_helper.o cc_helper.o mem_helper.o misc_helper.o
-obj-$(CONFIG_SOFTMMU) += machine.o
+obj-$(CONFIG_SOFTMMU) += ioinst.o
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/target-s390x/cc_helper.c b/target-s390x/cc_helper.c
index 19ef145..a6d60bf 100644
--- a/target-s390x/cc_helper.c
+++ b/target-s390x/cc_helper.c
@@ -20,6 +20,7 @@
 
 #include "cpu.h"
 #include "helper.h"
+#include "qemu/host-utils.h"
 
 /* #define DEBUG_HELPER */
 #ifdef DEBUG_HELPER
@@ -28,8 +29,7 @@
 #define HELPER_LOG(x...)
 #endif
 
-static inline uint32_t cc_calc_ltgt_32(CPUS390XState *env, int32_t src,
-                                       int32_t dst)
+static uint32_t cc_calc_ltgt_32(int32_t src, int32_t dst)
 {
     if (src == dst) {
         return 0;
@@ -40,13 +40,12 @@ static inline uint32_t cc_calc_ltgt_32(CPUS390XState *env, int32_t src,
     }
 }
 
-static inline uint32_t cc_calc_ltgt0_32(CPUS390XState *env, int32_t dst)
+static uint32_t cc_calc_ltgt0_32(int32_t dst)
 {
-    return cc_calc_ltgt_32(env, dst, 0);
+    return cc_calc_ltgt_32(dst, 0);
 }
 
-static inline uint32_t cc_calc_ltgt_64(CPUS390XState *env, int64_t src,
-                                       int64_t dst)
+static uint32_t cc_calc_ltgt_64(int64_t src, int64_t dst)
 {
     if (src == dst) {
         return 0;
@@ -57,13 +56,12 @@ static inline uint32_t cc_calc_ltgt_64(CPUS390XState *env, int64_t src,
     }
 }
 
-static inline uint32_t cc_calc_ltgt0_64(CPUS390XState *env, int64_t dst)
+static uint32_t cc_calc_ltgt0_64(int64_t dst)
 {
-    return cc_calc_ltgt_64(env, dst, 0);
+    return cc_calc_ltgt_64(dst, 0);
 }
 
-static inline uint32_t cc_calc_ltugtu_32(CPUS390XState *env, uint32_t src,
-                                         uint32_t dst)
+static uint32_t cc_calc_ltugtu_32(uint32_t src, uint32_t dst)
 {
     if (src == dst) {
         return 0;
@@ -74,8 +72,7 @@ static inline uint32_t cc_calc_ltugtu_32(CPUS390XState *env, uint32_t src,
     }
 }
 
-static inline uint32_t cc_calc_ltugtu_64(CPUS390XState *env, uint64_t src,
-                                         uint64_t dst)
+static uint32_t cc_calc_ltugtu_64(uint64_t src, uint64_t dst)
 {
     if (src == dst) {
         return 0;
@@ -86,13 +83,11 @@ static inline uint32_t cc_calc_ltugtu_64(CPUS390XState *env, uint64_t src,
     }
 }
 
-static inline uint32_t cc_calc_tm_32(CPUS390XState *env, uint32_t val,
-                                     uint32_t mask)
+static uint32_t cc_calc_tm_32(uint32_t val, uint32_t mask)
 {
-    uint16_t r = val & mask;
+    uint32_t r = val & mask;
 
-    HELPER_LOG("%s: val 0x%x mask 0x%x\n", __func__, val, mask);
-    if (r == 0 || mask == 0) {
+    if (r == 0) {
         return 0;
     } else if (r == mask) {
         return 3;
@@ -101,23 +96,17 @@ static inline uint32_t cc_calc_tm_32(CPUS390XState *env, uint32_t val,
     }
 }
 
-/* set condition code for test under mask */
-static inline uint32_t cc_calc_tm_64(CPUS390XState *env, uint64_t val,
-                                     uint32_t mask)
+static uint32_t cc_calc_tm_64(uint64_t val, uint64_t mask)
 {
-    uint16_t r = val & mask;
+    uint64_t r = val & mask;
 
-    HELPER_LOG("%s: val 0x%lx mask 0x%x r 0x%x\n", __func__, val, mask, r);
-    if (r == 0 || mask == 0) {
+    if (r == 0) {
         return 0;
     } else if (r == mask) {
         return 3;
     } else {
-        while (!(mask & 0x8000)) {
-            mask <<= 1;
-            val <<= 1;
-        }
-        if (val & 0x8000) {
+        int top = clz64(mask);
+        if ((int64_t)(val << top) < 0) {
             return 2;
         } else {
             return 1;
@@ -125,13 +114,12 @@ static inline uint32_t cc_calc_tm_64(CPUS390XState *env, uint64_t val,
     }
 }
 
-static inline uint32_t cc_calc_nz(CPUS390XState *env, uint64_t dst)
+static uint32_t cc_calc_nz(uint64_t dst)
 {
     return !!dst;
 }
 
-static inline uint32_t cc_calc_add_64(CPUS390XState *env, int64_t a1,
-                                      int64_t a2, int64_t ar)
+static uint32_t cc_calc_add_64(int64_t a1, int64_t a2, int64_t ar)
 {
     if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
         return 3; /* overflow */
@@ -146,26 +134,22 @@ static inline uint32_t cc_calc_add_64(CPUS390XState *env, int64_t a1,
     }
 }
 
-static inline uint32_t cc_calc_addu_64(CPUS390XState *env, uint64_t a1,
-                                       uint64_t a2, uint64_t ar)
+static uint32_t cc_calc_addu_64(uint64_t a1, uint64_t a2, uint64_t ar)
 {
-    if (ar == 0) {
-        if (a1) {
-            return 2;
-        } else {
-            return 0;
-        }
-    } else {
-        if (ar < a1 || ar < a2) {
-            return 3;
-        } else {
-            return 1;
-        }
-    }
+    return (ar != 0) + 2 * (ar < a1);
+}
+
+static uint32_t cc_calc_addc_64(uint64_t a1, uint64_t a2, uint64_t ar)
+{
+    /* Recover a2 + carry_in.  */
+    uint64_t a2c = ar - a1;
+    /* Check for a2+carry_in overflow, then a1+a2c overflow.  */
+    int carry_out = (a2c < a2) || (ar < a1);
+
+    return (ar != 0) + 2 * carry_out;
 }
 
-static inline uint32_t cc_calc_sub_64(CPUS390XState *env, int64_t a1,
-                                      int64_t a2, int64_t ar)
+static uint32_t cc_calc_sub_64(int64_t a1, int64_t a2, int64_t ar)
 {
     if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
         return 3; /* overflow */
@@ -180,8 +164,7 @@ static inline uint32_t cc_calc_sub_64(CPUS390XState *env, int64_t a1,
     }
 }
 
-static inline uint32_t cc_calc_subu_64(CPUS390XState *env, uint64_t a1,
-                                       uint64_t a2, uint64_t ar)
+static uint32_t cc_calc_subu_64(uint64_t a1, uint64_t a2, uint64_t ar)
 {
     if (ar == 0) {
         return 2;
@@ -194,7 +177,25 @@ static inline uint32_t cc_calc_subu_64(CPUS390XState *env, uint64_t a1,
     }
 }
 
-static inline uint32_t cc_calc_abs_64(CPUS390XState *env, int64_t dst)
+static uint32_t cc_calc_subb_64(uint64_t a1, uint64_t a2, uint64_t ar)
+{
+    /* We had borrow-in if normal subtraction isn't equal.  */
+    int borrow_in = ar - (a1 - a2);
+    int borrow_out;
+
+    /* If a2 was ULONG_MAX, and borrow_in, then a2 is logically 65 bits,
+       and we must have had borrow out.  */
+    if (borrow_in && a2 == (uint64_t)-1) {
+        borrow_out = 1;
+    } else {
+        a2 += borrow_in;
+        borrow_out = (a2 > a1);
+    }
+
+    return (ar != 0) + 2 * !borrow_out;
+}
+
+static uint32_t cc_calc_abs_64(int64_t dst)
 {
     if ((uint64_t)dst == 0x8000000000000000ULL) {
         return 3;
@@ -205,12 +206,12 @@ static inline uint32_t cc_calc_abs_64(CPUS390XState *env, int64_t dst)
     }
 }
 
-static inline uint32_t cc_calc_nabs_64(CPUS390XState *env, int64_t dst)
+static uint32_t cc_calc_nabs_64(int64_t dst)
 {
     return !!dst;
 }
 
-static inline uint32_t cc_calc_comp_64(CPUS390XState *env, int64_t dst)
+static uint32_t cc_calc_comp_64(int64_t dst)
 {
     if ((uint64_t)dst == 0x8000000000000000ULL) {
         return 3;
@@ -224,8 +225,7 @@ static inline uint32_t cc_calc_comp_64(CPUS390XState *env, int64_t dst)
 }
 
 
-static inline uint32_t cc_calc_add_32(CPUS390XState *env, int32_t a1,
-                                      int32_t a2, int32_t ar)
+static uint32_t cc_calc_add_32(int32_t a1, int32_t a2, int32_t ar)
 {
     if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
         return 3; /* overflow */
@@ -240,26 +240,22 @@ static inline uint32_t cc_calc_add_32(CPUS390XState *env, int32_t a1,
     }
 }
 
-static inline uint32_t cc_calc_addu_32(CPUS390XState *env, uint32_t a1,
-                                       uint32_t a2, uint32_t ar)
+static uint32_t cc_calc_addu_32(uint32_t a1, uint32_t a2, uint32_t ar)
 {
-    if (ar == 0) {
-        if (a1) {
-            return 2;
-        } else {
-            return 0;
-        }
-    } else {
-        if (ar < a1 || ar < a2) {
-            return 3;
-        } else {
-            return 1;
-        }
-    }
+    return (ar != 0) + 2 * (ar < a1);
+}
+
+static uint32_t cc_calc_addc_32(uint32_t a1, uint32_t a2, uint32_t ar)
+{
+    /* Recover a2 + carry_in.  */
+    uint32_t a2c = ar - a1;
+    /* Check for a2+carry_in overflow, then a1+a2c overflow.  */
+    int carry_out = (a2c < a2) || (ar < a1);
+
+    return (ar != 0) + 2 * carry_out;
 }
 
-static inline uint32_t cc_calc_sub_32(CPUS390XState *env, int32_t a1,
-                                      int32_t a2, int32_t ar)
+static uint32_t cc_calc_sub_32(int32_t a1, int32_t a2, int32_t ar)
 {
     if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
         return 3; /* overflow */
@@ -274,8 +270,7 @@ static inline uint32_t cc_calc_sub_32(CPUS390XState *env, int32_t a1,
     }
 }
 
-static inline uint32_t cc_calc_subu_32(CPUS390XState *env, uint32_t a1,
-                                       uint32_t a2, uint32_t ar)
+static uint32_t cc_calc_subu_32(uint32_t a1, uint32_t a2, uint32_t ar)
 {
     if (ar == 0) {
         return 2;
@@ -288,7 +283,25 @@ static inline uint32_t cc_calc_subu_32(CPUS390XState *env, uint32_t a1,
     }
 }
 
-static inline uint32_t cc_calc_abs_32(CPUS390XState *env, int32_t dst)
+static uint32_t cc_calc_subb_32(uint32_t a1, uint32_t a2, uint32_t ar)
+{
+    /* We had borrow-in if normal subtraction isn't equal.  */
+    int borrow_in = ar - (a1 - a2);
+    int borrow_out;
+
+    /* If a2 was UINT_MAX, and borrow_in, then a2 is logically 65 bits,
+       and we must have had borrow out.  */
+    if (borrow_in && a2 == (uint32_t)-1) {
+        borrow_out = 1;
+    } else {
+        a2 += borrow_in;
+        borrow_out = (a2 > a1);
+    }
+
+    return (ar != 0) + 2 * !borrow_out;
+}
+
+static uint32_t cc_calc_abs_32(int32_t dst)
 {
     if ((uint32_t)dst == 0x80000000UL) {
         return 3;
@@ -299,12 +312,12 @@ static inline uint32_t cc_calc_abs_32(CPUS390XState *env, int32_t dst)
     }
 }
 
-static inline uint32_t cc_calc_nabs_32(CPUS390XState *env, int32_t dst)
+static uint32_t cc_calc_nabs_32(int32_t dst)
 {
     return !!dst;
 }
 
-static inline uint32_t cc_calc_comp_32(CPUS390XState *env, int32_t dst)
+static uint32_t cc_calc_comp_32(int32_t dst)
 {
     if ((uint32_t)dst == 0x80000000UL) {
         return 3;
@@ -318,69 +331,80 @@ static inline uint32_t cc_calc_comp_32(CPUS390XState *env, int32_t dst)
 }
 
 /* calculate condition code for insert character under mask insn */
-static inline uint32_t cc_calc_icm_32(CPUS390XState *env, uint32_t mask,
-                                      uint32_t val)
+static uint32_t cc_calc_icm(uint64_t mask, uint64_t val)
 {
-    uint32_t cc;
-
-    HELPER_LOG("%s: mask 0x%x val %d\n", __func__, mask, val);
-    if (mask == 0xf) {
-        if (!val) {
-            return 0;
-        } else if (val & 0x80000000) {
+    if ((val & mask) == 0) {
+        return 0;
+    } else {
+        int top = clz64(mask);
+        if ((int64_t)(val << top) < 0) {
             return 1;
         } else {
             return 2;
         }
     }
+}
+
+static uint32_t cc_calc_sla_32(uint32_t src, int shift)
+{
+    uint32_t mask = ((1U << shift) - 1U) << (32 - shift);
+    uint32_t sign = 1U << 31;
+    uint32_t match;
+    int32_t r;
 
-    if (!val || !mask) {
-        cc = 0;
+    /* Check if the sign bit stays the same.  */
+    if (src & sign) {
+        match = mask;
     } else {
-        while (mask != 1) {
-            mask >>= 1;
-            val >>= 8;
-        }
-        if (val & 0x80) {
-            cc = 1;
-        } else {
-            cc = 2;
-        }
+        match = 0;
     }
-    return cc;
+    if ((src & mask) != match) {
+        /* Overflow.  */
+        return 3;
+    }
+
+    r = ((src << shift) & ~sign) | (src & sign);
+    if (r == 0) {
+        return 0;
+    } else if (r < 0) {
+        return 1;
+    }
+    return 2;
 }
 
-static inline uint32_t cc_calc_slag(CPUS390XState *env, uint64_t src,
-                                    uint64_t shift)
+static uint32_t cc_calc_sla_64(uint64_t src, int shift)
 {
     uint64_t mask = ((1ULL << shift) - 1ULL) << (64 - shift);
-    uint64_t match, r;
+    uint64_t sign = 1ULL << 63;
+    uint64_t match;
+    int64_t r;
 
-    /* check if the sign bit stays the same */
-    if (src & (1ULL << 63)) {
+    /* Check if the sign bit stays the same.  */
+    if (src & sign) {
         match = mask;
     } else {
         match = 0;
     }
-
     if ((src & mask) != match) {
-        /* overflow */
+        /* Overflow.  */
         return 3;
     }
 
-    r = ((src << shift) & ((1ULL << 63) - 1)) | (src & (1ULL << 63));
-
-    if ((int64_t)r == 0) {
+    r = ((src << shift) & ~sign) | (src & sign);
+    if (r == 0) {
         return 0;
-    } else if ((int64_t)r < 0) {
+    } else if (r < 0) {
         return 1;
     }
-
     return 2;
 }
 
+static uint32_t cc_calc_flogr(uint64_t dst)
+{
+    return dst ? 2 : 0;
+}
 
-static inline uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
+static uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
                                   uint64_t src, uint64_t dst, uint64_t vr)
 {
     uint32_t r = 0;
@@ -394,95 +418,110 @@ static inline uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
         r = cc_op;
         break;
     case CC_OP_LTGT0_32:
-        r = cc_calc_ltgt0_32(env, dst);
+        r = cc_calc_ltgt0_32(dst);
         break;
     case CC_OP_LTGT0_64:
-        r =  cc_calc_ltgt0_64(env, dst);
+        r =  cc_calc_ltgt0_64(dst);
         break;
     case CC_OP_LTGT_32:
-        r =  cc_calc_ltgt_32(env, src, dst);
+        r =  cc_calc_ltgt_32(src, dst);
         break;
     case CC_OP_LTGT_64:
-        r =  cc_calc_ltgt_64(env, src, dst);
+        r =  cc_calc_ltgt_64(src, dst);
         break;
     case CC_OP_LTUGTU_32:
-        r =  cc_calc_ltugtu_32(env, src, dst);
+        r =  cc_calc_ltugtu_32(src, dst);
         break;
     case CC_OP_LTUGTU_64:
-        r =  cc_calc_ltugtu_64(env, src, dst);
+        r =  cc_calc_ltugtu_64(src, dst);
         break;
     case CC_OP_TM_32:
-        r =  cc_calc_tm_32(env, src, dst);
+        r =  cc_calc_tm_32(src, dst);
         break;
     case CC_OP_TM_64:
-        r =  cc_calc_tm_64(env, src, dst);
+        r =  cc_calc_tm_64(src, dst);
         break;
     case CC_OP_NZ:
-        r =  cc_calc_nz(env, dst);
+        r =  cc_calc_nz(dst);
         break;
     case CC_OP_ADD_64:
-        r =  cc_calc_add_64(env, src, dst, vr);
+        r =  cc_calc_add_64(src, dst, vr);
         break;
     case CC_OP_ADDU_64:
-        r =  cc_calc_addu_64(env, src, dst, vr);
+        r =  cc_calc_addu_64(src, dst, vr);
+        break;
+    case CC_OP_ADDC_64:
+        r =  cc_calc_addc_64(src, dst, vr);
         break;
     case CC_OP_SUB_64:
-        r =  cc_calc_sub_64(env, src, dst, vr);
+        r =  cc_calc_sub_64(src, dst, vr);
         break;
     case CC_OP_SUBU_64:
-        r =  cc_calc_subu_64(env, src, dst, vr);
+        r =  cc_calc_subu_64(src, dst, vr);
+        break;
+    case CC_OP_SUBB_64:
+        r =  cc_calc_subb_64(src, dst, vr);
         break;
     case CC_OP_ABS_64:
-        r =  cc_calc_abs_64(env, dst);
+        r =  cc_calc_abs_64(dst);
         break;
     case CC_OP_NABS_64:
-        r =  cc_calc_nabs_64(env, dst);
+        r =  cc_calc_nabs_64(dst);
         break;
     case CC_OP_COMP_64:
-        r =  cc_calc_comp_64(env, dst);
+        r =  cc_calc_comp_64(dst);
         break;
 
     case CC_OP_ADD_32:
-        r =  cc_calc_add_32(env, src, dst, vr);
+        r =  cc_calc_add_32(src, dst, vr);
         break;
     case CC_OP_ADDU_32:
-        r =  cc_calc_addu_32(env, src, dst, vr);
+        r =  cc_calc_addu_32(src, dst, vr);
+        break;
+    case CC_OP_ADDC_32:
+        r =  cc_calc_addc_32(src, dst, vr);
         break;
     case CC_OP_SUB_32:
-        r =  cc_calc_sub_32(env, src, dst, vr);
+        r =  cc_calc_sub_32(src, dst, vr);
         break;
     case CC_OP_SUBU_32:
-        r =  cc_calc_subu_32(env, src, dst, vr);
+        r =  cc_calc_subu_32(src, dst, vr);
+        break;
+    case CC_OP_SUBB_32:
+        r =  cc_calc_subb_32(src, dst, vr);
         break;
     case CC_OP_ABS_32:
-        r =  cc_calc_abs_64(env, dst);
+        r =  cc_calc_abs_32(dst);
         break;
     case CC_OP_NABS_32:
-        r =  cc_calc_nabs_64(env, dst);
+        r =  cc_calc_nabs_32(dst);
         break;
     case CC_OP_COMP_32:
-        r =  cc_calc_comp_32(env, dst);
+        r =  cc_calc_comp_32(dst);
         break;
 
     case CC_OP_ICM:
-        r =  cc_calc_icm_32(env, src, dst);
+        r =  cc_calc_icm(src, dst);
         break;
-    case CC_OP_SLAG:
-        r =  cc_calc_slag(env, src, dst);
+    case CC_OP_SLA_32:
+        r =  cc_calc_sla_32(src, dst);
         break;
-
-    case CC_OP_LTGT_F32:
-        r = set_cc_f32(env, src, dst);
+    case CC_OP_SLA_64:
+        r =  cc_calc_sla_64(src, dst);
         break;
-    case CC_OP_LTGT_F64:
-        r = set_cc_f64(env, src, dst);
+    case CC_OP_FLOGR:
+        r = cc_calc_flogr(dst);
         break;
+
     case CC_OP_NZ_F32:
         r = set_cc_nz_f32(dst);
         break;
     case CC_OP_NZ_F64:
         r = set_cc_nz_f64(dst);
         break;
+    case CC_OP_NZ_F128:
+        r = set_cc_nz_f128(make_float128(src, dst));
+        break;
 
     default:
         cpu_abort(env, "Unknown CC operation: %s\n", cc_name(cc_op));
@@ -505,18 +544,6 @@ uint32_t HELPER(calc_cc)(CPUS390XState *env, uint32_t cc_op, uint64_t src,
     return do_calc_cc(env, cc_op, src, dst, vr);
 }
 
-/* insert psw mask and condition code into r1 */
-void HELPER(ipm)(CPUS390XState *env, uint32_t cc, uint32_t r1)
-{
-    uint64_t r = env->regs[r1];
-
-    r &= 0xffffffff00ffffffULL;
-    r |= (cc << 28) | ((env->psw.mask >> 40) & 0xf);
-    env->regs[r1] = r;
-    HELPER_LOG("%s: cc %d psw.mask 0x%lx r1 0x%lx\n", __func__,
-               cc, env->psw.mask, r);
-}
-
 #ifndef CONFIG_USER_ONLY
 void HELPER(load_psw)(CPUS390XState *env, uint64_t mask, uint64_t addr)
 {
diff --git a/target-s390x/cpu-qom.h b/target-s390x/cpu-qom.h
index 6fa55a8..d54e4a2 100644
--- a/target-s390x/cpu-qom.h
+++ b/target-s390x/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_S390_CPU_QOM_H
 #define QEMU_S390_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_S390_CPU "s390-cpu"
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index 619b202..d765e7b 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2009 Ulrich Hecht
  * Copyright (c) 2011 Alexander Graf
  * Copyright (c) 2012 SUSE LINUX Products GmbH
+ * Copyright (c) 2012 IBM Corp.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -18,12 +19,44 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ * Contributions after 2012-12-11 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
  */
 
 #include "cpu.h"
 #include "qemu-common.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
+#include "hw/hw.h"
+#ifndef CONFIG_USER_ONLY
+#include "sysemu/arch_init.h"
+#endif
+
+#define CR0_RESET       0xE0UL
+#define CR14_RESET      0xC2000000UL;
+
+/* generate CPU information for cpu -? */
+void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf)
+{
+#ifdef CONFIG_KVM
+    (*cpu_fprintf)(f, "s390 %16s\n", "host");
+#endif
+}
+
+#ifndef CONFIG_USER_ONLY
+CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
+{
+    CpuDefinitionInfoList *entry;
+    CpuDefinitionInfo *info;
+
+    info = g_malloc0(sizeof(*info));
+    info->name = g_strdup("host");
 
+    entry = g_malloc0(sizeof(*entry));
+    entry->value = info;
+
+    return entry;
+}
+#endif
 
 /* CPUClass::reset() */
 static void s390_cpu_reset(CPUState *s)
@@ -33,18 +66,37 @@ static void s390_cpu_reset(CPUState *s)
     CPUS390XState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
+    s390_del_running_cpu(cpu);
+
     scc->parent_reset(s);
 
     memset(env, 0, offsetof(CPUS390XState, breakpoints));
-    /* FIXME: reset vector? */
+
+    /* architectured initial values for CR 0 and 14 */
+    env->cregs[0] = CR0_RESET;
+    env->cregs[14] = CR14_RESET;
+    /* set halted to 1 to make sure we can add the cpu in
+     * s390_ipl_cpu code, where env->halted is set back to 0
+     * after incrementing the cpu counter */
+#if !defined(CONFIG_USER_ONLY)
+    env->halted = 1;
+#endif
     tlb_flush(env, 1);
-    s390_add_running_cpu(env);
 }
 
+#if !defined(CONFIG_USER_ONLY)
+static void s390_cpu_machine_reset_cb(void *opaque)
+{
+    S390CPU *cpu = opaque;
+
+    cpu_reset(CPU(cpu));
+}
+#endif
+
 static void s390_cpu_initfn(Object *obj)
 {
     S390CPU *cpu = S390_CPU(obj);
@@ -56,12 +108,17 @@ static void s390_cpu_initfn(Object *obj)
 
     cpu_exec_init(env);
 #if !defined(CONFIG_USER_ONLY)
+    qemu_register_reset(s390_cpu_machine_reset_cb, cpu);
     qemu_get_timedate(&tm, 0);
     env->tod_offset = TOD_UNIX_EPOCH +
                       (time2tod(mktimegm(&tm)) * 1000000000ULL);
     env->tod_basetime = 0;
     env->tod_timer = qemu_new_timer_ns(vm_clock, s390x_tod_timer, cpu);
     env->cpu_timer = qemu_new_timer_ns(vm_clock, s390x_cpu_timer, cpu);
+    /* set env->halted state to 1 to avoid decrementing the running
+     * cpu counter in s390_cpu_reset to a negative number at
+     * initial ipl */
+    env->halted = 1;
 #endif
     env->cpu_num = cpu_num++;
     env->ext_index = -1;
@@ -69,13 +126,30 @@ static void s390_cpu_initfn(Object *obj)
     cpu_reset(CPU(cpu));
 }
 
+static void s390_cpu_finalize(Object *obj)
+{
+#if !defined(CONFIG_USER_ONLY)
+    S390CPU *cpu = S390_CPU(obj);
+
+    qemu_unregister_reset(s390_cpu_machine_reset_cb, cpu);
+#endif
+}
+
+static const VMStateDescription vmstate_s390_cpu = {
+    .name = "cpu",
+    .unmigratable = 1,
+};
+
 static void s390_cpu_class_init(ObjectClass *oc, void *data)
 {
     S390CPUClass *scc = S390_CPU_CLASS(oc);
     CPUClass *cc = CPU_CLASS(scc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
 
     scc->parent_reset = cc->reset;
     cc->reset = s390_cpu_reset;
+
+    dc->vmsd = &vmstate_s390_cpu;
 }
 
 static const TypeInfo s390_cpu_type_info = {
@@ -83,6 +157,7 @@ static const TypeInfo s390_cpu_type_info = {
     .parent = TYPE_CPU,
     .instance_size = sizeof(S390CPU),
     .instance_init = s390_cpu_initfn,
+    .instance_finalize = s390_cpu_finalize,
     .abstract = false,
     .class_size = sizeof(S390CPUClass),
     .class_init = s390_cpu_class_init,
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 0f9a1f7..fa8dfe0 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -13,7 +13,10 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
+ * Contributions after 2012-10-29 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef CPU_S390X_H
@@ -28,15 +31,15 @@
 
 #define CPUArchState struct CPUS390XState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 #define TARGET_PAGE_BITS 12
 
 #define TARGET_PHYS_ADDR_SPACE_BITS 64
 #define TARGET_VIRT_ADDR_SPACE_BITS 64
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define NB_MMU_MODES 3
 
@@ -47,6 +50,11 @@
 #define MMU_USER_IDX 1
 
 #define MAX_EXT_QUEUE 16
+#define MAX_IO_QUEUE 16
+#define MAX_MCHK_QUEUE 16
+
+#define PSW_MCHK_MASK 0x0004000000000000
+#define PSW_IO_MASK 0x0200000000000000
 
 typedef struct PSW {
     uint64_t mask;
@@ -59,18 +67,32 @@ typedef struct ExtQueue {
     uint32_t param64;
 } ExtQueue;
 
-typedef struct CPUS390XState {
-    uint64_t regs[16];	/* GP registers */
+typedef struct IOIntQueue {
+    uint16_t id;
+    uint16_t nr;
+    uint32_t parm;
+    uint32_t word;
+} IOIntQueue;
 
-    uint32_t aregs[16];	/* access registers */
+typedef struct MchkQueue {
+    uint16_t type;
+} MchkQueue;
 
-    uint32_t fpc;	/* floating-point control register */
+typedef struct CPUS390XState {
+    uint64_t regs[16];     /* GP registers */
     CPU_DoubleU fregs[16]; /* FP registers */
+    uint32_t aregs[16];    /* access registers */
+
+    uint32_t fpc;          /* floating-point control register */
+    uint32_t cc_op;
+
     float_status fpu_status; /* passed to softfloat lib */
 
+    /* The low part of a 128-bit return, or remainder of a divide.  */
+    uint64_t retxl;
+
     PSW psw;
 
-    uint32_t cc_op;
     uint64_t cc_src;
     uint64_t cc_dst;
     uint64_t cc_vr;
@@ -79,17 +101,25 @@ typedef struct CPUS390XState {
     uint64_t psa;
 
     uint32_t int_pgm_code;
-    uint32_t int_pgm_ilc;
+    uint32_t int_pgm_ilen;
 
     uint32_t int_svc_code;
-    uint32_t int_svc_ilc;
+    uint32_t int_svc_ilen;
 
     uint64_t cregs[16]; /* control registers */
 
-    int pending_int;
     ExtQueue ext_queue[MAX_EXT_QUEUE];
+    IOIntQueue io_queue[MAX_IO_QUEUE][8];
+    MchkQueue mchk_queue[MAX_MCHK_QUEUE];
 
+    int pending_int;
     int ext_index;
+    int io_index[8];
+    int mchk_index;
+
+    uint64_t ckc;
+    uint64_t cputm;
+    uint32_t todpr;
 
     CPU_COMMON
 
@@ -113,10 +143,13 @@ static inline void cpu_clone_regs(CPUS390XState *env, target_ulong newsp)
     if (newsp) {
         env->regs[15] = newsp;
     }
-    env->regs[0] = 0;
+    env->regs[2] = 0;
 }
 #endif
 
+/* distinguish between 24 bit and 31 bit addressing */
+#define HIGH_ORDER_BIT 0x80000000
+
 /* Interrupt Codes */
 /* Program Interrupts */
 #define PGM_OPERATION                   0x0001
@@ -253,25 +286,31 @@ static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc,
              ((env->psw.mask & PSW_MASK_32) ? FLAG_MASK_32 : 0);
 }
 
-static inline int get_ilc(uint8_t opc)
+/* While the PoO talks about ILC (a number between 1-3) what is actually
+   stored in LowCore is shifted left one bit (an even between 2-6).  As
+   this is the actual length of the insn and therefore more useful, that
+   is what we want to pass around and manipulate.  To make sure that we
+   have applied this distinction universally, rename the "ILC" to "ILEN".  */
+static inline int get_ilen(uint8_t opc)
 {
     switch (opc >> 6) {
     case 0:
-        return 1;
+        return 2;
     case 1:
     case 2:
-        return 2;
-    case 3:
-        return 3;
+        return 4;
+    default:
+        return 6;
     }
-
-    return 0;
 }
 
-#define ILC_LATER       0x20
-#define ILC_LATER_INC   0x21
-#define ILC_LATER_INC_2 0x22
-
+#ifndef CONFIG_USER_ONLY
+/* In several cases of runtime exceptions, we havn't recorded the true
+   instruction length.  Use these codes when raising exceptions in order
+   to re-compute the length by examining the insn in memory.  */
+#define ILEN_LATER       0x20
+#define ILEN_LATER_INC   0x21
+#endif
 
 S390CPU *cpu_s390x_init(const char *cpu_model);
 void s390x_translate_init(void);
@@ -288,37 +327,56 @@ int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong address, int rw
                                 int mmu_idx);
 #define cpu_handle_mmu_fault cpu_s390x_handle_mmu_fault
 
+#include "ioinst.h"
 
 #ifndef CONFIG_USER_ONLY
+void *s390_cpu_physical_memory_map(CPUS390XState *env, hwaddr addr, hwaddr *len,
+                                   int is_write);
+void s390_cpu_physical_memory_unmap(CPUS390XState *env, void *addr, hwaddr len,
+                                    int is_write);
+static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb)
+{
+    hwaddr addr = 0;
+    uint8_t reg;
+
+    reg = ipb >> 28;
+    if (reg > 0) {
+        addr = env->regs[reg];
+    }
+    addr += (ipb >> 16) & 0xfff;
+
+    return addr;
+}
+
 void s390x_tod_timer(void *opaque);
 void s390x_cpu_timer(void *opaque);
 
-int s390_virtio_hypercall(CPUS390XState *env, uint64_t mem, uint64_t hypercall);
+int s390_virtio_hypercall(CPUS390XState *env);
 
 #ifdef CONFIG_KVM
-void kvm_s390_interrupt(CPUS390XState *env, int type, uint32_t code);
-void kvm_s390_virtio_irq(CPUS390XState *env, int config_change, uint64_t token);
-void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
+void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code);
+void kvm_s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token);
+void kvm_s390_interrupt_internal(S390CPU *cpu, int type, uint32_t parm,
                                  uint64_t parm64, int vm);
 #else
-static inline void kvm_s390_interrupt(CPUS390XState *env, int type, uint32_t code)
+static inline void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code)
 {
 }
 
-static inline void kvm_s390_virtio_irq(CPUS390XState *env, int config_change,
+static inline void kvm_s390_virtio_irq(S390CPU *cpu, int config_change,
                                        uint64_t token)
 {
 }
 
-static inline void kvm_s390_interrupt_internal(CPUS390XState *env, int type,
+static inline void kvm_s390_interrupt_internal(S390CPU *cpu, int type,
                                                uint32_t parm, uint64_t parm64,
                                                int vm)
 {
 }
 #endif
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr);
-void s390_add_running_cpu(CPUS390XState *env);
-unsigned s390_del_running_cpu(CPUS390XState *env);
+void s390_add_running_cpu(S390CPU *cpu);
+unsigned s390_del_running_cpu(S390CPU *cpu);
 
 /* service interrupts are floating therefore we must not pass an cpustate */
 void s390_sclp_extint(uint32_t parm);
@@ -327,11 +385,11 @@ void s390_sclp_extint(uint32_t parm);
 extern const hwaddr virtio_size;
 
 #else
-static inline void s390_add_running_cpu(CPUS390XState *env)
+static inline void s390_add_running_cpu(S390CPU *cpu)
 {
 }
 
-static inline unsigned s390_del_running_cpu(CPUS390XState *env)
+static inline unsigned s390_del_running_cpu(S390CPU *cpu)
 {
     return 0;
 }
@@ -339,6 +397,114 @@ static inline unsigned s390_del_running_cpu(CPUS390XState *env)
 void cpu_lock(void);
 void cpu_unlock(void);
 
+typedef struct SubchDev SubchDev;
+
+#ifndef CONFIG_USER_ONLY
+SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
+                         uint16_t schid);
+bool css_subch_visible(SubchDev *sch);
+void css_conditional_io_interrupt(SubchDev *sch);
+int css_do_stsch(SubchDev *sch, SCHIB *schib);
+bool css_schid_final(uint8_t cssid, uint8_t ssid, uint16_t schid);
+int css_do_msch(SubchDev *sch, SCHIB *schib);
+int css_do_xsch(SubchDev *sch);
+int css_do_csch(SubchDev *sch);
+int css_do_hsch(SubchDev *sch);
+int css_do_ssch(SubchDev *sch, ORB *orb);
+int css_do_tsch(SubchDev *sch, IRB *irb);
+int css_do_stcrw(CRW *crw);
+int css_do_tpi(IOIntCode *int_code, int lowcore);
+int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid, uint8_t l_chpid,
+                         int rfmt, void *buf);
+void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo);
+int css_enable_mcsse(void);
+int css_enable_mss(void);
+int css_do_rsch(SubchDev *sch);
+int css_do_rchp(uint8_t cssid, uint8_t chpid);
+bool css_present(uint8_t cssid);
+#else
+static inline SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
+                                       uint16_t schid)
+{
+    return NULL;
+}
+static inline bool css_subch_visible(SubchDev *sch)
+{
+    return false;
+}
+static inline void css_conditional_io_interrupt(SubchDev *sch)
+{
+}
+static inline int css_do_stsch(SubchDev *sch, SCHIB *schib)
+{
+    return -ENODEV;
+}
+static inline bool css_schid_final(uint8_t cssid, uint8_t ssid, uint16_t schid)
+{
+    return true;
+}
+static inline int css_do_msch(SubchDev *sch, SCHIB *schib)
+{
+    return -ENODEV;
+}
+static inline int css_do_xsch(SubchDev *sch)
+{
+    return -ENODEV;
+}
+static inline int css_do_csch(SubchDev *sch)
+{
+    return -ENODEV;
+}
+static inline int css_do_hsch(SubchDev *sch)
+{
+    return -ENODEV;
+}
+static inline int css_do_ssch(SubchDev *sch, ORB *orb)
+{
+    return -ENODEV;
+}
+static inline int css_do_tsch(SubchDev *sch, IRB *irb)
+{
+    return -ENODEV;
+}
+static inline int css_do_stcrw(CRW *crw)
+{
+    return 1;
+}
+static inline int css_do_tpi(IOIntCode *int_code, int lowcore)
+{
+    return 0;
+}
+static inline int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid,
+                                       int rfmt, uint8_t l_chpid, void *buf)
+{
+    return 0;
+}
+static inline void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo)
+{
+}
+static inline int css_enable_mss(void)
+{
+    return -EINVAL;
+}
+static inline int css_enable_mcsse(void)
+{
+    return -EINVAL;
+}
+static inline int css_do_rsch(SubchDev *sch)
+{
+    return -ENODEV;
+}
+static inline int css_do_rchp(uint8_t cssid, uint8_t chpid)
+{
+    return -ENODEV;
+}
+static inline bool css_present(uint8_t cssid)
+{
+    return false;
+}
+#endif
+
 static inline void cpu_set_tls(CPUS390XState *env, target_ulong newtls)
 {
     env->aregs[0] = newtls >> 32;
@@ -350,26 +516,22 @@ static inline void cpu_set_tls(CPUS390XState *env, target_ulong newtls)
 #define cpu_gen_code cpu_s390x_gen_code
 #define cpu_signal_handler cpu_s390x_signal_handler
 
-#include "exec-all.h"
+void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf);
+#define cpu_list s390_cpu_list
 
-#ifdef CONFIG_USER_ONLY
-
-#define EXCP_OPEX 1 /* operation exception (sigill) */
-#define EXCP_SVC 2 /* supervisor call (syscall) */
-#define EXCP_ADDR 5 /* addressing exception */
-#define EXCP_SPEC 6 /* specification exception */
-
-#else
+#include "exec/exec-all.h"
 
 #define EXCP_EXT 1 /* external interrupt */
 #define EXCP_SVC 2 /* supervisor call (syscall) */
 #define EXCP_PGM 3 /* program interruption */
-
-#endif /* CONFIG_USER_ONLY */
+#define EXCP_IO  7 /* I/O interrupt */
+#define EXCP_MCHK 8 /* machine check */
 
 #define INTERRUPT_EXT        (1 << 0)
 #define INTERRUPT_TOD        (1 << 1)
 #define INTERRUPT_CPUTIMER   (1 << 2)
+#define INTERRUPT_IO         (1 << 3)
+#define INTERRUPT_MCHK       (1 << 4)
 
 /* Program Status Word.  */
 #define S390_PSWM_REGNUM 0
@@ -430,79 +592,6 @@ static inline void cpu_set_tls(CPUS390XState *env, target_ulong newtls)
 /* Total.  */
 #define S390_NUM_REGS 51
 
-/* Pseudo registers -- PC and condition code.  */
-#define S390_PC_REGNUM S390_NUM_REGS
-#define S390_CC_REGNUM (S390_NUM_REGS+1)
-#define S390_NUM_PSEUDO_REGS 2
-#define S390_NUM_TOTAL_REGS (S390_NUM_REGS+2)
-
-
-
-/* Program Status Word.  */
-#define S390_PSWM_REGNUM 0
-#define S390_PSWA_REGNUM 1
-/* General Purpose Registers.  */
-#define S390_R0_REGNUM 2
-#define S390_R1_REGNUM 3
-#define S390_R2_REGNUM 4
-#define S390_R3_REGNUM 5
-#define S390_R4_REGNUM 6
-#define S390_R5_REGNUM 7
-#define S390_R6_REGNUM 8
-#define S390_R7_REGNUM 9
-#define S390_R8_REGNUM 10
-#define S390_R9_REGNUM 11
-#define S390_R10_REGNUM 12
-#define S390_R11_REGNUM 13
-#define S390_R12_REGNUM 14
-#define S390_R13_REGNUM 15
-#define S390_R14_REGNUM 16
-#define S390_R15_REGNUM 17
-/* Access Registers.  */
-#define S390_A0_REGNUM 18
-#define S390_A1_REGNUM 19
-#define S390_A2_REGNUM 20
-#define S390_A3_REGNUM 21
-#define S390_A4_REGNUM 22
-#define S390_A5_REGNUM 23
-#define S390_A6_REGNUM 24
-#define S390_A7_REGNUM 25
-#define S390_A8_REGNUM 26
-#define S390_A9_REGNUM 27
-#define S390_A10_REGNUM 28
-#define S390_A11_REGNUM 29
-#define S390_A12_REGNUM 30
-#define S390_A13_REGNUM 31
-#define S390_A14_REGNUM 32
-#define S390_A15_REGNUM 33
-/* Floating Point Control Word.  */
-#define S390_FPC_REGNUM 34
-/* Floating Point Registers.  */
-#define S390_F0_REGNUM 35
-#define S390_F1_REGNUM 36
-#define S390_F2_REGNUM 37
-#define S390_F3_REGNUM 38
-#define S390_F4_REGNUM 39
-#define S390_F5_REGNUM 40
-#define S390_F6_REGNUM 41
-#define S390_F7_REGNUM 42
-#define S390_F8_REGNUM 43
-#define S390_F9_REGNUM 44
-#define S390_F10_REGNUM 45
-#define S390_F11_REGNUM 46
-#define S390_F12_REGNUM 47
-#define S390_F13_REGNUM 48
-#define S390_F14_REGNUM 49
-#define S390_F15_REGNUM 50
-/* Total.  */
-#define S390_NUM_REGS 51
-
-/* Pseudo registers -- PC and condition code.  */
-#define S390_PC_REGNUM S390_NUM_REGS
-#define S390_CC_REGNUM (S390_NUM_REGS+1)
-#define S390_NUM_PSEUDO_REGS 2
-#define S390_NUM_TOTAL_REGS (S390_NUM_REGS+2)
-
 /* CC optimization */
 
 enum cc_op {
@@ -524,15 +613,19 @@ enum cc_op {
 
     CC_OP_ADD_64,               /* overflow on add (64bit) */
     CC_OP_ADDU_64,              /* overflow on unsigned add (64bit) */
+    CC_OP_ADDC_64,              /* overflow on unsigned add-carry (64bit) */
     CC_OP_SUB_64,               /* overflow on subtraction (64bit) */
     CC_OP_SUBU_64,              /* overflow on unsigned subtraction (64bit) */
+    CC_OP_SUBB_64,              /* overflow on unsigned sub-borrow (64bit) */
     CC_OP_ABS_64,               /* sign eval on abs (64bit) */
     CC_OP_NABS_64,              /* sign eval on nabs (64bit) */
 
     CC_OP_ADD_32,               /* overflow on add (32bit) */
     CC_OP_ADDU_32,              /* overflow on unsigned add (32bit) */
+    CC_OP_ADDC_32,              /* overflow on unsigned add-carry (32bit) */
     CC_OP_SUB_32,               /* overflow on subtraction (32bit) */
     CC_OP_SUBU_32,              /* overflow on unsigned subtraction (32bit) */
+    CC_OP_SUBB_32,              /* overflow on unsigned sub-borrow (32bit) */
     CC_OP_ABS_32,               /* sign eval on abs (64bit) */
     CC_OP_NABS_32,              /* sign eval on nabs (64bit) */
 
@@ -542,14 +635,14 @@ enum cc_op {
     CC_OP_TM_32,                /* test under mask (32bit) */
     CC_OP_TM_64,                /* test under mask (64bit) */
 
-    CC_OP_LTGT_F32,             /* FP compare (32bit) */
-    CC_OP_LTGT_F64,             /* FP compare (64bit) */
-
     CC_OP_NZ_F32,               /* FP dst != 0 (32bit) */
     CC_OP_NZ_F64,               /* FP dst != 0 (64bit) */
+    CC_OP_NZ_F128,              /* FP dst != 0 (128bit) */
 
     CC_OP_ICM,                  /* insert characters under mask */
-    CC_OP_SLAG,                 /* Calculate shift left signed */
+    CC_OP_SLA_32,               /* Calculate shift left signed (32bit) */
+    CC_OP_SLA_64,               /* Calculate shift left signed (64bit) */
+    CC_OP_FLOGR,                /* find leftmost one */
     CC_OP_MAX
 };
 
@@ -569,26 +662,31 @@ static const char *cc_names[] = {
     [CC_OP_LTGT0_64]  = "CC_OP_LTGT0_64",
     [CC_OP_ADD_64]    = "CC_OP_ADD_64",
     [CC_OP_ADDU_64]   = "CC_OP_ADDU_64",
+    [CC_OP_ADDC_64]   = "CC_OP_ADDC_64",
     [CC_OP_SUB_64]    = "CC_OP_SUB_64",
     [CC_OP_SUBU_64]   = "CC_OP_SUBU_64",
+    [CC_OP_SUBB_64]   = "CC_OP_SUBB_64",
     [CC_OP_ABS_64]    = "CC_OP_ABS_64",
     [CC_OP_NABS_64]   = "CC_OP_NABS_64",
     [CC_OP_ADD_32]    = "CC_OP_ADD_32",
     [CC_OP_ADDU_32]   = "CC_OP_ADDU_32",
+    [CC_OP_ADDC_32]   = "CC_OP_ADDC_32",
     [CC_OP_SUB_32]    = "CC_OP_SUB_32",
     [CC_OP_SUBU_32]   = "CC_OP_SUBU_32",
+    [CC_OP_SUBB_32]   = "CC_OP_SUBB_32",
     [CC_OP_ABS_32]    = "CC_OP_ABS_32",
     [CC_OP_NABS_32]   = "CC_OP_NABS_32",
     [CC_OP_COMP_32]   = "CC_OP_COMP_32",
     [CC_OP_COMP_64]   = "CC_OP_COMP_64",
     [CC_OP_TM_32]     = "CC_OP_TM_32",
     [CC_OP_TM_64]     = "CC_OP_TM_64",
-    [CC_OP_LTGT_F32]  = "CC_OP_LTGT_F32",
-    [CC_OP_LTGT_F64]  = "CC_OP_LTGT_F64",
     [CC_OP_NZ_F32]    = "CC_OP_NZ_F32",
     [CC_OP_NZ_F64]    = "CC_OP_NZ_F64",
+    [CC_OP_NZ_F128]   = "CC_OP_NZ_F128",
     [CC_OP_ICM]       = "CC_OP_ICM",
-    [CC_OP_SLAG]      = "CC_OP_SLAG",
+    [CC_OP_SLA_32]    = "CC_OP_SLA_32",
+    [CC_OP_SLA_64]    = "CC_OP_SLA_64",
+    [CC_OP_FLOGR]     = "CC_OP_FLOGR",
 };
 
 static inline const char *cc_name(int cc_op)
@@ -605,9 +703,9 @@ typedef struct LowCore
     uint32_t        ext_params;               /* 0x080 */
     uint16_t        cpu_addr;                 /* 0x084 */
     uint16_t        ext_int_code;             /* 0x086 */
-    uint16_t        svc_ilc;                  /* 0x088 */
+    uint16_t        svc_ilen;                 /* 0x088 */
     uint16_t        svc_code;                 /* 0x08a */
-    uint16_t        pgm_ilc;                  /* 0x08c */
+    uint16_t        pgm_ilen;                 /* 0x08c */
     uint16_t        pgm_code;                 /* 0x08e */
     uint32_t        data_exc_code;            /* 0x090 */
     uint16_t        mon_class_num;            /* 0x094 */
@@ -835,87 +933,6 @@ struct sysib_322 {
 #define SK_F                    (0x1 << 3)
 #define SK_ACC_MASK             (0xf << 4)
 
-
-/* EBCDIC handling */
-static const uint8_t ebcdic2ascii[] = {
-    0x00, 0x01, 0x02, 0x03, 0x07, 0x09, 0x07, 0x7F,
-    0x07, 0x07, 0x07, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-    0x10, 0x11, 0x12, 0x13, 0x07, 0x0A, 0x08, 0x07,
-    0x18, 0x19, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
-    0x07, 0x07, 0x1C, 0x07, 0x07, 0x0A, 0x17, 0x1B,
-    0x07, 0x07, 0x07, 0x07, 0x07, 0x05, 0x06, 0x07,
-    0x07, 0x07, 0x16, 0x07, 0x07, 0x07, 0x07, 0x04,
-    0x07, 0x07, 0x07, 0x07, 0x14, 0x15, 0x07, 0x1A,
-    0x20, 0xFF, 0x83, 0x84, 0x85, 0xA0, 0x07, 0x86,
-    0x87, 0xA4, 0x5B, 0x2E, 0x3C, 0x28, 0x2B, 0x21,
-    0x26, 0x82, 0x88, 0x89, 0x8A, 0xA1, 0x8C, 0x07,
-    0x8D, 0xE1, 0x5D, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
-    0x2D, 0x2F, 0x07, 0x8E, 0x07, 0x07, 0x07, 0x8F,
-    0x80, 0xA5, 0x07, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
-    0x07, 0x90, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
-    0x70, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
-    0x07, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
-    0x68, 0x69, 0xAE, 0xAF, 0x07, 0x07, 0x07, 0xF1,
-    0xF8, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70,
-    0x71, 0x72, 0xA6, 0xA7, 0x91, 0x07, 0x92, 0x07,
-    0xE6, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-    0x79, 0x7A, 0xAD, 0xAB, 0x07, 0x07, 0x07, 0x07,
-    0x9B, 0x9C, 0x9D, 0xFA, 0x07, 0x07, 0x07, 0xAC,
-    0xAB, 0x07, 0xAA, 0x7C, 0x07, 0x07, 0x07, 0x07,
-    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
-    0x48, 0x49, 0x07, 0x93, 0x94, 0x95, 0xA2, 0x07,
-    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
-    0x51, 0x52, 0x07, 0x96, 0x81, 0x97, 0xA3, 0x98,
-    0x5C, 0xF6, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
-    0x59, 0x5A, 0xFD, 0x07, 0x99, 0x07, 0x07, 0x07,
-    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
-    0x38, 0x39, 0x07, 0x07, 0x9A, 0x07, 0x07, 0x07,
-};
-
-static const uint8_t ascii2ebcdic [] = {
-    0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
-    0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
-    0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
-    0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
-    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
-    0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
-    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-    0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
-    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-    0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
-    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
-    0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
-    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-    0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
-    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
-    0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
-    0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
-};
-
-static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
-{
-    int i;
-
-    for (i = 0; i < len; i++) {
-        p[i] = ascii2ebcdic[(int)ascii[i]];
-    }
-}
-
 #define SIGP_SENSE             0x01
 #define SIGP_EXTERNAL_CALL     0x02
 #define SIGP_EMERGENCY         0x03
@@ -958,9 +975,11 @@ static inline uint64_t time2tod(uint64_t ns) {
     return (ns << 9) / 125;
 }
 
-static inline void cpu_inject_ext(CPUS390XState *env, uint32_t code, uint32_t param,
+static inline void cpu_inject_ext(S390CPU *cpu, uint32_t code, uint32_t param,
                                   uint64_t param64)
 {
+    CPUS390XState *env = &cpu->env;
+
     if (env->ext_index == MAX_EXT_QUEUE - 1) {
         /* ugh - can't queue anymore. Let's drop. */
         return;
@@ -977,6 +996,48 @@ static inline void cpu_inject_ext(CPUS390XState *env, uint32_t code, uint32_t pa
     cpu_interrupt(env, CPU_INTERRUPT_HARD);
 }
 
+static inline void cpu_inject_io(S390CPU *cpu, uint16_t subchannel_id,
+                                 uint16_t subchannel_number,
+                                 uint32_t io_int_parm, uint32_t io_int_word)
+{
+    CPUS390XState *env = &cpu->env;
+    int isc = IO_INT_WORD_ISC(io_int_word);
+
+    if (env->io_index[isc] == MAX_IO_QUEUE - 1) {
+        /* ugh - can't queue anymore. Let's drop. */
+        return;
+    }
+
+    env->io_index[isc]++;
+    assert(env->io_index[isc] < MAX_IO_QUEUE);
+
+    env->io_queue[env->io_index[isc]][isc].id = subchannel_id;
+    env->io_queue[env->io_index[isc]][isc].nr = subchannel_number;
+    env->io_queue[env->io_index[isc]][isc].parm = io_int_parm;
+    env->io_queue[env->io_index[isc]][isc].word = io_int_word;
+
+    env->pending_int |= INTERRUPT_IO;
+    cpu_interrupt(env, CPU_INTERRUPT_HARD);
+}
+
+static inline void cpu_inject_crw_mchk(S390CPU *cpu)
+{
+    CPUS390XState *env = &cpu->env;
+
+    if (env->mchk_index == MAX_MCHK_QUEUE - 1) {
+        /* ugh - can't queue anymore. Let's drop. */
+        return;
+    }
+
+    env->mchk_index++;
+    assert(env->mchk_index < MAX_MCHK_QUEUE);
+
+    env->mchk_queue[env->mchk_index].type = 1;
+
+    env->pending_int |= INTERRUPT_MCHK;
+    cpu_interrupt(env, CPU_INTERRUPT_HARD);
+}
+
 static inline bool cpu_has_work(CPUState *cpu)
 {
     CPUS390XState *env = &S390_CPU(cpu)->env;
@@ -991,12 +1052,61 @@ static inline void cpu_pc_from_tb(CPUS390XState *env, TranslationBlock* tb)
 }
 
 /* fpu_helper.c */
-uint32_t set_cc_f32(CPUS390XState *env, float32 v1, float32 v2);
-uint32_t set_cc_f64(CPUS390XState *env, float64 v1, float64 v2);
 uint32_t set_cc_nz_f32(float32 v);
 uint32_t set_cc_nz_f64(float64 v);
+uint32_t set_cc_nz_f128(float128 v);
 
 /* misc_helper.c */
-void program_interrupt(CPUS390XState *env, uint32_t code, int ilc);
+void program_interrupt(CPUS390XState *env, uint32_t code, int ilen);
+void QEMU_NORETURN runtime_exception(CPUS390XState *env, int excp,
+                                     uintptr_t retaddr);
+
+#include <sysemu/kvm.h>
+
+#ifdef CONFIG_KVM
+void kvm_s390_io_interrupt(S390CPU *cpu, uint16_t subchannel_id,
+                           uint16_t subchannel_nr, uint32_t io_int_parm,
+                           uint32_t io_int_word);
+void kvm_s390_crw_mchk(S390CPU *cpu);
+void kvm_s390_enable_css_support(S390CPU *cpu);
+#else
+static inline void kvm_s390_io_interrupt(S390CPU *cpu,
+                                        uint16_t subchannel_id,
+                                        uint16_t subchannel_nr,
+                                        uint32_t io_int_parm,
+                                        uint32_t io_int_word)
+{
+}
+static inline void kvm_s390_crw_mchk(S390CPU *cpu)
+{
+}
+static inline void kvm_s390_enable_css_support(S390CPU *cpu)
+{
+}
+#endif
+
+static inline void s390_io_interrupt(S390CPU *cpu,
+                                     uint16_t subchannel_id,
+                                     uint16_t subchannel_nr,
+                                     uint32_t io_int_parm,
+                                     uint32_t io_int_word)
+{
+    if (kvm_enabled()) {
+        kvm_s390_io_interrupt(cpu, subchannel_id, subchannel_nr, io_int_parm,
+                              io_int_word);
+    } else {
+        cpu_inject_io(cpu, subchannel_id, subchannel_nr, io_int_parm,
+                      io_int_word);
+    }
+}
+
+static inline void s390_crw_mchk(S390CPU *cpu)
+{
+    if (kvm_enabled()) {
+        kvm_s390_crw_mchk(cpu);
+    } else {
+        cpu_inject_crw_mchk(cpu);
+    }
+}
 
 #endif
diff --git a/target-s390x/fpu_helper.c b/target-s390x/fpu_helper.c
index ee9420d..94375b6 100644
--- a/target-s390x/fpu_helper.c
+++ b/target-s390x/fpu_helper.c
@@ -22,7 +22,7 @@
 #include "helper.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #endif
 
 /* #define DEBUG_HELPER */
@@ -32,6 +32,52 @@
 #define HELPER_LOG(x...)
 #endif
 
+#define RET128(F) (env->retxl = F.low, F.high)
+
+#define convert_bit(mask, from, to) \
+    (to < from                      \
+     ? (mask / (from / to)) & to    \
+     : (mask & from) * (to / from))
+
+static void ieee_exception(CPUS390XState *env, uint32_t dxc, uintptr_t retaddr)
+{
+    /* Install the DXC code.  */
+    env->fpc = (env->fpc & ~0xff00) | (dxc << 8);
+    /* Trap.  */
+    runtime_exception(env, PGM_DATA, retaddr);
+}
+
+/* Should be called after any operation that may raise IEEE exceptions.  */
+static void handle_exceptions(CPUS390XState *env, uintptr_t retaddr)
+{
+    unsigned s390_exc, qemu_exc;
+
+    /* Get the exceptions raised by the current operation.  Reset the
+       fpu_status contents so that the next operation has a clean slate.  */
+    qemu_exc = env->fpu_status.float_exception_flags;
+    if (qemu_exc == 0) {
+        return;
+    }
+    env->fpu_status.float_exception_flags = 0;
+
+    /* Convert softfloat exception bits to s390 exception bits.  */
+    s390_exc = 0;
+    s390_exc |= convert_bit(qemu_exc, float_flag_invalid, 0x80);
+    s390_exc |= convert_bit(qemu_exc, float_flag_divbyzero, 0x40);
+    s390_exc |= convert_bit(qemu_exc, float_flag_overflow, 0x20);
+    s390_exc |= convert_bit(qemu_exc, float_flag_underflow, 0x10);
+    s390_exc |= convert_bit(qemu_exc, float_flag_inexact, 0x08);
+
+    /* Install the exceptions that we raised.  */
+    env->fpc |= s390_exc << 16;
+
+    /* Send signals for enabled exceptions.  */
+    s390_exc &= env->fpc >> 24;
+    if (s390_exc) {
+        ieee_exception(env, s390_exc, retaddr);
+    }
+}
+
 static inline int float_comp_to_cc(CPUS390XState *env, int float_compare)
 {
     switch (float_compare) {
@@ -48,19 +94,6 @@ static inline int float_comp_to_cc(CPUS390XState *env, int float_compare)
     }
 }
 
-/* condition codes for binary FP ops */
-uint32_t set_cc_f32(CPUS390XState *env, float32 v1, float32 v2)
-{
-    return float_comp_to_cc(env, float32_compare_quiet(v1, v2,
-                                                       &env->fpu_status));
-}
-
-uint32_t set_cc_f64(CPUS390XState *env, float64 v1, float64 v2)
-{
-    return float_comp_to_cc(env, float64_compare_quiet(v1, v2,
-                                                       &env->fpu_status));
-}
-
 /* condition codes for unary FP ops */
 uint32_t set_cc_nz_f32(float32 v)
 {
@@ -88,7 +121,7 @@ uint32_t set_cc_nz_f64(float64 v)
     }
 }
 
-static uint32_t set_cc_nz_f128(float128 v)
+uint32_t set_cc_nz_f128(float128 v)
 {
     if (float128_is_any_nan(v)) {
         return 3;
@@ -101,433 +134,211 @@ static uint32_t set_cc_nz_f128(float128 v)
     }
 }
 
-/* convert 32-bit int to 64-bit float */
-void HELPER(cdfbr)(CPUS390XState *env, uint32_t f1, int32_t v2)
+/* 32-bit FP addition */
+uint64_t HELPER(aeb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    HELPER_LOG("%s: converting %d to f%d\n", __func__, v2, f1);
-    env->fregs[f1].d = int32_to_float64(v2, &env->fpu_status);
+    float32 ret = float32_add(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 32-bit int to 128-bit float */
-void HELPER(cxfbr)(CPUS390XState *env, uint32_t f1, int32_t v2)
+/* 64-bit FP addition */
+uint64_t HELPER(adb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    CPU_QuadU v1;
-
-    v1.q = int32_to_float128(v2, &env->fpu_status);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
+    float64 ret = float64_add(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 64-bit int to 32-bit float */
-void HELPER(cegbr)(CPUS390XState *env, uint32_t f1, int64_t v2)
+/* 128-bit FP addition */
+uint64_t HELPER(axb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
 {
-    HELPER_LOG("%s: converting %ld to f%d\n", __func__, v2, f1);
-    env->fregs[f1].l.upper = int64_to_float32(v2, &env->fpu_status);
+    float128 ret = float128_add(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* convert 64-bit int to 64-bit float */
-void HELPER(cdgbr)(CPUS390XState *env, uint32_t f1, int64_t v2)
+/* 32-bit FP subtraction */
+uint64_t HELPER(seb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    HELPER_LOG("%s: converting %ld to f%d\n", __func__, v2, f1);
-    env->fregs[f1].d = int64_to_float64(v2, &env->fpu_status);
+    float32 ret = float32_sub(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 64-bit int to 128-bit float */
-void HELPER(cxgbr)(CPUS390XState *env, uint32_t f1, int64_t v2)
+/* 64-bit FP subtraction */
+uint64_t HELPER(sdb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    CPU_QuadU x1;
-
-    x1.q = int64_to_float128(v2, &env->fpu_status);
-    HELPER_LOG("%s: converted %ld to 0x%lx and 0x%lx\n", __func__, v2,
-               x1.ll.upper, x1.ll.lower);
-    env->fregs[f1].ll = x1.ll.upper;
-    env->fregs[f1 + 2].ll = x1.ll.lower;
+    float64 ret = float64_sub(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 32-bit int to 32-bit float */
-void HELPER(cefbr)(CPUS390XState *env, uint32_t f1, int32_t v2)
+/* 128-bit FP subtraction */
+uint64_t HELPER(sxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
 {
-    env->fregs[f1].l.upper = int32_to_float32(v2, &env->fpu_status);
-    HELPER_LOG("%s: converting %d to 0x%d in f%d\n", __func__, v2,
-               env->fregs[f1].l.upper, f1);
+    float128 ret = float128_sub(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* 32-bit FP addition RR */
-uint32_t HELPER(aebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 32-bit FP division */
+uint64_t HELPER(deb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __func__,
-               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
+    float32 ret = float32_div(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP addition RR */
-uint32_t HELPER(adbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 64-bit FP division */
+uint64_t HELPER(ddb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    env->fregs[f1].d = float64_add(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%ld resulting in 0x%ld in f%d\n", __func__,
-               env->fregs[f2].d, env->fregs[f1].d, f1);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
+    float64 ret = float64_div(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 32-bit FP subtraction RR */
-uint32_t HELPER(sebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 128-bit FP division */
+uint64_t HELPER(dxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
 {
-    env->fregs[f1].l.upper = float32_sub(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __func__,
-               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
+    float128 ret = float128_div(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* 64-bit FP subtraction RR */
-uint32_t HELPER(sdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 32-bit FP multiplication */
+uint64_t HELPER(meeb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    env->fregs[f1].d = float64_sub(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-    HELPER_LOG("%s: subtracting 0x%ld resulting in 0x%ld in f%d\n",
-               __func__, env->fregs[f2].d, env->fregs[f1].d, f1);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
+    float32 ret = float32_mul(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 32-bit FP division RR */
-void HELPER(debr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 64-bit FP multiplication */
+uint64_t HELPER(mdb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    env->fregs[f1].l.upper = float32_div(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
+    float64 ret = float64_mul(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 128-bit FP division RR */
-void HELPER(dxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 64/32-bit FP multiplication */
+uint64_t HELPER(mdeb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-    CPU_QuadU res;
-
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    res.q = float128_div(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
+    float64 ret = float32_to_float64(f2, &env->fpu_status);
+    ret = float64_mul(f1, ret, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP multiplication RR */
-void HELPER(mdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 128-bit FP multiplication */
+uint64_t HELPER(mxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
 {
-    env->fregs[f1].d = float64_mul(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
+    float128 ret = float128_mul(make_float128(ah, al),
+                                make_float128(bh, bl),
+                                &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* 128-bit FP multiplication RR */
-void HELPER(mxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* 128/64-bit FP multiplication */
+uint64_t HELPER(mxdb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                      uint64_t f2)
 {
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-    CPU_QuadU res;
-
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    res.q = float128_mul(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
+    float128 ret = float64_to_float128(f2, &env->fpu_status);
+    ret = float128_mul(make_float128(ah, al), ret, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
 /* convert 32-bit float to 64-bit float */
-void HELPER(ldebr)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+uint64_t HELPER(ldeb)(CPUS390XState *env, uint64_t f2)
 {
-    env->fregs[r1].d = float32_to_float64(env->fregs[r2].l.upper,
-                                          &env->fpu_status);
+    float64 ret = float32_to_float64(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
 /* convert 128-bit float to 64-bit float */
-void HELPER(ldxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+uint64_t HELPER(ldxb)(CPUS390XState *env, uint64_t ah, uint64_t al)
 {
-    CPU_QuadU x2;
-
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].d = float128_to_float64(x2.q, &env->fpu_status);
-    HELPER_LOG("%s: to 0x%ld\n", __func__, env->fregs[f1].d);
+    float64 ret = float128_to_float64(make_float128(ah, al), &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
 /* convert 64-bit float to 128-bit float */
-void HELPER(lxdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU res;
-
-    res.q = float64_to_float128(env->fregs[f2].d, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* convert 64-bit float to 32-bit float */
-void HELPER(ledbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    float64 d2 = env->fregs[f2].d;
-
-    env->fregs[f1].l.upper = float64_to_float32(d2, &env->fpu_status);
-}
-
-/* convert 128-bit float to 32-bit float */
-void HELPER(lexbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x2;
-
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].l.upper = float128_to_float32(x2.q, &env->fpu_status);
-    HELPER_LOG("%s: to 0x%d\n", __func__, env->fregs[f1].l.upper);
-}
-
-/* absolute value of 32-bit float */
-uint32_t HELPER(lpebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    float32 v1;
-    float32 v2 = env->fregs[f2].d;
-
-    v1 = float32_abs(v2);
-    env->fregs[f1].d = v1;
-    return set_cc_nz_f32(v1);
-}
-
-/* absolute value of 64-bit float */
-uint32_t HELPER(lpdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    float64 v1;
-    float64 v2 = env->fregs[f2].d;
-
-    v1 = float64_abs(v2);
-    env->fregs[f1].d = v1;
-    return set_cc_nz_f64(v1);
-}
-
-/* absolute value of 128-bit float */
-uint32_t HELPER(lpxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    v1.q = float128_abs(v2.q);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-    return set_cc_nz_f128(v1.q);
-}
-
-/* load and test 64-bit float */
-uint32_t HELPER(ltdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = env->fregs[f2].d;
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* load and test 32-bit float */
-uint32_t HELPER(ltebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = env->fregs[f2].l.upper;
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* load and test 128-bit float */
-uint32_t HELPER(ltxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x;
-
-    x.ll.upper = env->fregs[f2].ll;
-    x.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].ll = x.ll.upper;
-    env->fregs[f1 + 2].ll = x.ll.lower;
-    return set_cc_nz_f128(x.q);
-}
-
-/* load complement of 32-bit float */
-uint32_t HELPER(lcebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_chs(env->fregs[f2].l.upper);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* load complement of 64-bit float */
-uint32_t HELPER(lcdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_chs(env->fregs[f2].d);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* load complement of 128-bit float */
-uint32_t HELPER(lcxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x1, x2;
-
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    x1.q = float128_chs(x2.q);
-    env->fregs[f1].ll = x1.ll.upper;
-    env->fregs[f1 + 2].ll = x1.ll.lower;
-    return set_cc_nz_f128(x1.q);
-}
-
-/* 32-bit FP addition RM */
-void HELPER(aeb)(CPUS390XState *env, uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-
-    v2.l = val;
-    HELPER_LOG("%s: adding 0x%d from f%d and 0x%d\n", __func__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_add(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP division RM */
-void HELPER(deb)(CPUS390XState *env, uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-
-    v2.l = val;
-    HELPER_LOG("%s: dividing 0x%d from f%d by 0x%d\n", __func__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_div(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP multiplication RM */
-void HELPER(meeb)(CPUS390XState *env, uint32_t f1, uint32_t val)
+uint64_t HELPER(lxdb)(CPUS390XState *env, uint64_t f2)
 {
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-
-    v2.l = val;
-    HELPER_LOG("%s: multiplying 0x%d from f%d and 0x%d\n", __func__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_mul(v1, v2.f, &env->fpu_status);
+    float128 ret = float64_to_float128(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* 32-bit FP compare RR */
-uint32_t HELPER(cebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* convert 32-bit float to 128-bit float */
+uint64_t HELPER(lxeb)(CPUS390XState *env, uint64_t f2)
 {
-    float32 v1 = env->fregs[f1].l.upper;
-    float32 v2 = env->fregs[f2].l.upper;
-
-    HELPER_LOG("%s: comparing 0x%d from f%d and 0x%d\n", __func__,
-               v1, f1, v2);
-    return set_cc_f32(env, v1, v2);
+    float128 ret = float32_to_float128(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* 64-bit FP compare RR */
-uint32_t HELPER(cdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    float64 v1 = env->fregs[f1].d;
-    float64 v2 = env->fregs[f2].d;
-
-    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%ld\n", __func__,
-               v1, f1, v2);
-    return set_cc_f64(env, v1, v2);
-}
-
-/* 128-bit FP compare RR */
-uint32_t HELPER(cxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-
-    return float_comp_to_cc(env, float128_compare_quiet(v1.q, v2.q,
-                                                   &env->fpu_status));
-}
-
-/* 64-bit FP compare RM */
-uint32_t HELPER(cdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+/* convert 64-bit float to 32-bit float */
+uint64_t HELPER(ledb)(CPUS390XState *env, uint64_t f2)
 {
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-
-    v2.ll = cpu_ldq_data(env, a2);
-    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%lx\n", __func__, v1,
-               f1, v2.d);
-    return set_cc_f64(env, v1, v2.d);
+    float32 ret = float64_to_float32(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP addition RM */
-uint32_t HELPER(adb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+/* convert 128-bit float to 32-bit float */
+uint64_t HELPER(lexb)(CPUS390XState *env, uint64_t ah, uint64_t al)
 {
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-
-    v2.ll = cpu_ldq_data(env, a2);
-    HELPER_LOG("%s: adding 0x%lx from f%d and 0x%lx\n", __func__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = v1 = float64_add(v1, v2.d, &env->fpu_status);
-    return set_cc_nz_f64(v1);
+    float32 ret = float128_to_float32(make_float128(ah, al), &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 32-bit FP subtraction RM */
-void HELPER(seb)(CPUS390XState *env, uint32_t f1, uint32_t val)
+/* 32-bit FP compare */
+uint32_t HELPER(ceb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-
-    v2.l = val;
-    env->fregs[f1].l.upper = float32_sub(v1, v2.f, &env->fpu_status);
+    int cmp = float32_compare_quiet(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return float_comp_to_cc(env, cmp);
 }
 
-/* 64-bit FP subtraction RM */
-uint32_t HELPER(sdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+/* 64-bit FP compare */
+uint32_t HELPER(cdb)(CPUS390XState *env, uint64_t f1, uint64_t f2)
 {
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-
-    v2.ll = cpu_ldq_data(env, a2);
-    env->fregs[f1].d = v1 = float64_sub(v1, v2.d, &env->fpu_status);
-    return set_cc_nz_f64(v1);
+    int cmp = float64_compare_quiet(f1, f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return float_comp_to_cc(env, cmp);
 }
 
-/* 64-bit FP multiplication RM */
-void HELPER(mdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+/* 128-bit FP compare */
+uint32_t HELPER(cxb)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                     uint64_t bh, uint64_t bl)
 {
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-
-    v2.ll = cpu_ldq_data(env, a2);
-    HELPER_LOG("%s: multiplying 0x%lx from f%d and 0x%ld\n", __func__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = float64_mul(v1, v2.d, &env->fpu_status);
+    int cmp = float128_compare_quiet(make_float128(ah, al),
+                                     make_float128(bh, bl),
+                                     &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return float_comp_to_cc(env, cmp);
 }
 
-/* 64-bit FP division RM */
-void HELPER(ddb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-
-    v2.ll = cpu_ldq_data(env, a2);
-    HELPER_LOG("%s: dividing 0x%lx from f%d by 0x%ld\n", __func__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = float64_div(v1, v2.d, &env->fpu_status);
-}
-
-static void set_round_mode(CPUS390XState *env, int m3)
+static int swap_round_mode(CPUS390XState *env, int m3)
 {
+    int ret = env->fpu_status.float_rounding_mode;
     switch (m3) {
     case 0:
         /* current mode */
@@ -551,232 +362,242 @@ static void set_round_mode(CPUS390XState *env, int m3)
         set_float_rounding_mode(float_round_down, &env->fpu_status);
         break;
     }
+    return ret;
 }
 
-/* convert 32-bit float to 64-bit int */
-uint32_t HELPER(cgebr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
-                       uint32_t m3)
+/* convert 64-bit int to 32-bit float */
+uint64_t HELPER(cegb)(CPUS390XState *env, int64_t v2, uint32_t m3)
 {
-    float32 v2 = env->fregs[f2].l.upper;
-
-    set_round_mode(env, m3);
-    env->regs[r1] = float32_to_int64(v2, &env->fpu_status);
-    return set_cc_nz_f32(v2);
+    int hold = swap_round_mode(env, m3);
+    float32 ret = int64_to_float32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 64-bit float to 64-bit int */
-uint32_t HELPER(cgdbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
-                       uint32_t m3)
+/* convert 64-bit int to 64-bit float */
+uint64_t HELPER(cdgb)(CPUS390XState *env, int64_t v2, uint32_t m3)
 {
-    float64 v2 = env->fregs[f2].d;
-
-    set_round_mode(env, m3);
-    env->regs[r1] = float64_to_int64(v2, &env->fpu_status);
-    return set_cc_nz_f64(v2);
+    int hold = swap_round_mode(env, m3);
+    float64 ret = int64_to_float64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 128-bit float to 64-bit int */
-uint32_t HELPER(cgxbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
-                       uint32_t m3)
+/* convert 64-bit int to 128-bit float */
+uint64_t HELPER(cxgb)(CPUS390XState *env, int64_t v2, uint32_t m3)
 {
-    CPU_QuadU v2;
-
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    set_round_mode(env, m3);
-    env->regs[r1] = float128_to_int64(v2.q, &env->fpu_status);
-    if (float128_is_any_nan(v2.q)) {
-        return 3;
-    } else if (float128_is_zero(v2.q)) {
-        return 0;
-    } else if (float128_is_neg(v2.q)) {
-        return 1;
-    } else {
-        return 2;
-    }
+    int hold = swap_round_mode(env, m3);
+    float128 ret = int64_to_float128(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* convert 32-bit float to 32-bit int */
-uint32_t HELPER(cfebr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
-                       uint32_t m3)
+/* convert 64-bit uint to 32-bit float */
+uint64_t HELPER(celgb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    float32 v2 = env->fregs[f2].l.upper;
-
-    set_round_mode(env, m3);
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-        float32_to_int32(v2, &env->fpu_status);
-    return set_cc_nz_f32(v2);
+    int hold = swap_round_mode(env, m3);
+    float32 ret = uint64_to_float32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 64-bit float to 32-bit int */
-uint32_t HELPER(cfdbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
-                       uint32_t m3)
+/* convert 64-bit uint to 64-bit float */
+uint64_t HELPER(cdlgb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    float64 v2 = env->fregs[f2].d;
-
-    set_round_mode(env, m3);
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-        float64_to_int32(v2, &env->fpu_status);
-    return set_cc_nz_f64(v2);
+    int hold = swap_round_mode(env, m3);
+    float64 ret = uint64_to_float64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 128-bit float to 32-bit int */
-uint32_t HELPER(cfxbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
-                       uint32_t m3)
+/* convert 64-bit uint to 128-bit float */
+uint64_t HELPER(cxlgb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    CPU_QuadU v2;
-
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-        float128_to_int32(v2.q, &env->fpu_status);
-    return set_cc_nz_f128(v2.q);
+    int hold = swap_round_mode(env, m3);
+    float128 ret = uint64_to_float128(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
 }
 
-/* load 32-bit FP zero */
-void HELPER(lzer)(CPUS390XState *env, uint32_t f1)
+/* convert 32-bit float to 64-bit int */
+uint64_t HELPER(cgeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    env->fregs[f1].l.upper = float32_zero;
+    int hold = swap_round_mode(env, m3);
+    int64_t ret = float32_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* load 64-bit FP zero */
-void HELPER(lzdr)(CPUS390XState *env, uint32_t f1)
+/* convert 64-bit float to 64-bit int */
+uint64_t HELPER(cgdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    env->fregs[f1].d = float64_zero;
+    int hold = swap_round_mode(env, m3);
+    int64_t ret = float64_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* load 128-bit FP zero */
-void HELPER(lzxr)(CPUS390XState *env, uint32_t f1)
+/* convert 128-bit float to 64-bit int */
+uint64_t HELPER(cgxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
 {
-    CPU_QuadU x;
-
-    x.q = float64_to_float128(float64_zero, &env->fpu_status);
-    env->fregs[f1].ll = x.ll.upper;
-    env->fregs[f1 + 1].ll = x.ll.lower;
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    int64_t ret = float128_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 128-bit FP subtraction RR */
-uint32_t HELPER(sxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* convert 32-bit float to 32-bit int */
+uint64_t HELPER(cfeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-    CPU_QuadU res;
-
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    res.q = float128_sub(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-    return set_cc_nz_f128(res.q);
+    int hold = swap_round_mode(env, m3);
+    int32_t ret = float32_to_int32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 128-bit FP addition RR */
-uint32_t HELPER(axbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* convert 64-bit float to 32-bit int */
+uint64_t HELPER(cfdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-    CPU_QuadU res;
-
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    res.q = float128_add(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-    return set_cc_nz_f128(res.q);
+    int hold = swap_round_mode(env, m3);
+    int32_t ret = float64_to_int32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 32-bit FP multiplication RR */
-void HELPER(meebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* convert 128-bit float to 32-bit int */
+uint64_t HELPER(cfxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
 {
-    env->fregs[f1].l.upper = float32_mul(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    int32_t ret = float128_to_int32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP division RR */
-void HELPER(ddbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* convert 32-bit float to 64-bit uint */
+uint64_t HELPER(clgeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    env->fregs[f1].d = float64_div(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
+    int hold = swap_round_mode(env, m3);
+    uint64_t ret;
+    v2 = float32_to_float64(v2, &env->fpu_status);
+    ret = float64_to_uint64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP multiply and add RM */
-void HELPER(madb)(CPUS390XState *env, uint32_t f1, uint64_t a2, uint32_t f3)
+/* convert 64-bit float to 64-bit uint */
+uint64_t HELPER(clgdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    CPU_DoubleU v2;
+    int hold = swap_round_mode(env, m3);
+    uint64_t ret = float64_to_uint64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
 
-    HELPER_LOG("%s: f1 %d a2 0x%lx f3 %d\n", __func__, f1, a2, f3);
-    v2.ll = cpu_ldq_data(env, a2);
-    env->fregs[f1].d = float64_add(env->fregs[f1].d,
-                                   float64_mul(v2.d, env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   &env->fpu_status);
+/* convert 128-bit float to 64-bit uint */
+uint64_t HELPER(clgxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    /* ??? Not 100% correct.  */
+    uint64_t ret = float128_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP multiply and add RR */
-void HELPER(madbr)(CPUS390XState *env, uint32_t f1, uint32_t f3, uint32_t f2)
+/* convert 32-bit float to 32-bit uint */
+uint64_t HELPER(clfeb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __func__, f1, f2, f3);
-    env->fregs[f1].d = float64_add(float64_mul(env->fregs[f2].d,
-                                               env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   env->fregs[f1].d, &env->fpu_status);
+    int hold = swap_round_mode(env, m3);
+    uint32_t ret = float32_to_uint32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 64-bit FP multiply and subtract RR */
-void HELPER(msdbr)(CPUS390XState *env, uint32_t f1, uint32_t f3, uint32_t f2)
+/* convert 64-bit float to 32-bit uint */
+uint64_t HELPER(clfdb)(CPUS390XState *env, uint64_t v2, uint32_t m3)
 {
-    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __func__, f1, f2, f3);
-    env->fregs[f1].d = float64_sub(float64_mul(env->fregs[f2].d,
-                                               env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   env->fregs[f1].d, &env->fpu_status);
+    int hold = swap_round_mode(env, m3);
+    uint32_t ret = float64_to_uint32(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* 32-bit FP multiply and add RR */
-void HELPER(maebr)(CPUS390XState *env, uint32_t f1, uint32_t f3, uint32_t f2)
+/* convert 128-bit float to 32-bit uint */
+uint64_t HELPER(clfxb)(CPUS390XState *env, uint64_t h, uint64_t l, uint32_t m3)
 {
-    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
-                                         float32_mul(env->fregs[f2].l.upper,
-                                                     env->fregs[f3].l.upper,
-                                                     &env->fpu_status),
-                                         &env->fpu_status);
+    int hold = swap_round_mode(env, m3);
+    float128 v2 = make_float128(h, l);
+    /* Not 100% correct.  */
+    uint32_t ret = float128_to_int64(v2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 32-bit float to 64-bit float */
-void HELPER(ldeb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+/* 32-bit FP multiply and add */
+uint64_t HELPER(maeb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
 {
-    uint32_t v2;
+    float32 ret = float32_muladd(f2, f3, f1, 0, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
 
-    v2 = cpu_ldl_data(env, a2);
-    env->fregs[f1].d = float32_to_float64(v2,
-                                          &env->fpu_status);
+/* 64-bit FP multiply and add */
+uint64_t HELPER(madb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
+{
+    float64 ret = float64_muladd(f2, f3, f1, 0, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
-/* convert 64-bit float to 128-bit float */
-void HELPER(lxdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+/* 32-bit FP multiply and subtract */
+uint64_t HELPER(mseb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
 {
-    CPU_DoubleU v2;
-    CPU_QuadU v1;
+    float32 ret = float32_muladd(f2, f3, f1, float_muladd_negate_c,
+                                 &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
 
-    v2.ll = cpu_ldq_data(env, a2);
-    v1.q = float64_to_float128(v2.d, &env->fpu_status);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
+/* 64-bit FP multiply and subtract */
+uint64_t HELPER(msdb)(CPUS390XState *env, uint64_t f1,
+                      uint64_t f2, uint64_t f3)
+{
+    float64 ret = float64_muladd(f2, f3, f1, float_muladd_negate_c,
+                                 &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
 }
 
 /* test data class 32-bit */
-uint32_t HELPER(tceb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
+uint32_t HELPER(tceb)(uint64_t f1, uint64_t m2)
 {
-    float32 v1 = env->fregs[f1].l.upper;
+    float32 v1 = f1;
     int neg = float32_is_neg(v1);
     uint32_t cc = 0;
 
-    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __func__, (long)v1, m2, neg);
     if ((float32_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
         (float32_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
         (float32_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
@@ -786,19 +607,16 @@ uint32_t HELPER(tceb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
         /* assume normalized number */
         cc = 1;
     }
-
     /* FIXME: denormalized? */
     return cc;
 }
 
 /* test data class 64-bit */
-uint32_t HELPER(tcdb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
+uint32_t HELPER(tcdb)(uint64_t v1, uint64_t m2)
 {
-    float64 v1 = env->fregs[f1].d;
     int neg = float64_is_neg(v1);
     uint32_t cc = 0;
 
-    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __func__, v1, m2, neg);
     if ((float64_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
         (float64_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
         (float64_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
@@ -813,20 +631,16 @@ uint32_t HELPER(tcdb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
 }
 
 /* test data class 128-bit */
-uint32_t HELPER(tcxb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
+uint32_t HELPER(tcxb)(uint64_t ah, uint64_t al, uint64_t m2)
 {
-    CPU_QuadU v1;
+    float128 v1 = make_float128(ah, al);
+    int neg = float128_is_neg(v1);
     uint32_t cc = 0;
-    int neg;
-
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
 
-    neg = float128_is_neg(v1.q);
-    if ((float128_is_zero(v1.q) && (m2 & (1 << (11-neg)))) ||
-        (float128_is_infinity(v1.q) && (m2 & (1 << (5-neg)))) ||
-        (float128_is_any_nan(v1.q) && (m2 & (1 << (3-neg)))) ||
-        (float128_is_signaling_nan(v1.q) && (m2 & (1 << (1-neg))))) {
+    if ((float128_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
+        (float128_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
+        (float128_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
+        (float128_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
         cc = 1;
     } else if (m2 & (1 << (9-neg))) {
         /* assume normalized number */
@@ -836,8 +650,64 @@ uint32_t HELPER(tcxb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
     return cc;
 }
 
-/* square root 64-bit RR */
-void HELPER(sqdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+/* square root 32-bit */
+uint64_t HELPER(sqeb)(CPUS390XState *env, uint64_t f2)
+{
+    float32 ret = float32_sqrt(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* square root 64-bit */
+uint64_t HELPER(sqdb)(CPUS390XState *env, uint64_t f2)
+{
+    float64 ret = float64_sqrt(f2, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* square root 128-bit */
+uint64_t HELPER(sqxb)(CPUS390XState *env, uint64_t ah, uint64_t al)
+{
+    float128 ret = float128_sqrt(make_float128(ah, al), &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
+static const int fpc_to_rnd[4] = {
+    float_round_nearest_even,
+    float_round_to_zero,
+    float_round_up,
+    float_round_down
+};
+
+/* set fpc */
+void HELPER(sfpc)(CPUS390XState *env, uint64_t fpc)
+{
+    /* Install everything in the main FPC.  */
+    env->fpc = fpc;
+
+    /* Install the rounding mode in the shadow fpu_status.  */
+    set_float_rounding_mode(fpc_to_rnd[fpc & 3], &env->fpu_status);
+}
+
+/* set fpc and signal */
+void HELPER(sfas)(CPUS390XState *env, uint64_t val)
 {
-    env->fregs[f1].d = float64_sqrt(env->fregs[f2].d, &env->fpu_status);
+    uint32_t signalling = env->fpc;
+    uint32_t source = val;
+    uint32_t s390_exc;
+
+    /* The contents of the source operand are placed in the FPC register;
+       then the flags in the FPC register are set to the logical OR of the
+       signalling flags and the source flags.  */
+    env->fpc = source | (signalling & 0x00ff0000);
+    set_float_rounding_mode(fpc_to_rnd[source & 3], &env->fpu_status);
+
+    /* If any signalling flag is 1 and the corresponding source mask
+       is also 1, a simulated-iee-exception trap occurs.  */
+    s390_exc = (signalling >> 16) & (source >> 24);
+    if (s390_exc) {
+        ieee_exception(env, s390_exc | 3, GETPC());
+    }
 }
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index b7b812a..7626831 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -19,10 +19,10 @@
  */
 
 #include "cpu.h"
-#include "gdbstub.h"
-#include "qemu-timer.h"
+#include "exec/gdbstub.h"
+#include "qemu/timer.h"
 #ifndef CONFIG_USER_ONLY
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #endif
 
 //#define DEBUG_S390
@@ -99,10 +99,10 @@ void do_interrupt(CPUS390XState *env)
 int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong address,
                                int rw, int mmu_idx)
 {
-    /* fprintf(stderr, "%s: address 0x%lx rw %d mmu_idx %d\n",
-       __func__, address, rw, mmu_idx); */
-    env->exception_index = EXCP_ADDR;
-    /* FIXME: find out how this works on a real machine */
+    env->exception_index = EXCP_PGM;
+    env->int_pgm_code = PGM_ADDRESSING;
+    /* On real machines this value is dropped into LowMem.  Since this
+       is userland, simply put this someplace that cpu_loop can find it.  */
     env->__excp_addr = address;
     return 1;
 }
@@ -111,11 +111,11 @@ int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong address,
 
 /* Ensure to exit the TB after this call! */
 static void trigger_pgm_exception(CPUS390XState *env, uint32_t code,
-                                  uint32_t ilc)
+                                  uint32_t ilen)
 {
     env->exception_index = EXCP_PGM;
     env->int_pgm_code = code;
-    env->int_pgm_ilc = ilc;
+    env->int_pgm_ilen = ilen;
 }
 
 static int trans_bits(CPUS390XState *env, uint64_t mode)
@@ -143,30 +143,30 @@ static int trans_bits(CPUS390XState *env, uint64_t mode)
 static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr,
                                uint64_t mode)
 {
-    int ilc = ILC_LATER_INC_2;
+    int ilen = ILEN_LATER_INC;
     int bits = trans_bits(env, mode) | 4;
 
     DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
 
     stq_phys(env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
-    trigger_pgm_exception(env, PGM_PROTECTION, ilc);
+    trigger_pgm_exception(env, PGM_PROTECTION, ilen);
 }
 
 static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr,
                                uint32_t type, uint64_t asc, int rw)
 {
-    int ilc = ILC_LATER;
+    int ilen = ILEN_LATER;
     int bits = trans_bits(env, asc);
 
+    /* Code accesses have an undefined ilc.  */
     if (rw == 2) {
-        /* code has is undefined ilc */
-        ilc = 2;
+        ilen = 2;
     }
 
     DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
 
     stq_phys(env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
-    trigger_pgm_exception(env, type, ilc);
+    trigger_pgm_exception(env, type, ilen);
 }
 
 static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
@@ -387,7 +387,7 @@ int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong orig_vaddr,
     int prot;
 
     DPRINTF("%s: address 0x%" PRIx64 " rw %d mmu_idx %d\n",
-            __func__, _vaddr, rw, mmu_idx);
+            __func__, orig_vaddr, rw, mmu_idx);
 
     orig_vaddr &= TARGET_PAGE_MASK;
     vaddr = orig_vaddr;
@@ -404,9 +404,9 @@ int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong orig_vaddr,
 
     /* check out of RAM access */
     if (raddr > (ram_size + virtio_size)) {
-        DPRINTF("%s: aaddr %" PRIx64 " > ram_size %" PRIx64 "\n", __func__,
-                (uint64_t)aaddr, (uint64_t)ram_size);
-        trigger_pgm_exception(env, PGM_ADDRESSING, ILC_LATER);
+        DPRINTF("%s: raddr %" PRIx64 " > ram_size %" PRIx64 "\n", __func__,
+                (uint64_t)raddr, (uint64_t)ram_size);
+        trigger_pgm_exception(env, PGM_ADDRESSING, ILEN_LATER);
         return 1;
     }
 
@@ -441,8 +441,9 @@ hwaddr cpu_get_phys_page_debug(CPUS390XState *env,
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr)
 {
     if (mask & PSW_MASK_WAIT) {
+        S390CPU *cpu = s390_env_get_cpu(env);
         if (!(mask & (PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK))) {
-            if (s390_del_running_cpu(env) == 0) {
+            if (s390_del_running_cpu(cpu) == 0) {
 #ifndef CONFIG_USER_ONLY
                 qemu_system_shutdown_request();
 #endif
@@ -454,38 +455,82 @@ void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr)
 
     env->psw.addr = addr;
     env->psw.mask = mask;
-    env->cc_op = (mask >> 13) & 3;
+    env->cc_op = (mask >> 44) & 3;
 }
 
 static uint64_t get_psw_mask(CPUS390XState *env)
 {
-    uint64_t r = env->psw.mask;
+    uint64_t r;
 
     env->cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst, env->cc_vr);
 
-    r &= ~(3ULL << 13);
+    r = env->psw.mask;
+    r &= ~PSW_MASK_CC;
     assert(!(env->cc_op & ~3));
-    r |= env->cc_op << 13;
+    r |= (uint64_t)env->cc_op << 44;
 
     return r;
 }
 
+static LowCore *cpu_map_lowcore(CPUS390XState *env)
+{
+    LowCore *lowcore;
+    hwaddr len = sizeof(LowCore);
+
+    lowcore = cpu_physical_memory_map(env->psa, &len, 1);
+
+    if (len < sizeof(LowCore)) {
+        cpu_abort(env, "Could not map lowcore\n");
+    }
+
+    return lowcore;
+}
+
+static void cpu_unmap_lowcore(LowCore *lowcore)
+{
+    cpu_physical_memory_unmap(lowcore, sizeof(LowCore), 1, sizeof(LowCore));
+}
+
+void *s390_cpu_physical_memory_map(CPUS390XState *env, hwaddr addr, hwaddr *len,
+                                   int is_write)
+{
+    hwaddr start = addr;
+
+    /* Mind the prefix area. */
+    if (addr < 8192) {
+        /* Map the lowcore. */
+        start += env->psa;
+        *len = MIN(*len, 8192 - addr);
+    } else if ((addr >= env->psa) && (addr < env->psa + 8192)) {
+        /* Map the 0 page. */
+        start -= env->psa;
+        *len = MIN(*len, 8192 - start);
+    }
+
+    return cpu_physical_memory_map(start, len, is_write);
+}
+
+void s390_cpu_physical_memory_unmap(CPUS390XState *env, void *addr, hwaddr len,
+                                    int is_write)
+{
+    cpu_physical_memory_unmap(addr, len, is_write, len);
+}
+
 static void do_svc_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    hwaddr len = TARGET_PAGE_SIZE;
 
-    lowcore = cpu_physical_memory_map(env->psa, &len, 1);
+    lowcore = cpu_map_lowcore(env);
 
     lowcore->svc_code = cpu_to_be16(env->int_svc_code);
-    lowcore->svc_ilc = cpu_to_be16(env->int_svc_ilc);
+    lowcore->svc_ilen = cpu_to_be16(env->int_svc_ilen);
     lowcore->svc_old_psw.mask = cpu_to_be64(get_psw_mask(env));
-    lowcore->svc_old_psw.addr = cpu_to_be64(env->psw.addr + (env->int_svc_ilc));
+    lowcore->svc_old_psw.addr = cpu_to_be64(env->psw.addr + env->int_svc_ilen);
     mask = be64_to_cpu(lowcore->svc_new_psw.mask);
     addr = be64_to_cpu(lowcore->svc_new_psw.addr);
 
-    cpu_physical_memory_unmap(lowcore, len, 1, len);
+    cpu_unmap_lowcore(lowcore);
 
     load_psw(env, mask, addr);
 }
@@ -494,39 +539,36 @@ static void do_program_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    hwaddr len = TARGET_PAGE_SIZE;
-    int ilc = env->int_pgm_ilc;
+    int ilen = env->int_pgm_ilen;
 
-    switch (ilc) {
-    case ILC_LATER:
-        ilc = get_ilc(cpu_ldub_code(env, env->psw.addr));
-        break;
-    case ILC_LATER_INC:
-        ilc = get_ilc(cpu_ldub_code(env, env->psw.addr));
-        env->psw.addr += ilc * 2;
+    switch (ilen) {
+    case ILEN_LATER:
+        ilen = get_ilen(cpu_ldub_code(env, env->psw.addr));
         break;
-    case ILC_LATER_INC_2:
-        ilc = get_ilc(cpu_ldub_code(env, env->psw.addr)) * 2;
-        env->psw.addr += ilc;
+    case ILEN_LATER_INC:
+        ilen = get_ilen(cpu_ldub_code(env, env->psw.addr));
+        env->psw.addr += ilen;
         break;
+    default:
+        assert(ilen == 2 || ilen == 4 || ilen == 6);
     }
 
-    qemu_log_mask(CPU_LOG_INT, "%s: code=0x%x ilc=%d\n",
-                  __func__, env->int_pgm_code, ilc);
+    qemu_log_mask(CPU_LOG_INT, "%s: code=0x%x ilen=%d\n",
+                  __func__, env->int_pgm_code, ilen);
 
-    lowcore = cpu_physical_memory_map(env->psa, &len, 1);
+    lowcore = cpu_map_lowcore(env);
 
-    lowcore->pgm_ilc = cpu_to_be16(ilc);
+    lowcore->pgm_ilen = cpu_to_be16(ilen);
     lowcore->pgm_code = cpu_to_be16(env->int_pgm_code);
     lowcore->program_old_psw.mask = cpu_to_be64(get_psw_mask(env));
     lowcore->program_old_psw.addr = cpu_to_be64(env->psw.addr);
     mask = be64_to_cpu(lowcore->program_new_psw.mask);
     addr = be64_to_cpu(lowcore->program_new_psw.addr);
 
-    cpu_physical_memory_unmap(lowcore, len, 1, len);
+    cpu_unmap_lowcore(lowcore);
 
     DPRINTF("%s: %x %x %" PRIx64 " %" PRIx64 "\n", __func__,
-            env->int_pgm_code, ilc, env->psw.mask,
+            env->int_pgm_code, ilen, env->psw.mask,
             env->psw.addr);
 
     load_psw(env, mask, addr);
@@ -538,7 +580,6 @@ static void do_ext_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    hwaddr len = TARGET_PAGE_SIZE;
     ExtQueue *q;
 
     if (!(env->psw.mask & PSW_MASK_EXT)) {
@@ -550,7 +591,7 @@ static void do_ext_interrupt(CPUS390XState *env)
     }
 
     q = &env->ext_queue[env->ext_index];
-    lowcore = cpu_physical_memory_map(env->psa, &len, 1);
+    lowcore = cpu_map_lowcore(env);
 
     lowcore->ext_int_code = cpu_to_be16(q->code);
     lowcore->ext_params = cpu_to_be32(q->param);
@@ -561,7 +602,7 @@ static void do_ext_interrupt(CPUS390XState *env)
     mask = be64_to_cpu(lowcore->external_new_psw.mask);
     addr = be64_to_cpu(lowcore->external_new_psw.addr);
 
-    cpu_physical_memory_unmap(lowcore, len, 1, len);
+    cpu_unmap_lowcore(lowcore);
 
     env->ext_index--;
     if (env->ext_index == -1) {
@@ -574,12 +615,147 @@ static void do_ext_interrupt(CPUS390XState *env)
     load_psw(env, mask, addr);
 }
 
+static void do_io_interrupt(CPUS390XState *env)
+{
+    LowCore *lowcore;
+    IOIntQueue *q;
+    uint8_t isc;
+    int disable = 1;
+    int found = 0;
+
+    if (!(env->psw.mask & PSW_MASK_IO)) {
+        cpu_abort(env, "I/O int w/o I/O mask\n");
+    }
+
+    for (isc = 0; isc < ARRAY_SIZE(env->io_index); isc++) {
+        uint64_t isc_bits;
+
+        if (env->io_index[isc] < 0) {
+            continue;
+        }
+        if (env->io_index[isc] > MAX_IO_QUEUE) {
+            cpu_abort(env, "I/O queue overrun for isc %d: %d\n",
+                      isc, env->io_index[isc]);
+        }
+
+        q = &env->io_queue[env->io_index[isc]][isc];
+        isc_bits = ISC_TO_ISC_BITS(IO_INT_WORD_ISC(q->word));
+        if (!(env->cregs[6] & isc_bits)) {
+            disable = 0;
+            continue;
+        }
+        if (!found) {
+            uint64_t mask, addr;
+
+            found = 1;
+            lowcore = cpu_map_lowcore(env);
+
+            lowcore->subchannel_id = cpu_to_be16(q->id);
+            lowcore->subchannel_nr = cpu_to_be16(q->nr);
+            lowcore->io_int_parm = cpu_to_be32(q->parm);
+            lowcore->io_int_word = cpu_to_be32(q->word);
+            lowcore->io_old_psw.mask = cpu_to_be64(get_psw_mask(env));
+            lowcore->io_old_psw.addr = cpu_to_be64(env->psw.addr);
+            mask = be64_to_cpu(lowcore->io_new_psw.mask);
+            addr = be64_to_cpu(lowcore->io_new_psw.addr);
+
+            cpu_unmap_lowcore(lowcore);
+
+            env->io_index[isc]--;
+
+            DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__,
+                    env->psw.mask, env->psw.addr);
+            load_psw(env, mask, addr);
+        }
+        if (env->io_index[isc] >= 0) {
+            disable = 0;
+        }
+        continue;
+    }
+
+    if (disable) {
+        env->pending_int &= ~INTERRUPT_IO;
+    }
+
+}
+
+static void do_mchk_interrupt(CPUS390XState *env)
+{
+    uint64_t mask, addr;
+    LowCore *lowcore;
+    MchkQueue *q;
+    int i;
+
+    if (!(env->psw.mask & PSW_MASK_MCHECK)) {
+        cpu_abort(env, "Machine check w/o mchk mask\n");
+    }
+
+    if (env->mchk_index < 0 || env->mchk_index > MAX_MCHK_QUEUE) {
+        cpu_abort(env, "Mchk queue overrun: %d\n", env->mchk_index);
+    }
+
+    q = &env->mchk_queue[env->mchk_index];
+
+    if (q->type != 1) {
+        /* Don't know how to handle this... */
+        cpu_abort(env, "Unknown machine check type %d\n", q->type);
+    }
+    if (!(env->cregs[14] & (1 << 28))) {
+        /* CRW machine checks disabled */
+        return;
+    }
+
+    lowcore = cpu_map_lowcore(env);
+
+    for (i = 0; i < 16; i++) {
+        lowcore->floating_pt_save_area[i] = cpu_to_be64(env->fregs[i].ll);
+        lowcore->gpregs_save_area[i] = cpu_to_be64(env->regs[i]);
+        lowcore->access_regs_save_area[i] = cpu_to_be32(env->aregs[i]);
+        lowcore->cregs_save_area[i] = cpu_to_be64(env->cregs[i]);
+    }
+    lowcore->prefixreg_save_area = cpu_to_be32(env->psa);
+    lowcore->fpt_creg_save_area = cpu_to_be32(env->fpc);
+    lowcore->tod_progreg_save_area = cpu_to_be32(env->todpr);
+    lowcore->cpu_timer_save_area[0] = cpu_to_be32(env->cputm >> 32);
+    lowcore->cpu_timer_save_area[1] = cpu_to_be32((uint32_t)env->cputm);
+    lowcore->clock_comp_save_area[0] = cpu_to_be32(env->ckc >> 32);
+    lowcore->clock_comp_save_area[1] = cpu_to_be32((uint32_t)env->ckc);
+
+    lowcore->mcck_interruption_code[0] = cpu_to_be32(0x00400f1d);
+    lowcore->mcck_interruption_code[1] = cpu_to_be32(0x40330000);
+    lowcore->mcck_old_psw.mask = cpu_to_be64(get_psw_mask(env));
+    lowcore->mcck_old_psw.addr = cpu_to_be64(env->psw.addr);
+    mask = be64_to_cpu(lowcore->mcck_new_psw.mask);
+    addr = be64_to_cpu(lowcore->mcck_new_psw.addr);
+
+    cpu_unmap_lowcore(lowcore);
+
+    env->mchk_index--;
+    if (env->mchk_index == -1) {
+        env->pending_int &= ~INTERRUPT_MCHK;
+    }
+
+    DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__,
+            env->psw.mask, env->psw.addr);
+
+    load_psw(env, mask, addr);
+}
+
 void do_interrupt(CPUS390XState *env)
 {
+    S390CPU *cpu = s390_env_get_cpu(env);
+
     qemu_log_mask(CPU_LOG_INT, "%s: %d at pc=%" PRIx64 "\n",
                   __func__, env->exception_index, env->psw.addr);
 
-    s390_add_running_cpu(env);
+    s390_add_running_cpu(cpu);
+    /* handle machine checks */
+    if ((env->psw.mask & PSW_MASK_MCHECK) &&
+        (env->exception_index == -1)) {
+        if (env->pending_int & INTERRUPT_MCHK) {
+            env->exception_index = EXCP_MCHK;
+        }
+    }
     /* handle external interrupts */
     if ((env->psw.mask & PSW_MASK_EXT) &&
         env->exception_index == -1) {
@@ -587,17 +763,24 @@ void do_interrupt(CPUS390XState *env)
             /* code is already in env */
             env->exception_index = EXCP_EXT;
         } else if (env->pending_int & INTERRUPT_TOD) {
-            cpu_inject_ext(env, 0x1004, 0, 0);
+            cpu_inject_ext(cpu, 0x1004, 0, 0);
             env->exception_index = EXCP_EXT;
             env->pending_int &= ~INTERRUPT_EXT;
             env->pending_int &= ~INTERRUPT_TOD;
         } else if (env->pending_int & INTERRUPT_CPUTIMER) {
-            cpu_inject_ext(env, 0x1005, 0, 0);
+            cpu_inject_ext(cpu, 0x1005, 0, 0);
             env->exception_index = EXCP_EXT;
             env->pending_int &= ~INTERRUPT_EXT;
             env->pending_int &= ~INTERRUPT_TOD;
         }
     }
+    /* handle I/O interrupts */
+    if ((env->psw.mask & PSW_MASK_IO) &&
+        (env->exception_index == -1)) {
+        if (env->pending_int & INTERRUPT_IO) {
+            env->exception_index = EXCP_IO;
+        }
+    }
 
     switch (env->exception_index) {
     case EXCP_PGM:
@@ -609,6 +792,12 @@ void do_interrupt(CPUS390XState *env)
     case EXCP_EXT:
         do_ext_interrupt(env);
         break;
+    case EXCP_IO:
+        do_io_interrupt(env);
+        break;
+    case EXCP_MCHK:
+        do_mchk_interrupt(env);
+        break;
     }
     env->exception_index = -1;
 
diff --git a/target-s390x/helper.h b/target-s390x/helper.h
index ac44eab..dd90d93 100644
--- a/target-s390x/helper.h
+++ b/target-s390x/helper.h
@@ -1,152 +1,120 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
-DEF_HELPER_2(exception, void, env, i32)
-DEF_HELPER_4(nc, i32, env, i32, i64, i64)
-DEF_HELPER_4(oc, i32, env, i32, i64, i64)
-DEF_HELPER_4(xc, i32, env, i32, i64, i64)
-DEF_HELPER_4(mvc, void, env, i32, i64, i64)
-DEF_HELPER_4(clc, i32, env, i32, i64, i64)
+DEF_HELPER_2(exception, noreturn, env, i32)
+DEF_HELPER_FLAGS_4(nc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(oc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(xc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(mvc, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(clc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
 DEF_HELPER_3(mvcl, i32, env, i32, i32)
-DEF_HELPER_FLAGS_1(set_cc_comp_s32, TCG_CALL_NO_RWG_SE, i32, s32)
-DEF_HELPER_FLAGS_1(set_cc_comp_s64, TCG_CALL_NO_RWG_SE, i32, s64)
-DEF_HELPER_FLAGS_2(set_cc_icm, TCG_CALL_NO_RWG_SE, i32, i32, i32)
-DEF_HELPER_4(clm, i32, env, i32, i32, i64)
-DEF_HELPER_4(stcm, void, env, i32, i32, i64)
-DEF_HELPER_3(mlg, void, env, i32, i64)
-DEF_HELPER_3(dlg, void, env, i32, i64)
-DEF_HELPER_FLAGS_3(set_cc_add64, TCG_CALL_NO_RWG_SE, i32, s64, s64, s64)
-DEF_HELPER_FLAGS_3(set_cc_addu64, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_3(set_cc_add32, TCG_CALL_NO_RWG_SE, i32, s32, s32, s32)
-DEF_HELPER_FLAGS_3(set_cc_addu32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(set_cc_sub64, TCG_CALL_NO_RWG_SE, i32, s64, s64, s64)
-DEF_HELPER_FLAGS_3(set_cc_subu64, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_3(set_cc_sub32, TCG_CALL_NO_RWG_SE, i32, s32, s32, s32)
-DEF_HELPER_FLAGS_3(set_cc_subu32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
-DEF_HELPER_4(srst, i32, env, i32, i32, i32)
-DEF_HELPER_4(clst, i32, env, i32, i32, i32)
+DEF_HELPER_FLAGS_4(clm, TCG_CALL_NO_WG, i32, env, i32, i32, i64)
+DEF_HELPER_FLAGS_3(mul128, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divs32, TCG_CALL_NO_WG, s64, env, s64, s64)
+DEF_HELPER_FLAGS_3(divu32, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divs64, TCG_CALL_NO_WG, s64, env, s64, s64)
+DEF_HELPER_FLAGS_4(divu64, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_4(srst, i64, env, i64, i64, i64)
+DEF_HELPER_4(clst, i64, env, i64, i64, i64)
 DEF_HELPER_4(mvpg, void, env, i64, i64, i64)
-DEF_HELPER_4(mvst, void, env, i32, i32, i32)
-DEF_HELPER_4(csg, i32, env, i32, i64, i32)
-DEF_HELPER_4(cdsg, i32, env, i32, i64, i32)
-DEF_HELPER_4(cs, i32, env, i32, i64, i32)
+DEF_HELPER_4(mvst, i64, env, i64, i64, i64)
 DEF_HELPER_5(ex, i32, env, i32, i64, i64, i64)
 DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_NO_RWG_SE, i32, s32)
 DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_NO_RWG_SE, s32, s32)
 DEF_HELPER_FLAGS_1(abs_i64, TCG_CALL_NO_RWG_SE, i64, s64)
 DEF_HELPER_FLAGS_1(nabs_i64, TCG_CALL_NO_RWG_SE, s64, s64)
-DEF_HELPER_4(stcmh, void, env, i32, i64, i32)
-DEF_HELPER_4(icmh, i32, env, i32, i64, i32)
-DEF_HELPER_3(ipm, void, env, i32, i32)
-DEF_HELPER_FLAGS_3(addc_u32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(set_cc_addc_u64, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
-DEF_HELPER_4(stam, void, env, i32, i64, i32)
-DEF_HELPER_4(lam, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(stam, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(lam, TCG_CALL_NO_WG, void, env, i32, i64, i32)
 DEF_HELPER_4(mvcle, i32, env, i32, i64, i32)
 DEF_HELPER_4(clcle, i32, env, i32, i64, i32)
-DEF_HELPER_4(slb, i32, env, i32, i32, i32)
-DEF_HELPER_5(slbg, i32, env, i32, i32, i64, i64)
-DEF_HELPER_3(cefbr, void, env, i32, s32)
-DEF_HELPER_3(cdfbr, void, env, i32, s32)
-DEF_HELPER_3(cxfbr, void, env, i32, s32)
-DEF_HELPER_3(cegbr, void, env, i32, s64)
-DEF_HELPER_3(cdgbr, void, env, i32, s64)
-DEF_HELPER_3(cxgbr, void, env, i32, s64)
-DEF_HELPER_3(adbr, i32, env, i32, i32)
-DEF_HELPER_3(aebr, i32, env, i32, i32)
-DEF_HELPER_3(sebr, i32, env, i32, i32)
-DEF_HELPER_3(sdbr, i32, env, i32, i32)
-DEF_HELPER_3(debr, void, env, i32, i32)
-DEF_HELPER_3(dxbr, void, env, i32, i32)
-DEF_HELPER_3(mdbr, void, env, i32, i32)
-DEF_HELPER_3(mxbr, void, env, i32, i32)
-DEF_HELPER_3(ldebr, void, env, i32, i32)
-DEF_HELPER_3(ldxbr, void, env, i32, i32)
-DEF_HELPER_3(lxdbr, void, env, i32, i32)
-DEF_HELPER_3(ledbr, void, env, i32, i32)
-DEF_HELPER_3(lexbr, void, env, i32, i32)
-DEF_HELPER_3(lpebr, i32, env, i32, i32)
-DEF_HELPER_3(lpdbr, i32, env, i32, i32)
-DEF_HELPER_3(lpxbr, i32, env, i32, i32)
-DEF_HELPER_3(ltebr, i32, env, i32, i32)
-DEF_HELPER_3(ltdbr, i32, env, i32, i32)
-DEF_HELPER_3(ltxbr, i32, env, i32, i32)
-DEF_HELPER_3(lcebr, i32, env, i32, i32)
-DEF_HELPER_3(lcdbr, i32, env, i32, i32)
-DEF_HELPER_3(lcxbr, i32, env, i32, i32)
-DEF_HELPER_3(aeb, void, env, i32, i32)
-DEF_HELPER_3(deb, void, env, i32, i32)
-DEF_HELPER_3(meeb, void, env, i32, i32)
-DEF_HELPER_3(cdb, i32, env, i32, i64)
-DEF_HELPER_3(adb, i32, env, i32, i64)
-DEF_HELPER_3(seb, void, env, i32, i32)
-DEF_HELPER_3(sdb, i32, env, i32, i64)
-DEF_HELPER_3(mdb, void, env, i32, i64)
-DEF_HELPER_3(ddb, void, env, i32, i64)
-DEF_HELPER_FLAGS_3(cebr, TCG_CALL_NO_SE, i32, env, i32, i32)
-DEF_HELPER_FLAGS_3(cdbr, TCG_CALL_NO_SE, i32, env, i32, i32)
-DEF_HELPER_FLAGS_3(cxbr, TCG_CALL_NO_SE, i32, env, i32, i32)
-DEF_HELPER_4(cgebr, i32, env, i32, i32, i32)
-DEF_HELPER_4(cgdbr, i32, env, i32, i32, i32)
-DEF_HELPER_4(cgxbr, i32, env, i32, i32, i32)
-DEF_HELPER_2(lzer, void, env, i32)
-DEF_HELPER_2(lzdr, void, env, i32)
-DEF_HELPER_2(lzxr, void, env, i32)
-DEF_HELPER_4(cfebr, i32, env, i32, i32, i32)
-DEF_HELPER_4(cfdbr, i32, env, i32, i32, i32)
-DEF_HELPER_4(cfxbr, i32, env, i32, i32, i32)
-DEF_HELPER_3(axbr, i32, env, i32, i32)
-DEF_HELPER_3(sxbr, i32, env, i32, i32)
-DEF_HELPER_3(meebr, void, env, i32, i32)
-DEF_HELPER_3(ddbr, void, env, i32, i32)
-DEF_HELPER_4(madb, void, env, i32, i64, i32)
-DEF_HELPER_4(maebr, void, env, i32, i32, i32)
-DEF_HELPER_4(madbr, void, env, i32, i32, i32)
-DEF_HELPER_4(msdbr, void, env, i32, i32, i32)
-DEF_HELPER_3(ldeb, void, env, i32, i64)
-DEF_HELPER_3(lxdb, void, env, i32, i64)
-DEF_HELPER_FLAGS_3(tceb, TCG_CALL_NO_SE, i32, env, i32, i64)
-DEF_HELPER_FLAGS_3(tcdb, TCG_CALL_NO_SE, i32, env, i32, i64)
-DEF_HELPER_FLAGS_3(tcxb, TCG_CALL_NO_SE, i32, env, i32, i64)
-DEF_HELPER_3(flogr, i32, env, i32, i64)
-DEF_HELPER_3(sqdbr, void, env, i32, i32)
+DEF_HELPER_3(cegb, i64, env, s64, i32)
+DEF_HELPER_3(cdgb, i64, env, s64, i32)
+DEF_HELPER_3(cxgb, i64, env, s64, i32)
+DEF_HELPER_3(celgb, i64, env, i64, i32)
+DEF_HELPER_3(cdlgb, i64, env, i64, i32)
+DEF_HELPER_3(cxlgb, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(aeb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(adb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(axb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(seb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(sdb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(sxb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(deb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(ddb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(dxb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(meeb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mdeb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mdb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_5(mxb, TCG_CALL_NO_WG, i64, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_4(mxdb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_2(ldeb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(ldxb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(lxdb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(lxeb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(ledb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(lexb, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(ceb, TCG_CALL_NO_WG_SE, i32, env, i64, i64)
+DEF_HELPER_FLAGS_3(cdb, TCG_CALL_NO_WG_SE, i32, env, i64, i64)
+DEF_HELPER_FLAGS_5(cxb, TCG_CALL_NO_WG_SE, i32, env, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cgeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(cgdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(cgxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(cfeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(cfdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(cfxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(clgeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(clgdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(clgxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(clfeb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(clfdb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(clfxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_4(maeb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(madb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(mseb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(msdb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_2(tceb, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_2(tcdb, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_3(tcxb, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_2(sqeb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(sqdb, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(sqxb, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_1(cvd, TCG_CALL_NO_RWG_SE, i64, s32)
-DEF_HELPER_4(unpk, void, env, i32, i64, i64)
-DEF_HELPER_4(tr, void, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(unpk, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(tr, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_4(cksm, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_5(calc_cc, TCG_CALL_NO_RWG_SE, i32, env, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_2(sfpc, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(sfas, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_1(popcnt, TCG_CALL_NO_RWG_SE, i64, i64)
 
-DEF_HELPER_3(servc, i32, env, i32, i64)
+#ifndef CONFIG_USER_ONLY
+DEF_HELPER_3(servc, i32, env, i64, i64)
 DEF_HELPER_4(diag, i64, env, i32, i64, i64)
-DEF_HELPER_3(load_psw, void, env, i64, i64)
-DEF_HELPER_1(program_interrupt, void, i32)
-DEF_HELPER_FLAGS_2(stidp, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_3(load_psw, noreturn, env, i64, i64)
 DEF_HELPER_FLAGS_2(spx, TCG_CALL_NO_RWG, void, env, i64)
-DEF_HELPER_FLAGS_1(sck, TCG_CALL_NO_RWG, i32, i64)
-DEF_HELPER_2(stck, i32, env, i64)
-DEF_HELPER_2(stcke, i32, env, i64)
+DEF_HELPER_FLAGS_1(stck, TCG_CALL_NO_RWG_SE, i64, env)
 DEF_HELPER_FLAGS_2(sckc, TCG_CALL_NO_RWG, void, env, i64)
-DEF_HELPER_FLAGS_2(stckc, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_1(stckc, TCG_CALL_NO_RWG, i64, env)
 DEF_HELPER_FLAGS_2(spt, TCG_CALL_NO_RWG, void, env, i64)
-DEF_HELPER_FLAGS_2(stpt, TCG_CALL_NO_RWG, void, env, i64)
-DEF_HELPER_4(stsi, i32, env, i64, i32, i32)
-DEF_HELPER_4(lctl, void, env, i32, i64, i32)
-DEF_HELPER_4(lctlg, void, env, i32, i64, i32)
-DEF_HELPER_4(stctl, void, env, i32, i64, i32)
-DEF_HELPER_4(stctg, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_1(stpt, TCG_CALL_NO_RWG, i64, env)
+DEF_HELPER_4(stsi, i32, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(lctl, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(lctlg, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(stctl, TCG_CALL_NO_WG, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_4(stctg, TCG_CALL_NO_WG, void, env, i32, i64, i32)
 DEF_HELPER_FLAGS_2(tprot, TCG_CALL_NO_RWG, i32, i64, i64)
 DEF_HELPER_FLAGS_2(iske, TCG_CALL_NO_RWG_SE, i64, env, i64)
-DEF_HELPER_FLAGS_3(sske, TCG_CALL_NO_RWG, void, env, i32, i64)
-DEF_HELPER_FLAGS_3(rrbe, TCG_CALL_NO_RWG, i32, env, i32, i64)
-DEF_HELPER_3(csp, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(sske, TCG_CALL_NO_RWG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(rrbe, TCG_CALL_NO_RWG, i32, env, i64)
+DEF_HELPER_3(csp, i32, env, i32, i64)
 DEF_HELPER_4(mvcs, i32, env, i64, i64, i64)
 DEF_HELPER_4(mvcp, i32, env, i64, i64, i64)
 DEF_HELPER_4(sigp, i32, env, i64, i32, i64)
-DEF_HELPER_2(sacf, void, env, i64)
+DEF_HELPER_FLAGS_2(sacf, TCG_CALL_NO_WG, void, env, i64)
 DEF_HELPER_FLAGS_3(ipte, TCG_CALL_NO_RWG, void, env, i64, i64)
 DEF_HELPER_FLAGS_1(ptlb, TCG_CALL_NO_RWG, void, env)
-DEF_HELPER_3(lra, i32, env, i64, i32)
-DEF_HELPER_3(stura, void, env, i64, i32)
-DEF_HELPER_3(cksm, void, env, i32, i32)
+DEF_HELPER_2(lra, i64, env, i64)
+DEF_HELPER_FLAGS_3(stura, TCG_CALL_NO_WG, void, env, i64, i64)
+#endif
 
-DEF_HELPER_FLAGS_5(calc_cc, TCG_CALL_NO_RWG_SE,
-                   i32, env, i32, i64, i64, i64)
-
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-s390x/insn-data.def b/target-s390x/insn-data.def
new file mode 100644
index 0000000..b42ebb6
--- /dev/null
+++ b/target-s390x/insn-data.def
@@ -0,0 +1,813 @@
+/* ADD */
+    C(0x1a00, AR,      RR_a,  Z,   r1, r2, new, r1_32, add, adds32)
+    C(0xb9f8, ARK,     RRF_a, DO,  r2, r3, new, r1_32, add, adds32)
+    C(0x5a00, A,       RX_a,  Z,   r1, m2_32s, new, r1_32, add, adds32)
+    C(0xe35a, AY,      RXY_a, LD,  r1, m2_32s, new, r1_32, add, adds32)
+    C(0xb908, AGR,     RRE,   Z,   r1, r2, r1, 0, add, adds64)
+    C(0xb918, AGFR,    RRE,   Z,   r1, r2_32s, r1, 0, add, adds64)
+    C(0xb9e8, AGRK,    RRF_a, DO,  r2, r3, r1, 0, add, adds64)
+    C(0xe308, AG,      RXY_a, Z,   r1, m2_64, r1, 0, add, adds64)
+    C(0xe318, AGF,     RXY_a, Z,   r1, m2_32s, r1, 0, add, adds64)
+    C(0xb30a, AEBR,    RRE,   Z,   e1, e2, new, e1, aeb, f32)
+    C(0xb31a, ADBR,    RRE,   Z,   f1_o, f2_o, f1, 0, adb, f64)
+    C(0xb34a, AXBR,    RRE,   Z,   0, x2_o, x1, 0, axb, f128)
+    C(0xed0a, AEB,     RXE,   Z,   e1, m2_32u, new, e1, aeb, f32)
+    C(0xed1a, ADB,     RXE,   Z,   f1_o, m2_64, f1, 0, adb, f64)
+/* ADD IMMEDIATE */
+    C(0xc209, AFI,     RIL_a, EI,  r1, i2, new, r1_32, add, adds32)
+    C(0xeb6a, ASI,     SIY,   GIE, m1_32s, i2, new, m1_32, add, adds32)
+    C(0xecd8, AHIK,    RIE_d, DO,  r3, i2, new, r1_32, add, adds32)
+    C(0xc208, AGFI,    RIL_a, EI,  r1, i2, r1, 0, add, adds64)
+    C(0xeb7a, AGSI,    SIY,   GIE, m1_64, i2, new, m1_64, add, adds64)
+    C(0xecd9, AGHIK,   RIE_d, DO,  r3, i2, r1, 0, add, adds64)
+/* ADD HALFWORD */
+    C(0x4a00, AH,      RX_a,  Z,   r1, m2_16s, new, r1_32, add, adds32)
+    C(0xe37a, AHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, add, adds32)
+/* ADD HALFWORD IMMEDIATE */
+    C(0xa70a, AHI,     RI_a,  Z,   r1, i2, new, r1_32, add, adds32)
+    C(0xa70b, AGHI,    RI_a,  Z,   r1, i2, r1, 0, add, adds64)
+
+/* ADD LOGICAL */
+    C(0x1e00, ALR,     RR_a,  Z,   r1, r2, new, r1_32, add, addu32)
+    C(0xb9fa, ALRK,    RRF_a, DO,  r2, r3, new, r1_32, add, addu32)
+    C(0x5e00, AL,      RX_a,  Z,   r1, m2_32u, new, r1_32, add, addu32)
+    C(0xe35e, ALY,     RXY_a, LD,  r1, m2_32u, new, r1_32, add, addu32)
+    C(0xb90a, ALGR,    RRE,   Z,   r1, r2, r1, 0, add, addu64)
+    C(0xb91a, ALGFR,   RRE,   Z,   r1, r2_32u, r1, 0, add, addu64)
+    C(0xb9ea, ALGRK,   RRF_a, DO,  r2, r3, r1, 0, add, addu64)
+    C(0xe30a, ALG,     RXY_a, Z,   r1, m2_64, r1, 0, add, addu64)
+    C(0xe31a, ALGF,    RXY_a, Z,   r1, m2_32u, r1, 0, add, addu64)
+/* ADD LOGICAL IMMEDIATE */
+    C(0xc20b, ALFI,    RIL_a, EI,  r1, i2_32u, new, r1_32, add, addu32)
+    C(0xc20a, ALGFI,   RIL_a, EI,  r1, i2_32u, r1, 0, add, addu64)
+/* ADD LOGICAL WITH SIGNED IMMEDIATE */
+    C(0xeb6e, ALSI,    SIY,   GIE, m1_32u, i2, new, m1_32, add, addu32)
+    C(0xecda, ALHSIK,  RIE_d, DO,  r3, i2, new, r1_32, add, addu32)
+    C(0xeb7e, ALGSI,   SIY,   GIE, m1_64, i2, new, m1_64, add, addu64)
+    C(0xecdb, ALGHSIK, RIE_d, DO,  r3, i2, r1, 0, add, addu64)
+/* ADD LOGICAL WITH CARRY */
+    C(0xb998, ALCR,    RRE,   Z,   r1, r2, new, r1_32, addc, addc32)
+    C(0xb988, ALCGR,   RRE,   Z,   r1, r2, r1, 0, addc, addc64)
+    C(0xe398, ALC,     RXY_a, Z,   r1, m2_32u, new, r1_32, addc, addc32)
+    C(0xe388, ALCG,    RXY_a, Z,   r1, m2_64, r1, 0, addc, addc64)
+
+/* AND */
+    C(0x1400, NR,      RR_a,  Z,   r1, r2, new, r1_32, and, nz32)
+    C(0xb9f4, NRK,     RRF_a, DO,  r2, r3, new, r1_32, and, nz32)
+    C(0x5400, N,       RX_a,  Z,   r1, m2_32s, new, r1_32, and, nz32)
+    C(0xe354, NY,      RXY_a, LD,  r1, m2_32s, new, r1_32, and, nz32)
+    C(0xb980, NGR,     RRE,   Z,   r1, r2, r1, 0, and, nz64)
+    C(0xb9e4, NGRK,    RRF_a, DO,  r2, r3, r1, 0, and, nz64)
+    C(0xe380, NG,      RXY_a, Z,   r1, m2_64, r1, 0, and, nz64)
+    C(0xd400, NC,      SS_a,  Z,   la1, a2, 0, 0, nc, 0)
+/* AND IMMEDIATE */
+    D(0xc00a, NIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, andi, 0, 0x2020)
+    D(0xc00b, NILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, andi, 0, 0x2000)
+    D(0xa504, NIHH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1030)
+    D(0xa505, NIHL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1020)
+    D(0xa506, NILH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1010)
+    D(0xa507, NILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1000)
+    C(0x9400, NI,      SI,    Z,   m1_8u, i2_8u, new, m1_8, and, nz64)
+    C(0xeb54, NIY,     SIY,   LD,  m1_8u, i2_8u, new, m1_8, and, nz64)
+
+/* BRANCH AND SAVE */
+    C(0x0d00, BASR,    RR_a,  Z,   0, r2_nz, r1, 0, bas, 0)
+    C(0x4d00, BAS,     RX_a,  Z,   0, a2, r1, 0, bas, 0)
+/* BRANCH RELATIVE AND SAVE */
+    C(0xa705, BRAS,    RI_b,  Z,   0, 0, r1, 0, basi, 0)
+    C(0xc005, BRASL,   RIL_b, Z,   0, 0, r1, 0, basi, 0)
+/* BRANCH ON CONDITION */
+    C(0x0700, BCR,     RR_b,  Z,   0, r2_nz, 0, 0, bc, 0)
+    C(0x4700, BC,      RX_b,  Z,   0, a2, 0, 0, bc, 0)
+/* BRANCH RELATIVE ON CONDITION */
+    C(0xa704, BRC,     RI_c,  Z,   0, 0, 0, 0, bc, 0)
+    C(0xc004, BRCL,    RIL_c, Z,   0, 0, 0, 0, bc, 0)
+/* BRANCH ON COUNT */
+    C(0x0600, BCTR,    RR_a,  Z,   0, r2_nz, 0, 0, bct32, 0)
+    C(0xb946, BCTGR,   RRE,   Z,   0, r2_nz, 0, 0, bct64, 0)
+    C(0x4600, BCT,     RX_a,  Z,   0, a2, 0, 0, bct32, 0)
+    C(0xe346, BCTG,    RXY_a, Z,   0, a2, 0, 0, bct64, 0)
+/* BRANCH RELATIVE ON COUNT */
+    C(0xa706, BRCT,    RI_b,  Z,   0, 0, 0, 0, bct32, 0)
+    C(0xa707, BRCTG,   RI_b,  Z,   0, 0, 0, 0, bct64, 0)
+/* BRANCH ON INDEX */
+    D(0x8600, BXH,     RS_a,  Z,   0, a2, 0, 0, bx32, 0, 0)
+    D(0x8700, BXLE,    RS_a,  Z,   0, a2, 0, 0, bx32, 0, 1)
+    D(0xeb44, BXHG,    RSY_a, Z,   0, a2, 0, 0, bx64, 0, 0)
+    D(0xeb45, BXLEG,   RSY_a, Z,   0, a2, 0, 0, bx64, 0, 1)
+/* BRANCH RELATIVE ON INDEX */
+    D(0x8400, BRXH,    RSI,   Z,   0, 0, 0, 0, bx32, 0, 0)
+    D(0x8500, BRXLE,   RSI,   Z,   0, 0, 0, 0, bx32, 0, 1)
+    D(0xec44, BRXHG,   RIE_e, Z,   0, 0, 0, 0, bx64, 0, 0)
+    D(0xec45, BRXHLE,  RIE_e, Z,   0, 0, 0, 0, bx64, 0, 1)
+
+/* CHECKSUM */
+    C(0xb241, CKSM,    RRE,   Z,   r1_o, ra2, new, r1_32, cksm, 0)
+
+/* COPY SIGN */
+    C(0xb372, CPSDR,   RRF_b, FPSSH, f3_o, f2_o, f1, 0, cps, 0)
+
+/* COMPARE */
+    C(0x1900, CR,      RR_a,  Z,   r1_o, r2_o, 0, 0, 0, cmps32)
+    C(0x5900, C,       RX_a,  Z,   r1_o, m2_32s, 0, 0, 0, cmps32)
+    C(0xe359, CY,      RXY_a, LD,  r1_o, m2_32s, 0, 0, 0, cmps32)
+    C(0xb920, CGR,     RRE,   Z,   r1_o, r2_o, 0, 0, 0, cmps64)
+    C(0xb930, CGFR,    RRE,   Z,   r1_o, r2_32s, 0, 0, 0, cmps64)
+    C(0xe320, CG,      RXY_a, Z,   r1_o, m2_64, 0, 0, 0, cmps64)
+    C(0xe330, CGF,     RXY_a, Z,   r1_o, m2_32s, 0, 0, 0, cmps64)
+    C(0xb309, CEBR,    RRE,   Z,   e1, e2, 0, 0, ceb, 0)
+    C(0xb319, CDBR,    RRE,   Z,   f1_o, f2_o, 0, 0, cdb, 0)
+    C(0xb349, CXBR,    RRE,   Z,   x1_o, x2_o, 0, 0, cxb, 0)
+    C(0xed09, CEB,     RXE,   Z,   e1, m2_32u, 0, 0, ceb, 0)
+    C(0xed19, CDB,     RXE,   Z,   f1_o, m2_64, 0, 0, cdb, 0)
+/* COMPARE IMMEDIATE */
+    C(0xc20d, CFI,     RIL_a, EI,  r1, i2, 0, 0, 0, cmps32)
+    C(0xc20c, CGFI,    RIL_a, EI,  r1, i2, 0, 0, 0, cmps64)
+/* COMPARE RELATIVE LONG */
+    C(0xc60d, CRL,     RIL_b, GIE, r1, mri2_32s, 0, 0, 0, cmps32)
+    C(0xc608, CGRL,    RIL_b, GIE, r1, mri2_64, 0, 0, 0, cmps64)
+    C(0xc60c, CGFRL,   RIL_b, GIE, r1, mri2_32s, 0, 0, 0, cmps64)
+/* COMPARE HALFWORD */
+    C(0x4900, CH,      RX_a,  Z,   r1_o, m2_16s, 0, 0, 0, cmps32)
+    C(0xe379, CHY,     RXY_a, LD,  r1_o, m2_16s, 0, 0, 0, cmps32)
+    C(0xe334, CGH,     RXY_a, GIE, r1_o, m2_16s, 0, 0, 0, cmps64)
+/* COMPARE HALFWORD IMMEDIATE */
+    C(0xa70e, CHI,     RI_a,  Z,   r1_o, i2, 0, 0, 0, cmps32)
+    C(0xa70f, CGHI,    RI_a,  Z,   r1_o, i2, 0, 0, 0, cmps64)
+    C(0xe554, CHHSI,   SIL,   GIE, m1_16s, i2, 0, 0, 0, cmps64)
+    C(0xe55c, CHSI,    SIL,   GIE, m1_32s, i2, 0, 0, 0, cmps64)
+    C(0xe558, CGHSI,   SIL,   GIE, m1_64, i2, 0, 0, 0, cmps64)
+/* COMPARE HALFWORD RELATIVE LONG */
+    C(0xc605, CHRL,    RIL_a, GIE, r1_o, mri2_32s, 0, 0, 0, cmps32)
+    C(0xc604, CGHRL,   RIL_a, GIE, r1_o, mri2_64, 0, 0, 0, cmps64)
+
+/* COMPARE LOGICAL */
+    C(0x1500, CLR,     RR_a,  Z,   r1, r2, 0, 0, 0, cmpu32)
+    C(0x5500, CL,      RX_a,  Z,   r1, m2_32s, 0, 0, 0, cmpu32)
+    C(0xe355, CLY,     RXY_a, LD,  r1, m2_32s, 0, 0, 0, cmpu32)
+    C(0xb921, CLGR,    RRE,   Z,   r1, r2, 0, 0, 0, cmpu64)
+    C(0xb931, CLGFR,   RRE,   Z,   r1, r2_32u, 0, 0, 0, cmpu64)
+    C(0xe321, CLG,     RXY_a, Z,   r1, m2_64, 0, 0, 0, cmpu64)
+    C(0xe331, CLGF,    RXY_a, Z,   r1, m2_32u, 0, 0, 0, cmpu64)
+    C(0xd500, CLC,     SS_a,  Z,   la1, a2, 0, 0, clc, 0)
+/* COMPARE LOGICAL IMMEDIATE */
+    C(0xc20f, CLFI,    RIL_a, EI,  r1, i2, 0, 0, 0, cmpu32)
+    C(0xc20e, CLGFI,   RIL_a, EI,  r1, i2_32u, 0, 0, 0, cmpu64)
+    C(0x9500, CLI,     SI,    Z,   m1_8u, i2_8u, 0, 0, 0, cmpu64)
+    C(0xeb55, CLIY,    SIY,   LD,  m1_8u, i2_8u, 0, 0, 0, cmpu64)
+    C(0xe555, CLHHSI,  SIL,   GIE, m1_16u, i2_16u, 0, 0, 0, cmpu64)
+    C(0xe55d, CLFHSI,  SIL,   GIE, m1_32u, i2_16u, 0, 0, 0, cmpu64)
+    C(0xe559, CLGHSI,  SIL,   GIE, m1_64, i2_16u, 0, 0, 0, cmpu64)
+/* COMPARE LOGICAL RELATIVE LONG */
+    C(0xc60f, CLRL,    RIL_b, GIE, r1_o, mri2_32u, 0, 0, 0, cmpu32)
+    C(0xc60a, CLGRL,   RIL_b, GIE, r1_o, mri2_64, 0, 0, 0, cmpu64)
+    C(0xc60e, CLGFRL,  RIL_b, GIE, r1_o, mri2_32u, 0, 0, 0, cmpu64)
+    C(0xc607, CLHRL,   RIL_b, GIE, r1_o, mri2_16u, 0, 0, 0, cmpu32)
+    C(0xc606, CLGHRL,  RIL_b, GIE, r1_o, mri2_16u, 0, 0, 0, cmpu64)
+/* COMPARE LOGICAL LONG EXTENDED */
+    C(0xa900, CLCLE,   RS_a,  Z,   0, a2, 0, 0, clcle, 0)
+/* COMPARE LOGICAL CHARACTERS UNDER MASK */
+    C(0xbd00, CLM,     RS_b,  Z,   r1_o, a2, 0, 0, clm, 0)
+    C(0xeb21, CLMY,    RSY_b, LD,  r1_o, a2, 0, 0, clm, 0)
+    C(0xeb20, CLMH,    RSY_b, Z,   r1_sr32, a2, 0, 0, clm, 0)
+/* COMPARE LOGICAL STRING */
+    C(0xb25d, CLST,    RRE,   Z,   r1_o, r2_o, 0, 0, clst, 0)
+
+/* COMPARE AND BRANCH */
+    D(0xecf6, CRB,     RRS,   GIE, r1_32s, r2_32s, 0, 0, cj, 0, 0)
+    D(0xece4, CGRB,    RRS,   GIE, r1_o, r2_o, 0, 0, cj, 0, 0)
+    D(0xec76, CRJ,     RIE_b, GIE, r1_32s, r2_32s, 0, 0, cj, 0, 0)
+    D(0xec64, CGRJ,    RIE_b, GIE, r1_o, r2_o, 0, 0, cj, 0, 0)
+    D(0xecfe, CIB,     RIS,   GIE, r1_32s, i2, 0, 0, cj, 0, 0)
+    D(0xecfc, CGIB,    RIS,   GIE, r1_o, i2, 0, 0, cj, 0, 0)
+    D(0xec7e, CIJ,     RIE_c, GIE, r1_32s, i2, 0, 0, cj, 0, 0)
+    D(0xec7c, CGIJ,    RIE_c, GIE, r1_o, i2, 0, 0, cj, 0, 0)
+/* COMPARE LOGICAL AND BRANCH */
+    D(0xecf7, CLRB,    RRS,   GIE, r1_32u, r2_32u, 0, 0, cj, 0, 1)
+    D(0xece5, CLGRB,   RRS,   GIE, r1_o, r2_o, 0, 0, cj, 0, 1)
+    D(0xec77, CLRJ,    RIE_b, GIE, r1_32u, r2_32u, 0, 0, cj, 0, 1)
+    D(0xec65, CLGRJ,   RIE_b, GIE, r1_o, r2_o, 0, 0, cj, 0, 1)
+    D(0xecff, CLIB,    RIS,   GIE, r1_32u, i2_8u, 0, 0, cj, 0, 1)
+    D(0xecfd, CLGIB,   RIS,   GIE, r1_o, i2_8u, 0, 0, cj, 0, 1)
+    D(0xec7f, CLIJ,    RIE_c, GIE, r1_32u, i2_8u, 0, 0, cj, 0, 1)
+    D(0xec7d, CLGIJ,   RIE_c, GIE, r1_o, i2_8u, 0, 0, cj, 0, 1)
+
+/* COMPARE AND SWAP */
+    D(0xba00, CS,      RS_a,  Z,   r3_32u, r1_32u, new, r1_32, cs, 0, 0)
+    D(0xeb14, CSY,     RSY_a, LD,  r3_32u, r1_32u, new, r1_32, cs, 0, 0)
+    D(0xeb30, CSG,     RSY_a, Z,   r3_o, r1_o, new, r1, cs, 0, 1)
+/* COMPARE DOUBLE AND SWAP */
+    D(0xbb00, CDS,     RS_a,  Z,   r3_D32, r1_D32, new, r1_D32, cs, 0, 1)
+    D(0xeb31, CDSY,    RSY_a, LD,  r3_D32, r1_D32, new, r1_D32, cs, 0, 1)
+    C(0xeb3e, CDSG,    RSY_a, Z,   0, 0, 0, 0, cdsg, 0)
+
+/* COMPARE AND TRAP */
+    D(0xb972, CRT,     RRF_c, GIE, r1_32s, r2_32s, 0, 0, ct, 0, 0)
+    D(0xb960, CGRT,    RRF_c, GIE, r1_o, r2_o, 0, 0, ct, 0, 0)
+    D(0xec72, CIT,     RIE_a, GIE, r1_32s, i2, 0, 0, ct, 0, 0)
+    D(0xec70, CGIT,    RIE_a, GIE, r1_o, i2, 0, 0, ct, 0, 0)
+/* COMPARE LOGICAL AND TRAP */
+    D(0xb973, CLRT,    RRF_c, GIE, r1_32u, r2_32u, 0, 0, ct, 0, 1)
+    D(0xb961, CLGRT,   RRF_c, GIE, r1_o, r2_o, 0, 0, ct, 0, 1)
+    D(0xec73, CLFIT,   RIE_a, GIE, r1_32u, i2_32u, 0, 0, ct, 0, 1)
+    D(0xec71, CLGIT,   RIE_a, GIE, r1_o, i2_32u, 0, 0, ct, 0, 0)
+
+/* CONVERT TO DECIMAL */
+    C(0x4e00, CVD,     RX_a,  Z,   r1_o, a2, 0, 0, cvd, 0)
+    C(0xe326, CVDY,    RXY_a, LD,  r1_o, a2, 0, 0, cvd, 0)
+/* CONVERT TO FIXED */
+    C(0xb398, CFEBR,   RRF_e, Z,   0, e2, new, r1_32, cfeb, 0)
+    C(0xb399, CFDBR,   RRF_e, Z,   0, f2_o, new, r1_32, cfdb, 0)
+    C(0xb39a, CFXBR,   RRF_e, Z,   0, x2_o, new, r1_32, cfxb, 0)
+    C(0xb3a8, CGEBR,   RRF_e, Z,   0, e2, r1, 0, cgeb, 0)
+    C(0xb3a9, CGDBR,   RRF_e, Z,   0, f2_o, r1, 0, cgdb, 0)
+    C(0xb3aa, CGXBR,   RRF_e, Z,   0, x2_o, r1, 0, cgxb, 0)
+/* CONVERT FROM FIXED */
+    C(0xb394, CEFBR,   RRF_e, Z,   0, r2_32s, new, e1, cegb, 0)
+    C(0xb395, CDFBR,   RRF_e, Z,   0, r2_32s, f1, 0, cdgb, 0)
+    C(0xb396, CXFBR,   RRF_e, Z,   0, r2_32s, x1, 0, cxgb, 0)
+    C(0xb3a4, CEGBR,   RRF_e, Z,   0, r2_o, new, e1, cegb, 0)
+    C(0xb3a5, CDGBR,   RRF_e, Z,   0, r2_o, f1, 0, cdgb, 0)
+    C(0xb3a6, CXGBR,   RRF_e, Z,   0, r2_o, x1, 0, cxgb, 0)
+/* CONVERT TO LOGICAL */
+    C(0xb39c, CLFEBR,  RRF_e, FPE, 0, e2, new, r1_32, clfeb, 0)
+    C(0xb39d, CLFDBR,  RRF_e, FPE, 0, f2_o, new, r1_32, clfdb, 0)
+    C(0xb39e, CLFXBR,  RRF_e, FPE, 0, x2_o, new, r1_32, clfxb, 0)
+    C(0xb3ac, CLGEBR,  RRF_e, FPE, 0, e2, r1, 0, clgeb, 0)
+    C(0xb3ad, CLGDBR,  RRF_e, FPE, 0, f2_o, r1, 0, clgdb, 0)
+    C(0xb3ae, CLGXBR,  RRF_e, FPE, 0, x2_o, r1, 0, clgxb, 0)
+/* CONVERT FROM LOGICAL */
+    C(0xb390, CELFBR,  RRF_e, FPE, 0, r2_32u, new, e1, celgb, 0)
+    C(0xb391, CDLFBR,  RRF_e, FPE, 0, r2_32u, f1, 0, cdlgb, 0)
+    C(0xb392, CXLFBR,  RRF_e, FPE, 0, r2_32u, x1, 0, cxlgb, 0)
+    C(0xb3a0, CELGBR,  RRF_e, FPE, 0, r2_o, new, e1, celgb, 0)
+    C(0xb3a1, CDLGBR,  RRF_e, FPE, 0, r2_o, f1, 0, cdlgb, 0)
+    C(0xb3a2, CXLGBR,  RRF_e, FPE, 0, r2_o, x1, 0, cxlgb, 0)
+
+/* DIVIDE */
+    C(0x1d00, DR,      RR_a,  Z,   r1_D32, r2_32s, new_P, r1_P32, divs32, 0)
+    C(0x5d00, D,       RX_a,  Z,   r1_D32, m2_32s, new_P, r1_P32, divs32, 0)
+    C(0xb30d, DEBR,    RRE,   Z,   e1, e2, new, e1, deb, 0)
+    C(0xb31d, DDBR,    RRE,   Z,   f1_o, f2_o, f1, 0, ddb, 0)
+    C(0xb34d, DXBR,    RRE,   Z,   0, x2_o, x1, 0, dxb, 0)
+    C(0xed0d, DEB,     RXE,   Z,   e1, m2_32u, new, e1, deb, 0)
+    C(0xed1d, DDB,     RXE,   Z,   f1_o, m2_64, f1, 0, ddb, 0)
+/* DIVIDE LOGICAL */
+    C(0xb997, DLR,     RRE,   Z,   r1_D32, r2_32u, new_P, r1_P32, divu32, 0)
+    C(0xe397, DL,      RXY_a, Z,   r1_D32, m2_32u, new_P, r1_P32, divu32, 0)
+    C(0xb987, DLGR,    RRE,   Z,   0, r2_o, r1_P, 0, divu64, 0)
+    C(0xe387, DLG,     RXY_a, Z,   0, m2_64, r1_P, 0, divu64, 0)
+/* DIVIDE SINGLE */
+    C(0xb90d, DSGR,    RRE,   Z,   r1p1, r2, r1_P, 0, divs64, 0)
+    C(0xb91d, DSGFR,   RRE,   Z,   r1p1, r2_32s, r1_P, 0, divs64, 0)
+    C(0xe30d, DSG,     RXY_a, Z,   r1p1, m2_64, r1_P, 0, divs64, 0)
+    C(0xe31d, DSGF,    RXY_a, Z,   r1p1, m2_32s, r1_P, 0, divs64, 0)
+
+/* EXCLUSIVE OR */
+    C(0x1700, XR,      RR_a,  Z,   r1, r2, new, r1_32, xor, nz32)
+    C(0xb9f7, XRK,     RRF_a, DO,  r2, r3, new, r1_32, xor, nz32)
+    C(0x5700, X,       RX_a,  Z,   r1, m2_32s, new, r1_32, xor, nz32)
+    C(0xe357, XY,      RXY_a, LD,  r1, m2_32s, new, r1_32, xor, nz32)
+    C(0xb982, XGR,     RRE,   Z,   r1, r2, r1, 0, xor, nz64)
+    C(0xb9e7, XGRK,    RRF_a, DO,  r2, r3, r1, 0, xor, nz64)
+    C(0xe382, XG,      RXY_a, Z,   r1, m2_64, r1, 0, xor, nz64)
+    C(0xd700, XC,      SS_a,  Z,   0, 0, 0, 0, xc, 0)
+/* EXCLUSIVE OR IMMEDIATE */
+    D(0xc006, XIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, xori, 0, 0x2020)
+    D(0xc007, XILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, xori, 0, 0x2000)
+    C(0x9700, XI,      SI,    Z,   m1_8u, i2_8u, new, m1_8, xor, nz64)
+    C(0xeb57, XIY,     SIY,   LD,  m1_8u, i2_8u, new, m1_8, xor, nz64)
+
+/* EXECUTE */
+    C(0x4400, EX,      RX_a,  Z,   r1_o, a2, 0, 0, ex, 0)
+/* EXECUTE RELATIVE LONG */
+    C(0xc600, EXRL,    RIL_b, EE,  r1_o, ri2, 0, 0, ex, 0)
+
+/* EXTRACT ACCESS */
+    C(0xb24f, EAR,     RRE,   Z,   0, 0, new, r1_32, ear, 0)
+/* EXTRACT FPC */
+    C(0xb38c, EFPC,    RRE,   Z,   0, 0, new, r1_32, efpc, 0)
+
+/* FIND LEFTMOST ONE */
+    C(0xb983, FLOGR,   RRE,   EI,  0, r2_o, r1_P, 0, flogr, 0)
+
+/* INSERT CHARACTER */
+    C(0x4300, IC,      RX_a,  Z,   0, m2_8u, 0, r1_8, mov2, 0)
+    C(0xe373, ICY,     RXY_a, LD,  0, m2_8u, 0, r1_8, mov2, 0)
+/* INSERT CHARACTERS UNDER MASK */
+    D(0xbf00, ICM,     RS_b,  Z,   0, a2, r1, 0, icm, 0, 0)
+    D(0xeb81, ICMY,    RSY_b, LD,  0, a2, r1, 0, icm, 0, 0)
+    D(0xeb80, ICMH,    RSY_b, Z,   0, a2, r1, 0, icm, 0, 32)
+/* INSERT IMMEDIATE */
+    D(0xc008, IIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, insi, 0, 0x2020)
+    D(0xc009, IILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, insi, 0, 0x2000)
+    D(0xa500, IIHH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1030)
+    D(0xa501, IIHL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1020)
+    D(0xa502, IILH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1010)
+    D(0xa503, IILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, insi, 0, 0x1000)
+/* INSERT PROGRAM MASK */
+    C(0xb222, IPM,     RRE,   Z,   0, 0, r1, 0, ipm, 0)
+
+/* LOAD */
+    C(0x1800, LR,      RR_a,  Z,   0, r2_o, 0, cond_r1r2_32, mov2, 0)
+    C(0x5800, L,       RX_a,  Z,   0, a2, new, r1_32, ld32s, 0)
+    C(0xe358, LY,      RXY_a, Z,   0, a2, new, r1_32, ld32s, 0)
+    C(0xb904, LGR,     RRE,   Z,   0, r2_o, 0, r1, mov2, 0)
+    C(0xb914, LGFR,    RRE,   Z,   0, r2_32s, 0, r1, mov2, 0)
+    C(0xe304, LG,      RXY_a, Z,   0, a2, r1, 0, ld64, 0)
+    C(0xe314, LGF,     RXY_a, Z,   0, a2, r1, 0, ld32s, 0)
+    C(0x2800, LDR,     RR_a,  Z,   0, f2_o, 0, f1, mov2, 0)
+    C(0x6800, LD,      RX_a,  Z,   0, m2_64, 0, f1, mov2, 0)
+    C(0xed65, LDY,     RXY_a, LD,  0, m2_64, 0, f1, mov2, 0)
+    C(0x3800, LER,     RR_a,  Z,   0, e2, 0, cond_e1e2, mov2, 0)
+    C(0x7800, LE,      RX_a,  Z,   0, m2_32u, 0, e1, mov2, 0)
+    C(0xed64, LEY,     RXY_a, LD,  0, m2_32u, 0, e1, mov2, 0)
+    C(0xb365, LXR,     RRE,   Z,   0, x2_o, 0, x1, movx, 0)
+/* LOAD IMMEDIATE */
+    C(0xc001, LGFI,    RIL_a, EI,  0, i2, 0, r1, mov2, 0)
+/* LOAD RELATIVE LONG */
+    C(0xc40d, LRL,     RIL_b, GIE, 0, ri2, new, r1_32, ld32s, 0)
+    C(0xc408, LGRL,    RIL_b, GIE, 0, ri2, r1, 0, ld64, 0)
+    C(0xc40c, LGFRL,   RIL_b, GIE, 0, ri2, r1, 0, ld32s, 0)
+/* LOAD ADDRESS */
+    C(0x4100, LA,      RX_a,  Z,   0, a2, 0, r1, mov2, 0)
+    C(0xe371, LAY,     RXY_a, LD,  0, a2, 0, r1, mov2, 0)
+/* LOAD ADDRESS RELATIVE LONG */
+    C(0xc000, LARL,    RIL_b, Z,   0, ri2, 0, r1, mov2, 0)
+/* LOAD AND TEST */
+    C(0x1200, LTR,     RR_a,  Z,   0, r2_o, 0, cond_r1r2_32, mov2, s32)
+    C(0xb902, LTGR,    RRE,   Z,   0, r2_o, 0, r1, mov2, s64)
+    C(0xb912, LTGFR,   RRE,   Z,   0, r2_32s, 0, r1, mov2, s64)
+    C(0xe312, LT,      RXY_a, EI,  0, a2, new, r1_32, ld32s, s64)
+    C(0xe302, LTG,     RXY_a, EI,  0, a2, r1, 0, ld64, s64)
+    C(0xe332, LTGF,    RXY_a, GIE, 0, a2, r1, 0, ld32s, s64)
+    C(0xb302, LTEBR,   RRE,   Z,   0, e2, 0, cond_e1e2, mov2, f32)
+    C(0xb312, LTDBR,   RRE,   Z,   0, f2_o, 0, f1, mov2, f64)
+    C(0xb342, LTXBR,   RRE,   Z,   0, x2_o, 0, x1, movx, f128)
+/* LOAD BYTE */
+    C(0xb926, LBR,     RRE,   EI,  0, r2_8s, 0, r1_32, mov2, 0)
+    C(0xb906, LGBR,    RRE,   EI,  0, r2_8s, 0, r1, mov2, 0)
+    C(0xe376, LB,      RXY_a, LD,  0, a2, new, r1_32, ld8s, 0)
+    C(0xe377, LGB,     RXY_a, LD,  0, a2, r1, 0, ld8s, 0)
+/* LOAD COMPLEMENT */
+    C(0x1300, LCR,     RR_a,  Z,   0, r2, new, r1_32, neg, neg32)
+    C(0xb903, LCGR,    RRE,   Z,   0, r2, r1, 0, neg, neg64)
+    C(0xb913, LCGFR,   RRE,   Z,   0, r2_32s, r1, 0, neg, neg64)
+    C(0xb303, LCEBR,   RRE,   Z,   0, e2, new, e1, negf32, f32)
+    C(0xb313, LCDBR,   RRE,   Z,   0, f2_o, f1, 0, negf64, f64)
+    C(0xb343, LCXBR,   RRE,   Z,   0, x2_o, x1, 0, negf128, f128)
+    C(0xb373, LCDFR,   RRE,   FPSSH, 0, f2_o, f1, 0, negf64, 0)
+/* LOAD HALFWORD */
+    C(0xb927, LHR,     RRE,   EI,  0, r2_16s, 0, r1_32, mov2, 0)
+    C(0xb907, LGHR,    RRE,   EI,  0, r2_16s, 0, r1, mov2, 0)
+    C(0x4800, LH,      RX_a,  Z,   0, a2, new, r1_32, ld16s, 0)
+    C(0xe378, LHY,     RXY_a, LD,  0, a2, new, r1_32, ld16s, 0)
+    C(0xe315, LGH,     RXY_a, Z,   0, a2, r1, 0, ld16s, 0)
+/* LOAD HALFWORD IMMEDIATE */
+    C(0xa708, LHI,     RI_a,  Z,   0, i2, 0, r1_32, mov2, 0)
+    C(0xa709, LGHI,    RI_a,  Z,   0, i2, 0, r1, mov2, 0)
+/* LOAD HALFWORD RELATIVE LONG */
+    C(0xc405, LHRL,    RIL_b, GIE, 0, ri2, new, r1_32, ld16s, 0)
+    C(0xc404, LGHRL,   RIL_b, GIE, 0, ri2, r1, 0, ld16s, 0)
+/* LOAD LOGICAL */
+    C(0xb916, LLGFR,   RRE,   Z,   0, r2_32u, 0, r1, mov2, 0)
+    C(0xe316, LLGF,    RXY_a, Z,   0, a2, r1, 0, ld32u, 0)
+/* LOAD LOGICAL RELATIVE LONG */
+    C(0xc40e, LLGFRL,  RIL_b, GIE, 0, ri2, r1, 0, ld32u, 0)
+/* LOAD LOGICAL CHARACTER */
+    C(0xb994, LLCR,    RRE,   EI,  0, r2_8u, 0, r1_32, mov2, 0)
+    C(0xb984, LLGCR,   RRE,   EI,  0, r2_8u, 0, r1, mov2, 0)
+    C(0xe394, LLC,     RXY_a, EI,  0, a2, new, r1_32, ld8u, 0)
+    C(0xe390, LLGC,    RXY_a, Z,   0, a2, r1, 0, ld8u, 0)
+/* LOAD LOGICAL HALFWORD */
+    C(0xb995, LLHR,    RRE,   EI,  0, r2_16u, 0, r1_32, mov2, 0)
+    C(0xb985, LLGHR,   RRE,   EI,  0, r2_16u, 0, r1, mov2, 0)
+    C(0xe395, LLH,     RXY_a, EI,  0, a2, new, r1_32, ld16u, 0)
+    C(0xe391, LLGH,    RXY_a, Z,   0, a2, r1, 0, ld16u, 0)
+/* LOAD LOGICAL HALFWORD RELATIVE LONG */
+    C(0xc402, LLHRL,   RIL_b, GIE, 0, ri2, new, r1_32, ld16u, 0)
+    C(0xc406, LLGHRL,  RIL_b, GIE, 0, ri2, r1, 0, ld16u, 0)
+/* LOAD LOGICAL IMMEDATE */
+    D(0xc00e, LLIHF,   RIL_a, EI, 0, i2_32u_shl, 0, r1, mov2, 0, 32)
+    D(0xc00f, LLILF,   RIL_a, EI, 0, i2_32u_shl, 0, r1, mov2, 0, 0)
+    D(0xa50c, LLIHH,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 48)
+    D(0xa50d, LLIHL,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 32)
+    D(0xa50e, LLILH,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 16)
+    D(0xa50f, LLILL,   RI_a,  Z,  0, i2_16u_shl, 0, r1, mov2, 0, 0)
+/* LOAD LOGICAL THIRTY ONE BITS */
+    C(0xb917, LLGTR,   RRE,   Z,  0, r2_o, r1, 0, llgt, 0)
+    C(0xe317, LLGT,    RXY_a, Z,  0, m2_32u, r1, 0, llgt, 0)
+/* LOAD FPR FROM GR */
+    C(0xb3c1, LDGR,    RRE,   FPRGR, 0, r2_o, 0, f1, mov2, 0)
+/* LOAD GR FROM FPR */
+    C(0xb3cd, LGDR,    RRE,   FPRGR, 0, f2_o, 0, r1, mov2, 0)
+/* LOAD NEGATIVE */
+    C(0x1100, LNR,     RR_a,  Z,   0, r2_32s, new, r1_32, nabs, nabs32)
+    C(0xb901, LNGR,    RRE,   Z,   0, r2, r1, 0, nabs, nabs64)
+    C(0xb911, LNGFR,   RRE,   Z,   0, r2_32s, r1, 0, nabs, nabs64)
+    C(0xb301, LNEBR,   RRE,   Z,   0, e2, new, e1, nabsf32, f32)
+    C(0xb311, LNDBR,   RRE,   Z,   0, f2_o, f1, 0, nabsf64, f64)
+    C(0xb341, LNXBR,   RRE,   Z,   0, x2_o, x1, 0, nabsf128, f128)
+/* LOAD ON CONDITION */
+    C(0xb9f2, LOCR,    RRF_c, LOC, r1, r2, new, r1_32, loc, 0)
+    C(0xb9e2, LOCGR,   RRF_c, LOC, r1, r2, r1, 0, loc, 0)
+    C(0xebf2, LOC,     RSY_b, LOC, r1, m2_32u, new, r1_32, loc, 0)
+    C(0xebe2, LOCG,    RSY_b, LOC, r1, m2_64, r1, 0, loc, 0)
+/* LOAD POSITIVE */
+    C(0x1000, LPR,     RR_a,  Z,   0, r2_32s, new, r1_32, abs, abs32)
+    C(0xb900, LPGR,    RRE,   Z,   0, r2, r1, 0, abs, abs64)
+    C(0xb910, LPGFR,   RRE,   Z,   0, r2_32s, r1, 0, abs, abs64)
+    C(0xb300, LPEBR,   RRE,   Z,   0, e2, new, e1, absf32, f32)
+    C(0xb310, LPDBR,   RRE,   Z,   0, f2_o, f1, 0, absf64, f64)
+    C(0xb340, LPXBR,   RRE,   Z,   0, x2_o, x1, 0, absf128, f128)
+/* LOAD REVERSED */
+    C(0xb91f, LRVR,    RRE,   Z,   0, r2_32u, new, r1_32, rev32, 0)
+    C(0xb90f, LRVGR,   RRE,   Z,   0, r2_o, r1, 0, rev64, 0)
+    C(0xe31f, LRVH,    RXY_a, Z,   0, m2_16u, new, r1_16, rev16, 0)
+    C(0xe31e, LRV,     RXY_a, Z,   0, m2_32u, new, r1_32, rev32, 0)
+    C(0xe30f, LRVG,    RXY_a, Z,   0, m2_64, r1, 0, rev64, 0)
+/* LOAD ZERO */
+    C(0xb374, LZER,    RRE,   Z,   0, 0, 0, e1, zero, 0)
+    C(0xb375, LZDR,    RRE,   Z,   0, 0, 0, f1, zero, 0)
+    C(0xb376, LZXR,    RRE,   Z,   0, 0, 0, x1, zero2, 0)
+
+/* LOAD FPC */
+    C(0xb29d, LFPC,    S,     Z,   0, m2_32u, 0, 0, sfpc, 0)
+/* LOAD FPC AND SIGNAL */
+    C(0xb2bd, LFAS,    S,     IEEEE_SIM, 0, m2_32u, 0, 0, sfas, 0)
+
+/* LOAD LENGTHENED */
+    C(0xb304, LDEBR,   RRE,   Z,   0, e2, f1, 0, ldeb, 0)
+    C(0xb305, LXDBR,   RRE,   Z,   0, f2_o, x1, 0, lxdb, 0)
+    C(0xb306, LXEBR,   RRE,   Z,   0, e2, x1, 0, lxeb, 0)
+    C(0xed04, LDEB,    RXE,   Z,   0, m2_32u, f1, 0, ldeb, 0)
+    C(0xed05, LXDB,    RXE,   Z,   0, m2_64, x1, 0, lxdb, 0)
+    C(0xed06, LXEB,    RXE,   Z,   0, m2_32u, x1, 0, lxeb, 0)
+/* LOAD ROUNDED */
+    C(0xb344, LEDBR,   RRE,   Z,   0, f2_o, new, e1, ledb, 0)
+    C(0xb345, LDXBR,   RRE,   Z,   0, x2_o, f1, 0, ldxb, 0)
+    C(0xb346, LEXBR,   RRE,   Z,   0, x2_o, new, e1, lexb, 0)
+
+/* LOAD MULTIPLE */
+    C(0x9800, LM,      RS_a,  Z,   0, a2, 0, 0, lm32, 0)
+    C(0xeb98, LMY,     RSY_a, LD,  0, a2, 0, 0, lm32, 0)
+    C(0xeb04, LMG,     RSY_a, Z,   0, a2, 0, 0, lm64, 0)
+/* LOAD MULTIPLE HIGH */
+    C(0xeb96, LMH,     RSY_a, Z,   0, a2, 0, 0, lmh, 0)
+/* LOAD ACCESS MULTIPLE */
+    C(0x9a00, LAM,     RS_a,  Z,   0, a2, 0, 0, lam, 0)
+    C(0xeb9a, LAMY,    RSY_a, LD,  0, a2, 0, 0, lam, 0)
+
+/* MOVE */
+    C(0xd200, MVC,     SS_a,  Z,   la1, a2, 0, 0, mvc, 0)
+    C(0xe544, MVHHI,   SIL,   GIE, la1, i2, 0, m1_16, mov2, 0)
+    C(0xe54c, MVHI,    SIL,   GIE, la1, i2, 0, m1_32, mov2, 0)
+    C(0xe548, MVGHI,   SIL,   GIE, la1, i2, 0, m1_64, mov2, 0)
+    C(0x9200, MVI,     SI,    Z,   la1, i2, 0, m1_8, mov2, 0)
+    C(0xeb52, MVIY,    SIY,   LD,  la1, i2, 0, m1_8, mov2, 0)
+/* MOVE LONG */
+    C(0x0e00, MVCL,    RR_a,  Z,   0, 0, 0, 0, mvcl, 0)
+/* MOVE LONG EXTENDED */
+    C(0xa800, MVCLE,   RS_a,  Z,   0, a2, 0, 0, mvcle, 0)
+/* MOVE PAGE */
+    C(0xb254, MVPG,    RRE,   Z,   r1_o, r2_o, 0, 0, mvpg, 0)
+/* MOVE STRING */
+    C(0xb255, MVST,    RRE,   Z,   r1_o, r2_o, 0, 0, mvst, 0)
+
+/* MULTIPLY */
+    C(0x1c00, MR,      RR_a,  Z,   r1p1_32s, r2_32s, new, r1_D32, mul, 0)
+    C(0x5c00, M,       RX_a,  Z,   r1p1_32s, m2_32s, new, r1_D32, mul, 0)
+    C(0xe35c, MFY,     RXY_a, GIE, r1p1_32s, m2_32s, new, r1_D32, mul, 0)
+    C(0xb317, MEEBR,   RRE,   Z,   e1, e2, new, e1, meeb, 0)
+    C(0xb31c, MDBR,    RRE,   Z,   f1_o, f2_o, f1, 0, mdb, 0)
+    C(0xb34c, MXBR,    RRE,   Z,   0, x2_o, x1, 0, mxb, 0)
+    C(0xb30c, MDEBR,   RRE,   Z,   f1_o, e2, f1, 0, mdeb, 0)
+    C(0xb307, MXDBR,   RRE,   Z,   0, f2_o, x1, 0, mxdb, 0)
+    C(0xed17, MEEB,    RXE,   Z,   e1, m2_32u, new, e1, meeb, 0)
+    C(0xed1c, MDB,     RXE,   Z,   f1_o, m2_64, f1, 0, mdb, 0)
+    C(0xed0c, MDEB,    RXE,   Z,   f1_o, m2_32u, f1, 0, mdeb, 0)
+    C(0xed07, MXDB,    RXE,   Z,   0, m2_64, x1, 0, mxdb, 0)
+/* MULTIPLY HALFWORD */
+    C(0x4c00, MH,      RX_a,  Z,   r1_o, m2_16s, new, r1_32, mul, 0)
+    C(0xe37c, MHY,     RXY_a, GIE, r1_o, m2_16s, new, r1_32, mul, 0)
+/* MULTIPLY HALFWORD IMMEDIATE */
+    C(0xa70c, MHI,     RI_a,  Z,   r1_o, i2, new, r1_32, mul, 0)
+    C(0xa70d, MGHI,    RI_a,  Z,   r1_o, i2, r1, 0, mul, 0)
+/* MULTIPLY LOGICAL */
+    C(0xb996, MLR,     RRE,   Z,   r1p1_32u, r2_32u, new, r1_D32, mul, 0)
+    C(0xe396, ML,      RXY_a, Z,   r1p1_32u, m2_32u, new, r1_D32, mul, 0)
+    C(0xb986, MLGR,    RRE,   Z,   r1p1, r2_o, r1_P, 0, mul128, 0)
+    C(0xe386, MLG,     RXY_a, Z,   r1p1, m2_64, r1_P, 0, mul128, 0)
+/* MULTIPLY SINGLE */
+    C(0xb252, MSR,     RRE,   Z,   r1_o, r2_o, new, r1_32, mul, 0)
+    C(0x7100, MS,      RX_a,  Z,   r1_o, m2_32s, new, r1_32, mul, 0)
+    C(0xe351, MSY,     RXY_a, LD,  r1_o, m2_32s, new, r1_32, mul, 0)
+    C(0xb90c, MSGR,    RRE,   Z,   r1_o, r2_o, r1, 0, mul, 0)
+    C(0xb91c, MSGFR,   RRE,   Z,   r1_o, r2_32s, r1, 0, mul, 0)
+    C(0xe30c, MSG,     RXY_a, Z,   r1_o, m2_64, r1, 0, mul, 0)
+    C(0xe31c, MSGF,    RXY_a, Z,   r1_o, m2_32s, r1, 0, mul, 0)
+/* MULTIPLY SINGLE IMMEDIATE */
+    C(0xc201, MSFI,    RIL_a, GIE, r1_o, i2, new, r1_32, mul, 0)
+    C(0xc200, MSGFI,   RIL_a, GIE, r1_o, i2, r1, 0, mul, 0)
+
+/* MULTIPLY AND ADD */
+    C(0xb30e, MAEBR,   RRD,   Z,   e1, e2, new, e1, maeb, 0)
+    C(0xb31e, MADBR,   RRD,   Z,   f1_o, f2_o, f1, 0, madb, 0)
+    C(0xed0e, MAEB,    RXF,   Z,   e1, m2_32u, new, e1, maeb, 0)
+    C(0xed1e, MADB,    RXF,   Z,   f1_o, m2_64, f1, 0, madb, 0)
+/* MULTIPLY AND SUBTRACT */
+    C(0xb30f, MSEBR,   RRD,   Z,   e1, e2, new, e1, mseb, 0)
+    C(0xb31f, MSDBR,   RRD,   Z,   f1_o, f2_o, f1, 0, msdb, 0)
+    C(0xed0f, MSEB,    RXF,   Z,   e1, m2_32u, new, e1, mseb, 0)
+    C(0xed1f, MSDB,    RXF,   Z,   f1_o, m2_64, f1, 0, msdb, 0)
+
+/* OR */
+    C(0x1600, OR,      RR_a,  Z,   r1, r2, new, r1_32, or, nz32)
+    C(0xb9f6, ORK,     RRF_a, DO,  r2, r3, new, r1_32, or, nz32)
+    C(0x5600, O,       RX_a,  Z,   r1, m2_32s, new, r1_32, or, nz32)
+    C(0xe356, OY,      RXY_a, LD,  r1, m2_32s, new, r1_32, or, nz32)
+    C(0xb981, OGR,     RRE,   Z,   r1, r2, r1, 0, or, nz64)
+    C(0xb9e6, OGRK,    RRF_a, DO,  r2, r3, r1, 0, or, nz64)
+    C(0xe381, OG,      RXY_a, Z,   r1, m2_64, r1, 0, or, nz64)
+    C(0xd600, OC,      SS_a,  Z,   la1, a2, 0, 0, oc, 0)
+/* OR IMMEDIATE */
+    D(0xc00c, OIHF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, ori, 0, 0x2020)
+    D(0xc00d, OILF,    RIL_a, EI,  r1_o, i2_32u, r1, 0, ori, 0, 0x2000)
+    D(0xa508, OIHH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1030)
+    D(0xa509, OIHL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1020)
+    D(0xa50a, OILH,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1010)
+    D(0xa50b, OILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1000)
+    C(0x9600, OI,      SI,    Z,   m1_8u, i2_8u, new, m1_8, or, nz64)
+    C(0xeb56, OIY,     SIY,   LD,  m1_8u, i2_8u, new, m1_8, or, nz64)
+
+/* PREFETCH */
+    /* Implemented as nops of course.  */
+    C(0xe336, PFD,     RXY_b, GIE, 0, 0, 0, 0, 0, 0)
+    C(0xc602, PFDRL,   RIL_c, GIE, 0, 0, 0, 0, 0, 0)
+
+/* POPULATION COUNT */
+    C(0xb9e1, POPCNT,  RRE,   PC,  0, r2_o, r1, 0, popcnt, nz64)
+
+/* ROTATE LEFT SINGLE LOGICAL */
+    C(0xeb1d, RLL,     RSY_a, Z,   r3_o, sh32, new, r1_32, rll32, 0)
+    C(0xeb1c, RLLG,    RSY_a, Z,   r3_o, sh64, r1, 0, rll64, 0)
+
+/* ROTATE THEN INSERT SELECTED BITS */
+    C(0xec55, RISBG,   RIE_f, GIE, 0, r2, r1, 0, risbg, s64)
+    C(0xec5d, RISBHG,  RIE_f, GIE, 0, r2, r1, 0, risbg, 0)
+    C(0xec51, RISBLG,  RIE_f, GIE, 0, r2, r1, 0, risbg, 0)
+/* ROTATE_THEN <OP> SELECTED BITS */
+    C(0xec54, RNSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
+    C(0xec56, ROSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
+    C(0xec57, RXSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
+
+/* SEARCH STRING */
+    C(0xb25e, SRST,    RRE,   Z,   r1_o, r2_o, 0, 0, srst, 0)
+
+/* SET ACCESS */
+    C(0xb24e, SAR,     RRE,   Z,   0, r2_o, 0, 0, sar, 0)
+/* SET FPC */
+    C(0xb384, SFPC,    RRE,   Z,   0, r1_o, 0, 0, sfpc, 0)
+/* SET FPC AND SIGNAL */
+    C(0xb385, SFASR,   RRE,   IEEEE_SIM, 0, r1_o, 0, 0, sfas, 0)
+/* SET BFP ROUNDING MODE */
+    C(0xb299, SRNM,    S,     Z,   0, 0, 0, 0, srnm, 0)
+    C(0xb2b8, SRNMB,   S,     FPE, 0, 0, 0, 0, srnm, 0)
+/* SET DFP ROUNDING MODE */
+    C(0xb2b9, SRNMT,   S,     DFP, 0, 0, 0, 0, srnm, 0)
+
+/* SHIFT LEFT SINGLE */
+    D(0x8b00, SLA,     RS_a,  Z,   r1, sh32, new, r1_32, sla, 0, 31)
+    D(0xebdd, SLAK,    RSY_a, DO,  r3, sh32, new, r1_32, sla, 0, 31)
+    D(0xeb0b, SLAG,    RSY_a, Z,   r3, sh64, r1, 0, sla, 0, 63)
+/* SHIFT LEFT SINGLE LOGICAL */
+    C(0x8900, SLL,     RS_a,  Z,   r1_o, sh32, new, r1_32, sll, 0)
+    C(0xebdf, SLLK,    RSY_a, DO,  r3_o, sh32, new, r1_32, sll, 0)
+    C(0xeb0d, SLLG,    RSY_a, Z,   r3_o, sh64, r1, 0, sll, 0)
+/* SHIFT RIGHT SINGLE */
+    C(0x8a00, SRA,     RS_a,  Z,   r1_32s, sh32, new, r1_32, sra, s32)
+    C(0xebdc, SRAK,    RSY_a, DO,  r3_32s, sh32, new, r1_32, sra, s32)
+    C(0xeb0a, SRAG,    RSY_a, Z,   r3_o, sh64, r1, 0, sra, s64)
+/* SHIFT RIGHT SINGLE LOGICAL */
+    C(0x8800, SRL,     RS_a,  Z,   r1_32u, sh32, new, r1_32, srl, 0)
+    C(0xebde, SRLK,    RSY_a, DO,  r3_32u, sh32, new, r1_32, srl, 0)
+    C(0xeb0c, SRLG,    RSY_a, Z,   r3_o, sh64, r1, 0, srl, 0)
+/* SHIFT LEFT DOUBLE */
+    D(0x8f00, SLDA,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, sla, 0, 31)
+/* SHIFT LEFT DOUBLE LOGICAL */
+    C(0x8d00, SLDL,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, sll, 0)
+/* SHIFT RIGHT DOUBLE */
+    C(0x8e00, SRDA,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, sra, s64)
+/* SHIFT RIGHT DOUBLE LOGICAL */
+    C(0x8c00, SRDL,    RS_a,  Z,   r1_D32, sh64, new, r1_D32, srl, 0)
+
+/* SQUARE ROOT */
+    C(0xb314, SQEBR,   RRE,   Z,   0, e2, new, e1, sqeb, 0)
+    C(0xb315, SQDBR,   RRE,   Z,   0, f2_o, f1, 0, sqdb, 0)
+    C(0xb316, SQXBR,   RRE,   Z,   0, x2_o, x1, 0, sqxb, 0)
+    C(0xed14, SQEB,    RXE,   Z,   0, m2_32u, new, e1, sqeb, 0)
+    C(0xed15, SQDB,    RXE,   Z,   0, m2_64, f1, 0, sqdb, 0)
+
+/* STORE */
+    C(0x5000, ST,      RX_a,  Z,   r1_o, a2, 0, 0, st32, 0)
+    C(0xe350, STY,     RXY_a, LD,  r1_o, a2, 0, 0, st32, 0)
+    C(0xe324, STG,     RXY_a, Z,   r1_o, a2, 0, 0, st64, 0)
+    C(0x6000, STD,     RX_a,  Z,   f1_o, a2, 0, 0, st64, 0)
+    C(0xed67, STDY,    RXY_a, LD,  f1_o, a2, 0, 0, st64, 0)
+    C(0x7000, STE,     RX_a,  Z,   e1, a2, 0, 0, st32, 0)
+    C(0xed66, STEY,    RXY_a, LD,  e1, a2, 0, 0, st32, 0)
+/* STORE RELATIVE LONG */
+    C(0xc40f, STRL,    RIL_b, GIE, r1_o, ri2, 0, 0, st32, 0)
+    C(0xc40b, STGRL,   RIL_b, GIE, r1_o, ri2, 0, 0, st64, 0)
+/* STORE CHARACTER */
+    C(0x4200, STC,     RX_a,  Z,   r1_o, a2, 0, 0, st8, 0)
+    C(0xe372, STCY,    RXY_a, LD,  r1_o, a2, 0, 0, st8, 0)
+/* STORE CHARACTERS UNDER MASK */
+    D(0xbe00, STCM,    RS_b,  Z,   r1_o, a2, 0, 0, stcm, 0, 0)
+    D(0xeb2d, STCMY,   RSY_b, LD,  r1_o, a2, 0, 0, stcm, 0, 0)
+    D(0xeb2c, STCMH,   RSY_b, LD,  r1_o, a2, 0, 0, stcm, 0, 32)
+/* STORE HALFWORD */
+    C(0x4000, STH,     RX_a,  Z,   r1_o, a2, 0, 0, st16, 0)
+    C(0xe370, STHY,    RXY_a, LD,  r1_o, a2, 0, 0, st16, 0)
+/* STORE HALFWORD RELATIVE LONG */
+    C(0xc407, STHRL,   RIL_b, GIE, r1_o, ri2, 0, 0, st16, 0)
+/* STORE ON CONDITION */
+    D(0xebf3, STOC,    RSY_b, LOC, 0, 0, 0, 0, soc, 0, 0)
+    D(0xebe3, STOCG,   RSY_b, LOC, 0, 0, 0, 0, soc, 0, 1)
+/* STORE REVERSED */
+    C(0xe33f, STRVH,   RXY_a, Z,   la2, r1_16u, new, m1_16, rev16, 0)
+    C(0xe33e, STRV,    RXY_a, Z,   la2, r1_32u, new, m1_32, rev32, 0)
+    C(0xe32f, STRVG,   RXY_a, Z,   la2, r1_o, new, m1_64, rev64, 0)
+
+/* STORE FPC */
+    C(0xb29c, STFPC,   S,     Z,   0, a2, new, m2_32, efpc, 0)
+
+/* STORE MULTIPLE */
+    D(0x9000, STM,     RS_a,  Z,   0, a2, 0, 0, stm, 0, 4)
+    D(0xeb90, STMY,    RSY_a, LD,  0, a2, 0, 0, stm, 0, 4)
+    D(0xeb24, STMG,    RSY_a, Z,   0, a2, 0, 0, stm, 0, 8)
+/* STORE MULTIPLE HIGH */
+    C(0xeb26, STMH,    RSY_a, Z,   0, a2, 0, 0, stmh, 0)
+/* STORE ACCESS MULTIPLE */
+    C(0x9b00, STAM,    RS_a,  Z,   0, a2, 0, 0, stam, 0)
+    C(0xeb9b, STAMY,   RSY_a, LD,  0, a2, 0, 0, stam, 0)
+
+/* SUBTRACT */
+    C(0x1b00, SR,      RR_a,  Z,   r1, r2, new, r1_32, sub, subs32)
+    C(0xb9f9, SRK,     RRF_a, DO,  r2, r3, new, r1_32, sub, subs32)
+    C(0x5b00, S,       RX_a,  Z,   r1, m2_32s, new, r1_32, sub, subs32)
+    C(0xe35b, SY,      RXY_a, LD,  r1, m2_32s, new, r1_32, sub, subs32)
+    C(0xb909, SGR,     RRE,   Z,   r1, r2, r1, 0, sub, subs64)
+    C(0xb919, SGFR,    RRE,   Z,   r1, r2_32s, r1, 0, sub, subs64)
+    C(0xb9e9, SGRK,    RRF_a, DO,  r2, r3, r1, 0, sub, subs64)
+    C(0xe309, SG,      RXY_a, Z,   r1, m2_64, r1, 0, sub, subs64)
+    C(0xe319, SGF,     RXY_a, Z,   r1, m2_32s, r1, 0, sub, subs64)
+    C(0xb30b, SEBR,    RRE,   Z,   e1, e2, new, e1, seb, f32)
+    C(0xb31b, SDBR,    RRE,   Z,   f1_o, f2_o, f1, 0, sdb, f64)
+    C(0xb34b, SXBR,    RRE,   Z,   0, x2_o, x1, 0, sxb, f128)
+    C(0xed0b, SEB,     RXE,   Z,   e1, m2_32u, new, e1, seb, f32)
+    C(0xed1b, SDB,     RXE,   Z,   f1_o, m2_64, f1, 0, sdb, f64)
+/* SUBTRACT HALFWORD */
+    C(0x4b00, SH,      RX_a,  Z,   r1, m2_16s, new, r1_32, sub, subs32)
+    C(0xe37b, SHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, sub, subs32)
+/* SUBTRACT LOGICAL */
+    C(0x1f00, SLR,     RR_a,  Z,   r1, r2, new, r1_32, sub, subu32)
+    C(0xb9fb, SLRK,    RRF_a, DO,  r2, r3, new, r1_32, sub, subu32)
+    C(0x5f00, SL,      RX_a,  Z,   r1, m2_32u, new, r1_32, sub, subu32)
+    C(0xe35f, SLY,     RXY_a, LD,  r1, m2_32u, new, r1_32, sub, subu32)
+    C(0xb90b, SLGR,    RRE,   Z,   r1, r2, r1, 0, sub, subu64)
+    C(0xb91b, SLGFR,   RRE,   Z,   r1, r2_32u, r1, 0, sub, subu64)
+    C(0xb9eb, SLGRK,   RRF_a, DO,  r2, r3, r1, 0, sub, subu64)
+    C(0xe30b, SLG,     RXY_a, Z,   r1, m2_64, r1, 0, sub, subu64)
+    C(0xe31b, SLGF,    RXY_a, Z,   r1, m2_32u, r1, 0, sub, subu64)
+/* SUBTRACT LOGICAL IMMEDIATE */
+    C(0xc205, SLFI,    RIL_a, EI,  r1, i2_32u, new, r1_32, sub, subu32)
+    C(0xc204, SLGFI,   RIL_a, EI,  r1, i2_32u, r1, 0, sub, subu64)
+/* SUBTRACT LOGICAL WITH BORROW */
+    C(0xb999, SLBR,    RRE,   Z,   r1, r2, new, r1_32, subb, subb32)
+    C(0xb989, SLBGR,   RRE,   Z,   r1, r2, r1, 0, subb, subb64)
+    C(0xe399, SLB,     RXY_a, Z,   r1, m2_32u, new, r1_32, subb, subb32)
+    C(0xe389, SLBG,    RXY_a, Z,   r1, m2_64, r1, 0, subb, subb64)
+
+/* SUPERVISOR CALL */
+    C(0x0a00, SVC,     I,     Z,   0, 0, 0, 0, svc, 0)
+
+/* TEST DATA CLASS */
+    C(0xed10, TCEB,    RXE,   Z,   e1, a2, 0, 0, tceb, 0)
+    C(0xed11, TCDB,    RXE,   Z,   f1_o, a2, 0, 0, tcdb, 0)
+    C(0xed12, TCXB,    RXE,   Z,   x1_o, a2, 0, 0, tcxb, 0)
+
+/* TEST UNDER MASK */
+    C(0x9100, TM,      SI,    Z,   m1_8u, i2_8u, 0, 0, 0, tm32)
+    C(0xeb51, TMY,     SIY,   LD,  m1_8u, i2_8u, 0, 0, 0, tm32)
+    D(0xa702, TMHH,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 48)
+    D(0xa703, TMHL,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 32)
+    D(0xa700, TMLH,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 16)
+    D(0xa701, TMLL,    RI_a,  Z,   r1_o, i2_16u_shl, 0, 0, 0, tm64, 0)
+
+/* TRANSLATE */
+    C(0xdc00, TR,      SS_a,  Z,   la1, a2, 0, 0, tr, 0)
+
+/* UNPACK */
+    /* Really format SS_b, but we pack both lengths into one argument
+       for the helper call, so we might as well leave one 8-bit field.  */
+    C(0xf300, UNPK,    SS_a,  Z,   la1, a2, 0, 0, unpk, 0)
+
+#ifndef CONFIG_USER_ONLY
+/* COMPARE AND SWAP AND PURGE */
+    C(0xb250, CSP,     RRE,   Z,   0, ra2, 0, 0, csp, 0)
+/* DIAGNOSE (KVM hypercall) */
+    C(0x8300, DIAG,    RX_a,  Z,   0, 0, 0, 0, diag, 0)
+/* INSERT STORAGE KEY EXTENDED */
+    C(0xb229, ISKE,    RRE,   Z,   0, r2_o, new, r1_8, iske, 0)
+/* INVALIDATE PAGE TABLE ENTRY */
+    C(0xb221, IPTE,    RRF_a, Z,   r1_o, r2_o, 0, 0, ipte, 0)
+/* LOAD CONTROL */
+    C(0xb700, LCTL,    RS_a,  Z,   0, a2, 0, 0, lctl, 0)
+    C(0xeb2f, LCTLG,   RSY_a, Z,   0, a2, 0, 0, lctlg, 0)
+/* LOAD PSW */
+    C(0x8200, LPSW,    S,     Z,   0, a2, 0, 0, lpsw, 0)
+/* LOAD PSW EXTENDED */
+    C(0xb2b2, LPSWE,   S,     Z,   0, a2, 0, 0, lpswe, 0)
+/* LOAD REAL ADDRESS */
+    C(0xb100, LRA,     RX_a,  Z,   0, a2, r1, 0, lra, 0)
+    C(0xe313, LRAY,    RXY_a, LD,  0, a2, r1, 0, lra, 0)
+    C(0xe303, LRAG,    RXY_a, Z,   0, a2, r1, 0, lra, 0)
+/* MOVE TO PRIMARY */
+    C(0xda00, MVCP,    SS_d,  Z,   la1, a2, 0, 0, mvcp, 0)
+/* MOVE TO SECONDARY */
+    C(0xdb00, MVCS,    SS_d,  Z,   la1, a2, 0, 0, mvcs, 0)
+/* PURGE TLB */
+    C(0xb20d, PTLB,    S,     Z,   0, 0, 0, 0, ptlb, 0)
+/* RESET REFERENCE BIT EXTENDED */
+    C(0xb22a, RRBE,    RRE,   Z,   0, r2_o, 0, 0, rrbe, 0)
+/* SERVICE CALL LOGICAL PROCESSOR (PV hypercall) */
+    C(0xb220, SERVC,   RRE,   Z,   r1_o, r2_o, 0, 0, servc, 0)
+/* SET ADDRESSING MODE */
+    /* We only do 64-bit, so accept this as a no-op.
+       Let SAM24 and SAM31 signal illegal instruction.  */
+    C(0x010e, SAM64,   E,     Z,   0, 0, 0, 0, 0, 0)
+/* SET ADDRESS SPACE CONTROL FAST */
+    C(0xb279, SACF,    S,     Z,   0, a2, 0, 0, sacf, 0)
+/* SET CLOCK */
+    /* ??? Not implemented - is it necessary? */
+    C(0xb204, SCK,     S,     Z,   0, 0, 0, 0, 0, 0)
+/* SET CLOCK COMPARATOR */
+    C(0xb206, SCKC,    S,     Z,   0, m2_64, 0, 0, sckc, 0)
+/* SET CPU TIMER */
+    C(0xb208, SPT,     S,     Z,   0, m2_64, 0, 0, spt, 0)
+/* SET PREFIX */
+    C(0xb210, SPX,     S,     Z,   0, m2_32u, 0, 0, spx, 0)
+/* SET PSW KEY FROM ADDRESS */
+    C(0xb20a, SPKA,    S,     Z,   0, a2, 0, 0, spka, 0)
+/* SET STORAGE KEY EXTENDED */
+    C(0xb22b, SSKE,    RRF_c, Z,   r1_o, r2_o, 0, 0, sske, 0)
+/* SET SYSTEM MASK */
+    C(0x8000, SSM,     S,     Z,   0, m2_8u, 0, 0, ssm, 0)
+/* SIGNAL PROCESSOR */
+    C(0xae00, SIGP,    RS_a,  Z,   r3_o, a2, 0, 0, sigp, 0)
+/* STORE CLOCK */
+    C(0xb205, STCK,    S,     Z,   la2, 0, new, m1_64, stck, 0)
+    C(0xb27c, STCKF,   S,     Z,   la2, 0, new, m1_64, stck, 0)
+/* STORE CLOCK EXTENDED */
+    C(0xb278, STCKE,   S,     Z,   0, a2, 0, 0, stcke, 0)
+/* STORE CLOCK COMPARATOR */
+    C(0xb207, STCKC,   S,     Z,   la2, 0, new, m1_64, stckc, 0)
+/* STORE CONTROL */
+    C(0xb600, STCTL,   RS_a,  Z,   0, a2, 0, 0, stctl, 0)
+    C(0xeb25, STCTG,   RSY_a, Z,   0, a2, 0, 0, stctg, 0)
+/* STORE CPU ADDRESS */
+    C(0xb212, STAP,    S,     Z,   la2, 0, new, m1_16, stap, 0)
+/* STORE CPU ID */
+    C(0xb202, STIDP,   S,     Z,   la2, 0, new, m1_64, stidp, 0)
+/* STORE CPU TIMER */
+    C(0xb209, STPT,    S,     Z,   la2, 0, new, m1_64, stpt, 0)
+/* STORE FACILITY LIST */
+    C(0xb2b1, STFL,    S,     Z,   0, 0, 0, 0, stfl, 0)
+/* STORE PREFIX */
+    C(0xb211, STPX,    S,     Z,   la2, 0, new, m1_32, stpx, 0)
+/* STORE SYSTEM INFORMATION */
+    C(0xb27d, STSI,    S,     Z,   0, a2, 0, 0, stsi, 0)
+/* STORE THEN AND SYSTEM MASK */
+    C(0xac00, STNSM,   SI,    Z,   la1, 0, 0, 0, stnosm, 0)
+/* STORE THEN OR SYSTEM MASK */
+    C(0xad00, STOSM,   SI,    Z,   la1, 0, 0, 0, stnosm, 0)
+/* STORE USING REAL ADDRESS */
+    C(0xb246, STURA,   RRE,   Z,   r1_o, r2_o, 0, 0, stura, 0)
+/* TEST PROTECTION */
+    C(0xe501, TPROT,   SSE,   Z,   la1, a2, 0, 0, tprot, 0)
+
+/* I/O Instructions.  For each we simply indicate non-operation.  */
+    C(0xb276, XSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb230, CSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb231, HSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb232, MSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb23b, RCHP,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb238, RSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb233, SSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb234, STSCH,   S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb235, TSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    /* ??? Not listed in PoO ninth edition, but there's a linux driver that
+       uses it: "A CHSC subchannel is usually present on LPAR only."  */
+    C(0xb25f, CHSC,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+#endif /* CONFIG_USER_ONLY */
diff --git a/target-s390x/insn-format.def b/target-s390x/insn-format.def
new file mode 100644
index 0000000..0e898b9
--- /dev/null
+++ b/target-s390x/insn-format.def
@@ -0,0 +1,55 @@
+/* Description of s390 insn formats.  */
+/* NAME   F1,          F2... */
+F0(E)
+F1(I,     I(1, 8, 8))
+F2(RI_a,  R(1, 8),     I(2,16,16))
+F2(RI_b,  R(1, 8),     I(2,16,16))
+F2(RI_c,  M(1, 8),     I(2,16,16))
+F3(RIE_a, R(1, 8),     I(2,16,16),  M(3,32))
+F4(RIE_b, R(1, 8),     R(2,12),     M(3,32),   I(4,16,16))
+F4(RIE_c, R(1, 8),     I(2,32, 8),  M(3,12),   I(4,16,16))
+F3(RIE_d, R(1, 8),     I(2,16,16),  R(3,12))
+F3(RIE_e, R(1, 8),     I(2,16,16),  R(3,12))
+F5(RIE_f, R(1, 8),     R(2,12),     I(3,16,8), I(4,24,8),  I(5,32,8))
+F2(RIL_a, R(1, 8),     I(2,16,32))
+F2(RIL_b, R(1, 8),     I(2,16,32))
+F2(RIL_c, M(1, 8),     I(2,16,32))
+F4(RIS,   R(1, 8),     I(2,32, 8),  M(3,12),   BD(4,16,20))
+/* ??? The PoO does not call out subtypes _a and _b for RR, as it does
+   for e.g. RX.  Our checking requires this for e.g. BCR.  */
+F2(RR_a,  R(1, 8),     R(2,12))
+F2(RR_b,  M(1, 8),     R(2,12))
+F2(RRE,   R(1,24),     R(2,28))
+F3(RRD,   R(1,16),     R(2,28),     R(3,24))
+F4(RRF_a, R(1,24),     R(2,28),     R(3,16),   M(4,20))
+F4(RRF_b, R(1,24),     R(2,28),     R(3,16),   M(4,20))
+F4(RRF_c, R(1,24),     R(2,28),     M(3,16),   M(4,20))
+F4(RRF_d, R(1,24),     R(2,28),     M(3,16),   M(4,20))
+F4(RRF_e, R(1,24),     R(2,28),     M(3,16),   M(4,20))
+F4(RRS,   R(1, 8),     R(2,12),     M(3,32),   BD(4,16,20))
+F3(RS_a,  R(1, 8),     BD(2,16,20), R(3,12))
+F3(RS_b,  R(1, 8),     BD(2,16,20), M(3,12))
+F3(RSI,   R(1, 8),     I(2,16,16),  R(3,12))
+F2(RSL,   L(1, 8, 4),  BD(1,16,20))
+F3(RSY_a, R(1, 8),     BDL(2),      R(3,12))
+F3(RSY_b, R(1, 8),     BDL(2),      M(3,12))
+F2(RX_a,  R(1, 8),     BXD(2))
+F2(RX_b,  M(1, 8),     BXD(2))
+F2(RXE,   R(1, 8),     BXD(2))
+F3(RXF,   R(1,32),     BXD(2),      R(3, 8))
+F2(RXY_a, R(1, 8),     BXDL(2))
+F2(RXY_b, M(1, 8),     BXDL(2))
+F1(S,     BD(2,16,20))
+F2(SI,    BD(1,16,20), I(2,8,8))
+F2(SIL,   BD(1,16,20), I(2,32,16))
+F2(SIY,   BDL(1),      I(2, 8, 8))
+F3(SS_a,  L(1, 8, 8),  BD(1,16,20), BD(2,32,36))
+F4(SS_b,  L(1, 8, 4),  BD(1,16,20), L(2,12,4),   BD(2,32,36))
+F4(SS_c,  L(1, 8, 4),  BD(1,16,20), BD(2,32,36), I(3,12, 4))
+/* ??? Odd man out.  The L1 field here is really a register, but the
+   easy way to compress the fields has R1 and B1 overlap.  */
+F4(SS_d,  L(1, 8, 4),  BD(1,16,20), BD(2,32,36), R(3,12))
+F4(SS_e,  R(1, 8),     BD(2,16,20), R(3,12),     BD(4,32,36))
+F3(SS_f,  BD(1,16,20), L(2,8,8),    BD(2,32,36))
+F2(SSE,   BD(1,16,20), BD(2,32,36))
+F3(SSF,   BD(1,16,20), BD(2,32,36), R(3,8))
diff --git a/target-s390x/int_helper.c b/target-s390x/int_helper.c
index f202a7e..6858301 100644
--- a/target-s390x/int_helper.c
+++ b/target-s390x/int_helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 
 /* #define DEBUG_HELPER */
@@ -30,46 +30,97 @@
 #endif
 
 /* 64/64 -> 128 unsigned multiplication */
-void HELPER(mlg)(CPUS390XState *env, uint32_t r1, uint64_t v2)
+uint64_t HELPER(mul128)(CPUS390XState *env, uint64_t v1, uint64_t v2)
 {
-#if HOST_LONG_BITS == 64 && defined(__GNUC__)
-    /* assuming 64-bit hosts have __uint128_t */
-    __uint128_t res = (__uint128_t)env->regs[r1 + 1];
+    uint64_t reth;
+    mulu64(&env->retxl, &reth, v1, v2);
+    return reth;
+}
 
-    res *= (__uint128_t)v2;
-    env->regs[r1] = (uint64_t)(res >> 64);
-    env->regs[r1 + 1] = (uint64_t)res;
-#else
-    mulu64(&env->regs[r1 + 1], &env->regs[r1], env->regs[r1 + 1], v2);
-#endif
+/* 64/32 -> 32 signed division */
+int64_t HELPER(divs32)(CPUS390XState *env, int64_t a, int64_t b64)
+{
+    int32_t ret, b = b64;
+    int64_t q;
+
+    if (b == 0) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    ret = q = a / b;
+    env->retxl = a % b;
+
+    /* Catch non-representable quotient.  */
+    if (ret != q) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    return ret;
 }
 
-/* 128 -> 64/64 unsigned division */
-void HELPER(dlg)(CPUS390XState *env, uint32_t r1, uint64_t v2)
+/* 64/32 -> 32 unsigned division */
+uint64_t HELPER(divu32)(CPUS390XState *env, uint64_t a, uint64_t b64)
+{
+    uint32_t ret, b = b64;
+    uint64_t q;
+
+    if (b == 0) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    ret = q = a / b;
+    env->retxl = a % b;
+
+    /* Catch non-representable quotient.  */
+    if (ret != q) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+
+    return ret;
+}
+
+/* 64/64 -> 64 signed division */
+int64_t HELPER(divs64)(CPUS390XState *env, int64_t a, int64_t b)
 {
-    uint64_t divisor = v2;
+    /* Catch divide by zero, and non-representable quotient (MIN / -1).  */
+    if (b == 0 || (b == -1 && a == (1ll << 63))) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+    env->retxl = a % b;
+    return a / b;
+}
 
-    if (!env->regs[r1]) {
+/* 128 -> 64/64 unsigned division */
+uint64_t HELPER(divu64)(CPUS390XState *env, uint64_t ah, uint64_t al,
+                        uint64_t b)
+{
+    uint64_t ret;
+    /* Signal divide by zero.  */
+    if (b == 0) {
+        runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+    }
+    if (ah == 0) {
         /* 64 -> 64/64 case */
-        env->regs[r1] = env->regs[r1 + 1] % divisor;
-        env->regs[r1 + 1] = env->regs[r1 + 1] / divisor;
-        return;
+        env->retxl = al % b;
+        ret = al / b;
     } else {
+        /* ??? Move i386 idivq helper to host-utils.  */
 #if HOST_LONG_BITS == 64 && defined(__GNUC__)
         /* assuming 64-bit hosts have __uint128_t */
-        __uint128_t dividend = (((__uint128_t)env->regs[r1]) << 64) |
-            (env->regs[r1 + 1]);
-        __uint128_t quotient = dividend / divisor;
-        __uint128_t remainder = dividend % divisor;
-
-        env->regs[r1 + 1] = quotient;
-        env->regs[r1] = remainder;
+        __uint128_t a = ((__uint128_t)ah << 64) | al;
+        __uint128_t q = a / b;
+        env->retxl = a % b;
+        ret = q;
+        if (ret != q) {
+            runtime_exception(env, PGM_FIXPT_DIVIDE, GETPC());
+        }
 #else
         /* 32-bit hosts would need special wrapper functionality - just abort if
            we encounter such a case; it's very unlikely anyways. */
         cpu_abort(env, "128 -> 64/64 division not implemented\n");
 #endif
     }
+    return ret;
 }
 
 /* absolute value 32-bit */
@@ -114,69 +165,10 @@ int64_t HELPER(nabs_i64)(int64_t val)
     }
 }
 
-/* add with carry 32-bit unsigned */
-uint32_t HELPER(addc_u32)(uint32_t cc, uint32_t v1, uint32_t v2)
+/* count leading zeros, for find leftmost one */
+uint64_t HELPER(clz)(uint64_t v)
 {
-    uint32_t res;
-
-    res = v1 + v2;
-    if (cc & 2) {
-        res++;
-    }
-
-    return res;
-}
-
-/* subtract unsigned v2 from v1 with borrow */
-uint32_t HELPER(slb)(CPUS390XState *env, uint32_t cc, uint32_t r1, uint32_t v2)
-{
-    uint32_t v1 = env->regs[r1];
-    uint32_t res = v1 + (~v2) + (cc >> 1);
-
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | res;
-    if (cc & 2) {
-        /* borrow */
-        return v1 ? 1 : 0;
-    } else {
-        return v1 ? 3 : 2;
-    }
-}
-
-/* subtract unsigned v2 from v1 with borrow */
-uint32_t HELPER(slbg)(CPUS390XState *env, uint32_t cc, uint32_t r1,
-                      uint64_t v1, uint64_t v2)
-{
-    uint64_t res = v1 + (~v2) + (cc >> 1);
-
-    env->regs[r1] = res;
-    if (cc & 2) {
-        /* borrow */
-        return v1 ? 1 : 0;
-    } else {
-        return v1 ? 3 : 2;
-    }
-}
-
-/* find leftmost one */
-uint32_t HELPER(flogr)(CPUS390XState *env, uint32_t r1, uint64_t v2)
-{
-    uint64_t res = 0;
-    uint64_t ov2 = v2;
-
-    while (!(v2 & 0x8000000000000000ULL) && v2) {
-        v2 <<= 1;
-        res++;
-    }
-
-    if (!v2) {
-        env->regs[r1] = 64;
-        env->regs[r1 + 1] = 0;
-        return 0;
-    } else {
-        env->regs[r1] = res;
-        env->regs[r1 + 1] = ov2 & ~(0x8000000000000000ULL >> res);
-        return 2;
-    }
+    return clz64(v);
 }
 
 uint64_t HELPER(cvd)(int32_t bin)
@@ -199,3 +191,15 @@ uint64_t HELPER(cvd)(int32_t bin)
 
     return dec;
 }
+
+uint64_t HELPER(popcnt)(uint64_t r2)
+{
+    uint64_t ret = 0;
+    int i;
+
+    for (i = 0; i < 64; i += 8) {
+        uint64_t t = ctpop32((r2 >> i) & 0xff);
+        ret |= t << i;
+    }
+    return ret;
+}
diff --git a/target-s390x/interrupt.c b/target-s390x/interrupt.c
index c1b034f..6d6580d 100644
--- a/target-s390x/interrupt.c
+++ b/target-s390x/interrupt.c
@@ -8,7 +8,7 @@
  */
 
 #include "cpu.h"
-#include "kvm.h"
+#include "sysemu/kvm.h"
 
 #if !defined(CONFIG_USER_ONLY)
 /* service interrupts are floating therefore we must not pass an cpustate */
@@ -19,11 +19,12 @@ void s390_sclp_extint(uint32_t parm)
 
     if (kvm_enabled()) {
 #ifdef CONFIG_KVM
-        kvm_s390_interrupt_internal(env, KVM_S390_INT_SERVICE, parm, 0, 1);
+        kvm_s390_interrupt_internal(dummy_cpu, KVM_S390_INT_SERVICE, parm,
+                                    0, 1);
 #endif
     } else {
         env->psw.addr += 4;
-        cpu_inject_ext(env, EXT_SERVICE, parm, 0);
+        cpu_inject_ext(dummy_cpu, EXT_SERVICE, parm, 0);
     }
 }
 #endif
diff --git a/target-s390x/ioinst.c b/target-s390x/ioinst.c
new file mode 100644
index 0000000..e3531f3
--- /dev/null
+++ b/target-s390x/ioinst.c
@@ -0,0 +1,761 @@
+/*
+ * I/O instructions for S/390
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include <sys/types.h>
+
+#include "cpu.h"
+#include "ioinst.h"
+#include "trace.h"
+
+int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
+                                 int *schid)
+{
+    if (!IOINST_SCHID_ONE(value)) {
+        return -EINVAL;
+    }
+    if (!IOINST_SCHID_M(value)) {
+        if (IOINST_SCHID_CSSID(value)) {
+            return -EINVAL;
+        }
+        *cssid = 0;
+        *m = 0;
+    } else {
+        *cssid = IOINST_SCHID_CSSID(value);
+        *m = 1;
+    }
+    *ssid = IOINST_SCHID_SSID(value);
+    *schid = IOINST_SCHID_NR(value);
+    return 0;
+}
+
+int ioinst_handle_xsch(CPUS390XState *env, uint64_t reg1)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    int ret = -ENODEV;
+    int cc;
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("xsch", cssid, ssid, schid);
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_xsch(sch);
+    }
+    switch (ret) {
+    case -ENODEV:
+        cc = 3;
+        break;
+    case -EBUSY:
+        cc = 2;
+        break;
+    case 0:
+        cc = 0;
+        break;
+    default:
+        cc = 1;
+        break;
+    }
+
+    return cc;
+}
+
+int ioinst_handle_csch(CPUS390XState *env, uint64_t reg1)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    int ret = -ENODEV;
+    int cc;
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("csch", cssid, ssid, schid);
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_csch(sch);
+    }
+    if (ret == -ENODEV) {
+        cc = 3;
+    } else {
+        cc = 0;
+    }
+    return cc;
+}
+
+int ioinst_handle_hsch(CPUS390XState *env, uint64_t reg1)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    int ret = -ENODEV;
+    int cc;
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("hsch", cssid, ssid, schid);
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_hsch(sch);
+    }
+    switch (ret) {
+    case -ENODEV:
+        cc = 3;
+        break;
+    case -EBUSY:
+        cc = 2;
+        break;
+    case 0:
+        cc = 0;
+        break;
+    default:
+        cc = 1;
+        break;
+    }
+
+    return cc;
+}
+
+static int ioinst_schib_valid(SCHIB *schib)
+{
+    if ((schib->pmcw.flags & PMCW_FLAGS_MASK_INVALID) ||
+        (schib->pmcw.chars & PMCW_CHARS_MASK_INVALID)) {
+        return 0;
+    }
+    /* Disallow extended measurements for now. */
+    if (schib->pmcw.chars & PMCW_CHARS_MASK_XMWME) {
+        return 0;
+    }
+    return 1;
+}
+
+int ioinst_handle_msch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    SCHIB *schib;
+    uint64_t addr;
+    int ret = -ENODEV;
+    int cc;
+    hwaddr len = sizeof(*schib);
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("msch", cssid, ssid, schid);
+    addr = decode_basedisp_s(env, ipb);
+    schib = s390_cpu_physical_memory_map(env, addr, &len, 0);
+    if (!schib || len != sizeof(*schib)) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        cc = -EIO;
+        goto out;
+    }
+    if (!ioinst_schib_valid(schib)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        cc = -EIO;
+        goto out;
+    }
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_msch(sch, schib);
+    }
+    switch (ret) {
+    case -ENODEV:
+        cc = 3;
+        break;
+    case -EBUSY:
+        cc = 2;
+        break;
+    case 0:
+        cc = 0;
+        break;
+    default:
+        cc = 1;
+        break;
+    }
+out:
+    s390_cpu_physical_memory_unmap(env, schib, len, 0);
+    return cc;
+}
+
+static void copy_orb_from_guest(ORB *dest, const ORB *src)
+{
+    dest->intparm = be32_to_cpu(src->intparm);
+    dest->ctrl0 = be16_to_cpu(src->ctrl0);
+    dest->lpm = src->lpm;
+    dest->ctrl1 = src->ctrl1;
+    dest->cpa = be32_to_cpu(src->cpa);
+}
+
+static int ioinst_orb_valid(ORB *orb)
+{
+    if ((orb->ctrl0 & ORB_CTRL0_MASK_INVALID) ||
+        (orb->ctrl1 & ORB_CTRL1_MASK_INVALID)) {
+        return 0;
+    }
+    if ((orb->cpa & HIGH_ORDER_BIT) != 0) {
+        return 0;
+    }
+    return 1;
+}
+
+int ioinst_handle_ssch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    ORB *orig_orb, orb;
+    uint64_t addr;
+    int ret = -ENODEV;
+    int cc;
+    hwaddr len = sizeof(*orig_orb);
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("ssch", cssid, ssid, schid);
+    addr = decode_basedisp_s(env, ipb);
+    orig_orb = s390_cpu_physical_memory_map(env, addr, &len, 0);
+    if (!orig_orb || len != sizeof(*orig_orb)) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        cc = -EIO;
+        goto out;
+    }
+    copy_orb_from_guest(&orb, orig_orb);
+    if (!ioinst_orb_valid(&orb)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        cc = -EIO;
+        goto out;
+    }
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_ssch(sch, &orb);
+    }
+    switch (ret) {
+    case -ENODEV:
+        cc = 3;
+        break;
+    case -EBUSY:
+        cc = 2;
+        break;
+    case 0:
+        cc = 0;
+        break;
+    default:
+        cc = 1;
+        break;
+    }
+
+out:
+    s390_cpu_physical_memory_unmap(env, orig_orb, len, 0);
+    return cc;
+}
+
+int ioinst_handle_stcrw(CPUS390XState *env, uint32_t ipb)
+{
+    CRW *crw;
+    uint64_t addr;
+    int cc;
+    hwaddr len = sizeof(*crw);
+
+    addr = decode_basedisp_s(env, ipb);
+    crw = s390_cpu_physical_memory_map(env, addr, &len, 1);
+    if (!crw || len != sizeof(*crw)) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        cc = -EIO;
+        goto out;
+    }
+    cc = css_do_stcrw(crw);
+    /* 0 - crw stored, 1 - zeroes stored */
+out:
+    s390_cpu_physical_memory_unmap(env, crw, len, 1);
+    return cc;
+}
+
+int ioinst_handle_stsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    uint64_t addr;
+    int cc;
+    SCHIB *schib;
+    hwaddr len = sizeof(*schib);
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("stsch", cssid, ssid, schid);
+    addr = decode_basedisp_s(env, ipb);
+    schib = s390_cpu_physical_memory_map(env, addr, &len, 1);
+    if (!schib || len != sizeof(*schib)) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        cc = -EIO;
+        goto out;
+    }
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch) {
+        if (css_subch_visible(sch)) {
+            css_do_stsch(sch, schib);
+            cc = 0;
+        } else {
+            /* Indicate no more subchannels in this css/ss */
+            cc = 3;
+        }
+    } else {
+        if (css_schid_final(cssid, ssid, schid)) {
+            cc = 3; /* No more subchannels in this css/ss */
+        } else {
+            /* Store an empty schib. */
+            memset(schib, 0, sizeof(*schib));
+            cc = 0;
+        }
+    }
+out:
+    s390_cpu_physical_memory_unmap(env, schib, len, 1);
+    return cc;
+}
+
+int ioinst_handle_tsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    IRB *irb;
+    uint64_t addr;
+    int ret = -ENODEV;
+    int cc;
+    hwaddr len = sizeof(*irb);
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("tsch", cssid, ssid, schid);
+    addr = decode_basedisp_s(env, ipb);
+    irb = s390_cpu_physical_memory_map(env, addr, &len, 1);
+    if (!irb || len != sizeof(*irb)) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        cc = -EIO;
+        goto out;
+    }
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_tsch(sch, irb);
+        /* 0 - status pending, 1 - not status pending */
+        cc = ret;
+    } else {
+        cc = 3;
+    }
+out:
+    s390_cpu_physical_memory_unmap(env, irb, sizeof(*irb), 1);
+    return cc;
+}
+
+typedef struct ChscReq {
+    uint16_t len;
+    uint16_t command;
+    uint32_t param0;
+    uint32_t param1;
+    uint32_t param2;
+} QEMU_PACKED ChscReq;
+
+typedef struct ChscResp {
+    uint16_t len;
+    uint16_t code;
+    uint32_t param;
+    char data[0];
+} QEMU_PACKED ChscResp;
+
+#define CHSC_MIN_RESP_LEN 0x0008
+
+#define CHSC_SCPD 0x0002
+#define CHSC_SCSC 0x0010
+#define CHSC_SDA  0x0031
+
+#define CHSC_SCPD_0_M 0x20000000
+#define CHSC_SCPD_0_C 0x10000000
+#define CHSC_SCPD_0_FMT 0x0f000000
+#define CHSC_SCPD_0_CSSID 0x00ff0000
+#define CHSC_SCPD_0_RFMT 0x00000f00
+#define CHSC_SCPD_0_RES 0xc000f000
+#define CHSC_SCPD_1_RES 0xffffff00
+#define CHSC_SCPD_01_CHPID 0x000000ff
+static void ioinst_handle_chsc_scpd(ChscReq *req, ChscResp *res)
+{
+    uint16_t len = be16_to_cpu(req->len);
+    uint32_t param0 = be32_to_cpu(req->param0);
+    uint32_t param1 = be32_to_cpu(req->param1);
+    uint16_t resp_code;
+    int rfmt;
+    uint16_t cssid;
+    uint8_t f_chpid, l_chpid;
+    int desc_size;
+    int m;
+
+    rfmt = (param0 & CHSC_SCPD_0_RFMT) >> 8;
+    if ((rfmt == 0) ||  (rfmt == 1)) {
+        rfmt = !!(param0 & CHSC_SCPD_0_C);
+    }
+    if ((len != 0x0010) || (param0 & CHSC_SCPD_0_RES) ||
+        (param1 & CHSC_SCPD_1_RES) || req->param2) {
+        resp_code = 0x0003;
+        goto out_err;
+    }
+    if (param0 & CHSC_SCPD_0_FMT) {
+        resp_code = 0x0007;
+        goto out_err;
+    }
+    cssid = (param0 & CHSC_SCPD_0_CSSID) >> 16;
+    m = param0 & CHSC_SCPD_0_M;
+    if (cssid != 0) {
+        if (!m || !css_present(cssid)) {
+            resp_code = 0x0008;
+            goto out_err;
+        }
+    }
+    f_chpid = param0 & CHSC_SCPD_01_CHPID;
+    l_chpid = param1 & CHSC_SCPD_01_CHPID;
+    if (l_chpid < f_chpid) {
+        resp_code = 0x0003;
+        goto out_err;
+    }
+    /* css_collect_chp_desc() is endian-aware */
+    desc_size = css_collect_chp_desc(m, cssid, f_chpid, l_chpid, rfmt,
+                                     &res->data);
+    res->code = cpu_to_be16(0x0001);
+    res->len = cpu_to_be16(8 + desc_size);
+    res->param = cpu_to_be32(rfmt);
+    return;
+
+  out_err:
+    res->code = cpu_to_be16(resp_code);
+    res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
+    res->param = cpu_to_be32(rfmt);
+}
+
+#define CHSC_SCSC_0_M 0x20000000
+#define CHSC_SCSC_0_FMT 0x000f0000
+#define CHSC_SCSC_0_CSSID 0x0000ff00
+#define CHSC_SCSC_0_RES 0xdff000ff
+static void ioinst_handle_chsc_scsc(ChscReq *req, ChscResp *res)
+{
+    uint16_t len = be16_to_cpu(req->len);
+    uint32_t param0 = be32_to_cpu(req->param0);
+    uint8_t cssid;
+    uint16_t resp_code;
+    uint32_t general_chars[510];
+    uint32_t chsc_chars[508];
+
+    if (len != 0x0010) {
+        resp_code = 0x0003;
+        goto out_err;
+    }
+
+    if (param0 & CHSC_SCSC_0_FMT) {
+        resp_code = 0x0007;
+        goto out_err;
+    }
+    cssid = (param0 & CHSC_SCSC_0_CSSID) >> 8;
+    if (cssid != 0) {
+        if (!(param0 & CHSC_SCSC_0_M) || !css_present(cssid)) {
+            resp_code = 0x0008;
+            goto out_err;
+        }
+    }
+    if ((param0 & CHSC_SCSC_0_RES) || req->param1 || req->param2) {
+        resp_code = 0x0003;
+        goto out_err;
+    }
+    res->code = cpu_to_be16(0x0001);
+    res->len = cpu_to_be16(4080);
+    res->param = 0;
+
+    memset(general_chars, 0, sizeof(general_chars));
+    memset(chsc_chars, 0, sizeof(chsc_chars));
+
+    general_chars[0] = cpu_to_be32(0x03000000);
+    general_chars[1] = cpu_to_be32(0x00059000);
+
+    chsc_chars[0] = cpu_to_be32(0x40000000);
+    chsc_chars[3] = cpu_to_be32(0x00040000);
+
+    memcpy(res->data, general_chars, sizeof(general_chars));
+    memcpy(res->data + sizeof(general_chars), chsc_chars, sizeof(chsc_chars));
+    return;
+
+  out_err:
+    res->code = cpu_to_be16(resp_code);
+    res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
+    res->param = 0;
+}
+
+#define CHSC_SDA_0_FMT 0x0f000000
+#define CHSC_SDA_0_OC 0x0000ffff
+#define CHSC_SDA_0_RES 0xf0ff0000
+#define CHSC_SDA_OC_MCSSE 0x0
+#define CHSC_SDA_OC_MSS 0x2
+static void ioinst_handle_chsc_sda(ChscReq *req, ChscResp *res)
+{
+    uint16_t resp_code = 0x0001;
+    uint16_t len = be16_to_cpu(req->len);
+    uint32_t param0 = be32_to_cpu(req->param0);
+    uint16_t oc;
+    int ret;
+
+    if ((len != 0x0400) || (param0 & CHSC_SDA_0_RES)) {
+        resp_code = 0x0003;
+        goto out;
+    }
+
+    if (param0 & CHSC_SDA_0_FMT) {
+        resp_code = 0x0007;
+        goto out;
+    }
+
+    oc = param0 & CHSC_SDA_0_OC;
+    switch (oc) {
+    case CHSC_SDA_OC_MCSSE:
+        ret = css_enable_mcsse();
+        if (ret == -EINVAL) {
+            resp_code = 0x0101;
+            goto out;
+        }
+        break;
+    case CHSC_SDA_OC_MSS:
+        ret = css_enable_mss();
+        if (ret == -EINVAL) {
+            resp_code = 0x0101;
+            goto out;
+        }
+        break;
+    default:
+        resp_code = 0x0003;
+        goto out;
+    }
+
+out:
+    res->code = cpu_to_be16(resp_code);
+    res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
+    res->param = 0;
+}
+
+static void ioinst_handle_chsc_unimplemented(ChscResp *res)
+{
+    res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
+    res->code = cpu_to_be16(0x0004);
+    res->param = 0;
+}
+
+int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb)
+{
+    ChscReq *req;
+    ChscResp *res;
+    uint64_t addr;
+    int reg;
+    uint16_t len;
+    uint16_t command;
+    hwaddr map_size = TARGET_PAGE_SIZE;
+    int ret = 0;
+
+    trace_ioinst("chsc");
+    reg = (ipb >> 20) & 0x00f;
+    addr = env->regs[reg];
+    /* Page boundary? */
+    if (addr & 0xfff) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        return -EIO;
+    }
+    req = s390_cpu_physical_memory_map(env, addr, &map_size, 1);
+    if (!req || map_size != TARGET_PAGE_SIZE) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        ret = -EIO;
+        goto out;
+    }
+    len = be16_to_cpu(req->len);
+    /* Length field valid? */
+    if ((len < 16) || (len > 4088) || (len & 7)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        ret = -EIO;
+        goto out;
+    }
+    memset((char *)req + len, 0, TARGET_PAGE_SIZE - len);
+    res = (void *)((char *)req + len);
+    command = be16_to_cpu(req->command);
+    trace_ioinst_chsc_cmd(command, len);
+    switch (command) {
+    case CHSC_SCSC:
+        ioinst_handle_chsc_scsc(req, res);
+        break;
+    case CHSC_SCPD:
+        ioinst_handle_chsc_scpd(req, res);
+        break;
+    case CHSC_SDA:
+        ioinst_handle_chsc_sda(req, res);
+        break;
+    default:
+        ioinst_handle_chsc_unimplemented(res);
+        break;
+    }
+
+out:
+    s390_cpu_physical_memory_unmap(env, req, map_size, 1);
+    return ret;
+}
+
+int ioinst_handle_tpi(CPUS390XState *env, uint32_t ipb)
+{
+    uint64_t addr;
+    int lowcore;
+    IOIntCode *int_code;
+    hwaddr len, orig_len;
+    int ret;
+
+    trace_ioinst("tpi");
+    addr = decode_basedisp_s(env, ipb);
+    lowcore = addr ? 0 : 1;
+    len = lowcore ? 8 /* two words */ : 12 /* three words */;
+    orig_len = len;
+    int_code = s390_cpu_physical_memory_map(env, addr, &len, 1);
+    if (!int_code || (len != orig_len)) {
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        ret = -EIO;
+        goto out;
+    }
+    ret = css_do_tpi(int_code, lowcore);
+out:
+    s390_cpu_physical_memory_unmap(env, int_code, len, 1);
+    return ret;
+}
+
+#define SCHM_REG1_RES(_reg) (_reg & 0x000000000ffffffc)
+#define SCHM_REG1_MBK(_reg) ((_reg & 0x00000000f0000000) >> 28)
+#define SCHM_REG1_UPD(_reg) ((_reg & 0x0000000000000002) >> 1)
+#define SCHM_REG1_DCT(_reg) (_reg & 0x0000000000000001)
+
+int ioinst_handle_schm(CPUS390XState *env, uint64_t reg1, uint64_t reg2,
+                       uint32_t ipb)
+{
+    uint8_t mbk;
+    int update;
+    int dct;
+
+    trace_ioinst("schm");
+
+    if (SCHM_REG1_RES(reg1)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+
+    mbk = SCHM_REG1_MBK(reg1);
+    update = SCHM_REG1_UPD(reg1);
+    dct = SCHM_REG1_DCT(reg1);
+
+    if (update && (reg2 & 0x0000000000000fff)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+
+    css_do_schm(mbk, update, dct, update ? reg2 : 0);
+
+    return 0;
+}
+
+int ioinst_handle_rsch(CPUS390XState *env, uint64_t reg1)
+{
+    int cssid, ssid, schid, m;
+    SubchDev *sch;
+    int ret = -ENODEV;
+    int cc;
+
+    if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    trace_ioinst_sch_id("rsch", cssid, ssid, schid);
+    sch = css_find_subch(m, cssid, ssid, schid);
+    if (sch && css_subch_visible(sch)) {
+        ret = css_do_rsch(sch);
+    }
+    switch (ret) {
+    case -ENODEV:
+        cc = 3;
+        break;
+    case -EINVAL:
+        cc = 2;
+        break;
+    case 0:
+        cc = 0;
+        break;
+    default:
+        cc = 1;
+        break;
+    }
+
+    return cc;
+
+}
+
+#define RCHP_REG1_RES(_reg) (_reg & 0x00000000ff00ff00)
+#define RCHP_REG1_CSSID(_reg) ((_reg & 0x0000000000ff0000) >> 16)
+#define RCHP_REG1_CHPID(_reg) (_reg & 0x00000000000000ff)
+int ioinst_handle_rchp(CPUS390XState *env, uint64_t reg1)
+{
+    int cc;
+    uint8_t cssid;
+    uint8_t chpid;
+    int ret;
+
+    if (RCHP_REG1_RES(reg1)) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+
+    cssid = RCHP_REG1_CSSID(reg1);
+    chpid = RCHP_REG1_CHPID(reg1);
+
+    trace_ioinst_chp_id("rchp", cssid, chpid);
+
+    ret = css_do_rchp(cssid, chpid);
+
+    switch (ret) {
+    case -ENODEV:
+        cc = 3;
+        break;
+    case -EBUSY:
+        cc = 2;
+        break;
+    case 0:
+        cc = 0;
+        break;
+    default:
+        /* Invalid channel subsystem. */
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+
+    return cc;
+}
+
+#define SAL_REG1_INVALID(_reg) (_reg & 0x0000000080000000)
+int ioinst_handle_sal(CPUS390XState *env, uint64_t reg1)
+{
+    /* We do not provide address limit checking, so let's suppress it. */
+    if (SAL_REG1_INVALID(reg1) || reg1 & 0x000000000000ffff) {
+        program_interrupt(env, PGM_OPERAND, 2);
+        return -EIO;
+    }
+    return 0;
+}
diff --git a/target-s390x/ioinst.h b/target-s390x/ioinst.h
new file mode 100644
index 0000000..7bed291
--- /dev/null
+++ b/target-s390x/ioinst.h
@@ -0,0 +1,233 @@
+/*
+ * S/390 channel I/O instructions
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+*/
+
+#ifndef IOINST_S390X_H
+#define IOINST_S390X_H
+/*
+ * Channel I/O related definitions, as defined in the Principles
+ * Of Operation (and taken from the Linux implementation).
+ */
+
+/* subchannel status word (command mode only) */
+typedef struct SCSW {
+    uint16_t flags;
+    uint16_t ctrl;
+    uint32_t cpa;
+    uint8_t dstat;
+    uint8_t cstat;
+    uint16_t count;
+} QEMU_PACKED SCSW;
+
+#define SCSW_FLAGS_MASK_KEY 0xf000
+#define SCSW_FLAGS_MASK_SCTL 0x0800
+#define SCSW_FLAGS_MASK_ESWF 0x0400
+#define SCSW_FLAGS_MASK_CC 0x0300
+#define SCSW_FLAGS_MASK_FMT 0x0080
+#define SCSW_FLAGS_MASK_PFCH 0x0040
+#define SCSW_FLAGS_MASK_ISIC 0x0020
+#define SCSW_FLAGS_MASK_ALCC 0x0010
+#define SCSW_FLAGS_MASK_SSI 0x0008
+#define SCSW_FLAGS_MASK_ZCC 0x0004
+#define SCSW_FLAGS_MASK_ECTL 0x0002
+#define SCSW_FLAGS_MASK_PNO 0x0001
+
+#define SCSW_CTRL_MASK_FCTL 0x7000
+#define SCSW_CTRL_MASK_ACTL 0x0fe0
+#define SCSW_CTRL_MASK_STCTL 0x001f
+
+#define SCSW_FCTL_CLEAR_FUNC 0x1000
+#define SCSW_FCTL_HALT_FUNC 0x2000
+#define SCSW_FCTL_START_FUNC 0x4000
+
+#define SCSW_ACTL_SUSP 0x0020
+#define SCSW_ACTL_DEVICE_ACTIVE 0x0040
+#define SCSW_ACTL_SUBCH_ACTIVE 0x0080
+#define SCSW_ACTL_CLEAR_PEND 0x0100
+#define SCSW_ACTL_HALT_PEND  0x0200
+#define SCSW_ACTL_START_PEND 0x0400
+#define SCSW_ACTL_RESUME_PEND 0x0800
+
+#define SCSW_STCTL_STATUS_PEND 0x0001
+#define SCSW_STCTL_SECONDARY 0x0002
+#define SCSW_STCTL_PRIMARY 0x0004
+#define SCSW_STCTL_INTERMEDIATE 0x0008
+#define SCSW_STCTL_ALERT 0x0010
+
+#define SCSW_DSTAT_ATTENTION     0x80
+#define SCSW_DSTAT_STAT_MOD      0x40
+#define SCSW_DSTAT_CU_END        0x20
+#define SCSW_DSTAT_BUSY          0x10
+#define SCSW_DSTAT_CHANNEL_END   0x08
+#define SCSW_DSTAT_DEVICE_END    0x04
+#define SCSW_DSTAT_UNIT_CHECK    0x02
+#define SCSW_DSTAT_UNIT_EXCEP    0x01
+
+#define SCSW_CSTAT_PCI           0x80
+#define SCSW_CSTAT_INCORR_LEN    0x40
+#define SCSW_CSTAT_PROG_CHECK    0x20
+#define SCSW_CSTAT_PROT_CHECK    0x10
+#define SCSW_CSTAT_DATA_CHECK    0x08
+#define SCSW_CSTAT_CHN_CTRL_CHK  0x04
+#define SCSW_CSTAT_INTF_CTRL_CHK 0x02
+#define SCSW_CSTAT_CHAIN_CHECK   0x01
+
+/* path management control word */
+typedef struct PMCW {
+    uint32_t intparm;
+    uint16_t flags;
+    uint16_t devno;
+    uint8_t  lpm;
+    uint8_t  pnom;
+    uint8_t  lpum;
+    uint8_t  pim;
+    uint16_t mbi;
+    uint8_t  pom;
+    uint8_t  pam;
+    uint8_t  chpid[8];
+    uint32_t chars;
+} QEMU_PACKED PMCW;
+
+#define PMCW_FLAGS_MASK_QF 0x8000
+#define PMCW_FLAGS_MASK_W 0x4000
+#define PMCW_FLAGS_MASK_ISC 0x3800
+#define PMCW_FLAGS_MASK_ENA 0x0080
+#define PMCW_FLAGS_MASK_LM 0x0060
+#define PMCW_FLAGS_MASK_MME 0x0018
+#define PMCW_FLAGS_MASK_MP 0x0004
+#define PMCW_FLAGS_MASK_TF 0x0002
+#define PMCW_FLAGS_MASK_DNV 0x0001
+#define PMCW_FLAGS_MASK_INVALID 0x0700
+
+#define PMCW_CHARS_MASK_ST 0x00e00000
+#define PMCW_CHARS_MASK_MBFC 0x00000004
+#define PMCW_CHARS_MASK_XMWME 0x00000002
+#define PMCW_CHARS_MASK_CSENSE 0x00000001
+#define PMCW_CHARS_MASK_INVALID 0xff1ffff8
+
+/* subchannel information block */
+typedef struct SCHIB {
+    PMCW pmcw;
+    SCSW scsw;
+    uint64_t mba;
+    uint8_t mda[4];
+} QEMU_PACKED SCHIB;
+
+/* interruption response block */
+typedef struct IRB {
+    SCSW scsw;
+    uint32_t esw[5];
+    uint32_t ecw[8];
+    uint32_t emw[8];
+} QEMU_PACKED IRB;
+
+/* operation request block */
+typedef struct ORB {
+    uint32_t intparm;
+    uint16_t ctrl0;
+    uint8_t lpm;
+    uint8_t ctrl1;
+    uint32_t cpa;
+} QEMU_PACKED ORB;
+
+#define ORB_CTRL0_MASK_KEY 0xf000
+#define ORB_CTRL0_MASK_SPND 0x0800
+#define ORB_CTRL0_MASK_STR 0x0400
+#define ORB_CTRL0_MASK_MOD 0x0200
+#define ORB_CTRL0_MASK_SYNC 0x0100
+#define ORB_CTRL0_MASK_FMT 0x0080
+#define ORB_CTRL0_MASK_PFCH 0x0040
+#define ORB_CTRL0_MASK_ISIC 0x0020
+#define ORB_CTRL0_MASK_ALCC 0x0010
+#define ORB_CTRL0_MASK_SSIC 0x0008
+#define ORB_CTRL0_MASK_C64 0x0002
+#define ORB_CTRL0_MASK_I2K 0x0001
+#define ORB_CTRL0_MASK_INVALID 0x0004
+
+#define ORB_CTRL1_MASK_ILS 0x80
+#define ORB_CTRL1_MASK_MIDAW 0x40
+#define ORB_CTRL1_MASK_ORBX 0x01
+#define ORB_CTRL1_MASK_INVALID 0x3e
+
+/* channel command word (type 1) */
+typedef struct CCW1 {
+    uint8_t cmd_code;
+    uint8_t flags;
+    uint16_t count;
+    uint32_t cda;
+} QEMU_PACKED CCW1;
+
+#define CCW_FLAG_DC              0x80
+#define CCW_FLAG_CC              0x40
+#define CCW_FLAG_SLI             0x20
+#define CCW_FLAG_SKIP            0x10
+#define CCW_FLAG_PCI             0x08
+#define CCW_FLAG_IDA             0x04
+#define CCW_FLAG_SUSPEND         0x02
+
+#define CCW_CMD_NOOP             0x03
+#define CCW_CMD_BASIC_SENSE      0x04
+#define CCW_CMD_TIC              0x08
+#define CCW_CMD_SENSE_ID         0xe4
+
+typedef struct CRW {
+    uint16_t flags;
+    uint16_t rsid;
+} QEMU_PACKED CRW;
+
+#define CRW_FLAGS_MASK_S 0x4000
+#define CRW_FLAGS_MASK_R 0x2000
+#define CRW_FLAGS_MASK_C 0x1000
+#define CRW_FLAGS_MASK_RSC 0x0f00
+#define CRW_FLAGS_MASK_A 0x0080
+#define CRW_FLAGS_MASK_ERC 0x003f
+
+#define CRW_ERC_INIT 0x02
+#define CRW_ERC_IPI  0x04
+
+#define CRW_RSC_SUBCH 0x3
+#define CRW_RSC_CHP   0x4
+
+/* I/O interruption code */
+typedef struct IOIntCode {
+    uint32_t subsys_id;
+    uint32_t intparm;
+    uint32_t interrupt_id;
+} QEMU_PACKED IOIntCode;
+
+/* schid disintegration */
+#define IOINST_SCHID_ONE(_schid)   ((_schid & 0x00010000) >> 16)
+#define IOINST_SCHID_M(_schid)     ((_schid & 0x00080000) >> 19)
+#define IOINST_SCHID_CSSID(_schid) ((_schid & 0xff000000) >> 24)
+#define IOINST_SCHID_SSID(_schid)  ((_schid & 0x00060000) >> 17)
+#define IOINST_SCHID_NR(_schid)    (_schid & 0x0000ffff)
+
+#define IO_INT_WORD_ISC(_int_word) ((_int_word & 0x38000000) >> 24)
+#define ISC_TO_ISC_BITS(_isc)      ((0x80 >> _isc) << 24)
+
+int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
+                                 int *schid);
+int ioinst_handle_xsch(CPUS390XState *env, uint64_t reg1);
+int ioinst_handle_csch(CPUS390XState *env, uint64_t reg1);
+int ioinst_handle_hsch(CPUS390XState *env, uint64_t reg1);
+int ioinst_handle_msch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
+int ioinst_handle_ssch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
+int ioinst_handle_stcrw(CPUS390XState *env, uint32_t ipb);
+int ioinst_handle_stsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
+int ioinst_handle_tsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
+int ioinst_handle_chsc(CPUS390XState *env, uint32_t ipb);
+int ioinst_handle_tpi(CPUS390XState *env, uint32_t ipb);
+int ioinst_handle_schm(CPUS390XState *env, uint64_t reg1, uint64_t reg2,
+                       uint32_t ipb);
+int ioinst_handle_rsch(CPUS390XState *env, uint64_t reg1);
+int ioinst_handle_rchp(CPUS390XState *env, uint64_t reg1);
+int ioinst_handle_sal(CPUS390XState *env, uint64_t reg1);
+
+#endif
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 94de764..3929771 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -2,6 +2,7 @@
  * QEMU S390x KVM implementation
  *
  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ * Copyright IBM Corp. 2012
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -13,7 +14,10 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
- * You should have received a copy of the GNU Lesser General Public
+ * Contributions after 2012-10-29 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
@@ -25,11 +29,11 @@
 #include <asm/ptrace.h>
 
 #include "qemu-common.h"
-#include "qemu-timer.h"
-#include "sysemu.h"
-#include "kvm.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "cpu.h"
-#include "device_tree.h"
+#include "sysemu/device_tree.h"
 
 /* #define DEBUG_KVM */
 
@@ -43,9 +47,29 @@
 
 #define IPA0_DIAG                       0x8300
 #define IPA0_SIGP                       0xae00
-#define IPA0_PRIV                       0xb200
+#define IPA0_B2                         0xb200
+#define IPA0_B9                         0xb900
+#define IPA0_EB                         0xeb00
 
 #define PRIV_SCLP_CALL                  0x20
+#define PRIV_CSCH                       0x30
+#define PRIV_HSCH                       0x31
+#define PRIV_MSCH                       0x32
+#define PRIV_SSCH                       0x33
+#define PRIV_STSCH                      0x34
+#define PRIV_TSCH                       0x35
+#define PRIV_TPI                        0x36
+#define PRIV_SAL                        0x37
+#define PRIV_RSCH                       0x38
+#define PRIV_STCRW                      0x39
+#define PRIV_STCPS                      0x3a
+#define PRIV_RCHP                       0x3b
+#define PRIV_SCHM                       0x3c
+#define PRIV_CHSC                       0x5f
+#define PRIV_SIGA                       0x74
+#define PRIV_XSCH                       0x76
+#define PRIV_SQBS                       0x8a
+#define PRIV_EQBS                       0x9c
 #define DIAG_KVM_HYPERCALL              0x500
 #define DIAG_KVM_BREAKPOINT             0x501
 
@@ -72,43 +96,57 @@ int kvm_arch_init(KVMState *s)
     return 0;
 }
 
-int kvm_arch_init_vcpu(CPUS390XState *env)
+unsigned long kvm_arch_vcpu_id(CPUState *cpu)
+{
+    return cpu->cpu_index;
+}
+
+int kvm_arch_init_vcpu(CPUState *cpu)
 {
     int ret = 0;
 
-    if (kvm_vcpu_ioctl(env, KVM_S390_INITIAL_RESET, NULL) < 0) {
+    if (kvm_vcpu_ioctl(cpu, KVM_S390_INITIAL_RESET, NULL) < 0) {
         perror("cannot init reset vcpu");
     }
 
     return ret;
 }
 
-void kvm_arch_reset_vcpu(CPUS390XState *env)
+void kvm_arch_reset_vcpu(CPUState *cpu)
 {
-    /* FIXME: add code to reset vcpu. */
+    /* The initial reset call is needed here to reset in-kernel
+     * vcpu data that we can't access directly from QEMU
+     * (i.e. with older kernels which don't support sync_regs/ONE_REG).
+     * Before this ioctl cpu_synchronize_state() is called in common kvm
+     * code (kvm-all) */
+    if (kvm_vcpu_ioctl(cpu, KVM_S390_INITIAL_RESET, NULL)) {
+        perror("Can't reset vcpu\n");
+    }
 }
 
-int kvm_arch_put_registers(CPUS390XState *env, int level)
+int kvm_arch_put_registers(CPUState *cs, int level)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
     struct kvm_sregs sregs;
     struct kvm_regs regs;
     int ret;
     int i;
 
     /* always save the PSW  and the GPRS*/
-    env->kvm_run->psw_addr = env->psw.addr;
-    env->kvm_run->psw_mask = env->psw.mask;
+    cs->kvm_run->psw_addr = env->psw.addr;
+    cs->kvm_run->psw_mask = env->psw.mask;
 
-    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
         for (i = 0; i < 16; i++) {
-            env->kvm_run->s.regs.gprs[i] = env->regs[i];
-            env->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
+            cs->kvm_run->s.regs.gprs[i] = env->regs[i];
+            cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
         }
     } else {
         for (i = 0; i < 16; i++) {
             regs.gprs[i] = env->regs[i];
         }
-        ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_REGS, &regs);
         if (ret < 0) {
             return ret;
         }
@@ -120,53 +158,55 @@ int kvm_arch_put_registers(CPUS390XState *env, int level)
     }
 
     if (cap_sync_regs &&
-        env->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
-        env->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
         for (i = 0; i < 16; i++) {
-            env->kvm_run->s.regs.acrs[i] = env->aregs[i];
-            env->kvm_run->s.regs.crs[i] = env->cregs[i];
+            cs->kvm_run->s.regs.acrs[i] = env->aregs[i];
+            cs->kvm_run->s.regs.crs[i] = env->cregs[i];
         }
-        env->kvm_run->kvm_dirty_regs |= KVM_SYNC_ACRS;
-        env->kvm_run->kvm_dirty_regs |= KVM_SYNC_CRS;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_ACRS;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_CRS;
     } else {
         for (i = 0; i < 16; i++) {
             sregs.acrs[i] = env->aregs[i];
             sregs.crs[i] = env->cregs[i];
         }
-        ret = kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
         if (ret < 0) {
             return ret;
         }
     }
 
     /* Finally the prefix */
-    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
-        env->kvm_run->s.regs.prefix = env->psa;
-        env->kvm_run->kvm_dirty_regs |= KVM_SYNC_PREFIX;
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+        cs->kvm_run->s.regs.prefix = env->psa;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_PREFIX;
     } else {
         /* prefix is only supported via sync regs */
     }
     return 0;
 }
 
-int kvm_arch_get_registers(CPUS390XState *env)
+int kvm_arch_get_registers(CPUState *cs)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
     struct kvm_sregs sregs;
     struct kvm_regs regs;
     int ret;
     int i;
 
     /* get the PSW */
-    env->psw.addr = env->kvm_run->psw_addr;
-    env->psw.mask = env->kvm_run->psw_mask;
+    env->psw.addr = cs->kvm_run->psw_addr;
+    env->psw.mask = cs->kvm_run->psw_mask;
 
     /* the GPRS */
-    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
         for (i = 0; i < 16; i++) {
-            env->regs[i] = env->kvm_run->s.regs.gprs[i];
+            env->regs[i] = cs->kvm_run->s.regs.gprs[i];
         }
     } else {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_REGS, &regs);
         if (ret < 0) {
             return ret;
         }
@@ -177,14 +217,14 @@ int kvm_arch_get_registers(CPUS390XState *env)
 
     /* The ACRS and CRS */
     if (cap_sync_regs &&
-        env->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
-        env->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
+        cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
         for (i = 0; i < 16; i++) {
-            env->aregs[i] = env->kvm_run->s.regs.acrs[i];
-            env->cregs[i] = env->kvm_run->s.regs.crs[i];
+            env->aregs[i] = cs->kvm_run->s.regs.acrs[i];
+            env->cregs[i] = cs->kvm_run->s.regs.crs[i];
         }
     } else {
-        ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
         if (ret < 0) {
             return ret;
         }
@@ -195,8 +235,8 @@ int kvm_arch_get_registers(CPUS390XState *env)
     }
 
     /* Finally the prefix */
-    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
-        env->psa = env->kvm_run->s.regs.prefix;
+    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+        env->psa = cs->kvm_run->s.regs.prefix;
     } else {
         /* no prefix without sync regs */
     }
@@ -239,8 +279,10 @@ void *kvm_arch_vmalloc(ram_addr_t size)
     }
 }
 
-int kvm_arch_insert_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
     static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
 
     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
@@ -250,8 +292,10 @@ int kvm_arch_insert_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *
     return 0;
 }
 
-int kvm_arch_remove_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *bp)
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
     uint8_t t[4];
     static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
 
@@ -266,26 +310,28 @@ int kvm_arch_remove_sw_breakpoint(CPUS390XState *env, struct kvm_sw_breakpoint *
     return 0;
 }
 
-void kvm_arch_pre_run(CPUS390XState *env, struct kvm_run *run)
+void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
 {
 }
 
-void kvm_arch_post_run(CPUS390XState *env, struct kvm_run *run)
+void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
 }
 
-int kvm_arch_process_async_events(CPUS390XState *env)
+int kvm_arch_process_async_events(CPUState *cs)
 {
-    return env->halted;
+    S390CPU *cpu = S390_CPU(cs);
+    return cpu->env.halted;
 }
 
-void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
+void kvm_s390_interrupt_internal(S390CPU *cpu, int type, uint32_t parm,
                                  uint64_t parm64, int vm)
 {
+    CPUState *cs = CPU(cpu);
     struct kvm_s390_interrupt kvmint;
     int r;
 
-    if (!env->kvm_state) {
+    if (!cs->kvm_state) {
         return;
     }
 
@@ -294,9 +340,9 @@ void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
     kvmint.parm64 = parm64;
 
     if (vm) {
-        r = kvm_vm_ioctl(env->kvm_state, KVM_S390_INTERRUPT, &kvmint);
+        r = kvm_vm_ioctl(cs->kvm_state, KVM_S390_INTERRUPT, &kvmint);
     } else {
-        r = kvm_vcpu_ioctl(env, KVM_S390_INTERRUPT, &kvmint);
+        r = kvm_vcpu_ioctl(cs, KVM_S390_INTERRUPT, &kvmint);
     }
 
     if (r < 0) {
@@ -305,34 +351,38 @@ void kvm_s390_interrupt_internal(CPUS390XState *env, int type, uint32_t parm,
     }
 }
 
-void kvm_s390_virtio_irq(CPUS390XState *env, int config_change, uint64_t token)
+void kvm_s390_virtio_irq(S390CPU *cpu, int config_change, uint64_t token)
 {
-    kvm_s390_interrupt_internal(env, KVM_S390_INT_VIRTIO, config_change,
+    kvm_s390_interrupt_internal(cpu, KVM_S390_INT_VIRTIO, config_change,
                                 token, 1);
 }
 
-void kvm_s390_interrupt(CPUS390XState *env, int type, uint32_t code)
+void kvm_s390_interrupt(S390CPU *cpu, int type, uint32_t code)
 {
-    kvm_s390_interrupt_internal(env, type, code, 0, 0);
+    kvm_s390_interrupt_internal(cpu, type, code, 0, 0);
 }
 
-static void enter_pgmcheck(CPUS390XState *env, uint16_t code)
+static void enter_pgmcheck(S390CPU *cpu, uint16_t code)
 {
-    kvm_s390_interrupt(env, KVM_S390_PROGRAM_INT, code);
+    kvm_s390_interrupt(cpu, KVM_S390_PROGRAM_INT, code);
 }
 
-static inline void setcc(CPUS390XState *env, uint64_t cc)
+static inline void setcc(S390CPU *cpu, uint64_t cc)
 {
-    env->kvm_run->psw_mask &= ~(3ull << 44);
-    env->kvm_run->psw_mask |= (cc & 3) << 44;
+    CPUS390XState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+
+    cs->kvm_run->psw_mask &= ~(3ull << 44);
+    cs->kvm_run->psw_mask |= (cc & 3) << 44;
 
     env->psw.mask &= ~(3ul << 44);
     env->psw.mask |= (cc & 3) << 44;
 }
 
-static int kvm_sclp_service_call(CPUS390XState *env, struct kvm_run *run,
+static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
                                  uint16_t ipbh0)
 {
+    CPUS390XState *env = &cpu->env;
     uint32_t sccb;
     uint64_t code;
     int r = 0;
@@ -343,26 +393,147 @@ static int kvm_sclp_service_call(CPUS390XState *env, struct kvm_run *run,
 
     r = sclp_service_call(sccb, code);
     if (r < 0) {
-        enter_pgmcheck(env, -r);
+        enter_pgmcheck(cpu, -r);
     }
-    setcc(env, r);
+    setcc(cpu, r);
 
     return 0;
 }
 
-static int handle_priv(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
+static int kvm_handle_css_inst(S390CPU *cpu, struct kvm_run *run,
+                               uint8_t ipa0, uint8_t ipa1, uint8_t ipb)
+{
+    int r = 0;
+    int no_cc = 0;
+    CPUS390XState *env = &cpu->env;
+
+    if (ipa0 != 0xb2) {
+        /* Not handled for now. */
+        return -1;
+    }
+    cpu_synchronize_state(env);
+    switch (ipa1) {
+    case PRIV_XSCH:
+        r = ioinst_handle_xsch(env, env->regs[1]);
+        break;
+    case PRIV_CSCH:
+        r = ioinst_handle_csch(env, env->regs[1]);
+        break;
+    case PRIV_HSCH:
+        r = ioinst_handle_hsch(env, env->regs[1]);
+        break;
+    case PRIV_MSCH:
+        r = ioinst_handle_msch(env, env->regs[1], run->s390_sieic.ipb);
+        break;
+    case PRIV_SSCH:
+        r = ioinst_handle_ssch(env, env->regs[1], run->s390_sieic.ipb);
+        break;
+    case PRIV_STCRW:
+        r = ioinst_handle_stcrw(env, run->s390_sieic.ipb);
+        break;
+    case PRIV_STSCH:
+        r = ioinst_handle_stsch(env, env->regs[1], run->s390_sieic.ipb);
+        break;
+    case PRIV_TSCH:
+        /* We should only get tsch via KVM_EXIT_S390_TSCH. */
+        fprintf(stderr, "Spurious tsch intercept\n");
+        break;
+    case PRIV_CHSC:
+        r = ioinst_handle_chsc(env, run->s390_sieic.ipb);
+        break;
+    case PRIV_TPI:
+        /* This should have been handled by kvm already. */
+        fprintf(stderr, "Spurious tpi intercept\n");
+        break;
+    case PRIV_SCHM:
+        no_cc = 1;
+        r = ioinst_handle_schm(env, env->regs[1], env->regs[2],
+                               run->s390_sieic.ipb);
+        break;
+    case PRIV_RSCH:
+        r = ioinst_handle_rsch(env, env->regs[1]);
+        break;
+    case PRIV_RCHP:
+        r = ioinst_handle_rchp(env, env->regs[1]);
+        break;
+    case PRIV_STCPS:
+        /* We do not provide this instruction, it is suppressed. */
+        no_cc = 1;
+        r = 0;
+        break;
+    case PRIV_SAL:
+        no_cc = 1;
+        r = ioinst_handle_sal(env, env->regs[1]);
+        break;
+    default:
+        r = -1;
+        break;
+    }
+
+    if (r >= 0) {
+        if (!no_cc) {
+            setcc(cpu, r);
+        }
+        r = 0;
+    } else if (r < -1) {
+        r = 0;
+    }
+    return r;
+}
+
+static int is_ioinst(uint8_t ipa0, uint8_t ipa1, uint8_t ipb)
+{
+    int ret = 0;
+    uint16_t ipa = (ipa0 << 8) | ipa1;
+
+    switch (ipa) {
+    case IPA0_B2 | PRIV_CSCH:
+    case IPA0_B2 | PRIV_HSCH:
+    case IPA0_B2 | PRIV_MSCH:
+    case IPA0_B2 | PRIV_SSCH:
+    case IPA0_B2 | PRIV_STSCH:
+    case IPA0_B2 | PRIV_TPI:
+    case IPA0_B2 | PRIV_SAL:
+    case IPA0_B2 | PRIV_RSCH:
+    case IPA0_B2 | PRIV_STCRW:
+    case IPA0_B2 | PRIV_STCPS:
+    case IPA0_B2 | PRIV_RCHP:
+    case IPA0_B2 | PRIV_SCHM:
+    case IPA0_B2 | PRIV_CHSC:
+    case IPA0_B2 | PRIV_SIGA:
+    case IPA0_B2 | PRIV_XSCH:
+    case IPA0_B9 | PRIV_EQBS:
+    case IPA0_EB | PRIV_SQBS:
+        ret = 1;
+        break;
+    }
+
+    return ret;
+}
+
+static int handle_priv(S390CPU *cpu, struct kvm_run *run,
+                       uint8_t ipa0, uint8_t ipa1)
 {
     int r = 0;
     uint16_t ipbh0 = (run->s390_sieic.ipb & 0xffff0000) >> 16;
+    uint8_t ipb = run->s390_sieic.ipb & 0xff;
 
     dprintf("KVM: PRIV: %d\n", ipa1);
     switch (ipa1) {
         case PRIV_SCLP_CALL:
-            r = kvm_sclp_service_call(env, run, ipbh0);
+            r = kvm_sclp_service_call(cpu, run, ipbh0);
             break;
         default:
-            dprintf("KVM: unknown PRIV: 0x%x\n", ipa1);
-            r = -1;
+            if (is_ioinst(ipa0, ipa1, ipb)) {
+                r = kvm_handle_css_inst(cpu, run, ipa0, ipa1, ipb);
+                if (r == -1) {
+                    setcc(cpu, 3);
+                    r = 0;
+                }
+            } else {
+                dprintf("KVM: unknown PRIV: 0x%x\n", ipa1);
+                r = -1;
+            }
             break;
     }
 
@@ -372,7 +543,7 @@ static int handle_priv(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
 static int handle_hypercall(CPUS390XState *env, struct kvm_run *run)
 {
     cpu_synchronize_state(env);
-    env->regs[2] = s390_virtio_hypercall(env, env->regs[2], env->regs[1]);
+    env->regs[2] = s390_virtio_hypercall(env);
 
     return 0;
 }
@@ -399,12 +570,10 @@ static int handle_diag(CPUS390XState *env, struct kvm_run *run, int ipb_code)
 
 static int s390_cpu_restart(S390CPU *cpu)
 {
-    CPUS390XState *env = &cpu->env;
-
-    kvm_s390_interrupt(env, KVM_S390_RESTART, 0);
-    s390_add_running_cpu(env);
+    kvm_s390_interrupt(cpu, KVM_S390_RESTART, 0);
+    s390_add_running_cpu(cpu);
     qemu_cpu_kick(CPU(cpu));
-    dprintf("DONE: SIGP cpu restart: %p\n", env);
+    dprintf("DONE: SIGP cpu restart: %p\n", &cpu->env);
     return 0;
 }
 
@@ -415,12 +584,13 @@ static int s390_store_status(CPUS390XState *env, uint32_t parameter)
     return -1;
 }
 
-static int s390_cpu_initial_reset(CPUS390XState *env)
+static int s390_cpu_initial_reset(S390CPU *cpu)
 {
+    CPUS390XState *env = &cpu->env;
     int i;
 
-    s390_del_running_cpu(env);
-    if (kvm_vcpu_ioctl(env, KVM_S390_INITIAL_RESET, NULL) < 0) {
+    s390_del_running_cpu(cpu);
+    if (kvm_vcpu_ioctl(CPU(cpu), KVM_S390_INITIAL_RESET, NULL) < 0) {
         perror("cannot init reset vcpu");
     }
 
@@ -434,8 +604,9 @@ static int s390_cpu_initial_reset(CPUS390XState *env)
     return 0;
 }
 
-static int handle_sigp(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
+static int handle_sigp(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
 {
+    CPUS390XState *env = &cpu->env;
     uint8_t order_code;
     uint32_t parameter;
     uint16_t cpu_addr;
@@ -479,7 +650,7 @@ static int handle_sigp(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
             /* make the caller panic */
             return -1;
         case SIGP_INITIAL_CPU_RESET:
-            r = s390_cpu_initial_reset(target_env);
+            r = s390_cpu_initial_reset(target_cpu);
             break;
         default:
             fprintf(stderr, "KVM: unknown SIGP: 0x%x\n", order_code);
@@ -487,12 +658,13 @@ static int handle_sigp(CPUS390XState *env, struct kvm_run *run, uint8_t ipa1)
     }
 
 out:
-    setcc(env, r ? 3 : 0);
+    setcc(cpu, r ? 3 : 0);
     return 0;
 }
 
-static int handle_instruction(CPUS390XState *env, struct kvm_run *run)
+static int handle_instruction(S390CPU *cpu, struct kvm_run *run)
 {
+    CPUS390XState *env = &cpu->env;
     unsigned int ipa0 = (run->s390_sieic.ipa & 0xff00);
     uint8_t ipa1 = run->s390_sieic.ipa & 0x00ff;
     int ipb_code = (run->s390_sieic.ipb & 0x0fff0000) >> 16;
@@ -500,50 +672,53 @@ static int handle_instruction(CPUS390XState *env, struct kvm_run *run)
 
     dprintf("handle_instruction 0x%x 0x%x\n", run->s390_sieic.ipa, run->s390_sieic.ipb);
     switch (ipa0) {
-        case IPA0_PRIV:
-            r = handle_priv(env, run, ipa1);
-            break;
-        case IPA0_DIAG:
-            r = handle_diag(env, run, ipb_code);
-            break;
-        case IPA0_SIGP:
-            r = handle_sigp(env, run, ipa1);
-            break;
+    case IPA0_B2:
+    case IPA0_B9:
+    case IPA0_EB:
+        r = handle_priv(cpu, run, ipa0 >> 8, ipa1);
+        break;
+    case IPA0_DIAG:
+        r = handle_diag(env, run, ipb_code);
+        break;
+    case IPA0_SIGP:
+        r = handle_sigp(cpu, run, ipa1);
+        break;
     }
 
     if (r < 0) {
-        enter_pgmcheck(env, 0x0001);
+        enter_pgmcheck(cpu, 0x0001);
     }
     return 0;
 }
 
-static bool is_special_wait_psw(CPUS390XState *env)
+static bool is_special_wait_psw(CPUState *cs)
 {
     /* signal quiesce */
-    return env->kvm_run->psw_addr == 0xfffUL;
+    return cs->kvm_run->psw_addr == 0xfffUL;
 }
 
-static int handle_intercept(CPUS390XState *env)
+static int handle_intercept(S390CPU *cpu)
 {
-    struct kvm_run *run = env->kvm_run;
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
     int icpt_code = run->s390_sieic.icptcode;
     int r = 0;
 
     dprintf("intercept: 0x%x (at 0x%lx)\n", icpt_code,
-            (long)env->kvm_run->psw_addr);
+            (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
-            r = handle_instruction(env, run);
+            r = handle_instruction(cpu, run);
             break;
         case ICPT_WAITPSW:
-            if (s390_del_running_cpu(env) == 0 &&
-                is_special_wait_psw(env)) {
+            if (s390_del_running_cpu(cpu) == 0 &&
+                is_special_wait_psw(cs)) {
                 qemu_system_shutdown_request();
             }
             r = EXCP_HALTED;
             break;
         case ICPT_CPU_STOP:
-            if (s390_del_running_cpu(env) == 0) {
+            if (s390_del_running_cpu(cpu) == 0) {
                 qemu_system_shutdown_request();
             }
             r = EXCP_HALTED;
@@ -565,17 +740,58 @@ static int handle_intercept(CPUS390XState *env)
     return r;
 }
 
-int kvm_arch_handle_exit(CPUS390XState *env, struct kvm_run *run)
+static int handle_tsch(S390CPU *cpu)
 {
+    CPUS390XState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
+    int ret;
+
+    cpu_synchronize_state(env);
+    ret = ioinst_handle_tsch(env, env->regs[1], run->s390_tsch.ipb);
+    if (ret >= 0) {
+        /* Success; set condition code. */
+        setcc(cpu, ret);
+        ret = 0;
+    } else if (ret < -1) {
+        /*
+         * Failure.
+         * If an I/O interrupt had been dequeued, we have to reinject it.
+         */
+        if (run->s390_tsch.dequeued) {
+            uint16_t subchannel_id = run->s390_tsch.subchannel_id;
+            uint16_t subchannel_nr = run->s390_tsch.subchannel_nr;
+            uint32_t io_int_parm = run->s390_tsch.io_int_parm;
+            uint32_t io_int_word = run->s390_tsch.io_int_word;
+            uint32_t type = ((subchannel_id & 0xff00) << 24) |
+                ((subchannel_id & 0x00060) << 22) | (subchannel_nr << 16);
+
+            kvm_s390_interrupt_internal(cpu, type,
+                                        ((uint32_t)subchannel_id << 16)
+                                        | subchannel_nr,
+                                        ((uint64_t)io_int_parm << 32)
+                                        | io_int_word, 1);
+        }
+        ret = 0;
+    }
+    return ret;
+}
+
+int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
+{
+    S390CPU *cpu = S390_CPU(cs);
     int ret = 0;
 
     switch (run->exit_reason) {
         case KVM_EXIT_S390_SIEIC:
-            ret = handle_intercept(env);
+            ret = handle_intercept(cpu);
             break;
         case KVM_EXIT_S390_RESET:
             qemu_system_reset_request();
             break;
+        case KVM_EXIT_S390_TSCH:
+            ret = handle_tsch(cpu);
+            break;
         default:
             fprintf(stderr, "Unknown KVM exit: %d\n", run->exit_reason);
             break;
@@ -587,12 +803,12 @@ int kvm_arch_handle_exit(CPUS390XState *env, struct kvm_run *run)
     return ret;
 }
 
-bool kvm_arch_stop_on_emulation_error(CPUS390XState *env)
+bool kvm_arch_stop_on_emulation_error(CPUState *cpu)
 {
     return true;
 }
 
-int kvm_arch_on_sigbus_vcpu(CPUS390XState *env, int code, void *addr)
+int kvm_arch_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
 {
     return 1;
 }
@@ -601,3 +817,33 @@ int kvm_arch_on_sigbus(int code, void *addr)
 {
     return 1;
 }
+
+void kvm_s390_io_interrupt(S390CPU *cpu, uint16_t subchannel_id,
+                           uint16_t subchannel_nr, uint32_t io_int_parm,
+                           uint32_t io_int_word)
+{
+    uint32_t type;
+
+    type = ((subchannel_id & 0xff00) << 24) |
+        ((subchannel_id & 0x00060) << 22) | (subchannel_nr << 16);
+    kvm_s390_interrupt_internal(cpu, type,
+                                ((uint32_t)subchannel_id << 16) | subchannel_nr,
+                                ((uint64_t)io_int_parm << 32) | io_int_word, 1);
+}
+
+void kvm_s390_crw_mchk(S390CPU *cpu)
+{
+    kvm_s390_interrupt_internal(cpu, KVM_S390_MCHK, 1 << 28,
+                                0x00400f1d40330000, 1);
+}
+
+void kvm_s390_enable_css_support(S390CPU *cpu)
+{
+    struct kvm_enable_cap cap = {};
+    int r;
+
+    /* Activate host kernel channel subsystem support. */
+    cap.cap = KVM_CAP_S390_CSS_SUPPORT;
+    r = kvm_vcpu_ioctl(CPU(cpu), KVM_ENABLE_CAP, &cap);
+    assert(r == 0);
+}
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
index 6ebc22d..372334b 100644
--- a/target-s390x/mem_helper.c
+++ b/target-s390x/mem_helper.c
@@ -24,21 +24,21 @@
 /*****************************************************************************/
 /* Softmmu support */
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -47,19 +47,13 @@
 void tlb_fill(CPUS390XState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
     int ret;
 
     ret = cpu_s390x_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret != 0)) {
         if (likely(retaddr)) {
             /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (likely(tb)) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
@@ -310,214 +304,142 @@ uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
     return cc;
 }
 
-/* store character under mask */
-void HELPER(stcm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
-                  uint64_t addr)
+static inline uint64_t fix_address(CPUS390XState *env, uint64_t a)
 {
-    uint8_t r;
-
-    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%lx\n", __func__, r1, mask,
-               addr);
-    while (mask) {
-        if (mask & 8) {
-            r = (r1 & 0xff000000UL) >> 24;
-            cpu_stb_data(env, addr, r);
-            HELPER_LOG("mask 0x%x %02x (0x%lx) ", mask, r, addr);
-            addr++;
-        }
-        mask = (mask << 1) & 0xf;
-        r1 <<= 8;
+    /* 31-Bit mode */
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        a &= 0x7fffffff;
     }
-    HELPER_LOG("\n");
+    return a;
 }
 
 static inline uint64_t get_address(CPUS390XState *env, int x2, int b2, int d2)
 {
     uint64_t r = d2;
-
     if (x2) {
         r += env->regs[x2];
     }
-
     if (b2) {
         r += env->regs[b2];
     }
-
-    /* 31-Bit mode */
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        r &= 0x7fffffff;
-    }
-
-    return r;
+    return fix_address(env, r);
 }
 
 static inline uint64_t get_address_31fix(CPUS390XState *env, int reg)
 {
-    uint64_t r = env->regs[reg];
-
-    /* 31-Bit mode */
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        r &= 0x7fffffff;
-    }
-
-    return r;
+    return fix_address(env, env->regs[reg]);
 }
 
 /* search string (c is byte to search, r2 is string, r1 end of string) */
-uint32_t HELPER(srst)(CPUS390XState *env, uint32_t c, uint32_t r1, uint32_t r2)
+uint64_t HELPER(srst)(CPUS390XState *env, uint64_t r0, uint64_t end,
+                      uint64_t str)
 {
-    uint64_t i;
-    uint32_t cc = 2;
-    uint64_t str = get_address_31fix(env, r2);
-    uint64_t end = get_address_31fix(env, r1);
-
-    HELPER_LOG("%s: c %d *r1 0x%" PRIx64 " *r2 0x%" PRIx64 "\n", __func__,
-               c, env->regs[r1], env->regs[r2]);
-
-    for (i = str; i != end; i++) {
-        if (cpu_ldub_data(env, i) == c) {
-            env->regs[r1] = i;
-            cc = 1;
-            break;
+    uint32_t len;
+    uint8_t v, c = r0;
+
+    str = fix_address(env, str);
+    end = fix_address(env, end);
+
+    /* Assume for now that R2 is unmodified.  */
+    env->retxl = str;
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    for (len = 0; len < 0x2000; ++len) {
+        if (str + len == end) {
+            /* Character not found.  R1 & R2 are unmodified.  */
+            env->cc_op = 2;
+            return end;
+        }
+        v = cpu_ldub_data(env, str + len);
+        if (v == c) {
+            /* Character found.  Set R1 to the location; R2 is unmodified.  */
+            env->cc_op = 1;
+            return str + len;
         }
     }
 
-    return cc;
+    /* CPU-determined bytes processed.  Advance R2 to next byte to process.  */
+    env->retxl = str + len;
+    env->cc_op = 3;
+    return end;
 }
 
 /* unsigned string compare (c is string terminator) */
-uint32_t HELPER(clst)(CPUS390XState *env, uint32_t c, uint32_t r1, uint32_t r2)
+uint64_t HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
 {
-    uint64_t s1 = get_address_31fix(env, r1);
-    uint64_t s2 = get_address_31fix(env, r2);
-    uint8_t v1, v2;
-    uint32_t cc;
+    uint32_t len;
 
     c = c & 0xff;
-#ifdef CONFIG_USER_ONLY
-    if (!c) {
-        HELPER_LOG("%s: comparing '%s' and '%s'\n",
-                   __func__, (char *)g2h(s1), (char *)g2h(s2));
-    }
-#endif
-    for (;;) {
-        v1 = cpu_ldub_data(env, s1);
-        v2 = cpu_ldub_data(env, s2);
-        if ((v1 == c || v2 == c) || (v1 != v2)) {
-            break;
+    s1 = fix_address(env, s1);
+    s2 = fix_address(env, s2);
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    for (len = 0; len < 0x2000; ++len) {
+        uint8_t v1 = cpu_ldub_data(env, s1 + len);
+        uint8_t v2 = cpu_ldub_data(env, s2 + len);
+        if (v1 == v2) {
+            if (v1 == c) {
+                /* Equal.  CC=0, and don't advance the registers.  */
+                env->cc_op = 0;
+                env->retxl = s2;
+                return s1;
+            }
+        } else {
+            /* Unequal.  CC={1,2}, and advance the registers.  Note that
+               the terminator need not be zero, but the string that contains
+               the terminator is by definition "low".  */
+            env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
+            env->retxl = s2 + len;
+            return s1 + len;
         }
-        s1++;
-        s2++;
     }
 
-    if (v1 == v2) {
-        cc = 0;
-    } else {
-        cc = (v1 < v2) ? 1 : 2;
-        /* FIXME: 31-bit mode! */
-        env->regs[r1] = s1;
-        env->regs[r2] = s2;
-    }
-    return cc;
+    /* CPU-determined bytes equal; advance the registers.  */
+    env->cc_op = 3;
+    env->retxl = s2 + len;
+    return s1 + len;
 }
 
 /* move page */
 void HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint64_t r1, uint64_t r2)
 {
     /* XXX missing r0 handling */
+    env->cc_op = 0;
 #ifdef CONFIG_USER_ONLY
-    int i;
-
-    for (i = 0; i < TARGET_PAGE_SIZE; i++) {
-        cpu_stb_data(env, r1 + i, cpu_ldub_data(env, r2 + i));
-    }
+    memmove(g2h(r1), g2h(r2), TARGET_PAGE_SIZE);
 #else
     mvc_fast_memmove(env, TARGET_PAGE_SIZE, r1, r2);
 #endif
 }
 
 /* string copy (c is string terminator) */
-void HELPER(mvst)(CPUS390XState *env, uint32_t c, uint32_t r1, uint32_t r2)
+uint64_t HELPER(mvst)(CPUS390XState *env, uint64_t c, uint64_t d, uint64_t s)
 {
-    uint64_t dest = get_address_31fix(env, r1);
-    uint64_t src = get_address_31fix(env, r2);
-    uint8_t v;
+    uint32_t len;
 
     c = c & 0xff;
-#ifdef CONFIG_USER_ONLY
-    if (!c) {
-        HELPER_LOG("%s: copy '%s' to 0x%lx\n", __func__, (char *)g2h(src),
-                   dest);
-    }
-#endif
-    for (;;) {
-        v = cpu_ldub_data(env, src);
-        cpu_stb_data(env, dest, v);
+    d = fix_address(env, d);
+    s = fix_address(env, s);
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    for (len = 0; len < 0x2000; ++len) {
+        uint8_t v = cpu_ldub_data(env, s + len);
+        cpu_stb_data(env, d + len, v);
         if (v == c) {
-            break;
+            /* Complete.  Set CC=1 and advance R1.  */
+            env->cc_op = 1;
+            env->retxl = s;
+            return d + len;
         }
-        src++;
-        dest++;
     }
-    env->regs[r1] = dest; /* FIXME: 31-bit mode! */
-}
 
-/* compare and swap 64-bit */
-uint32_t HELPER(csg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2 = cpu_ldq_data(env, a2);
-
-    if (env->regs[r1] == v2) {
-        cc = 0;
-        cpu_stq_data(env, a2, env->regs[r3]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2;
-    }
-    return cc;
-}
-
-/* compare double and swap 64-bit */
-uint32_t HELPER(cdsg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2_hi = cpu_ldq_data(env, a2);
-    uint64_t v2_lo = cpu_ldq_data(env, a2 + 8);
-    uint64_t v1_hi = env->regs[r1];
-    uint64_t v1_lo = env->regs[r1 + 1];
-
-    if ((v1_hi == v2_hi) && (v1_lo == v2_lo)) {
-        cc = 0;
-        cpu_stq_data(env, a2, env->regs[r3]);
-        cpu_stq_data(env, a2 + 8, env->regs[r3 + 1]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2_hi;
-        env->regs[r1 + 1] = v2_lo;
-    }
-
-    return cc;
-}
-
-/* compare and swap 32-bit */
-uint32_t HELPER(cs)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint32_t v2 = cpu_ldl_data(env, a2);
-
-    HELPER_LOG("%s: r1 %d a2 0x%lx r3 %d\n", __func__, r1, a2, r3);
-    if (((uint32_t)env->regs[r1]) == v2) {
-        cc = 0;
-        cpu_stl_data(env, a2, (uint32_t)env->regs[r3]);
-    } else {
-        cc = 1;
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | v2;
-    }
-    return cc;
+    /* Incomplete.  Set CC=3 and signal to advance R1 and R2.  */
+    env->cc_op = 3;
+    env->retxl = s + len;
+    return d + len;
 }
 
 static uint32_t helper_icm(CPUS390XState *env, uint32_t r1, uint64_t address,
@@ -600,7 +522,7 @@ uint32_t HELPER(ex)(CPUS390XState *env, uint32_t cc, uint64_t v1,
         HELPER_LOG("%s: svc %ld via execute\n", __func__, (insn | v1) & 0xff);
         env->psw.addr = ret - 4;
         env->int_svc_code = (insn | v1) & 0xff;
-        env->int_svc_ilc = 4;
+        env->int_svc_ilen = 4;
         helper_exception(env, EXCP_SVC);
     } else if ((insn & 0xff00) == 0xbf00) {
         uint32_t insn2, r1, r3, b2, d2;
@@ -619,55 +541,6 @@ uint32_t HELPER(ex)(CPUS390XState *env, uint32_t cc, uint64_t v1,
     return cc;
 }
 
-/* store character under mask high operates on the upper half of r1 */
-void HELPER(stcmh)(CPUS390XState *env, uint32_t r1, uint64_t address,
-                   uint32_t mask)
-{
-    int pos = 56; /* top of the upper half of r1 */
-
-    while (mask) {
-        if (mask & 8) {
-            cpu_stb_data(env, address, (env->regs[r1] >> pos) & 0xff);
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-    }
-}
-
-/* insert character under mask high; same as icm, but operates on the
-   upper half of r1 */
-uint32_t HELPER(icmh)(CPUS390XState *env, uint32_t r1, uint64_t address,
-                      uint32_t mask)
-{
-    int pos = 56; /* top of the upper half of r1 */
-    uint64_t rmask = 0xff00000000000000ULL;
-    uint8_t val = 0;
-    int ccd = 0;
-    uint32_t cc = 0;
-
-    while (mask) {
-        if (mask & 8) {
-            env->regs[r1] &= ~rmask;
-            val = cpu_ldub_data(env, address);
-            if ((val & 0x80) && !ccd) {
-                cc = 1;
-            }
-            ccd = 1;
-            if (val && cc == 0) {
-                cc = 2;
-            }
-            env->regs[r1] |= (uint64_t)val << pos;
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-        rmask >>= 8;
-    }
-
-    return cc;
-}
-
 /* load access registers r1 to r3 from memory at a2 */
 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 {
@@ -828,42 +701,49 @@ uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
 }
 
 /* checksum */
-void HELPER(cksm)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+uint64_t HELPER(cksm)(CPUS390XState *env, uint64_t r1,
+                      uint64_t src, uint64_t src_len)
 {
-    uint64_t src = get_address_31fix(env, r2);
-    uint64_t src_len = env->regs[(r2 + 1) & 15];
-    uint64_t cksm = (uint32_t)env->regs[r1];
+    uint64_t max_len, len;
+    uint64_t cksm = (uint32_t)r1;
 
-    while (src_len >= 4) {
-        cksm += cpu_ldl_data(env, src);
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, lets cap at 8k.  */
+    max_len = (src_len > 0x2000 ? 0x2000 : src_len);
 
-        /* move to next word */
-        src_len -= 4;
-        src += 4;
+    /* Process full words as available.  */
+    for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
+        cksm += (uint32_t)cpu_ldl_data(env, src);
     }
 
-    switch (src_len) {
-    case 0:
-        break;
+    switch (max_len - len) {
     case 1:
         cksm += cpu_ldub_data(env, src) << 24;
+        len += 1;
         break;
     case 2:
         cksm += cpu_lduw_data(env, src) << 16;
+        len += 2;
         break;
     case 3:
         cksm += cpu_lduw_data(env, src) << 16;
         cksm += cpu_ldub_data(env, src + 2) << 8;
+        len += 3;
         break;
     }
 
-    /* indicate we've processed everything */
-    env->regs[r2] = src + src_len;
-    env->regs[(r2 + 1) & 15] = 0;
+    /* Fold the carry from the checksum.  Note that we can see carry-out
+       during folding more than once (but probably not more than twice).  */
+    while (cksm > 0xffffffffull) {
+        cksm = (uint32_t)cksm + (cksm >> 32);
+    }
+
+    /* Indicate whether or not we've processed everything.  */
+    env->cc_op = (len == src_len ? 0 : 3);
 
-    /* store result */
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-        ((uint32_t)cksm + (cksm >> 32));
+    /* Return both cksm and processed length.  */
+    env->retxl = cksm;
+    return len;
 }
 
 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
@@ -1013,7 +893,7 @@ uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
 }
 
 /* set storage key extended */
-void HELPER(sske)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 {
     uint64_t addr = get_address(env, 0, 0, r2);
 
@@ -1025,7 +905,7 @@ void HELPER(sske)(CPUS390XState *env, uint32_t r1, uint64_t r2)
 }
 
 /* reset reference bit extended */
-uint32_t HELPER(rrbe)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
 {
     uint8_t re;
     uint8_t key;
@@ -1051,16 +931,16 @@ uint32_t HELPER(rrbe)(CPUS390XState *env, uint32_t r1, uint64_t r2)
 }
 
 /* compare and swap and purge */
-uint32_t HELPER(csp)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+uint32_t HELPER(csp)(CPUS390XState *env, uint32_t r1, uint64_t r2)
 {
     uint32_t cc;
     uint32_t o1 = env->regs[r1];
-    uint64_t a2 = get_address_31fix(env, r2) & ~3ULL;
+    uint64_t a2 = r2 & ~3ULL;
     uint32_t o2 = cpu_ldl_data(env, a2);
 
     if (o1 == o2) {
         cpu_stl_data(env, a2, env->regs[(r1 + 1) & 15]);
-        if (env->regs[r2] & 0x3) {
+        if (r2 & 0x3) {
             /* flush TLB / ALB */
             tlb_flush(env, 1);
         }
@@ -1160,13 +1040,13 @@ void HELPER(ptlb)(CPUS390XState *env)
 }
 
 /* store using real address */
-void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint32_t v1)
+void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1)
 {
-    stw_phys(get_address(env, 0, 0, addr), v1);
+    stw_phys(get_address(env, 0, 0, addr), (uint32_t)v1);
 }
 
 /* load real address */
-uint32_t HELPER(lra)(CPUS390XState *env, uint64_t addr, uint32_t r1)
+uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
 {
     uint32_t cc = 0;
     int old_exc = env->exception_index;
@@ -1190,14 +1070,7 @@ uint32_t HELPER(lra)(CPUS390XState *env, uint64_t addr, uint32_t r1)
     }
     env->exception_index = old_exc;
 
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-            (ret & 0xffffffffULL);
-    } else {
-        env->regs[r1] = ret;
-    }
-
-    return cc;
+    env->cc_op = cc;
+    return ret;
 }
-
 #endif
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index 38d8f2a..09301d0 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -19,19 +19,19 @@
  */
 
 #include "cpu.h"
-#include "memory.h"
-#include "host-utils.h"
+#include "exec/memory.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
 #include <string.h>
-#include "kvm.h"
-#include "qemu-timer.h"
+#include "sysemu/kvm.h"
+#include "qemu/timer.h"
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
-#include "sysemu.h"
+#include "exec/softmmu_exec.h"
+#include "sysemu/sysemu.h"
 #endif
 
 /* #define DEBUG_HELPER */
@@ -41,7 +41,27 @@
 #define HELPER_LOG(x...)
 #endif
 
-/* raise an exception */
+/* Raise an exception dynamically from a helper function.  */
+void QEMU_NORETURN runtime_exception(CPUS390XState *env, int excp,
+                                     uintptr_t retaddr)
+{
+    int t;
+
+    env->exception_index = EXCP_PGM;
+    env->int_pgm_code = excp;
+
+    /* Use the (ultimate) callers address to find the insn that trapped.  */
+    cpu_restore_state(env, retaddr);
+
+    /* Advance past the insn.  */
+    t = cpu_ldub_code(env, env->psw.addr);
+    env->int_pgm_ilen = t = get_ilen(t);
+    env->psw.addr += 2 * t;
+
+    cpu_loop_exit(env);
+}
+
+/* Raise an exception statically from a TB.  */
 void HELPER(exception)(CPUS390XState *env, uint32_t excp)
 {
     HELPER_LOG("%s: exception %d\n", __func__, excp);
@@ -50,29 +70,108 @@ void HELPER(exception)(CPUS390XState *env, uint32_t excp)
 }
 
 #ifndef CONFIG_USER_ONLY
-void program_interrupt(CPUS390XState *env, uint32_t code, int ilc)
+
+/* EBCDIC handling */
+static const uint8_t ebcdic2ascii[] = {
+    0x00, 0x01, 0x02, 0x03, 0x07, 0x09, 0x07, 0x7F,
+    0x07, 0x07, 0x07, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    0x10, 0x11, 0x12, 0x13, 0x07, 0x0A, 0x08, 0x07,
+    0x18, 0x19, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x07, 0x07, 0x1C, 0x07, 0x07, 0x0A, 0x17, 0x1B,
+    0x07, 0x07, 0x07, 0x07, 0x07, 0x05, 0x06, 0x07,
+    0x07, 0x07, 0x16, 0x07, 0x07, 0x07, 0x07, 0x04,
+    0x07, 0x07, 0x07, 0x07, 0x14, 0x15, 0x07, 0x1A,
+    0x20, 0xFF, 0x83, 0x84, 0x85, 0xA0, 0x07, 0x86,
+    0x87, 0xA4, 0x5B, 0x2E, 0x3C, 0x28, 0x2B, 0x21,
+    0x26, 0x82, 0x88, 0x89, 0x8A, 0xA1, 0x8C, 0x07,
+    0x8D, 0xE1, 0x5D, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
+    0x2D, 0x2F, 0x07, 0x8E, 0x07, 0x07, 0x07, 0x8F,
+    0x80, 0xA5, 0x07, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+    0x07, 0x90, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+    0x70, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+    0x07, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+    0x68, 0x69, 0xAE, 0xAF, 0x07, 0x07, 0x07, 0xF1,
+    0xF8, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70,
+    0x71, 0x72, 0xA6, 0xA7, 0x91, 0x07, 0x92, 0x07,
+    0xE6, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+    0x79, 0x7A, 0xAD, 0xAB, 0x07, 0x07, 0x07, 0x07,
+    0x9B, 0x9C, 0x9D, 0xFA, 0x07, 0x07, 0x07, 0xAC,
+    0xAB, 0x07, 0xAA, 0x7C, 0x07, 0x07, 0x07, 0x07,
+    0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+    0x48, 0x49, 0x07, 0x93, 0x94, 0x95, 0xA2, 0x07,
+    0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
+    0x51, 0x52, 0x07, 0x96, 0x81, 0x97, 0xA3, 0x98,
+    0x5C, 0xF6, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
+    0x59, 0x5A, 0xFD, 0x07, 0x99, 0x07, 0x07, 0x07,
+    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+    0x38, 0x39, 0x07, 0x07, 0x9A, 0x07, 0x07, 0x07,
+};
+
+static const uint8_t ascii2ebcdic[] = {
+    0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
+    0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+    0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
+    0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
+    0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
+    0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+    0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+    0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+    0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+    0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
+    0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
+    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+    0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+    0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
+    0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+    0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
+};
+
+static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
+{
+    int i;
+
+    for (i = 0; i < len; i++) {
+        p[i] = ascii2ebcdic[(uint8_t)ascii[i]];
+    }
+}
+
+void program_interrupt(CPUS390XState *env, uint32_t code, int ilen)
 {
     qemu_log_mask(CPU_LOG_INT, "program interrupt at %#" PRIx64 "\n",
                   env->psw.addr);
 
     if (kvm_enabled()) {
 #ifdef CONFIG_KVM
-        kvm_s390_interrupt(env, KVM_S390_PROGRAM_INT, code);
+        kvm_s390_interrupt(s390_env_get_cpu(env), KVM_S390_PROGRAM_INT, code);
 #endif
     } else {
         env->int_pgm_code = code;
-        env->int_pgm_ilc = ilc;
+        env->int_pgm_ilen = ilen;
         env->exception_index = EXCP_PGM;
         cpu_loop_exit(env);
     }
 }
 
 /* SCLP service call */
-uint32_t HELPER(servc)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+uint32_t HELPER(servc)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 {
-    int r;
-
-    r = sclp_service_call(r1, r2);
+    int r = sclp_service_call(r1, r2);
     if (r < 0) {
         program_interrupt(env, -r, 4);
         return 0;
@@ -89,7 +188,7 @@ uint64_t HELPER(diag)(CPUS390XState *env, uint32_t num, uint64_t mem,
     switch (num) {
     case 0x500:
         /* KVM hypercall */
-        r = s390_virtio_hypercall(env, mem, code);
+        r = s390_virtio_hypercall(env);
         break;
     case 0x44:
         /* yield */
@@ -105,38 +204,22 @@ uint64_t HELPER(diag)(CPUS390XState *env, uint32_t num, uint64_t mem,
     }
 
     if (r) {
-        program_interrupt(env, PGM_OPERATION, ILC_LATER_INC);
+        program_interrupt(env, PGM_OPERATION, ILEN_LATER_INC);
     }
 
     return r;
 }
 
-/* Store CPU ID */
-void HELPER(stidp)(CPUS390XState *env, uint64_t a1)
-{
-    cpu_stq_data(env, a1, env->cpu_num);
-}
-
 /* Set Prefix */
 void HELPER(spx)(CPUS390XState *env, uint64_t a1)
 {
-    uint32_t prefix;
-
-    prefix = cpu_ldl_data(env, a1);
-    env->psa = prefix & 0xfffff000;
+    uint32_t prefix = a1 & 0x7fffe000;
+    env->psa = prefix;
     qemu_log("prefix: %#x\n", prefix);
     tlb_flush_page(env, 0);
     tlb_flush_page(env, TARGET_PAGE_SIZE);
 }
 
-/* Set Clock */
-uint32_t HELPER(sck)(uint64_t a1)
-{
-    /* XXX not implemented - is it necessary? */
-
-    return 0;
-}
-
 static inline uint64_t clock_value(CPUS390XState *env)
 {
     uint64_t time;
@@ -148,32 +231,14 @@ static inline uint64_t clock_value(CPUS390XState *env)
 }
 
 /* Store Clock */
-uint32_t HELPER(stck)(CPUS390XState *env, uint64_t a1)
+uint64_t HELPER(stck)(CPUS390XState *env)
 {
-    cpu_stq_data(env, a1, clock_value(env));
-
-    return 0;
-}
-
-/* Store Clock Extended */
-uint32_t HELPER(stcke)(CPUS390XState *env, uint64_t a1)
-{
-    cpu_stb_data(env, a1, 0);
-    /* basically the same value as stck */
-    cpu_stq_data(env, a1 + 1, clock_value(env) | env->cpu_num);
-    /* more fine grained than stck */
-    cpu_stq_data(env, a1 + 9, 0);
-    /* XXX programmable fields */
-    cpu_stw_data(env, a1 + 17, 0);
-
-    return 0;
+    return clock_value(env);
 }
 
 /* Set Clock Comparator */
-void HELPER(sckc)(CPUS390XState *env, uint64_t a1)
+void HELPER(sckc)(CPUS390XState *env, uint64_t time)
 {
-    uint64_t time = cpu_ldq_data(env, a1);
-
     if (time == -1ULL) {
         return;
     }
@@ -187,17 +252,15 @@ void HELPER(sckc)(CPUS390XState *env, uint64_t a1)
 }
 
 /* Store Clock Comparator */
-void HELPER(stckc)(CPUS390XState *env, uint64_t a1)
+uint64_t HELPER(stckc)(CPUS390XState *env)
 {
     /* XXX implement */
-    cpu_stq_data(env, a1, 0);
+    return 0;
 }
 
 /* Set CPU Timer */
-void HELPER(spt)(CPUS390XState *env, uint64_t a1)
+void HELPER(spt)(CPUS390XState *env, uint64_t time)
 {
-    uint64_t time = cpu_ldq_data(env, a1);
-
     if (time == -1ULL) {
         return;
     }
@@ -209,15 +272,15 @@ void HELPER(spt)(CPUS390XState *env, uint64_t a1)
 }
 
 /* Store CPU Timer */
-void HELPER(stpt)(CPUS390XState *env, uint64_t a1)
+uint64_t HELPER(stpt)(CPUS390XState *env)
 {
     /* XXX implement */
-    cpu_stq_data(env, a1, 0);
+    return 0;
 }
 
 /* Store System Information */
-uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0, uint32_t r0,
-                      uint32_t r1)
+uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0,
+                      uint64_t r0, uint64_t r1)
 {
     int cc = 0;
     int sel1, sel2;
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 993f207..a57296c 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -18,7 +18,6 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-/* #define DEBUG_ILLEGAL_INSTRUCTIONS */
 /* #define DEBUG_INLINE_BRANCHES */
 #define S390X_DEBUG_DISAS
 /* #define S390X_DEBUG_DISAS_VERBOSE */
@@ -30,41 +29,54 @@
 #endif
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
+#include "qemu/host-utils.h"
 
 /* global register indexes */
 static TCGv_ptr cpu_env;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 #include "helper.h"
 #define GEN_HELPER 1
 #include "helper.h"
 
+
+/* Information that (most) every instruction needs to manipulate.  */
 typedef struct DisasContext DisasContext;
+typedef struct DisasInsn DisasInsn;
+typedef struct DisasFields DisasFields;
+
 struct DisasContext {
-    uint64_t pc;
-    int is_jmp;
-    enum cc_op cc_op;
     struct TranslationBlock *tb;
+    const DisasInsn *insn;
+    DisasFields *fields;
+    uint64_t pc, next_pc;
+    enum cc_op cc_op;
+    bool singlestep_enabled;
 };
 
-#define DISAS_EXCP 4
+/* Information carried about a condition to be evaluated.  */
+typedef struct {
+    TCGCond cond:8;
+    bool is_64;
+    bool g1;
+    bool g2;
+    union {
+        struct { TCGv_i64 a, b; } s64;
+        struct { TCGv_i32 a, b; } s32;
+    } u;
+} DisasCompare;
 
-static void gen_op_calc_cc(DisasContext *s);
+#define DISAS_EXCP 4
 
 #ifdef DEBUG_INLINE_BRANCHES
 static uint64_t inline_branch_hit[CC_OP_MAX];
 static uint64_t inline_branch_miss[CC_OP_MAX];
 #endif
 
-static inline void debug_insn(uint64_t insn)
-{
-    LOG_DISAS("insn: 0x%" PRIx64 "\n", insn);
-}
-
-static inline uint64_t pc_to_link_info(DisasContext *s, uint64_t pc)
+static uint64_t pc_to_link_info(DisasContext *s, uint64_t pc)
 {
     if (!(s->tb->flags & FLAG_MASK_64)) {
         if (s->tb->flags & FLAG_MASK_32) {
@@ -97,7 +109,7 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
     }
 
     for (i = 0; i < 16; i++) {
-        cpu_fprintf(f, "F%02d=%016" PRIx64, i, *(uint64_t *)&env->fregs[i]);
+        cpu_fprintf(f, "F%02d=%016" PRIx64, i, env->fregs[i].ll);
         if ((i % 4) == 3) {
             cpu_fprintf(f, "\n");
         } else {
@@ -134,21 +146,22 @@ static TCGv_i64 cc_src;
 static TCGv_i64 cc_dst;
 static TCGv_i64 cc_vr;
 
-static char cpu_reg_names[10*3 + 6*4];
+static char cpu_reg_names[32][4];
 static TCGv_i64 regs[16];
+static TCGv_i64 fregs[16];
 
 static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
 
 void s390x_translate_init(void)
 {
     int i;
-    size_t cpu_reg_names_size = sizeof(cpu_reg_names);
-    char *p;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    psw_addr = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, psw.addr),
+    psw_addr = tcg_global_mem_new_i64(TCG_AREG0,
+                                      offsetof(CPUS390XState, psw.addr),
                                       "psw_addr");
-    psw_mask = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, psw.mask),
+    psw_mask = tcg_global_mem_new_i64(TCG_AREG0,
+                                      offsetof(CPUS390XState, psw.mask),
                                       "psw_mask");
 
     cc_op = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUS390XState, cc_op),
@@ -160,116 +173,87 @@ void s390x_translate_init(void)
     cc_vr = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_vr),
                                    "cc_vr");
 
-    p = cpu_reg_names;
     for (i = 0; i < 16; i++) {
-        snprintf(p, cpu_reg_names_size, "r%d", i);
+        snprintf(cpu_reg_names[i], sizeof(cpu_reg_names[0]), "r%d", i);
         regs[i] = tcg_global_mem_new(TCG_AREG0,
-                                     offsetof(CPUS390XState, regs[i]), p);
-        p += (i < 10) ? 3 : 4;
-        cpu_reg_names_size -= (i < 10) ? 3 : 4;
+                                     offsetof(CPUS390XState, regs[i]),
+                                     cpu_reg_names[i]);
     }
-}
 
-static inline TCGv_i64 load_reg(int reg)
-{
-    TCGv_i64 r = tcg_temp_new_i64();
-    tcg_gen_mov_i64(r, regs[reg]);
-    return r;
-}
-
-static inline TCGv_i64 load_freg(int reg)
-{
-    TCGv_i64 r = tcg_temp_new_i64();
-    tcg_gen_ld_i64(r, cpu_env, offsetof(CPUS390XState, fregs[reg].d));
-    return r;
-}
+    for (i = 0; i < 16; i++) {
+        snprintf(cpu_reg_names[i + 16], sizeof(cpu_reg_names[0]), "f%d", i);
+        fregs[i] = tcg_global_mem_new(TCG_AREG0,
+                                      offsetof(CPUS390XState, fregs[i].d),
+                                      cpu_reg_names[i + 16]);
+    }
 
-static inline TCGv_i32 load_freg32(int reg)
-{
-    TCGv_i32 r = tcg_temp_new_i32();
-    tcg_gen_ld_i32(r, cpu_env, offsetof(CPUS390XState, fregs[reg].l.upper));
-    return r;
+    /* register helpers */
+#define GEN_HELPER 2
+#include "helper.h"
 }
 
-static inline TCGv_i32 load_reg32(int reg)
+static TCGv_i64 load_reg(int reg)
 {
-    TCGv_i32 r = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(r, regs[reg]);
+    TCGv_i64 r = tcg_temp_new_i64();
+    tcg_gen_mov_i64(r, regs[reg]);
     return r;
 }
 
-static inline TCGv_i64 load_reg32_i64(int reg)
+static TCGv_i64 load_freg32_i64(int reg)
 {
     TCGv_i64 r = tcg_temp_new_i64();
-    tcg_gen_ext32s_i64(r, regs[reg]);
+    tcg_gen_shri_i64(r, fregs[reg], 32);
     return r;
 }
 
-static inline void store_reg(int reg, TCGv_i64 v)
+static void store_reg(int reg, TCGv_i64 v)
 {
     tcg_gen_mov_i64(regs[reg], v);
 }
 
-static inline void store_freg(int reg, TCGv_i64 v)
+static void store_freg(int reg, TCGv_i64 v)
 {
-    tcg_gen_st_i64(v, cpu_env, offsetof(CPUS390XState, fregs[reg].d));
+    tcg_gen_mov_i64(fregs[reg], v);
 }
 
-static inline void store_reg32(int reg, TCGv_i32 v)
+static void store_reg32_i64(int reg, TCGv_i64 v)
 {
-#if HOST_LONG_BITS == 32
-    tcg_gen_mov_i32(TCGV_LOW(regs[reg]), v);
-#else
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    tcg_gen_extu_i32_i64(tmp, v);
     /* 32 bit register writes keep the upper half */
-    tcg_gen_deposit_i64(regs[reg], regs[reg], tmp, 0, 32);
-    tcg_temp_free_i64(tmp);
-#endif
-}
-
-static inline void store_reg32_i64(int reg, TCGv_i64 v)
-{
-    /* 32 bit register writes keep the upper half */
-#if HOST_LONG_BITS == 32
-    tcg_gen_mov_i32(TCGV_LOW(regs[reg]), TCGV_LOW(v));
-#else
     tcg_gen_deposit_i64(regs[reg], regs[reg], v, 0, 32);
-#endif
 }
 
-static inline void store_reg16(int reg, TCGv_i32 v)
+static void store_reg32h_i64(int reg, TCGv_i64 v)
 {
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    tcg_gen_extu_i32_i64(tmp, v);
-    /* 16 bit register writes keep the upper bytes */
-    tcg_gen_deposit_i64(regs[reg], regs[reg], tmp, 0, 16);
-    tcg_temp_free_i64(tmp);
+    tcg_gen_deposit_i64(regs[reg], regs[reg], v, 32, 32);
 }
 
-static inline void store_reg8(int reg, TCGv_i64 v)
+static void store_freg32_i64(int reg, TCGv_i64 v)
 {
-    /* 8 bit register writes keep the upper bytes */
-    tcg_gen_deposit_i64(regs[reg], regs[reg], v, 0, 8);
+    tcg_gen_deposit_i64(fregs[reg], fregs[reg], v, 32, 32);
 }
 
-static inline void store_freg32(int reg, TCGv_i32 v)
+static void return_low128(TCGv_i64 dest)
 {
-    tcg_gen_st_i32(v, cpu_env, offsetof(CPUS390XState, fregs[reg].l.upper));
+    tcg_gen_ld_i64(dest, cpu_env, offsetof(CPUS390XState, retxl));
 }
 
-static inline void update_psw_addr(DisasContext *s)
+static void update_psw_addr(DisasContext *s)
 {
     /* psw.addr */
     tcg_gen_movi_i64(psw_addr, s->pc);
 }
 
-static inline void potential_page_fault(DisasContext *s)
+static void update_cc_op(DisasContext *s)
+{
+    if (s->cc_op != CC_OP_DYNAMIC && s->cc_op != CC_OP_STATIC) {
+        tcg_gen_movi_i32(cc_op, s->cc_op);
+    }
+}
+
+static void potential_page_fault(DisasContext *s)
 {
-#ifndef CONFIG_USER_ONLY
     update_psw_addr(s);
-    gen_op_calc_cc(s);
-#endif
+    update_cc_op(s);
 }
 
 static inline uint64_t ld_code2(CPUS390XState *env, uint64_t pc)
@@ -279,18 +263,15 @@ static inline uint64_t ld_code2(CPUS390XState *env, uint64_t pc)
 
 static inline uint64_t ld_code4(CPUS390XState *env, uint64_t pc)
 {
-    return (uint64_t)cpu_ldl_code(env, pc);
+    return (uint64_t)(uint32_t)cpu_ldl_code(env, pc);
 }
 
 static inline uint64_t ld_code6(CPUS390XState *env, uint64_t pc)
 {
-    uint64_t opc;
-    opc = (uint64_t)cpu_lduw_code(env, pc) << 32;
-    opc |= (uint64_t)(uint32_t)cpu_ldl_code(env, pc + 2);
-    return opc;
+    return (ld_code2(env, pc) << 32) | ld_code4(env, pc + 2);
 }
 
-static inline int get_mem_index(DisasContext *s)
+static int get_mem_index(DisasContext *s)
 {
     switch (s->tb->flags & FLAG_MASK_ASC) {
     case PSW_ASC_PRIMARY >> 32:
@@ -305,179 +286,114 @@ static inline int get_mem_index(DisasContext *s)
     }
 }
 
-static inline void gen_debug(DisasContext *s)
+static void gen_exception(int excp)
 {
-    TCGv_i32 tmp = tcg_const_i32(EXCP_DEBUG);
-    update_psw_addr(s);
-    gen_op_calc_cc(s);
+    TCGv_i32 tmp = tcg_const_i32(excp);
     gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_EXCP;
-}
-
-#ifdef CONFIG_USER_ONLY
-
-static void gen_illegal_opcode(CPUS390XState *env, DisasContext *s, int ilc)
-{
-    TCGv_i32 tmp = tcg_const_i32(EXCP_SPEC);
-    update_psw_addr(s);
-    gen_op_calc_cc(s);
-    gen_helper_exception(cpu_env, tmp);
-    tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_EXCP;
-}
-
-#else /* CONFIG_USER_ONLY */
-
-static void debug_print_inst(CPUS390XState *env, DisasContext *s, int ilc)
-{
-#ifdef DEBUG_ILLEGAL_INSTRUCTIONS
-    uint64_t inst = 0;
-
-    switch (ilc & 3) {
-    case 1:
-        inst = ld_code2(env, s->pc);
-        break;
-    case 2:
-        inst = ld_code4(env, s->pc);
-        break;
-    case 3:
-        inst = ld_code6(env, s->pc);
-        break;
-    }
-
-    fprintf(stderr, "Illegal instruction [%d at %016" PRIx64 "]: 0x%016"
-            PRIx64 "\n", ilc, s->pc, inst);
-#endif
 }
 
-static void gen_program_exception(CPUS390XState *env, DisasContext *s, int ilc,
-                                  int code)
+static void gen_program_exception(DisasContext *s, int code)
 {
     TCGv_i32 tmp;
 
-    debug_print_inst(env, s, ilc);
-
-    /* remember what pgm exeption this was */
+    /* Remember what pgm exeption this was.  */
     tmp = tcg_const_i32(code);
     tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUS390XState, int_pgm_code));
     tcg_temp_free_i32(tmp);
 
-    tmp = tcg_const_i32(ilc);
-    tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUS390XState, int_pgm_ilc));
+    tmp = tcg_const_i32(s->next_pc - s->pc);
+    tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUS390XState, int_pgm_ilen));
     tcg_temp_free_i32(tmp);
 
-    /* advance past instruction */
-    s->pc += (ilc * 2);
+    /* Advance past instruction.  */
+    s->pc = s->next_pc;
     update_psw_addr(s);
 
-    /* save off cc */
-    gen_op_calc_cc(s);
-
-    /* trigger exception */
-    tmp = tcg_const_i32(EXCP_PGM);
-    gen_helper_exception(cpu_env, tmp);
-    tcg_temp_free_i32(tmp);
+    /* Save off cc.  */
+    update_cc_op(s);
 
-    /* end TB here */
-    s->is_jmp = DISAS_EXCP;
-}
-
-
-static void gen_illegal_opcode(CPUS390XState *env, DisasContext *s, int ilc)
-{
-    gen_program_exception(env, s, ilc, PGM_SPECIFICATION);
+    /* Trigger exception.  */
+    gen_exception(EXCP_PGM);
 }
 
-static void gen_privileged_exception(CPUS390XState *env, DisasContext *s,
-                                     int ilc)
+static inline void gen_illegal_opcode(DisasContext *s)
 {
-    gen_program_exception(env, s, ilc, PGM_PRIVILEGED);
+    gen_program_exception(s, PGM_SPECIFICATION);
 }
 
-static void check_privileged(CPUS390XState *env, DisasContext *s, int ilc)
+static inline void check_privileged(DisasContext *s)
 {
     if (s->tb->flags & (PSW_MASK_PSTATE >> 32)) {
-        gen_privileged_exception(env, s, ilc);
+        gen_program_exception(s, PGM_PRIVILEGED);
     }
 }
 
-#endif /* CONFIG_USER_ONLY */
-
 static TCGv_i64 get_address(DisasContext *s, int x2, int b2, int d2)
 {
-    TCGv_i64 tmp;
+    TCGv_i64 tmp = tcg_temp_new_i64();
+    bool need_31 = !(s->tb->flags & FLAG_MASK_64);
 
-    /* 31-bitify the immediate part; register contents are dealt with below */
-    if (!(s->tb->flags & FLAG_MASK_64)) {
-        d2 &= 0x7fffffffUL;
-    }
+    /* Note that d2 is limited to 20 bits, signed.  If we crop negative
+       displacements early we create larger immedate addends.  */
 
-    if (x2) {
-        if (d2) {
-            tmp = tcg_const_i64(d2);
-            tcg_gen_add_i64(tmp, tmp, regs[x2]);
-        } else {
-            tmp = load_reg(x2);
-        }
-        if (b2) {
-            tcg_gen_add_i64(tmp, tmp, regs[b2]);
-        }
+    /* Note that addi optimizes the imm==0 case.  */
+    if (b2 && x2) {
+        tcg_gen_add_i64(tmp, regs[b2], regs[x2]);
+        tcg_gen_addi_i64(tmp, tmp, d2);
     } else if (b2) {
-        if (d2) {
-            tmp = tcg_const_i64(d2);
-            tcg_gen_add_i64(tmp, tmp, regs[b2]);
-        } else {
-            tmp = load_reg(b2);
-        }
+        tcg_gen_addi_i64(tmp, regs[b2], d2);
+    } else if (x2) {
+        tcg_gen_addi_i64(tmp, regs[x2], d2);
     } else {
-        tmp = tcg_const_i64(d2);
+        if (need_31) {
+            d2 &= 0x7fffffff;
+            need_31 = false;
+        }
+        tcg_gen_movi_i64(tmp, d2);
     }
-
-    /* 31-bit mode mask if there are values loaded from registers */
-    if (!(s->tb->flags & FLAG_MASK_64) && (x2 || b2)) {
-        tcg_gen_andi_i64(tmp, tmp, 0x7fffffffUL);
+    if (need_31) {
+        tcg_gen_andi_i64(tmp, tmp, 0x7fffffff);
     }
 
     return tmp;
 }
 
-static void gen_op_movi_cc(DisasContext *s, uint32_t val)
+static inline bool live_cc_data(DisasContext *s)
 {
-    s->cc_op = CC_OP_CONST0 + val;
+    return (s->cc_op != CC_OP_DYNAMIC
+            && s->cc_op != CC_OP_STATIC
+            && s->cc_op > 3);
 }
 
-static void gen_op_update1_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 dst)
+static inline void gen_op_movi_cc(DisasContext *s, uint32_t val)
 {
-    tcg_gen_discard_i64(cc_src);
-    tcg_gen_mov_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = op;
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_src);
+        tcg_gen_discard_i64(cc_dst);
+        tcg_gen_discard_i64(cc_vr);
+    }
+    s->cc_op = CC_OP_CONST0 + val;
 }
 
-static void gen_op_update1_cc_i32(DisasContext *s, enum cc_op op, TCGv_i32 dst)
+static void gen_op_update1_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 dst)
 {
-    tcg_gen_discard_i64(cc_src);
-    tcg_gen_extu_i32_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_src);
+        tcg_gen_discard_i64(cc_vr);
+    }
+    tcg_gen_mov_i64(cc_dst, dst);
     s->cc_op = op;
 }
 
 static void gen_op_update2_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 src,
                                   TCGv_i64 dst)
 {
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_vr);
+    }
     tcg_gen_mov_i64(cc_src, src);
     tcg_gen_mov_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = op;
-}
-
-static void gen_op_update2_cc_i32(DisasContext *s, enum cc_op op, TCGv_i32 src,
-                                  TCGv_i32 dst)
-{
-    tcg_gen_extu_i32_i64(cc_src, src);
-    tcg_gen_extu_i32_i64(cc_dst, dst);
-    tcg_gen_discard_i64(cc_vr);
     s->cc_op = op;
 }
 
@@ -490,214 +406,71 @@ static void gen_op_update3_cc_i64(DisasContext *s, enum cc_op op, TCGv_i64 src,
     s->cc_op = op;
 }
 
-static void gen_op_update3_cc_i32(DisasContext *s, enum cc_op op, TCGv_i32 src,
-                                  TCGv_i32 dst, TCGv_i32 vr)
-{
-    tcg_gen_extu_i32_i64(cc_src, src);
-    tcg_gen_extu_i32_i64(cc_dst, dst);
-    tcg_gen_extu_i32_i64(cc_vr, vr);
-    s->cc_op = op;
-}
-
-static inline void set_cc_nz_u32(DisasContext *s, TCGv_i32 val)
-{
-    gen_op_update1_cc_i32(s, CC_OP_NZ, val);
-}
-
-static inline void set_cc_nz_u64(DisasContext *s, TCGv_i64 val)
+static void set_cc_nz_u64(DisasContext *s, TCGv_i64 val)
 {
     gen_op_update1_cc_i64(s, CC_OP_NZ, val);
 }
 
-static inline void cmp_32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2,
-                          enum cc_op cond)
-{
-    gen_op_update2_cc_i32(s, cond, v1, v2);
-}
-
-static inline void cmp_64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2,
-                          enum cc_op cond)
-{
-    gen_op_update2_cc_i64(s, cond, v1, v2);
-}
-
-static inline void cmp_s32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2)
-{
-    cmp_32(s, v1, v2, CC_OP_LTGT_32);
-}
-
-static inline void cmp_u32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2)
-{
-    cmp_32(s, v1, v2, CC_OP_LTUGTU_32);
-}
-
-static inline void cmp_s32c(DisasContext *s, TCGv_i32 v1, int32_t v2)
-{
-    /* XXX optimize for the constant? put it in s? */
-    TCGv_i32 tmp = tcg_const_i32(v2);
-    cmp_32(s, v1, tmp, CC_OP_LTGT_32);
-    tcg_temp_free_i32(tmp);
-}
-
-static inline void cmp_u32c(DisasContext *s, TCGv_i32 v1, uint32_t v2)
-{
-    TCGv_i32 tmp = tcg_const_i32(v2);
-    cmp_32(s, v1, tmp, CC_OP_LTUGTU_32);
-    tcg_temp_free_i32(tmp);
-}
-
-static inline void cmp_s64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2)
-{
-    cmp_64(s, v1, v2, CC_OP_LTGT_64);
-}
-
-static inline void cmp_u64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2)
-{
-    cmp_64(s, v1, v2, CC_OP_LTUGTU_64);
-}
-
-static inline void cmp_s64c(DisasContext *s, TCGv_i64 v1, int64_t v2)
-{
-    TCGv_i64 tmp = tcg_const_i64(v2);
-    cmp_s64(s, v1, tmp);
-    tcg_temp_free_i64(tmp);
-}
-
-static inline void cmp_u64c(DisasContext *s, TCGv_i64 v1, uint64_t v2)
-{
-    TCGv_i64 tmp = tcg_const_i64(v2);
-    cmp_u64(s, v1, tmp);
-    tcg_temp_free_i64(tmp);
-}
-
-static inline void set_cc_s32(DisasContext *s, TCGv_i32 val)
-{
-    gen_op_update1_cc_i32(s, CC_OP_LTGT0_32, val);
-}
-
-static inline void set_cc_s64(DisasContext *s, TCGv_i64 val)
-{
-    gen_op_update1_cc_i64(s, CC_OP_LTGT0_64, val);
-}
-
-static void set_cc_add64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2, TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_ADD_64, v1, v2, vr);
-}
-
-static void set_cc_addu64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2,
-                          TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_ADDU_64, v1, v2, vr);
-}
-
-static void set_cc_sub64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2, TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_SUB_64, v1, v2, vr);
-}
-
-static void set_cc_subu64(DisasContext *s, TCGv_i64 v1, TCGv_i64 v2,
-                          TCGv_i64 vr)
-{
-    gen_op_update3_cc_i64(s, CC_OP_SUBU_64, v1, v2, vr);
-}
-
-static void set_cc_abs64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_ABS_64, v1);
-}
-
-static void set_cc_nabs64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_NABS_64, v1);
-}
-
-static void set_cc_add32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2, TCGv_i32 vr)
-{
-    gen_op_update3_cc_i32(s, CC_OP_ADD_32, v1, v2, vr);
-}
-
-static void set_cc_addu32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2,
-                          TCGv_i32 vr)
+static void gen_set_cc_nz_f32(DisasContext *s, TCGv_i64 val)
 {
-    gen_op_update3_cc_i32(s, CC_OP_ADDU_32, v1, v2, vr);
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F32, val);
 }
 
-static void set_cc_sub32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2, TCGv_i32 vr)
+static void gen_set_cc_nz_f64(DisasContext *s, TCGv_i64 val)
 {
-    gen_op_update3_cc_i32(s, CC_OP_SUB_32, v1, v2, vr);
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F64, val);
 }
 
-static void set_cc_subu32(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2,
-                          TCGv_i32 vr)
+static void gen_set_cc_nz_f128(DisasContext *s, TCGv_i64 vh, TCGv_i64 vl)
 {
-    gen_op_update3_cc_i32(s, CC_OP_SUBU_32, v1, v2, vr);
-}
-
-static void set_cc_abs32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_ABS_32, v1);
-}
-
-static void set_cc_nabs32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_NABS_32, v1);
-}
-
-static void set_cc_comp32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_COMP_32, v1);
-}
-
-static void set_cc_comp64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_COMP_64, v1);
-}
-
-static void set_cc_icm(DisasContext *s, TCGv_i32 v1, TCGv_i32 v2)
-{
-    gen_op_update2_cc_i32(s, CC_OP_ICM, v1, v2);
-}
-
-static void set_cc_cmp_f32_i64(DisasContext *s, TCGv_i32 v1, TCGv_i64 v2)
-{
-    tcg_gen_extu_i32_i64(cc_src, v1);
-    tcg_gen_mov_i64(cc_dst, v2);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = CC_OP_LTGT_F32;
-}
-
-static void gen_set_cc_nz_f32(DisasContext *s, TCGv_i32 v1)
-{
-    gen_op_update1_cc_i32(s, CC_OP_NZ_F32, v1);
+    gen_op_update2_cc_i64(s, CC_OP_NZ_F128, vh, vl);
 }
 
 /* CC value is in env->cc_op */
-static inline void set_cc_static(DisasContext *s)
+static void set_cc_static(DisasContext *s)
 {
-    tcg_gen_discard_i64(cc_src);
-    tcg_gen_discard_i64(cc_dst);
-    tcg_gen_discard_i64(cc_vr);
-    s->cc_op = CC_OP_STATIC;
-}
-
-static inline void gen_op_set_cc_op(DisasContext *s)
-{
-    if (s->cc_op != CC_OP_DYNAMIC && s->cc_op != CC_OP_STATIC) {
-        tcg_gen_movi_i32(cc_op, s->cc_op);
+    if (live_cc_data(s)) {
+        tcg_gen_discard_i64(cc_src);
+        tcg_gen_discard_i64(cc_dst);
+        tcg_gen_discard_i64(cc_vr);
     }
-}
-
-static inline void gen_update_cc_op(DisasContext *s)
-{
-    gen_op_set_cc_op(s);
+    s->cc_op = CC_OP_STATIC;
 }
 
 /* calculates cc into cc_op */
 static void gen_op_calc_cc(DisasContext *s)
 {
-    TCGv_i32 local_cc_op = tcg_const_i32(s->cc_op);
-    TCGv_i64 dummy = tcg_const_i64(0);
+    TCGv_i32 local_cc_op;
+    TCGv_i64 dummy;
+
+    TCGV_UNUSED_I32(local_cc_op);
+    TCGV_UNUSED_I64(dummy);
+    switch (s->cc_op) {
+    default:
+        dummy = tcg_const_i64(0);
+        /* FALLTHRU */
+    case CC_OP_ADD_64:
+    case CC_OP_ADDU_64:
+    case CC_OP_ADDC_64:
+    case CC_OP_SUB_64:
+    case CC_OP_SUBU_64:
+    case CC_OP_SUBB_64:
+    case CC_OP_ADD_32:
+    case CC_OP_ADDU_32:
+    case CC_OP_ADDC_32:
+    case CC_OP_SUB_32:
+    case CC_OP_SUBU_32:
+    case CC_OP_SUBB_32:
+        local_cc_op = tcg_const_i32(s->cc_op);
+        break;
+    case CC_OP_CONST0:
+    case CC_OP_CONST1:
+    case CC_OP_CONST2:
+    case CC_OP_CONST3:
+    case CC_OP_STATIC:
+    case CC_OP_DYNAMIC:
+        break;
+    }
 
     switch (s->cc_op) {
     case CC_OP_CONST0:
@@ -721,6 +494,7 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_COMP_64:
     case CC_OP_NZ_F32:
     case CC_OP_NZ_F64:
+    case CC_OP_FLOGR:
         /* 1 argument */
         gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, dummy, cc_dst, dummy);
         break;
@@ -731,20 +505,24 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_LTUGTU_64:
     case CC_OP_TM_32:
     case CC_OP_TM_64:
-    case CC_OP_LTGT_F32:
-    case CC_OP_LTGT_F64:
-    case CC_OP_SLAG:
+    case CC_OP_SLA_32:
+    case CC_OP_SLA_64:
+    case CC_OP_NZ_F128:
         /* 2 arguments */
         gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, cc_src, cc_dst, dummy);
         break;
     case CC_OP_ADD_64:
     case CC_OP_ADDU_64:
+    case CC_OP_ADDC_64:
     case CC_OP_SUB_64:
     case CC_OP_SUBU_64:
+    case CC_OP_SUBB_64:
     case CC_OP_ADD_32:
     case CC_OP_ADDU_32:
+    case CC_OP_ADDC_32:
     case CC_OP_SUB_32:
     case CC_OP_SUBU_32:
+    case CC_OP_SUBB_32:
         /* 3 arguments */
         gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, cc_src, cc_dst, cc_vr);
         break;
@@ -756,4358 +534,4204 @@ static void gen_op_calc_cc(DisasContext *s)
         tcg_abort();
     }
 
-    tcg_temp_free_i32(local_cc_op);
+    if (!TCGV_IS_UNUSED_I32(local_cc_op)) {
+        tcg_temp_free_i32(local_cc_op);
+    }
+    if (!TCGV_IS_UNUSED_I64(dummy)) {
+        tcg_temp_free_i64(dummy);
+    }
 
     /* We now have cc in cc_op as constant */
     set_cc_static(s);
 }
 
-static inline void decode_rr(DisasContext *s, uint64_t insn, int *r1, int *r2)
-{
-    debug_insn(insn);
-
-    *r1 = (insn >> 4) & 0xf;
-    *r2 = insn & 0xf;
-}
-
-static inline TCGv_i64 decode_rx(DisasContext *s, uint64_t insn, int *r1,
-                                 int *x2, int *b2, int *d2)
+static int use_goto_tb(DisasContext *s, uint64_t dest)
 {
-    debug_insn(insn);
-
-    *r1 = (insn >> 20) & 0xf;
-    *x2 = (insn >> 16) & 0xf;
-    *b2 = (insn >> 12) & 0xf;
-    *d2 = insn & 0xfff;
-
-    return get_address(s, *x2, *b2, *d2);
-}
-
-static inline void decode_rs(DisasContext *s, uint64_t insn, int *r1, int *r3,
-                             int *b2, int *d2)
-{
-    debug_insn(insn);
-
-    *r1 = (insn >> 20) & 0xf;
-    /* aka m3 */
-    *r3 = (insn >> 16) & 0xf;
-    *b2 = (insn >> 12) & 0xf;
-    *d2 = insn & 0xfff;
-}
-
-static inline TCGv_i64 decode_si(DisasContext *s, uint64_t insn, int *i2,
-                                 int *b1, int *d1)
-{
-    debug_insn(insn);
-
-    *i2 = (insn >> 16) & 0xff;
-    *b1 = (insn >> 12) & 0xf;
-    *d1 = insn & 0xfff;
-
-    return get_address(s, 0, *b1, *d1);
-}
-
-static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong pc)
-{
-    TranslationBlock *tb;
-
-    gen_update_cc_op(s);
-
-    tb = s->tb;
     /* NOTE: we handle the case where the TB spans two pages here */
-    if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) ||
-        (pc & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))  {
-        /* jump to same page: we can use a direct jump */
-        tcg_gen_goto_tb(tb_num);
-        tcg_gen_movi_i64(psw_addr, pc);
-        tcg_gen_exit_tb((tcg_target_long)tb + tb_num);
-    } else {
-        /* jump to another page: currently not optimized */
-        tcg_gen_movi_i64(psw_addr, pc);
-        tcg_gen_exit_tb(0);
-    }
+    return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK)
+             || (dest & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))
+            && !s->singlestep_enabled
+            && !(s->tb->cflags & CF_LAST_IO));
 }
 
-static inline void account_noninline_branch(DisasContext *s, int cc_op)
+static void account_noninline_branch(DisasContext *s, int cc_op)
 {
 #ifdef DEBUG_INLINE_BRANCHES
     inline_branch_miss[cc_op]++;
 #endif
 }
 
-static inline void account_inline_branch(DisasContext *s)
+static void account_inline_branch(DisasContext *s, int cc_op)
 {
 #ifdef DEBUG_INLINE_BRANCHES
-    inline_branch_hit[s->cc_op]++;
+    inline_branch_hit[cc_op]++;
 #endif
 }
 
-static void gen_jcc(DisasContext *s, uint32_t mask, int skip)
+/* Table of mask values to comparison codes, given a comparison as input.
+   For such, CC=3 should not be possible.  */
+static const TCGCond ltgt_cond[16] = {
+    TCG_COND_NEVER,  TCG_COND_NEVER,     /*    |    |    | x */
+    TCG_COND_GT,     TCG_COND_GT,        /*    |    | GT | x */
+    TCG_COND_LT,     TCG_COND_LT,        /*    | LT |    | x */
+    TCG_COND_NE,     TCG_COND_NE,        /*    | LT | GT | x */
+    TCG_COND_EQ,     TCG_COND_EQ,        /* EQ |    |    | x */
+    TCG_COND_GE,     TCG_COND_GE,        /* EQ |    | GT | x */
+    TCG_COND_LE,     TCG_COND_LE,        /* EQ | LT |    | x */
+    TCG_COND_ALWAYS, TCG_COND_ALWAYS,    /* EQ | LT | GT | x */
+};
+
+/* Table of mask values to comparison codes, given a logic op as input.
+   For such, only CC=0 and CC=1 should be possible.  */
+static const TCGCond nz_cond[16] = {
+    TCG_COND_NEVER, TCG_COND_NEVER,      /*    |    | x | x */
+    TCG_COND_NEVER, TCG_COND_NEVER,
+    TCG_COND_NE, TCG_COND_NE,            /*    | NE | x | x */
+    TCG_COND_NE, TCG_COND_NE,
+    TCG_COND_EQ, TCG_COND_EQ,            /* EQ |    | x | x */
+    TCG_COND_EQ, TCG_COND_EQ,
+    TCG_COND_ALWAYS, TCG_COND_ALWAYS,    /* EQ | NE | x | x */
+    TCG_COND_ALWAYS, TCG_COND_ALWAYS,
+};
+
+/* Interpret MASK in terms of S->CC_OP, and fill in C with all the
+   details required to generate a TCG comparison.  */
+static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
 {
-    TCGv_i32 tmp, tmp2, r;
-    TCGv_i64 tmp64;
-    int old_cc_op;
+    TCGCond cond;
+    enum cc_op old_cc_op = s->cc_op;
+
+    if (mask == 15 || mask == 0) {
+        c->cond = (mask ? TCG_COND_ALWAYS : TCG_COND_NEVER);
+        c->u.s32.a = cc_op;
+        c->u.s32.b = cc_op;
+        c->g1 = c->g2 = true;
+        c->is_64 = false;
+        return;
+    }
 
-    switch (s->cc_op) {
+    /* Find the TCG condition for the mask + cc op.  */
+    switch (old_cc_op) {
     case CC_OP_LTGT0_32:
-        tmp = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp, cc_dst);
-        switch (mask) {
-        case 0x8 | 0x4: /* dst <= 0 */
-            tcg_gen_brcondi_i32(TCG_COND_GT, tmp, 0, skip);
-            break;
-        case 0x8 | 0x2: /* dst >= 0 */
-            tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, skip);
-            break;
-        case 0x8: /* dst == 0 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, skip);
-            break;
-        case 0x7: /* dst != 0 */
-        case 0x6: /* dst != 0 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, skip);
-            break;
-        case 0x4: /* dst < 0 */
-            tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, skip);
-            break;
-        case 0x2: /* dst > 0 */
-            tcg_gen_brcondi_i32(TCG_COND_LE, tmp, 0, skip);
-            break;
-        default:
-            tcg_temp_free_i32(tmp);
-            goto do_dynamic;
-        }
-        account_inline_branch(s);
-        tcg_temp_free_i32(tmp);
-        break;
     case CC_OP_LTGT0_64:
-        switch (mask) {
-        case 0x8 | 0x4: /* dst <= 0 */
-            tcg_gen_brcondi_i64(TCG_COND_GT, cc_dst, 0, skip);
-            break;
-        case 0x8 | 0x2: /* dst >= 0 */
-            tcg_gen_brcondi_i64(TCG_COND_LT, cc_dst, 0, skip);
-            break;
-        case 0x8: /* dst == 0 */
-            tcg_gen_brcondi_i64(TCG_COND_NE, cc_dst, 0, skip);
-            break;
-        case 0x7: /* dst != 0 */
-        case 0x6: /* dst != 0 */
-            tcg_gen_brcondi_i64(TCG_COND_EQ, cc_dst, 0, skip);
-            break;
-        case 0x4: /* dst < 0 */
-            tcg_gen_brcondi_i64(TCG_COND_GE, cc_dst, 0, skip);
-            break;
-        case 0x2: /* dst > 0 */
-            tcg_gen_brcondi_i64(TCG_COND_LE, cc_dst, 0, skip);
-            break;
-        default:
+    case CC_OP_LTGT_32:
+    case CC_OP_LTGT_64:
+        cond = ltgt_cond[mask];
+        if (cond == TCG_COND_NEVER) {
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTGT_32:
-        tmp = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp, cc_src);
-        tcg_gen_trunc_i64_i32(tmp2, cc_dst);
-        switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i32(TCG_COND_GT, tmp, tmp2, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i32(TCG_COND_LT, tmp, tmp2, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i32(TCG_COND_NE, tmp, tmp2, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i32(TCG_COND_EQ, tmp, tmp2, skip);
-            break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i32(TCG_COND_GE, tmp, tmp2, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i32(TCG_COND_LE, tmp, tmp2, skip);
-            break;
-        default:
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
+
+    case CC_OP_LTUGTU_32:
+    case CC_OP_LTUGTU_64:
+        cond = tcg_unsigned_cond(ltgt_cond[mask]);
+        if (cond == TCG_COND_NEVER) {
             goto do_dynamic;
         }
-        account_inline_branch(s);
-        tcg_temp_free_i32(tmp);
-        tcg_temp_free_i32(tmp2);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTGT_64:
-        switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i64(TCG_COND_GT, cc_src, cc_dst, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i64(TCG_COND_LT, cc_src, cc_dst, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i64(TCG_COND_NE, cc_src, cc_dst, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i64(TCG_COND_EQ, cc_src, cc_dst, skip);
-            break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i64(TCG_COND_GE, cc_src, cc_dst, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i64(TCG_COND_LE, cc_src, cc_dst, skip);
-            break;
-        default:
+
+    case CC_OP_NZ:
+        cond = nz_cond[mask];
+        if (cond == TCG_COND_NEVER) {
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTUGTU_32:
-        tmp = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp, cc_src);
-        tcg_gen_trunc_i64_i32(tmp2, cc_dst);
+
+    case CC_OP_TM_32:
+    case CC_OP_TM_64:
         switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i32(TCG_COND_GTU, tmp, tmp2, skip);
-            break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i32(TCG_COND_LTU, tmp, tmp2, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i32(TCG_COND_NE, tmp, tmp2, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i32(TCG_COND_EQ, tmp, tmp2, skip);
+        case 8:
+            cond = TCG_COND_EQ;
             break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i32(TCG_COND_GEU, tmp, tmp2, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i32(TCG_COND_LEU, tmp, tmp2, skip);
+        case 4 | 2 | 1:
+            cond = TCG_COND_NE;
             break;
         default:
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
             goto do_dynamic;
         }
-        account_inline_branch(s);
-        tcg_temp_free_i32(tmp);
-        tcg_temp_free_i32(tmp2);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_LTUGTU_64:
+
+    case CC_OP_ICM:
         switch (mask) {
-        case 0x8 | 0x4: /* src <= dst */
-            tcg_gen_brcond_i64(TCG_COND_GTU, cc_src, cc_dst, skip);
+        case 8:
+            cond = TCG_COND_EQ;
             break;
-        case 0x8 | 0x2: /* src >= dst */
-            tcg_gen_brcond_i64(TCG_COND_LTU, cc_src, cc_dst, skip);
-            break;
-        case 0x8: /* src == dst */
-            tcg_gen_brcond_i64(TCG_COND_NE, cc_src, cc_dst, skip);
-            break;
-        case 0x7: /* src != dst */
-        case 0x6: /* src != dst */
-            tcg_gen_brcond_i64(TCG_COND_EQ, cc_src, cc_dst, skip);
-            break;
-        case 0x4: /* src < dst */
-            tcg_gen_brcond_i64(TCG_COND_GEU, cc_src, cc_dst, skip);
-            break;
-        case 0x2: /* src > dst */
-            tcg_gen_brcond_i64(TCG_COND_LEU, cc_src, cc_dst, skip);
+        case 4 | 2 | 1:
+        case 4 | 2:
+            cond = TCG_COND_NE;
             break;
         default:
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_NZ:
-        switch (mask) {
-        /* dst == 0 || dst != 0 */
-        case 0x8 | 0x4:
-        case 0x8 | 0x4 | 0x2:
-        case 0x8 | 0x4 | 0x2 | 0x1:
-        case 0x8 | 0x4 | 0x1:
-            break;
-        /* dst == 0 */
-        case 0x8:
-        case 0x8 | 0x2:
-        case 0x8 | 0x2 | 0x1:
-        case 0x8 | 0x1:
-            tcg_gen_brcondi_i64(TCG_COND_NE, cc_dst, 0, skip);
+
+    case CC_OP_FLOGR:
+        switch (mask & 0xa) {
+        case 8: /* src == 0 -> no one bit found */
+            cond = TCG_COND_EQ;
             break;
-        /* dst != 0 */
-        case 0x4:
-        case 0x4 | 0x2:
-        case 0x4 | 0x2 | 0x1:
-        case 0x4 | 0x1:
-            tcg_gen_brcondi_i64(TCG_COND_EQ, cc_dst, 0, skip);
+        case 2: /* src != 0 -> one bit found */
+            cond = TCG_COND_NE;
             break;
         default:
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_TM_32:
-        tmp = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i32();
 
-        tcg_gen_trunc_i64_i32(tmp, cc_src);
-        tcg_gen_trunc_i64_i32(tmp2, cc_dst);
-        tcg_gen_and_i32(tmp, tmp, tmp2);
+    case CC_OP_ADDU_32:
+    case CC_OP_ADDU_64:
         switch (mask) {
-        case 0x8: /* val & mask == 0 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, skip);
+        case 8 | 2: /* vr == 0 */
+            cond = TCG_COND_EQ;
+            break;
+        case 4 | 1: /* vr != 0 */
+            cond = TCG_COND_NE;
             break;
-        case 0x4 | 0x2 | 0x1: /* val & mask != 0 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, skip);
+        case 8 | 4: /* no carry -> vr >= src */
+            cond = TCG_COND_GEU;
+            break;
+        case 2 | 1: /* carry -> vr < src */
+            cond = TCG_COND_LTU;
             break;
         default:
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
             goto do_dynamic;
         }
-        tcg_temp_free_i32(tmp);
-        tcg_temp_free_i32(tmp2);
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_TM_64:
-        tmp64 = tcg_temp_new_i64();
 
-        tcg_gen_and_i64(tmp64, cc_src, cc_dst);
-        switch (mask) {
-        case 0x8: /* val & mask == 0 */
-            tcg_gen_brcondi_i64(TCG_COND_NE, tmp64, 0, skip);
+    case CC_OP_SUBU_32:
+    case CC_OP_SUBU_64:
+        /* Note that CC=0 is impossible; treat it as dont-care.  */
+        switch (mask & 7) {
+        case 2: /* zero -> op1 == op2 */
+            cond = TCG_COND_EQ;
             break;
-        case 0x4 | 0x2 | 0x1: /* val & mask != 0 */
-            tcg_gen_brcondi_i64(TCG_COND_EQ, tmp64, 0, skip);
+        case 4 | 1: /* !zero -> op1 != op2 */
+            cond = TCG_COND_NE;
             break;
-        default:
-            tcg_temp_free_i64(tmp64);
-            goto do_dynamic;
-        }
-        tcg_temp_free_i64(tmp64);
-        account_inline_branch(s);
-        break;
-    case CC_OP_ICM:
-        switch (mask) {
-        case 0x8: /* val == 0 */
-            tcg_gen_brcondi_i64(TCG_COND_NE, cc_dst, 0, skip);
+        case 4: /* borrow (!carry) -> op1 < op2 */
+            cond = TCG_COND_LTU;
             break;
-        case 0x4 | 0x2 | 0x1: /* val != 0 */
-        case 0x4 | 0x2: /* val != 0 */
-            tcg_gen_brcondi_i64(TCG_COND_EQ, cc_dst, 0, skip);
+        case 2 | 1: /* !borrow (carry) -> op1 >= op2 */
+            cond = TCG_COND_GEU;
             break;
         default:
             goto do_dynamic;
         }
-        account_inline_branch(s);
+        account_inline_branch(s, old_cc_op);
         break;
-    case CC_OP_STATIC:
-        old_cc_op = s->cc_op;
-        goto do_dynamic_nocccalc;
-    case CC_OP_DYNAMIC:
+
     default:
-do_dynamic:
-        old_cc_op = s->cc_op;
-        /* calculate cc value */
+    do_dynamic:
+        /* Calculate cc value.  */
         gen_op_calc_cc(s);
+        /* FALLTHRU */
 
-do_dynamic_nocccalc:
-        /* jump based on cc */
+    case CC_OP_STATIC:
+        /* Jump based on CC.  We'll load up the real cond below;
+           the assignment here merely avoids a compiler warning.  */
         account_noninline_branch(s, old_cc_op);
+        old_cc_op = CC_OP_STATIC;
+        cond = TCG_COND_NEVER;
+        break;
+    }
+
+    /* Load up the arguments of the comparison.  */
+    c->is_64 = true;
+    c->g1 = c->g2 = false;
+    switch (old_cc_op) {
+    case CC_OP_LTGT0_32:
+        c->is_64 = false;
+        c->u.s32.a = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_dst);
+        c->u.s32.b = tcg_const_i32(0);
+        break;
+    case CC_OP_LTGT_32:
+    case CC_OP_LTUGTU_32:
+    case CC_OP_SUBU_32:
+        c->is_64 = false;
+        c->u.s32.a = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_src);
+        c->u.s32.b = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.b, cc_dst);
+        break;
+
+    case CC_OP_LTGT0_64:
+    case CC_OP_NZ:
+    case CC_OP_FLOGR:
+        c->u.s64.a = cc_dst;
+        c->u.s64.b = tcg_const_i64(0);
+        c->g1 = true;
+        break;
+    case CC_OP_LTGT_64:
+    case CC_OP_LTUGTU_64:
+    case CC_OP_SUBU_64:
+        c->u.s64.a = cc_src;
+        c->u.s64.b = cc_dst;
+        c->g1 = c->g2 = true;
+        break;
 
+    case CC_OP_TM_32:
+    case CC_OP_TM_64:
+    case CC_OP_ICM:
+        c->u.s64.a = tcg_temp_new_i64();
+        c->u.s64.b = tcg_const_i64(0);
+        tcg_gen_and_i64(c->u.s64.a, cc_src, cc_dst);
+        break;
+
+    case CC_OP_ADDU_32:
+        c->is_64 = false;
+        c->u.s32.a = tcg_temp_new_i32();
+        c->u.s32.b = tcg_temp_new_i32();
+        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_vr);
+        if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
+            tcg_gen_movi_i32(c->u.s32.b, 0);
+        } else {
+            tcg_gen_trunc_i64_i32(c->u.s32.b, cc_src);
+        }
+        break;
+
+    case CC_OP_ADDU_64:
+        c->u.s64.a = cc_vr;
+        c->g1 = true;
+        if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
+            c->u.s64.b = tcg_const_i64(0);
+        } else {
+            c->u.s64.b = cc_src;
+            c->g2 = true;
+        }
+        break;
+
+    case CC_OP_STATIC:
+        c->is_64 = false;
+        c->u.s32.a = cc_op;
+        c->g1 = true;
         switch (mask) {
-        case 0x8 | 0x4 | 0x2 | 0x1:
-            /* always true */
-            break;
         case 0x8 | 0x4 | 0x2: /* cc != 3 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 3, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(3);
             break;
         case 0x8 | 0x4 | 0x1: /* cc != 2 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 2, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(2);
             break;
         case 0x8 | 0x2 | 0x1: /* cc != 1 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 1, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(1);
             break;
-        case 0x8 | 0x2: /* cc == 0 || cc == 2 */
-            tmp = tcg_temp_new_i32();
-            tcg_gen_andi_i32(tmp, cc_op, 1);
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, skip);
-            tcg_temp_free_i32(tmp);
+        case 0x8 | 0x2: /* cc == 0 || cc == 2 => (cc & 1) == 0 */
+            cond = TCG_COND_EQ;
+            c->g1 = false;
+            c->u.s32.a = tcg_temp_new_i32();
+            c->u.s32.b = tcg_const_i32(0);
+            tcg_gen_andi_i32(c->u.s32.a, cc_op, 1);
             break;
         case 0x8 | 0x4: /* cc < 2 */
-            tcg_gen_brcondi_i32(TCG_COND_GEU, cc_op, 2, skip);
+            cond = TCG_COND_LTU;
+            c->u.s32.b = tcg_const_i32(2);
             break;
         case 0x8: /* cc == 0 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 0, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(0);
             break;
         case 0x4 | 0x2 | 0x1: /* cc != 0 */
-            tcg_gen_brcondi_i32(TCG_COND_EQ, cc_op, 0, skip);
+            cond = TCG_COND_NE;
+            c->u.s32.b = tcg_const_i32(0);
             break;
-        case 0x4 | 0x1: /* cc == 1 || cc == 3 */
-            tmp = tcg_temp_new_i32();
-            tcg_gen_andi_i32(tmp, cc_op, 1);
-            tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, skip);
-            tcg_temp_free_i32(tmp);
+        case 0x4 | 0x1: /* cc == 1 || cc == 3 => (cc & 1) != 0 */
+            cond = TCG_COND_NE;
+            c->g1 = false;
+            c->u.s32.a = tcg_temp_new_i32();
+            c->u.s32.b = tcg_const_i32(0);
+            tcg_gen_andi_i32(c->u.s32.a, cc_op, 1);
             break;
         case 0x4: /* cc == 1 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 1, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(1);
             break;
         case 0x2 | 0x1: /* cc > 1 */
-            tcg_gen_brcondi_i32(TCG_COND_LEU, cc_op, 1, skip);
+            cond = TCG_COND_GTU;
+            c->u.s32.b = tcg_const_i32(1);
             break;
         case 0x2: /* cc == 2 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 2, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(2);
             break;
         case 0x1: /* cc == 3 */
-            tcg_gen_brcondi_i32(TCG_COND_NE, cc_op, 3, skip);
+            cond = TCG_COND_EQ;
+            c->u.s32.b = tcg_const_i32(3);
             break;
-        default: /* cc is masked by something else */
-            tmp = tcg_const_i32(3);
-            /* 3 - cc */
-            tcg_gen_sub_i32(tmp, tmp, cc_op);
-            tmp2 = tcg_const_i32(1);
-            /* 1 << (3 - cc) */
-            tcg_gen_shl_i32(tmp2, tmp2, tmp);
-            r = tcg_const_i32(mask);
-            /* mask & (1 << (3 - cc)) */
-            tcg_gen_and_i32(r, r, tmp2);
-            tcg_temp_free_i32(tmp);
-            tcg_temp_free_i32(tmp2);
-
-            tcg_gen_brcondi_i32(TCG_COND_EQ, r, 0, skip);
-            tcg_temp_free_i32(r);
+        default:
+            /* CC is masked by something else: (8 >> cc) & mask.  */
+            cond = TCG_COND_NE;
+            c->g1 = false;
+            c->u.s32.a = tcg_const_i32(8);
+            c->u.s32.b = tcg_const_i32(0);
+            tcg_gen_shr_i32(c->u.s32.a, c->u.s32.a, cc_op);
+            tcg_gen_andi_i32(c->u.s32.a, c->u.s32.a, mask);
             break;
         }
         break;
+
+    default:
+        abort();
     }
+    c->cond = cond;
 }
 
-static void gen_bcr(DisasContext *s, uint32_t mask, TCGv_i64 target,
-                    uint64_t offset)
+static void free_compare(DisasCompare *c)
 {
-    int skip;
-
-    if (mask == 0xf) {
-        /* unconditional */
-        tcg_gen_mov_i64(psw_addr, target);
-        tcg_gen_exit_tb(0);
-    } else if (mask == 0) {
-        /* ignore cc and never match */
-        gen_goto_tb(s, 0, offset + 2);
-    } else {
-        TCGv_i64 new_addr = tcg_temp_local_new_i64();
-
-        tcg_gen_mov_i64(new_addr, target);
-        skip = gen_new_label();
-        gen_jcc(s, mask, skip);
-        tcg_gen_mov_i64(psw_addr, new_addr);
-        tcg_temp_free_i64(new_addr);
-        tcg_gen_exit_tb(0);
-        gen_set_label(skip);
-        tcg_temp_free_i64(new_addr);
-        gen_goto_tb(s, 1, offset + 2);
+    if (!c->g1) {
+        if (c->is_64) {
+            tcg_temp_free_i64(c->u.s64.a);
+        } else {
+            tcg_temp_free_i32(c->u.s32.a);
+        }
+    }
+    if (!c->g2) {
+        if (c->is_64) {
+            tcg_temp_free_i64(c->u.s64.b);
+        } else {
+            tcg_temp_free_i32(c->u.s32.b);
+        }
     }
 }
 
-static void gen_brc(uint32_t mask, DisasContext *s, int32_t offset)
+/* ====================================================================== */
+/* Define the insn format enumeration.  */
+#define F0(N)                         FMT_##N,
+#define F1(N, X1)                     F0(N)
+#define F2(N, X1, X2)                 F0(N)
+#define F3(N, X1, X2, X3)             F0(N)
+#define F4(N, X1, X2, X3, X4)         F0(N)
+#define F5(N, X1, X2, X3, X4, X5)     F0(N)
+
+typedef enum {
+#include "insn-format.def"
+} DisasFormat;
+
+#undef F0
+#undef F1
+#undef F2
+#undef F3
+#undef F4
+#undef F5
+
+/* Define a structure to hold the decoded fields.  We'll store each inside
+   an array indexed by an enum.  In order to conserve memory, we'll arrange
+   for fields that do not exist at the same time to overlap, thus the "C"
+   for compact.  For checking purposes there is an "O" for original index
+   as well that will be applied to availability bitmaps.  */
+
+enum DisasFieldIndexO {
+    FLD_O_r1,
+    FLD_O_r2,
+    FLD_O_r3,
+    FLD_O_m1,
+    FLD_O_m3,
+    FLD_O_m4,
+    FLD_O_b1,
+    FLD_O_b2,
+    FLD_O_b4,
+    FLD_O_d1,
+    FLD_O_d2,
+    FLD_O_d4,
+    FLD_O_x2,
+    FLD_O_l1,
+    FLD_O_l2,
+    FLD_O_i1,
+    FLD_O_i2,
+    FLD_O_i3,
+    FLD_O_i4,
+    FLD_O_i5
+};
+
+enum DisasFieldIndexC {
+    FLD_C_r1 = 0,
+    FLD_C_m1 = 0,
+    FLD_C_b1 = 0,
+    FLD_C_i1 = 0,
+
+    FLD_C_r2 = 1,
+    FLD_C_b2 = 1,
+    FLD_C_i2 = 1,
+
+    FLD_C_r3 = 2,
+    FLD_C_m3 = 2,
+    FLD_C_i3 = 2,
+
+    FLD_C_m4 = 3,
+    FLD_C_b4 = 3,
+    FLD_C_i4 = 3,
+    FLD_C_l1 = 3,
+
+    FLD_C_i5 = 4,
+    FLD_C_d1 = 4,
+
+    FLD_C_d2 = 5,
+
+    FLD_C_d4 = 6,
+    FLD_C_x2 = 6,
+    FLD_C_l2 = 6,
+
+    NUM_C_FIELD = 7
+};
+
+struct DisasFields {
+    unsigned op:8;
+    unsigned op2:8;
+    unsigned presentC:16;
+    unsigned int presentO;
+    int c[NUM_C_FIELD];
+};
+
+/* This is the way fields are to be accessed out of DisasFields.  */
+#define have_field(S, F)  have_field1((S), FLD_O_##F)
+#define get_field(S, F)   get_field1((S), FLD_O_##F, FLD_C_##F)
+
+static bool have_field1(const DisasFields *f, enum DisasFieldIndexO c)
+{
+    return (f->presentO >> c) & 1;
+}
+
+static int get_field1(const DisasFields *f, enum DisasFieldIndexO o,
+                      enum DisasFieldIndexC c)
+{
+    assert(have_field1(f, o));
+    return f->c[c];
+}
+
+/* Describe the layout of each field in each format.  */
+typedef struct DisasField {
+    unsigned int beg:8;
+    unsigned int size:8;
+    unsigned int type:2;
+    unsigned int indexC:6;
+    enum DisasFieldIndexO indexO:8;
+} DisasField;
+
+typedef struct DisasFormatInfo {
+    DisasField op[NUM_C_FIELD];
+} DisasFormatInfo;
+
+#define R(N, B)       {  B,  4, 0, FLD_C_r##N, FLD_O_r##N }
+#define M(N, B)       {  B,  4, 0, FLD_C_m##N, FLD_O_m##N }
+#define BD(N, BB, BD) { BB,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { BD, 12, 0, FLD_C_d##N, FLD_O_d##N }
+#define BXD(N)        { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { 12,  4, 0, FLD_C_x##N, FLD_O_x##N }, \
+                      { 20, 12, 0, FLD_C_d##N, FLD_O_d##N }
+#define BDL(N)        { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { 20, 20, 2, FLD_C_d##N, FLD_O_d##N }
+#define BXDL(N)       { 16,  4, 0, FLD_C_b##N, FLD_O_b##N }, \
+                      { 12,  4, 0, FLD_C_x##N, FLD_O_x##N }, \
+                      { 20, 20, 2, FLD_C_d##N, FLD_O_d##N }
+#define I(N, B, S)    {  B,  S, 1, FLD_C_i##N, FLD_O_i##N }
+#define L(N, B, S)    {  B,  S, 0, FLD_C_l##N, FLD_O_l##N }
+
+#define F0(N)                     { { } },
+#define F1(N, X1)                 { { X1 } },
+#define F2(N, X1, X2)             { { X1, X2 } },
+#define F3(N, X1, X2, X3)         { { X1, X2, X3 } },
+#define F4(N, X1, X2, X3, X4)     { { X1, X2, X3, X4 } },
+#define F5(N, X1, X2, X3, X4, X5) { { X1, X2, X3, X4, X5 } },
+
+static const DisasFormatInfo format_info[] = {
+#include "insn-format.def"
+};
+
+#undef F0
+#undef F1
+#undef F2
+#undef F3
+#undef F4
+#undef F5
+#undef R
+#undef M
+#undef BD
+#undef BXD
+#undef BDL
+#undef BXDL
+#undef I
+#undef L
+
+/* Generally, we'll extract operands into this structures, operate upon
+   them, and store them back.  See the "in1", "in2", "prep", "wout" sets
+   of routines below for more details.  */
+typedef struct {
+    bool g_out, g_out2, g_in1, g_in2;
+    TCGv_i64 out, out2, in1, in2;
+    TCGv_i64 addr1;
+} DisasOps;
+
+/* Instructions can place constraints on their operands, raising specification
+   exceptions if they are violated.  To make this easy to automate, each "in1",
+   "in2", "prep", "wout" helper will have a SPEC_<name> define that equals one
+   of the following, or 0.  To make this easy to document, we'll put the
+   SPEC_<name> defines next to <name>.  */
+
+#define SPEC_r1_even    1
+#define SPEC_r2_even    2
+#define SPEC_r3_even    4
+#define SPEC_r1_f128    8
+#define SPEC_r2_f128    16
+
+/* Return values from translate_one, indicating the state of the TB.  */
+typedef enum {
+    /* Continue the TB.  */
+    NO_EXIT,
+    /* We have emitted one or more goto_tb.  No fixup required.  */
+    EXIT_GOTO_TB,
+    /* We are not using a goto_tb (for whatever reason), but have updated
+       the PC (for whatever reason), so there's no need to do it again on
+       exiting the TB.  */
+    EXIT_PC_UPDATED,
+    /* We are exiting the TB, but have neither emitted a goto_tb, nor
+       updated the PC for the next instruction to be executed.  */
+    EXIT_PC_STALE,
+    /* We are ending the TB with a noreturn function call, e.g. longjmp.
+       No following code will be executed.  */
+    EXIT_NORETURN,
+} ExitStatus;
+
+typedef enum DisasFacility {
+    FAC_Z,                  /* zarch (default) */
+    FAC_CASS,               /* compare and swap and store */
+    FAC_CASS2,              /* compare and swap and store 2*/
+    FAC_DFP,                /* decimal floating point */
+    FAC_DFPR,               /* decimal floating point rounding */
+    FAC_DO,                 /* distinct operands */
+    FAC_EE,                 /* execute extensions */
+    FAC_EI,                 /* extended immediate */
+    FAC_FPE,                /* floating point extension */
+    FAC_FPSSH,              /* floating point support sign handling */
+    FAC_FPRGR,              /* FPR-GR transfer */
+    FAC_GIE,                /* general instructions extension */
+    FAC_HFP_MA,             /* HFP multiply-and-add/subtract */
+    FAC_HW,                 /* high-word */
+    FAC_IEEEE_SIM,          /* IEEE exception sumilation */
+    FAC_LOC,                /* load/store on condition */
+    FAC_LD,                 /* long displacement */
+    FAC_PC,                 /* population count */
+    FAC_SCF,                /* store clock fast */
+    FAC_SFLE,               /* store facility list extended */
+} DisasFacility;
+
+struct DisasInsn {
+    unsigned opc:16;
+    DisasFormat fmt:8;
+    DisasFacility fac:8;
+    unsigned spec:8;
+
+    const char *name;
+
+    void (*help_in1)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_in2)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_prep)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_wout)(DisasContext *, DisasFields *, DisasOps *);
+    void (*help_cout)(DisasContext *, DisasOps *);
+    ExitStatus (*help_op)(DisasContext *, DisasOps *);
+
+    uint64_t data;
+};
+
+/* ====================================================================== */
+/* Miscelaneous helpers, used by several operations.  */
+
+static void help_l2_shift(DisasContext *s, DisasFields *f,
+                          DisasOps *o, int mask)
 {
-    int skip;
+    int b2 = get_field(f, b2);
+    int d2 = get_field(f, d2);
 
-    if (mask == 0xf) {
-        /* unconditional */
-        gen_goto_tb(s, 0, s->pc + offset);
-    } else if (mask == 0) {
-        /* ignore cc and never match */
-        gen_goto_tb(s, 0, s->pc + 4);
+    if (b2 == 0) {
+        o->in2 = tcg_const_i64(d2 & mask);
     } else {
-        skip = gen_new_label();
-        gen_jcc(s, mask, skip);
-        gen_goto_tb(s, 0, s->pc + offset);
-        gen_set_label(skip);
-        gen_goto_tb(s, 1, s->pc + 4);
+        o->in2 = get_address(s, 0, b2, d2);
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
     }
-    s->is_jmp = DISAS_TB_JUMP;
 }
 
-static void gen_op_mvc(DisasContext *s, int l, TCGv_i64 s1, TCGv_i64 s2)
+static ExitStatus help_goto_direct(DisasContext *s, uint64_t dest)
 {
-    TCGv_i64 tmp, tmp2;
-    int i;
-    int l_memset = gen_new_label();
-    int l_out = gen_new_label();
-    TCGv_i64 dest = tcg_temp_local_new_i64();
-    TCGv_i64 src = tcg_temp_local_new_i64();
-    TCGv_i32 vl;
+    if (dest == s->next_pc) {
+        return NO_EXIT;
+    }
+    if (use_goto_tb(s, dest)) {
+        update_cc_op(s);
+        tcg_gen_goto_tb(0);
+        tcg_gen_movi_i64(psw_addr, dest);
+        tcg_gen_exit_tb((tcg_target_long)s->tb);
+        return EXIT_GOTO_TB;
+    } else {
+        tcg_gen_movi_i64(psw_addr, dest);
+        return EXIT_PC_UPDATED;
+    }
+}
 
-    /* Find out if we should use the inline version of mvc */
-    switch (l) {
-    case 0:
-    case 1:
-    case 2:
-    case 3:
-    case 4:
-    case 5:
-    case 6:
-    case 7:
-    case 11:
-    case 15:
-        /* use inline */
-        break;
-    default:
-        /* Fall back to helper */
-        vl = tcg_const_i32(l);
-        potential_page_fault(s);
-        gen_helper_mvc(cpu_env, vl, s1, s2);
-        tcg_temp_free_i32(vl);
-        return;
+static ExitStatus help_branch(DisasContext *s, DisasCompare *c,
+                              bool is_imm, int imm, TCGv_i64 cdest)
+{
+    ExitStatus ret;
+    uint64_t dest = s->pc + 2 * imm;
+    int lab;
+
+    /* Take care of the special cases first.  */
+    if (c->cond == TCG_COND_NEVER) {
+        ret = NO_EXIT;
+        goto egress;
+    }
+    if (is_imm) {
+        if (dest == s->next_pc) {
+            /* Branch to next.  */
+            ret = NO_EXIT;
+            goto egress;
+        }
+        if (c->cond == TCG_COND_ALWAYS) {
+            ret = help_goto_direct(s, dest);
+            goto egress;
+        }
+    } else {
+        if (TCGV_IS_UNUSED_I64(cdest)) {
+            /* E.g. bcr %r0 -> no branch.  */
+            ret = NO_EXIT;
+            goto egress;
+        }
+        if (c->cond == TCG_COND_ALWAYS) {
+            tcg_gen_mov_i64(psw_addr, cdest);
+            ret = EXIT_PC_UPDATED;
+            goto egress;
+        }
     }
 
-    tcg_gen_mov_i64(dest, s1);
-    tcg_gen_mov_i64(src, s2);
+    if (use_goto_tb(s, s->next_pc)) {
+        if (is_imm && use_goto_tb(s, dest)) {
+            /* Both exits can use goto_tb.  */
+            update_cc_op(s);
 
-    if (!(s->tb->flags & FLAG_MASK_64)) {
-        /* XXX what if we overflow while moving? */
-        tcg_gen_andi_i64(dest, dest, 0x7fffffffUL);
-        tcg_gen_andi_i64(src, src, 0x7fffffffUL);
+            lab = gen_new_label();
+            if (c->is_64) {
+                tcg_gen_brcond_i64(c->cond, c->u.s64.a, c->u.s64.b, lab);
+            } else {
+                tcg_gen_brcond_i32(c->cond, c->u.s32.a, c->u.s32.b, lab);
+            }
+
+            /* Branch not taken.  */
+            tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(psw_addr, s->next_pc);
+            tcg_gen_exit_tb((tcg_target_long)s->tb + 0);
+
+            /* Branch taken.  */
+            gen_set_label(lab);
+            tcg_gen_goto_tb(1);
+            tcg_gen_movi_i64(psw_addr, dest);
+            tcg_gen_exit_tb((tcg_target_long)s->tb + 1);
+
+            ret = EXIT_GOTO_TB;
+        } else {
+            /* Fallthru can use goto_tb, but taken branch cannot.  */
+            /* Store taken branch destination before the brcond.  This
+               avoids having to allocate a new local temp to hold it.
+               We'll overwrite this in the not taken case anyway.  */
+            if (!is_imm) {
+                tcg_gen_mov_i64(psw_addr, cdest);
+            }
+
+            lab = gen_new_label();
+            if (c->is_64) {
+                tcg_gen_brcond_i64(c->cond, c->u.s64.a, c->u.s64.b, lab);
+            } else {
+                tcg_gen_brcond_i32(c->cond, c->u.s32.a, c->u.s32.b, lab);
+            }
+
+            /* Branch not taken.  */
+            update_cc_op(s);
+            tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(psw_addr, s->next_pc);
+            tcg_gen_exit_tb((tcg_target_long)s->tb + 0);
+
+            gen_set_label(lab);
+            if (is_imm) {
+                tcg_gen_movi_i64(psw_addr, dest);
+            }
+            ret = EXIT_PC_UPDATED;
+        }
+    } else {
+        /* Fallthru cannot use goto_tb.  This by itself is vanishingly rare.
+           Most commonly we're single-stepping or some other condition that
+           disables all use of goto_tb.  Just update the PC and exit.  */
+
+        TCGv_i64 next = tcg_const_i64(s->next_pc);
+        if (is_imm) {
+            cdest = tcg_const_i64(dest);
+        }
+
+        if (c->is_64) {
+            tcg_gen_movcond_i64(c->cond, psw_addr, c->u.s64.a, c->u.s64.b,
+                                cdest, next);
+        } else {
+            TCGv_i32 t0 = tcg_temp_new_i32();
+            TCGv_i64 t1 = tcg_temp_new_i64();
+            TCGv_i64 z = tcg_const_i64(0);
+            tcg_gen_setcond_i32(c->cond, t0, c->u.s32.a, c->u.s32.b);
+            tcg_gen_extu_i32_i64(t1, t0);
+            tcg_temp_free_i32(t0);
+            tcg_gen_movcond_i64(TCG_COND_NE, psw_addr, t1, z, cdest, next);
+            tcg_temp_free_i64(t1);
+            tcg_temp_free_i64(z);
+        }
+
+        if (is_imm) {
+            tcg_temp_free_i64(cdest);
+        }
+        tcg_temp_free_i64(next);
+
+        ret = EXIT_PC_UPDATED;
     }
 
-    tmp = tcg_temp_new_i64();
-    tcg_gen_addi_i64(tmp, src, 1);
-    tcg_gen_brcond_i64(TCG_COND_EQ, dest, tmp, l_memset);
-    tcg_temp_free_i64(tmp);
+ egress:
+    free_compare(c);
+    return ret;
+}
 
-    switch (l) {
-    case 0:
-        tmp = tcg_temp_new_i64();
+/* ====================================================================== */
+/* The operations.  These perform the bulk of the work for any insn,
+   usually after the operands have been loaded and output initialized.  */
 
-        tcg_gen_qemu_ld8u(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st8(tmp, dest, get_mem_index(s));
+static ExitStatus op_abs(DisasContext *s, DisasOps *o)
+{
+    gen_helper_abs_i64(o->out, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    case 1:
-        tmp = tcg_temp_new_i64();
+static ExitStatus op_absf32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffull);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld16u(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st16(tmp, dest, get_mem_index(s));
+static ExitStatus op_absf64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffffffffffull);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    case 3:
-        tmp = tcg_temp_new_i64();
+static ExitStatus op_absf128(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in1, 0x7fffffffffffffffull);
+    tcg_gen_mov_i64(o->out2, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld32u(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st32(tmp, dest, get_mem_index(s));
+static ExitStatus op_add(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_add_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    case 4:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
-
-        tcg_gen_qemu_ld32u(tmp, src, get_mem_index(s));
-        tcg_gen_addi_i64(src, src, 4);
-        tcg_gen_qemu_ld8u(tmp2, src, get_mem_index(s));
-        tcg_gen_qemu_st32(tmp, dest, get_mem_index(s));
-        tcg_gen_addi_i64(dest, dest, 4);
-        tcg_gen_qemu_st8(tmp2, dest, get_mem_index(s));
-
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 7:
-        tmp = tcg_temp_new_i64();
+static ExitStatus op_addc(DisasContext *s, DisasOps *o)
+{
+    DisasCompare cmp;
+    TCGv_i64 carry;
+
+    tcg_gen_add_i64(o->out, o->in1, o->in2);
+
+    /* The carry flag is the msb of CC, therefore the branch mask that would
+       create that comparison is 3.  Feeding the generated comparison to
+       setcond produces the carry flag that we desire.  */
+    disas_jcc(s, &cmp, 3);
+    carry = tcg_temp_new_i64();
+    if (cmp.is_64) {
+        tcg_gen_setcond_i64(cmp.cond, carry, cmp.u.s64.a, cmp.u.s64.b);
+    } else {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_setcond_i32(cmp.cond, t, cmp.u.s32.a, cmp.u.s32.b);
+        tcg_gen_extu_i32_i64(carry, t);
+        tcg_temp_free_i32(t);
+    }
+    free_compare(&cmp);
 
-        tcg_gen_qemu_ld64(tmp, src, get_mem_index(s));
-        tcg_gen_qemu_st64(tmp, dest, get_mem_index(s));
+    tcg_gen_add_i64(o->out, o->out, carry);
+    tcg_temp_free_i64(carry);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        break;
-    default:
-        /* The inline version can become too big for too uneven numbers, only
-           use it on known good lengths */
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_const_i64(8);
-        for (i = 0; (i + 7) <= l; i += 8) {
-            tcg_gen_qemu_ld64(tmp, src, get_mem_index(s));
-            tcg_gen_qemu_st64(tmp, dest, get_mem_index(s));
-
-            tcg_gen_add_i64(src, src, tmp2);
-            tcg_gen_add_i64(dest, dest, tmp2);
-        }
+static ExitStatus op_aeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_aeb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp2);
-        tmp2 = tcg_const_i64(1);
+static ExitStatus op_adb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_adb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        for (; i <= l; i++) {
-            tcg_gen_qemu_ld8u(tmp, src, get_mem_index(s));
-            tcg_gen_qemu_st8(tmp, dest, get_mem_index(s));
+static ExitStatus op_axb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_axb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
 
-            tcg_gen_add_i64(src, src, tmp2);
-            tcg_gen_add_i64(dest, dest, tmp2);
-        }
+static ExitStatus op_and(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_and_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp);
-        break;
+static ExitStatus op_andi(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    uint64_t mask = ((1ull << size) - 1) << shift;
+
+    assert(!o->g_in2);
+    tcg_gen_shli_i64(o->in2, o->in2, shift);
+    tcg_gen_ori_i64(o->in2, o->in2, ~mask);
+    tcg_gen_and_i64(o->out, o->in1, o->in2);
+
+    /* Produce the CC from only the bits manipulated.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_bas(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_movi_i64(o->out, pc_to_link_info(s, s->next_pc));
+    if (!TCGV_IS_UNUSED_I64(o->in2)) {
+        tcg_gen_mov_i64(psw_addr, o->in2);
+        return EXIT_PC_UPDATED;
+    } else {
+        return NO_EXIT;
     }
+}
 
-    tcg_gen_br(l_out);
+static ExitStatus op_basi(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_movi_i64(o->out, pc_to_link_info(s, s->next_pc));
+    return help_goto_direct(s, s->pc + 2 * get_field(s->fields, i2));
+}
 
-    gen_set_label(l_memset);
-    /* memset case (dest == (src + 1)) */
+static ExitStatus op_bc(DisasContext *s, DisasOps *o)
+{
+    int m1 = get_field(s->fields, m1);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+
+    disas_jcc(s, &c, m1);
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
+
+static ExitStatus op_bct32(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+    TCGv_i64 t;
+
+    c.cond = TCG_COND_NE;
+    c.is_64 = false;
+    c.g1 = false;
+    c.g2 = false;
+
+    t = tcg_temp_new_i64();
+    tcg_gen_subi_i64(t, regs[r1], 1);
+    store_reg32_i64(r1, t);
+    c.u.s32.a = tcg_temp_new_i32();
+    c.u.s32.b = tcg_const_i32(0);
+    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_temp_free_i64(t);
+
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
+
+static ExitStatus op_bct64(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+
+    c.cond = TCG_COND_NE;
+    c.is_64 = true;
+    c.g1 = true;
+    c.g2 = false;
 
-    tmp = tcg_temp_new_i64();
-    tmp2 = tcg_temp_new_i64();
-    /* fill tmp with the byte */
-    tcg_gen_qemu_ld8u(tmp, src, get_mem_index(s));
-    tcg_gen_shli_i64(tmp2, tmp, 8);
-    tcg_gen_or_i64(tmp, tmp, tmp2);
-    tcg_gen_shli_i64(tmp2, tmp, 16);
-    tcg_gen_or_i64(tmp, tmp, tmp2);
-    tcg_gen_shli_i64(tmp2, tmp, 32);
-    tcg_gen_or_i64(tmp, tmp, tmp2);
-    tcg_temp_free_i64(tmp2);
+    tcg_gen_subi_i64(regs[r1], regs[r1], 1);
+    c.u.s64.a = regs[r1];
+    c.u.s64.b = tcg_const_i64(0);
 
-    tmp2 = tcg_const_i64(8);
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
 
-    for (i = 0; (i + 7) <= l; i += 8) {
-        tcg_gen_qemu_st64(tmp, dest, get_mem_index(s));
-        tcg_gen_addi_i64(dest, dest, 8);
+static ExitStatus op_bx32(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+    TCGv_i64 t;
+
+    c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT);
+    c.is_64 = false;
+    c.g1 = false;
+    c.g2 = false;
+
+    t = tcg_temp_new_i64();
+    tcg_gen_add_i64(t, regs[r1], regs[r3]);
+    c.u.s32.a = tcg_temp_new_i32();
+    c.u.s32.b = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_trunc_i64_i32(c.u.s32.b, regs[r3 | 1]);
+    store_reg32_i64(r1, t);
+    tcg_temp_free_i64(t);
+
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
+
+static ExitStatus op_bx64(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    bool is_imm = have_field(s->fields, i2);
+    int imm = is_imm ? get_field(s->fields, i2) : 0;
+    DisasCompare c;
+
+    c.cond = (s->insn->data ? TCG_COND_LE : TCG_COND_GT);
+    c.is_64 = true;
+
+    if (r1 == (r3 | 1)) {
+        c.u.s64.b = load_reg(r3 | 1);
+        c.g2 = false;
+    } else {
+        c.u.s64.b = regs[r3 | 1];
+        c.g2 = true;
     }
 
-    tcg_temp_free_i64(tmp2);
-    tmp2 = tcg_const_i64(1);
+    tcg_gen_add_i64(regs[r1], regs[r1], regs[r3]);
+    c.u.s64.a = regs[r1];
+    c.g1 = true;
+
+    return help_branch(s, &c, is_imm, imm, o->in2);
+}
 
-    for (; i <= l; i++) {
-        tcg_gen_qemu_st8(tmp, dest, get_mem_index(s));
-        tcg_gen_addi_i64(dest, dest, 1);
+static ExitStatus op_cj(DisasContext *s, DisasOps *o)
+{
+    int imm, m3 = get_field(s->fields, m3);
+    bool is_imm;
+    DisasCompare c;
+
+    c.cond = ltgt_cond[m3];
+    if (s->insn->data) {
+        c.cond = tcg_unsigned_cond(c.cond);
     }
+    c.is_64 = c.g1 = c.g2 = true;
+    c.u.s64.a = o->in1;
+    c.u.s64.b = o->in2;
 
-    tcg_temp_free_i64(tmp2);
-    tcg_temp_free_i64(tmp);
+    is_imm = have_field(s->fields, i4);
+    if (is_imm) {
+        imm = get_field(s->fields, i4);
+    } else {
+        imm = 0;
+        o->out = get_address(s, 0, get_field(s->fields, b4),
+                             get_field(s->fields, d4));
+    }
 
-    gen_set_label(l_out);
+    return help_branch(s, &c, is_imm, imm, o->out);
+}
 
-    tcg_temp_free(dest);
-    tcg_temp_free(src);
+static ExitStatus op_ceb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ceb(cc_op, cpu_env, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
 }
 
-static void gen_op_clc(DisasContext *s, int l, TCGv_i64 s1, TCGv_i64 s2)
+static ExitStatus op_cdb(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp;
-    TCGv_i64 tmp2;
-    TCGv_i32 vl;
+    gen_helper_cdb(cc_op, cpu_env, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 
-    /* check for simple 32bit or 64bit match */
-    switch (l) {
-    case 0:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_cxb(cc_op, cpu_env, o->out, o->out2, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld8u(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld8u(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_cfeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cfeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    case 1:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cfdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cfdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld16u(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld16u(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_cfxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cfxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    case 3:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cgeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cgeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld32u(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld32u(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_cgdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cgdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    case 7:
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
+static ExitStatus op_cgxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cgxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_gen_qemu_ld64(tmp, s1, get_mem_index(s));
-        tcg_gen_qemu_ld64(tmp2, s2, get_mem_index(s));
-        cmp_u64(s, tmp, tmp2);
+static ExitStatus op_clfeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clfeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
 
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        return;
-    }
+static ExitStatus op_clfdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clfdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clfxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clfxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clgeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clgeb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f32(s, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clgdb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clgdb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f64(s, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clgxb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_clgxb(o->out, cpu_env, o->in1, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    gen_set_cc_nz_f128(s, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cegb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cegb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cdgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cdgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cxgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cxgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_celgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_celgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cdlgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cdlgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cxlgb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_cxlgb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cksm(DisasContext *s, DisasOps *o)
+{
+    int r2 = get_field(s->fields, r2);
+    TCGv_i64 len = tcg_temp_new_i64();
 
     potential_page_fault(s);
-    vl = tcg_const_i32(l);
-    gen_helper_clc(cc_op, cpu_env, vl, s1, s2);
-    tcg_temp_free_i32(vl);
+    gen_helper_cksm(len, cpu_env, o->in1, o->in2, regs[r2 + 1]);
     set_cc_static(s);
+    return_low128(o->out);
+
+    tcg_gen_add_i64(regs[r2], regs[r2], len);
+    tcg_gen_sub_i64(regs[r2 + 1], regs[r2 + 1], len);
+    tcg_temp_free_i64(len);
+
+    return NO_EXIT;
 }
 
-static void disas_e3(CPUS390XState *env, DisasContext* s, int op, int r1,
-                     int x2, int b2, int d2)
+static ExitStatus op_clc(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 addr, tmp, tmp2, tmp3, tmp4;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
+    int l = get_field(s->fields, l1);
+    TCGv_i32 vl;
 
-    LOG_DISAS("disas_e3: op 0x%x r1 %d x2 %d b2 %d d2 %d\n",
-              op, r1, x2, b2, d2);
-    addr = get_address(s, x2, b2, d2);
-    switch (op) {
-    case 0x2: /* LTG R1,D2(X2,B2) [RXY] */
-    case 0x4: /* lg r1,d2(x2,b2) */
-        tcg_gen_qemu_ld64(regs[r1], addr, get_mem_index(s));
-        if (op == 0x2) {
-            set_cc_s64(s, regs[r1]);
-        }
-        break;
-    case 0x12: /* LT R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        store_reg32(r1, tmp32_1);
-        set_cc_s32(s, tmp32_1);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xc: /* MSG      R1,D2(X2,B2)     [RXY] */
-    case 0x1c: /* MSGF     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0xc) {
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        } else {
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        }
-        tcg_gen_mul_i64(regs[r1], regs[r1], tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0xd: /* DSG      R1,D2(X2,B2)     [RXY] */
-    case 0x1d: /* DSGF      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0x1d) {
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        } else {
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        }
-        tmp4 = load_reg(r1 + 1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_div_i64(tmp3, tmp4, tmp2);
-        store_reg(r1 + 1, tmp3);
-        tcg_gen_rem_i64(tmp3, tmp4, tmp2);
-        store_reg(r1, tmp3);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x8: /* AG      R1,D2(X2,B2)     [RXY] */
-    case 0xa: /* ALG      R1,D2(X2,B2)     [RXY] */
-    case 0x18: /* AGF       R1,D2(X2,B2)     [RXY] */
-    case 0x1a: /* ALGF      R1,D2(X2,B2)     [RXY] */
-        if (op == 0x1a) {
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        } else if (op == 0x18) {
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        } else {
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        }
-        tmp4 = load_reg(r1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_add_i64(tmp3, tmp4, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x8:
-        case 0x18:
-            set_cc_add64(s, tmp4, tmp2, tmp3);
-            break;
-        case 0xa:
-        case 0x1a:
-            set_cc_addu64(s, tmp4, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x9: /* SG      R1,D2(X2,B2)     [RXY] */
-    case 0xb: /* SLG      R1,D2(X2,B2)     [RXY] */
-    case 0x19: /* SGF      R1,D2(X2,B2)     [RXY] */
-    case 0x1b: /* SLGF     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0x19) {
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        } else if (op == 0x1b) {
-            tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        } else {
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        }
-        tmp4 = load_reg(r1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_sub_i64(tmp3, tmp4, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x9:
-        case 0x19:
-            set_cc_sub64(s, tmp4, tmp2, tmp3);
-            break;
-        case 0xb:
-        case 0x1b:
-            set_cc_subu64(s, tmp4, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0xf: /* LRVG     R1,D2(X2,B2)     [RXE] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        tcg_gen_bswap64_i64(tmp2, tmp2);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x14: /* LGF      R1,D2(X2,B2)     [RXY] */
-    case 0x16: /* LLGF      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        if (op == 0x14) {
-            tcg_gen_ext32s_i64(tmp2, tmp2);
-        }
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x15: /* LGH     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, addr, get_mem_index(s));
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x17: /* LLGT      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_andi_i64(tmp2, tmp2, 0x7fffffffULL);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x1e: /* LRV R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_gen_bswap32_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x1f: /* LRVH R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld16u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_gen_bswap16_i32(tmp32_1, tmp32_1);
-        store_reg16(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x20: /* CG      R1,D2(X2,B2)     [RXY] */
-    case 0x21: /* CLG      R1,D2(X2,B2) */
-    case 0x30: /* CGF       R1,D2(X2,B2)     [RXY] */
-    case 0x31: /* CLGF      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        switch (op) {
-        case 0x20:
-        case 0x21:
-            tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-            break;
-        case 0x30:
-            tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-            break;
-        case 0x31:
-            tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-            break;
-        default:
-            tcg_abort();
-        }
-        switch (op) {
-        case 0x20:
-        case 0x30:
-            cmp_s64(s, regs[r1], tmp2);
-            break;
-        case 0x21:
-        case 0x31:
-            cmp_u64(s, regs[r1], tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x24: /* stg r1, d2(x2,b2) */
-        tcg_gen_qemu_st64(regs[r1], addr, get_mem_index(s));
-        break;
-    case 0x3e: /* STRV R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_bswap32_i32(tmp32_1, tmp32_1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_gen_qemu_st32(tmp2, addr, get_mem_index(s));
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x50: /* STY  R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_gen_qemu_st32(tmp2, addr, get_mem_index(s));
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x57: /* XY R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_gen_xor_i32(tmp32_2, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_2);
-        set_cc_nz_u32(s, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x58: /* LY R1,D2(X2,B2) [RXY] */
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp3, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp3);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x5a: /* AY R1,D2(X2,B2) [RXY] */
-    case 0x5b: /* SY R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32s(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        tcg_temp_free_i64(tmp2);
-        switch (op) {
-        case 0x5a:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        case 0x5b:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_3);
-        switch (op) {
-        case 0x5a:
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5b:
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x71: /* LAY R1,D2(X2,B2) [RXY] */
-        store_reg(r1, addr);
-        break;
-    case 0x72: /* STCY R1,D2(X2,B2) [RXY] */
-        tmp32_1 = load_reg32(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-        tcg_gen_qemu_st8(tmp2, addr, get_mem_index(s));
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x73: /* ICY R1,D2(X2,B2) [RXY] */
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp3, addr, get_mem_index(s));
-        store_reg8(r1, tmp3);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x76: /* LB R1,D2(X2,B2) [RXY] */
-    case 0x77: /* LGB R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8s(tmp2, addr, get_mem_index(s));
-        switch (op) {
-        case 0x76:
-            tcg_gen_ext8s_i64(tmp2, tmp2);
-            store_reg32_i64(r1, tmp2);
-            break;
-        case 0x77:
-            tcg_gen_ext8s_i64(tmp2, tmp2);
-            store_reg(r1, tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x78: /* LHY R1,D2(X2,B2) [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x80: /* NG      R1,D2(X2,B2)     [RXY] */
-    case 0x81: /* OG      R1,D2(X2,B2)     [RXY] */
-    case 0x82: /* XG      R1,D2(X2,B2)     [RXY] */
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp3, addr, get_mem_index(s));
-        switch (op) {
-        case 0x80:
-            tcg_gen_and_i64(regs[r1], regs[r1], tmp3);
-            break;
-        case 0x81:
-            tcg_gen_or_i64(regs[r1], regs[r1], tmp3);
-            break;
-        case 0x82:
-            tcg_gen_xor_i64(regs[r1], regs[r1], tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        set_cc_nz_u64(s, regs[r1]);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x86: /* MLG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        gen_helper_mlg(cpu_env, tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x87: /* DLG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        gen_helper_dlg(cpu_env, tmp32_1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x88: /* ALCG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        tcg_gen_extu_i32_i64(tmp3, cc_op);
-        tcg_gen_shri_i64(tmp3, tmp3, 1);
-        tcg_gen_andi_i64(tmp3, tmp3, 1);
-        tcg_gen_add_i64(tmp3, tmp2, tmp3);
-        tcg_gen_add_i64(tmp3, regs[r1], tmp3);
-        store_reg(r1, tmp3);
-        set_cc_addu64(s, regs[r1], tmp2, tmp3);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x89: /* SLBG      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_slbg(cc_op, cpu_env, cc_op, tmp32_1, regs[r1], tmp2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x90: /* LLGC      R1,D2(X2,B2)     [RXY] */
-        tcg_gen_qemu_ld8u(regs[r1], addr, get_mem_index(s));
-        break;
-    case 0x91: /* LLGH      R1,D2(X2,B2)     [RXY] */
-        tcg_gen_qemu_ld16u(regs[r1], addr, get_mem_index(s));
-        break;
-    case 0x94: /* LLC     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x95: /* LLH     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16u(tmp2, addr, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x96: /* ML      R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    switch (l + 1) {
+    case 1:
+        tcg_gen_qemu_ld8u(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld8u(cc_dst, o->in2, get_mem_index(s));
         break;
-    case 0x97: /* DL     R1,D2(X2,B2)     [RXY] */
-        /* reg(r1) = reg(r1, r1+1) % ld32(addr) */
-        /* reg(r1+1) = reg(r1, r1+1) / ld32(addr) */
-        tmp = load_reg(r1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp2, tmp2);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp3);
-
-        tcg_gen_rem_i64(tmp3, tmp, tmp2);
-        tcg_gen_div_i64(tmp, tmp, tmp2);
-        store_reg32_i64((r1 + 1) & 15, tmp);
-        store_reg32_i64(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    case 2:
+        tcg_gen_qemu_ld16u(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld16u(cc_dst, o->in2, get_mem_index(s));
         break;
-    case 0x98: /* ALC     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_addc_u32(tmp32_3, cc_op, tmp32_1, tmp32_2);
-        set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-        store_reg32(r1, tmp32_3);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+    case 4:
+        tcg_gen_qemu_ld32u(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld32u(cc_dst, o->in2, get_mem_index(s));
         break;
-    case 0x99: /* SLB     R1,D2(X2,B2)     [RXY] */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_slb(cc_op, cpu_env, cc_op, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+    case 8:
+        tcg_gen_qemu_ld64(cc_src, o->addr1, get_mem_index(s));
+        tcg_gen_qemu_ld64(cc_dst, o->in2, get_mem_index(s));
         break;
     default:
-        LOG_DISAS("illegal e3 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 3);
-        break;
+        potential_page_fault(s);
+        vl = tcg_const_i32(l);
+        gen_helper_clc(cc_op, cpu_env, vl, o->addr1, o->in2);
+        tcg_temp_free_i32(vl);
+        set_cc_static(s);
+        return NO_EXIT;
     }
+    gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, cc_src, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clcle(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_clcle(cc_op, cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clm(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    TCGv_i32 t1 = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(t1, o->in1);
+    potential_page_fault(s);
+    gen_helper_clm(cc_op, cpu_env, t1, m3, o->in2);
+    set_cc_static(s);
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_clst(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_clst(o->in1, cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return_low128(o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cps(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t = tcg_temp_new_i64();
+    tcg_gen_andi_i64(t, o->in1, 0x8000000000000000ull);
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffffffffffffull);
+    tcg_gen_or_i64(o->out, o->out, t);
+    tcg_temp_free_i64(t);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cs(DisasContext *s, DisasOps *o)
+{
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    int is_64 = s->insn->data;
+    TCGv_i64 addr, mem, cc, z;
+
+    /* Note that in1 = R3 (new value) and
+       in2 = (zero-extended) R1 (expected value).  */
+
+    /* Load the memory into the (temporary) output.  While the PoO only talks
+       about moving the memory to R1 on inequality, if we include equality it
+       means that R1 is equal to the memory in all conditions.  */
+    addr = get_address(s, 0, b2, d2);
+    if (is_64) {
+        tcg_gen_qemu_ld64(o->out, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_ld32u(o->out, addr, get_mem_index(s));
+    }
+
+    /* Are the memory and expected values (un)equal?  Note that this setcond
+       produces the output CC value, thus the NE sense of the test.  */
+    cc = tcg_temp_new_i64();
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, o->in2, o->out);
+
+    /* If the memory and expected values are equal (CC==0), copy R3 to MEM.
+       Recall that we are allowed to unconditionally issue the store (and
+       thus any possible write trap), so (re-)store the original contents
+       of MEM in case of inequality.  */
+    z = tcg_const_i64(0);
+    mem = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, mem, cc, z, o->in1, o->out);
+    if (is_64) {
+        tcg_gen_qemu_st64(mem, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_st32(mem, addr, get_mem_index(s));
+    }
+    tcg_temp_free_i64(z);
+    tcg_temp_free_i64(mem);
     tcg_temp_free_i64(addr);
+
+    /* Store CC back to cc_op.  Wait until after the store so that any
+       exception gets the old cc_op value.  */
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(cc);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
+{
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    TCGv_i64 addrh, addrl, memh, meml, outh, outl, cc, z;
+
+    /* Note that R1:R1+1 = expected value and R3:R3+1 = new value.  */
+
+    addrh = get_address(s, 0, b2, d2);
+    addrl = get_address(s, 0, b2, d2 + 8);
+    outh = tcg_temp_new_i64();
+    outl = tcg_temp_new_i64();
+
+    tcg_gen_qemu_ld64(outh, addrh, get_mem_index(s));
+    tcg_gen_qemu_ld64(outl, addrl, get_mem_index(s));
+
+    /* Fold the double-word compare with arithmetic.  */
+    cc = tcg_temp_new_i64();
+    z = tcg_temp_new_i64();
+    tcg_gen_xor_i64(cc, outh, regs[r1]);
+    tcg_gen_xor_i64(z, outl, regs[r1 + 1]);
+    tcg_gen_or_i64(cc, cc, z);
+    tcg_gen_movi_i64(z, 0);
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, cc, z);
+
+    memh = tcg_temp_new_i64();
+    meml = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, memh, cc, z, regs[r3], outh);
+    tcg_gen_movcond_i64(TCG_COND_EQ, meml, cc, z, regs[r3 + 1], outl);
+    tcg_temp_free_i64(z);
+
+    tcg_gen_qemu_st64(memh, addrh, get_mem_index(s));
+    tcg_gen_qemu_st64(meml, addrl, get_mem_index(s));
+    tcg_temp_free_i64(memh);
+    tcg_temp_free_i64(meml);
+    tcg_temp_free_i64(addrh);
+    tcg_temp_free_i64(addrl);
+
+    /* Save back state now that we've passed all exceptions.  */
+    tcg_gen_mov_i64(regs[r1], outh);
+    tcg_gen_mov_i64(regs[r1 + 1], outl);
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(outh);
+    tcg_temp_free_i64(outl);
+    tcg_temp_free_i64(cc);
+    set_cc_static(s);
+    return NO_EXIT;
 }
 
 #ifndef CONFIG_USER_ONLY
-static void disas_e5(CPUS390XState *env, DisasContext* s, uint64_t insn)
+static ExitStatus op_csp(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2;
-    int op = (insn >> 32) & 0xff;
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    check_privileged(s);
+    gen_helper_csp(cc_op, cpu_env, r1, o->in2);
+    tcg_temp_free_i32(r1);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+#endif
 
-    tmp = get_address(s, 0, (insn >> 28) & 0xf, (insn >> 16) & 0xfff);
-    tmp2 = get_address(s, 0, (insn >> 12) & 0xf, insn & 0xfff);
+static ExitStatus op_cvd(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i32 t2 = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(t2, o->in1);
+    gen_helper_cvd(t1, t2);
+    tcg_temp_free_i32(t2);
+    tcg_gen_qemu_st64(t1, o->in2, get_mem_index(s));
+    tcg_temp_free_i64(t1);
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_e5: insn %" PRIx64 "\n", insn);
-    switch (op) {
-    case 0x01: /* TPROT    D1(B1),D2(B2)  [SSE] */
-        /* Test Protection */
-        potential_page_fault(s);
-        gen_helper_tprot(cc_op, tmp, tmp2);
-        set_cc_static(s);
-        break;
-    default:
-        LOG_DISAS("illegal e5 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 3);
-        break;
+static ExitStatus op_ct(DisasContext *s, DisasOps *o)
+{
+    int m3 = get_field(s->fields, m3);
+    int lab = gen_new_label();
+    TCGv_i32 t;
+    TCGCond c;
+
+    c = tcg_invert_cond(ltgt_cond[m3]);
+    if (s->insn->data) {
+        c = tcg_unsigned_cond(c);
     }
+    tcg_gen_brcond_i64(c, o->in1, o->in2, lab);
 
-    tcg_temp_free_i64(tmp);
-    tcg_temp_free_i64(tmp2);
+    /* Set DXC to 0xff.  */
+    t = tcg_temp_new_i32();
+    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_gen_ori_i32(t, t, 0xff00);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_temp_free_i32(t);
+
+    /* Trap.  */
+    gen_program_exception(s, PGM_DATA);
+
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_diag(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 tmp;
+
+    check_privileged(s);
+    potential_page_fault(s);
+
+    /* We pretend the format is RX_a so that D2 is the field we want.  */
+    tmp = tcg_const_i32(get_field(s->fields, d2) & 0xfff);
+    gen_helper_diag(regs[2], cpu_env, tmp, regs[2], regs[1]);
+    tcg_temp_free_i32(tmp);
+    return NO_EXIT;
 }
 #endif
 
-static void disas_eb(CPUS390XState *env, DisasContext *s, int op, int r1,
-                     int r3, int b2, int d2)
+static ExitStatus op_divs32(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3, tmp4;
-    TCGv_i32 tmp32_1, tmp32_2;
-    int i, stm_len;
-    int ilc = 3;
+    gen_helper_divs32(o->out2, cpu_env, o->in1, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_eb: op 0x%x r1 %d r3 %d b2 %d d2 0x%x\n",
-              op, r1, r3, b2, d2);
-    switch (op) {
-    case 0xc: /* SRLG     R1,R3,D2(B2)     [RSY] */
-    case 0xd: /* SLLG     R1,R3,D2(B2)     [RSY] */
-    case 0xa: /* SRAG     R1,R3,D2(B2)     [RSY] */
-    case 0xb: /* SLAG     R1,R3,D2(B2)     [RSY] */
-    case 0x1c: /* RLLG     R1,R3,D2(B2)     [RSY] */
-        if (b2) {
-            tmp = get_address(s, 0, b2, d2);
-            tcg_gen_andi_i64(tmp, tmp, 0x3f);
-        } else {
-            tmp = tcg_const_i64(d2 & 0x3f);
-        }
-        switch (op) {
-        case 0xc:
-            tcg_gen_shr_i64(regs[r1], regs[r3], tmp);
-            break;
-        case 0xd:
-            tcg_gen_shl_i64(regs[r1], regs[r3], tmp);
-            break;
-        case 0xa:
-            tcg_gen_sar_i64(regs[r1], regs[r3], tmp);
-            break;
-        case 0xb:
-            tmp2 = tcg_temp_new_i64();
-            tmp3 = tcg_temp_new_i64();
-            gen_op_update2_cc_i64(s, CC_OP_SLAG, regs[r3], tmp);
-            tcg_gen_shl_i64(tmp2, regs[r3], tmp);
-            /* override sign bit with source sign */
-            tcg_gen_andi_i64(tmp2, tmp2, ~0x8000000000000000ULL);
-            tcg_gen_andi_i64(tmp3, regs[r3], 0x8000000000000000ULL);
-            tcg_gen_or_i64(regs[r1], tmp2, tmp3);
-            tcg_temp_free_i64(tmp2);
-            tcg_temp_free_i64(tmp3);
-            break;
-        case 0x1c:
-            tcg_gen_rotl_i64(regs[r1], regs[r3], tmp);
-            break;
-        default:
-            tcg_abort();
-            break;
-        }
-        if (op == 0xa) {
-            set_cc_s64(s, regs[r1]);
-        }
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x1d: /* RLL    R1,R3,D2(B2)        [RSY] */
-        if (b2) {
-            tmp = get_address(s, 0, b2, d2);
-            tcg_gen_andi_i64(tmp, tmp, 0x3f);
-        } else {
-            tmp = tcg_const_i64(d2 & 0x3f);
-        }
-        tmp32_1 = tcg_temp_new_i32();
-        tmp32_2 = load_reg32(r3);
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        switch (op) {
-        case 0x1d:
-            tcg_gen_rotl_i32(tmp32_1, tmp32_2, tmp32_1);
-            break;
-        default:
-            tcg_abort();
-            break;
-        }
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+static ExitStatus op_divu32(DisasContext *s, DisasOps *o)
+{
+    gen_helper_divu32(o->out2, cpu_env, o->in1, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
+
+static ExitStatus op_divs64(DisasContext *s, DisasOps *o)
+{
+    gen_helper_divs64(o->out2, cpu_env, o->in1, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
+
+static ExitStatus op_divu64(DisasContext *s, DisasOps *o)
+{
+    gen_helper_divu64(o->out2, cpu_env, o->out, o->out2, o->in2);
+    return_low128(o->out);
+    return NO_EXIT;
+}
+
+static ExitStatus op_deb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_deb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ddb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ddb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_dxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_dxb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ear(DisasContext *s, DisasOps *o)
+{
+    int r2 = get_field(s->fields, r2);
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, aregs[r2]));
+    return NO_EXIT;
+}
+
+static ExitStatus op_efpc(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, fpc));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ex(DisasContext *s, DisasOps *o)
+{
+    /* ??? Perhaps a better way to implement EXECUTE is to set a bit in
+       tb->flags, (ab)use the tb->cs_base field as the address of
+       the template in memory, and grab 8 bits of tb->flags/cflags for
+       the contents of the register.  We would then recognize all this
+       in gen_intermediate_code_internal, generating code for exactly
+       one instruction.  This new TB then gets executed normally.
+
+       On the other hand, this seems to be mostly used for modifying
+       MVC inside of memcpy, which needs a helper call anyway.  So
+       perhaps this doesn't bear thinking about any further.  */
+
+    TCGv_i64 tmp;
+
+    update_psw_addr(s);
+    update_cc_op(s);
+
+    tmp = tcg_const_i64(s->next_pc);
+    gen_helper_ex(cc_op, cpu_env, cc_op, o->in1, o->in2, tmp);
+    tcg_temp_free_i64(tmp);
+
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_flogr(DisasContext *s, DisasOps *o)
+{
+    /* We'll use the original input for cc computation, since we get to
+       compare that against 0, which ought to be better than comparing
+       the real output against 64.  It also lets cc_dst be a convenient
+       temporary during our computation.  */
+    gen_op_update1_cc_i64(s, CC_OP_FLOGR, o->in2);
+
+    /* R1 = IN ? CLZ(IN) : 64.  */
+    gen_helper_clz(o->out, o->in2);
+
+    /* R1+1 = IN & ~(found bit).  Note that we may attempt to shift this
+       value by 64, which is undefined.  But since the shift is 64 iff the
+       input is zero, we still get the correct result after and'ing.  */
+    tcg_gen_movi_i64(o->out2, 0x8000000000000000ull);
+    tcg_gen_shr_i64(o->out2, o->out2, o->out);
+    tcg_gen_andc_i64(o->out2, cc_dst, o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_icm(DisasContext *s, DisasOps *o)
+{
+    int m3 = get_field(s->fields, m3);
+    int pos, len, base = s->insn->data;
+    TCGv_i64 tmp = tcg_temp_new_i64();
+    uint64_t ccm;
+
+    switch (m3) {
+    case 0xf:
+        /* Effectively a 32-bit load.  */
+        tcg_gen_qemu_ld32u(tmp, o->in2, get_mem_index(s));
+        len = 32;
+        goto one_insert;
+
+    case 0xc:
+    case 0x6:
+    case 0x3:
+        /* Effectively a 16-bit load.  */
+        tcg_gen_qemu_ld16u(tmp, o->in2, get_mem_index(s));
+        len = 16;
+        goto one_insert;
+
+    case 0x8:
+    case 0x4:
+    case 0x2:
+    case 0x1:
+        /* Effectively an 8-bit load.  */
+        tcg_gen_qemu_ld8u(tmp, o->in2, get_mem_index(s));
+        len = 8;
+        goto one_insert;
+
+    one_insert:
+        pos = base + ctz32(m3) * 8;
+        tcg_gen_deposit_i64(o->out, o->out, tmp, pos, len);
+        ccm = ((1ull << len) - 1) << pos;
         break;
-    case 0x4:  /* LMG      R1,R3,D2(B2)     [RSE] */
-    case 0x24: /* STMG     R1,R3,D2(B2)     [RSE] */
-        stm_len = 8;
-        goto do_mh;
-    case 0x26: /* STMH     R1,R3,D2(B2)     [RSE] */
-    case 0x96: /* LMH      R1,R3,D2(B2)     [RSE] */
-        stm_len = 4;
-do_mh:
-        /* Apparently, unrolling lmg/stmg of any size gains performance -
-           even for very long ones... */
-        tmp = get_address(s, 0, b2, d2);
-        tmp3 = tcg_const_i64(stm_len);
-        tmp4 = tcg_const_i64(op == 0x26 ? 32 : 4);
-        for (i = r1;; i = (i + 1) % 16) {
-            switch (op) {
-            case 0x4:
-                tcg_gen_qemu_ld64(regs[i], tmp, get_mem_index(s));
-                break;
-            case 0x96:
-                tmp2 = tcg_temp_new_i64();
-#if HOST_LONG_BITS == 32
-                tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-                tcg_gen_trunc_i64_i32(TCGV_HIGH(regs[i]), tmp2);
-#else
-                tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-                tcg_gen_shl_i64(tmp2, tmp2, tmp4);
-                tcg_gen_ext32u_i64(regs[i], regs[i]);
-                tcg_gen_or_i64(regs[i], regs[i], tmp2);
-#endif
-                tcg_temp_free_i64(tmp2);
-                break;
-            case 0x24:
-                tcg_gen_qemu_st64(regs[i], tmp, get_mem_index(s));
-                break;
-            case 0x26:
-                tmp2 = tcg_temp_new_i64();
-                tcg_gen_shr_i64(tmp2, regs[i], tmp4);
-                tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-                tcg_temp_free_i64(tmp2);
-                break;
-            default:
-                tcg_abort();
-            }
-            if (i == r3) {
-                break;
+
+    default:
+        /* This is going to be a sequence of loads and inserts.  */
+        pos = base + 32 - 8;
+        ccm = 0;
+        while (m3) {
+            if (m3 & 0x8) {
+                tcg_gen_qemu_ld8u(tmp, o->in2, get_mem_index(s));
+                tcg_gen_addi_i64(o->in2, o->in2, 1);
+                tcg_gen_deposit_i64(o->out, o->out, tmp, pos, 8);
+                ccm |= 0xff << pos;
             }
-            tcg_gen_add_i64(tmp, tmp, tmp3);
+            m3 = (m3 << 1) & 0xf;
+            pos -= 8;
         }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x2c: /* STCMH R1,M3,D2(B2) [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stcmh(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
         break;
+    }
+
+    tcg_gen_movi_i64(tmp, ccm);
+    gen_op_update2_cc_i64(s, CC_OP_ICM, tmp, o->out);
+    tcg_temp_free_i64(tmp);
+    return NO_EXIT;
+}
+
+static ExitStatus op_insi(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    tcg_gen_deposit_i64(o->out, o->in1, o->in2, shift, size);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ipm(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1;
+
+    gen_op_calc_cc(s);
+    tcg_gen_andi_i64(o->out, o->out, ~0xff000000ull);
+
+    t1 = tcg_temp_new_i64();
+    tcg_gen_shli_i64(t1, psw_mask, 20);
+    tcg_gen_shri_i64(t1, t1, 36);
+    tcg_gen_or_i64(o->out, o->out, t1);
+
+    tcg_gen_extu_i32_i64(t1, cc_op);
+    tcg_gen_shli_i64(t1, t1, 28);
+    tcg_gen_or_i64(o->out, o->out, t1);
+    tcg_temp_free_i64(t1);
+    return NO_EXIT;
+}
+
 #ifndef CONFIG_USER_ONLY
-    case 0x2f: /* LCTLG     R1,R3,D2(B2)     [RSE] */
-        /* Load Control */
-        check_privileged(env, s, ilc);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_lctlg(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x25: /* STCTG     R1,R3,D2(B2)     [RSE] */
-        /* Store Control */
-        check_privileged(env, s, ilc);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stctg(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
+static ExitStatus op_ipte(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_ipte(cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_iske(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_iske(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
 #endif
-    case 0x30: /* CSG     R1,R3,D2(B2)     [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        /* XXX rewrite in tcg */
-        gen_helper_csg(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x3e: /* CDSG R1,R3,D2(B2) [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        /* XXX rewrite in tcg */
-        gen_helper_cdsg(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x51: /* TMY D1(B1),I2 [SIY] */
-        tmp = get_address(s, 0, b2, d2); /* SIY -> this is the destination */
-        tmp2 = tcg_const_i64((r1 << 4) | r3);
-        tcg_gen_qemu_ld8u(tmp, tmp, get_mem_index(s));
-        /* yes, this is a 32 bit operation with 64 bit tcg registers, because
-           that incurs less conversions */
-        cmp_64(s, tmp, tmp2, CC_OP_TM_32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x52: /* MVIY D1(B1),I2 [SIY] */
-        tmp = get_address(s, 0, b2, d2); /* SIY -> this is the destination */
-        tmp2 = tcg_const_i64((r1 << 4) | r3);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x55: /* CLIY D1(B1),I2 [SIY] */
-        tmp3 = get_address(s, 0, b2, d2); /* SIY -> this is the 1st operand */
-        tmp = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld8u(tmp, tmp3, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        cmp_u32c(s, tmp32_1, (r1 << 4) | r3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x80: /* ICMH      R1,M3,D2(B2)     [RSY] */
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        /* XXX split CC calculation out */
-        gen_helper_icmh(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    default:
-        LOG_DISAS("illegal eb operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, ilc);
-        break;
-    }
+
+static ExitStatus op_ldeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ldeb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
 }
 
-static void disas_ed(CPUS390XState *env, DisasContext *s, int op, int r1,
-                     int x2, int b2, int d2, int r1b)
+static ExitStatus op_ledb(DisasContext *s, DisasOps *o)
 {
-    TCGv_i32 tmp_r1, tmp32;
-    TCGv_i64 addr, tmp;
-    addr = get_address(s, x2, b2, d2);
-    tmp_r1 = tcg_const_i32(r1);
-    switch (op) {
-    case 0x4: /* LDEB R1,D2(X2,B2) [RXE] */
-        potential_page_fault(s);
-        gen_helper_ldeb(cpu_env, tmp_r1, addr);
-        break;
-    case 0x5: /* LXDB R1,D2(X2,B2) [RXE] */
-        potential_page_fault(s);
-        gen_helper_lxdb(cpu_env, tmp_r1, addr);
-        break;
-    case 0x9: /* CEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = load_freg32(r1);
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        set_cc_cmp_f32_i64(s, tmp32, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0xa: /* AEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_aeb(cpu_env, tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-
-        tmp32 = load_freg32(r1);
-        gen_set_cc_nz_f32(s, tmp32);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0xb: /* SEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_seb(cpu_env, tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-
-        tmp32 = load_freg32(r1);
-        gen_set_cc_nz_f32(s, tmp32);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0xd: /* DEB    R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_deb(cpu_env, tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0x10: /* TCEB   R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_tceb(cc_op, cpu_env, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x11: /* TCDB   R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_tcdb(cc_op, cpu_env, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x12: /* TCXB   R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_tcxb(cc_op, cpu_env, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x17: /* MEEB   R1,D2(X2,B2)       [RXE] */
-        tmp = tcg_temp_new_i64();
-        tmp32 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_meeb(cpu_env, tmp_r1, tmp32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32);
-        break;
-    case 0x19: /* CDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_cdb(cc_op, cpu_env, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x1a: /* ADB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_adb(cc_op, cpu_env, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x1b: /* SDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_sdb(cc_op, cpu_env, tmp_r1, addr);
-        set_cc_static(s);
-        break;
-    case 0x1c: /* MDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_mdb(cpu_env, tmp_r1, addr);
-        break;
-    case 0x1d: /* DDB    R1,D2(X2,B2)       [RXE] */
-        potential_page_fault(s);
-        gen_helper_ddb(cpu_env, tmp_r1, addr);
-        break;
-    case 0x1e: /* MADB  R1,R3,D2(X2,B2) [RXF] */
-        /* for RXF insns, r1 is R3 and r1b is R1 */
-        tmp32 = tcg_const_i32(r1b);
-        potential_page_fault(s);
-        gen_helper_madb(cpu_env, tmp32, addr, tmp_r1);
-        tcg_temp_free_i32(tmp32);
-        break;
-    default:
-        LOG_DISAS("illegal ed operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 3);
-        return;
+    gen_helper_ledb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ldxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_ldxb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lexb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_lexb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lxdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_lxdb(o->out, cpu_env, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lxeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_lxeb(o->out, cpu_env, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_llgt(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffff);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld8s(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld8s(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld8u(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld8u(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld16s(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld16s(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld16u(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld16u(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld32s(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld32s(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld32u(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld32u(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_ld64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_ld64(o->out, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_loc(DisasContext *s, DisasOps *o)
+{
+    DisasCompare c;
+
+    disas_jcc(s, &c, get_field(s->fields, m3));
+
+    if (c.is_64) {
+        tcg_gen_movcond_i64(c.cond, o->out, c.u.s64.a, c.u.s64.b,
+                            o->in2, o->in1);
+        free_compare(&c);
+    } else {
+        TCGv_i32 t32 = tcg_temp_new_i32();
+        TCGv_i64 t, z;
+
+        tcg_gen_setcond_i32(c.cond, t32, c.u.s32.a, c.u.s32.b);
+        free_compare(&c);
+
+        t = tcg_temp_new_i64();
+        tcg_gen_extu_i32_i64(t, t32);
+        tcg_temp_free_i32(t32);
+
+        z = tcg_const_i64(0);
+        tcg_gen_movcond_i64(TCG_COND_NE, o->out, t, z, o->in2, o->in1);
+        tcg_temp_free_i64(t);
+        tcg_temp_free_i64(z);
     }
-    tcg_temp_free_i32(tmp_r1);
-    tcg_temp_free_i64(addr);
+
+    return NO_EXIT;
 }
 
-static void disas_a5(CPUS390XState *env, DisasContext *s, int op, int r1,
-                     int i2)
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_lctl(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2;
-    TCGv_i32 tmp32;
-    LOG_DISAS("disas_a5: op 0x%x r1 %d i2 0x%x\n", op, r1, i2);
-    switch (op) {
-    case 0x0: /* IIHH     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 48, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x1: /* IIHL     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 32, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x2: /* IILH     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 16, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x3: /* IILL     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        tcg_gen_deposit_i64(regs[r1], regs[r1], tmp, 0, 16);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x4: /* NIHH     R1,I2     [RI] */
-    case 0x8: /* OIHH     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x4:
-            tmp2 = tcg_const_i64((((uint64_t)i2) << 48)
-                               | 0x0000ffffffffffffULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0x8:
-            tmp2 = tcg_const_i64(((uint64_t)i2) << 48);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp2, tmp, 48);
-        tcg_gen_trunc_i64_i32(tmp32, tmp2);
-        set_cc_nz_u32(s, tmp32);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x5: /* NIHL     R1,I2     [RI] */
-    case 0x9: /* OIHL     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x5:
-            tmp2 = tcg_const_i64((((uint64_t)i2) << 32)
-                               | 0xffff0000ffffffffULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0x9:
-            tmp2 = tcg_const_i64(((uint64_t)i2) << 32);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp2, tmp, 32);
-        tcg_gen_trunc_i64_i32(tmp32, tmp2);
-        tcg_gen_andi_i32(tmp32, tmp32, 0xffff);
-        set_cc_nz_u32(s, tmp32);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x6: /* NILH     R1,I2     [RI] */
-    case 0xa: /* OILH     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x6:
-            tmp2 = tcg_const_i64((((uint64_t)i2) << 16)
-                               | 0xffffffff0000ffffULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0xa:
-            tmp2 = tcg_const_i64(((uint64_t)i2) << 16);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lctl(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lctlg(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lctlg(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+static ExitStatus op_lra(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lra(o->out, cpu_env, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lpsw(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1, t2;
+
+    check_privileged(s);
+
+    t1 = tcg_temp_new_i64();
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(o->in2, o->in2, 4);
+    tcg_gen_qemu_ld32u(t2, o->in2, get_mem_index(s));
+    /* Convert the 32-bit PSW_MASK into the 64-bit PSW_MASK.  */
+    tcg_gen_shli_i64(t1, t1, 32);
+    gen_helper_load_psw(cpu_env, t1, t2);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    return EXIT_NORETURN;
+}
+
+static ExitStatus op_lpswe(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 t1, t2;
+
+    check_privileged(s);
+
+    t1 = tcg_temp_new_i64();
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld64(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(o->in2, o->in2, 8);
+    tcg_gen_qemu_ld64(t2, o->in2, get_mem_index(s));
+    gen_helper_load_psw(cpu_env, t1, t2);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    return EXIT_NORETURN;
+}
+#endif
+
+static ExitStatus op_lam(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_lam(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lm32(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_const_i64(4);
+
+    while (1) {
+        tcg_gen_qemu_ld32u(t, o->in2, get_mem_index(s));
+        store_reg32_i64(r1, t);
+        if (r1 == r3) {
             break;
-        default:
-            tcg_abort();
         }
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp, tmp, 16);
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        tcg_gen_andi_i32(tmp32, tmp32, 0xffff);
-        set_cc_nz_u32(s, tmp32);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x7: /* NILL     R1,I2     [RI] */
-    case 0xb: /* OILL     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp32 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x7:
-            tmp2 = tcg_const_i64(i2 | 0xffffffffffff0000ULL);
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0xb:
-            tmp2 = tcg_const_i64(i2);
-            tcg_gen_or_i64(tmp, tmp, tmp2);
+        tcg_gen_add_i64(o->in2, o->in2, t4);
+        r1 = (r1 + 1) & 15;
+    }
+
+    tcg_temp_free_i64(t);
+    tcg_temp_free_i64(t4);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lmh(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_const_i64(4);
+
+    while (1) {
+        tcg_gen_qemu_ld32u(t, o->in2, get_mem_index(s));
+        store_reg32h_i64(r1, t);
+        if (r1 == r3) {
             break;
-        default:
-            tcg_abort();
         }
-        store_reg(r1, tmp);
-        tcg_gen_trunc_i64_i32(tmp32, tmp);
-        tcg_gen_andi_i32(tmp32, tmp32, 0xffff);
-        set_cc_nz_u32(s, tmp32);        /* signedness should not matter here */
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xc: /* LLIHH     R1,I2     [RI] */
-        tmp = tcg_const_i64( ((uint64_t)i2) << 48 );
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xd: /* LLIHL     R1,I2     [RI] */
-        tmp = tcg_const_i64( ((uint64_t)i2) << 32 );
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xe: /* LLILH     R1,I2     [RI] */
-        tmp = tcg_const_i64( ((uint64_t)i2) << 16 );
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xf: /* LLILL     R1,I2     [RI] */
-        tmp = tcg_const_i64(i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    default:
-        LOG_DISAS("illegal a5 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 2);
-        return;
+        tcg_gen_add_i64(o->in2, o->in2, t4);
+        r1 = (r1 + 1) & 15;
     }
+
+    tcg_temp_free_i64(t);
+    tcg_temp_free_i64(t4);
+    return NO_EXIT;
 }
 
-static void disas_a7(CPUS390XState *env, DisasContext *s, int op, int r1,
-                     int i2)
+static ExitStatus op_lm64(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
-    int l1;
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t8 = tcg_const_i64(8);
 
-    LOG_DISAS("disas_a7: op 0x%x r1 %d i2 0x%x\n", op, r1, i2);
-    switch (op) {
-    case 0x0: /* TMLH or TMH     R1,I2     [RI] */
-    case 0x1: /* TMLL or TML     R1,I2     [RI] */
-    case 0x2: /* TMHH     R1,I2     [RI] */
-    case 0x3: /* TMHL     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tmp2 = tcg_const_i64((uint16_t)i2);
-        switch (op) {
-        case 0x0:
-            tcg_gen_shri_i64(tmp, tmp, 16);
-            break;
-        case 0x1:
-            break;
-        case 0x2:
-            tcg_gen_shri_i64(tmp, tmp, 48);
-            break;
-        case 0x3:
-            tcg_gen_shri_i64(tmp, tmp, 32);
+    while (1) {
+        tcg_gen_qemu_ld64(regs[r1], o->in2, get_mem_index(s));
+        if (r1 == r3) {
             break;
         }
-        tcg_gen_andi_i64(tmp, tmp, 0xffff);
-        cmp_64(s, tmp, tmp2, CC_OP_TM_64);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x4: /* brc m1, i2 */
-        gen_brc(r1, s, i2 * 2LL);
-        return;
-    case 0x5: /* BRAS     R1,I2     [RI] */
-        tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 4));
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        gen_goto_tb(s, 0, s->pc + i2 * 2LL);
-        s->is_jmp = DISAS_TB_JUMP;
-        break;
-    case 0x6: /* BRCT     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
-        store_reg32(r1, tmp32_1);
-        gen_update_cc_op(s);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp32_1, 0, l1);
-        gen_goto_tb(s, 0, s->pc + (i2 * 2LL));
-        gen_set_label(l1);
-        gen_goto_tb(s, 1, s->pc + 4);
-        s->is_jmp = DISAS_TB_JUMP;
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x7: /* BRCTG     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tcg_gen_subi_i64(tmp, tmp, 1);
-        store_reg(r1, tmp);
-        gen_update_cc_op(s);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i64(TCG_COND_EQ, tmp, 0, l1);
-        gen_goto_tb(s, 0, s->pc + (i2 * 2LL));
-        gen_set_label(l1);
-        gen_goto_tb(s, 1, s->pc + 4);
-        s->is_jmp = DISAS_TB_JUMP;
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x8: /* lhi r1, i2 */
-        tmp32_1 = tcg_const_i32(i2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x9: /* lghi r1, i2 */
-        tmp = tcg_const_i64(i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xa: /* AHI     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_const_i32(i2);
+        tcg_gen_add_i64(o->in2, o->in2, t8);
+        r1 = (r1 + 1) & 15;
+    }
 
-        if (i2 < 0) {
-            tcg_gen_subi_i32(tmp32_2, tmp32_1, -i2);
-        } else {
-            tcg_gen_add_i32(tmp32_2, tmp32_1, tmp32_3);
-        }
+    tcg_temp_free_i64(t8);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mov2(DisasContext *s, DisasOps *o)
+{
+    o->out = o->in2;
+    o->g_out = o->g_in2;
+    TCGV_UNUSED_I64(o->in2);
+    o->g_in2 = false;
+    return NO_EXIT;
+}
+
+static ExitStatus op_movx(DisasContext *s, DisasOps *o)
+{
+    o->out = o->in1;
+    o->out2 = o->in2;
+    o->g_out = o->g_in1;
+    o->g_out2 = o->g_in2;
+    TCGV_UNUSED_I64(o->in1);
+    TCGV_UNUSED_I64(o->in2);
+    o->g_in1 = o->g_in2 = false;
+    return NO_EXIT;
+}
 
-        store_reg32(r1, tmp32_2);
-        set_cc_add32(s, tmp32_1, tmp32_3, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+static ExitStatus op_mvc(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_mvc(cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvcl(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2));
+    potential_page_fault(s);
+    gen_helper_mvcl(cc_op, cpu_env, r1, r2);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvcle(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_mvcle(cc_op, cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_mvcp(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, l1);
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_mvcp(cc_op, cpu_env, regs[r1], o->addr1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvcs(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, l1);
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_mvcs(cc_op, cpu_env, regs[r1], o->addr1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+#endif
+
+static ExitStatus op_mvpg(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_mvpg(cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mvst(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_mvst(o->in1, cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return_low128(o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mul(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_mul_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mul128(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mul128(o->out, cpu_env, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_meeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_meeb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mdeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mdeb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mdb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mxb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mxdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_mxdb(o->out, cpu_env, o->out, o->out2, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_maeb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 r3 = load_freg32_i64(get_field(s->fields, r3));
+    gen_helper_maeb(o->out, cpu_env, o->in1, o->in2, r3);
+    tcg_temp_free_i64(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_madb(DisasContext *s, DisasOps *o)
+{
+    int r3 = get_field(s->fields, r3);
+    gen_helper_madb(o->out, cpu_env, o->in1, o->in2, fregs[r3]);
+    return NO_EXIT;
+}
+
+static ExitStatus op_mseb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 r3 = load_freg32_i64(get_field(s->fields, r3));
+    gen_helper_mseb(o->out, cpu_env, o->in1, o->in2, r3);
+    tcg_temp_free_i64(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_msdb(DisasContext *s, DisasOps *o)
+{
+    int r3 = get_field(s->fields, r3);
+    gen_helper_msdb(o->out, cpu_env, o->in1, o->in2, fregs[r3]);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabs(DisasContext *s, DisasOps *o)
+{
+    gen_helper_nabs_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabsf32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ori_i64(o->out, o->in2, 0x80000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabsf64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ori_i64(o->out, o->in2, 0x8000000000000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nabsf128(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ori_i64(o->out, o->in1, 0x8000000000000000ull);
+    tcg_gen_mov_i64(o->out2, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_nc(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_nc(cc_op, cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_neg(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_neg_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_negf32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xori_i64(o->out, o->in2, 0x80000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_negf64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xori_i64(o->out, o->in2, 0x8000000000000000ull);
+    return NO_EXIT;
+}
+
+static ExitStatus op_negf128(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xori_i64(o->out, o->in1, 0x8000000000000000ull);
+    tcg_gen_mov_i64(o->out2, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_oc(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_oc(cc_op, cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_or(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_or_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ori(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    uint64_t mask = ((1ull << size) - 1) << shift;
+
+    assert(!o->g_in2);
+    tcg_gen_shli_i64(o->in2, o->in2, shift);
+    tcg_gen_or_i64(o->out, o->in1, o->in2);
+
+    /* Produce the CC from only the bits manipulated.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_popcnt(DisasContext *s, DisasOps *o)
+{
+    gen_helper_popcnt(o->out, o->in2);
+    return NO_EXIT;
+}
+
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_ptlb(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_ptlb(cpu_env);
+    return NO_EXIT;
+}
+#endif
+
+static ExitStatus op_risbg(DisasContext *s, DisasOps *o)
+{
+    int i3 = get_field(s->fields, i3);
+    int i4 = get_field(s->fields, i4);
+    int i5 = get_field(s->fields, i5);
+    int do_zero = i4 & 0x80;
+    uint64_t mask, imask, pmask;
+    int pos, len, rot;
+
+    /* Adjust the arguments for the specific insn.  */
+    switch (s->fields->op2) {
+    case 0x55: /* risbg */
+        i3 &= 63;
+        i4 &= 63;
+        pmask = ~0;
+        break;
+    case 0x5d: /* risbhg */
+        i3 &= 31;
+        i4 &= 31;
+        pmask = 0xffffffff00000000ull;
+        break;
+    case 0x51: /* risblg */
+        i3 &= 31;
+        i4 &= 31;
+        pmask = 0x00000000ffffffffull;
         break;
-    case 0xb: /* aghi r1, i2 */
-        tmp = load_reg(r1);
-        tmp2 = tcg_const_i64(i2);
+    default:
+        abort();
+    }
 
-        if (i2 < 0) {
-            tcg_gen_subi_i64(regs[r1], tmp, -i2);
+    /* MASK is the set of bits to be inserted from R2.
+       Take care for I3/I4 wraparound.  */
+    mask = pmask >> i3;
+    if (i3 <= i4) {
+        mask ^= pmask >> i4 >> 1;
+    } else {
+        mask |= ~(pmask >> i4 >> 1);
+    }
+    mask &= pmask;
+
+    /* IMASK is the set of bits to be kept from R1.  In the case of the high/low
+       insns, we need to keep the other half of the register.  */
+    imask = ~mask | ~pmask;
+    if (do_zero) {
+        if (s->fields->op2 == 0x55) {
+            imask = 0;
         } else {
-            tcg_gen_add_i64(regs[r1], tmp, tmp2);
+            imask = ~pmask;
         }
-        set_cc_add64(s, tmp, tmp2, regs[r1]);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0xc: /* MHI     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_muli_i32(tmp32_1, tmp32_1, i2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xd: /* MGHI     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        tcg_gen_muli_i64(tmp, tmp, i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
+    }
+
+    /* In some cases we can implement this with deposit, which can be more
+       efficient on some hosts.  */
+    if (~mask == imask && i3 <= i4) {
+        if (s->fields->op2 == 0x5d) {
+            i3 += 32, i4 += 32;
+        }
+        /* Note that we rotate the bits to be inserted to the lsb, not to
+           the position as described in the PoO.  */
+        len = i4 - i3 + 1;
+        pos = 63 - i4;
+        rot = (i5 - pos) & 63;
+    } else {
+        pos = len = -1;
+        rot = i5 & 63;
+    }
+
+    /* Rotate the input as necessary.  */
+    tcg_gen_rotli_i64(o->in2, o->in2, rot);
+
+    /* Insert the selected bits into the output.  */
+    if (pos >= 0) {
+        tcg_gen_deposit_i64(o->out, o->out, o->in2, pos, len);
+    } else if (imask == 0) {
+        tcg_gen_andi_i64(o->out, o->in2, mask);
+    } else {
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
+        tcg_gen_andi_i64(o->out, o->out, imask);
+        tcg_gen_or_i64(o->out, o->out, o->in2);
+    }
+    return NO_EXIT;
+}
+
+static ExitStatus op_rosbg(DisasContext *s, DisasOps *o)
+{
+    int i3 = get_field(s->fields, i3);
+    int i4 = get_field(s->fields, i4);
+    int i5 = get_field(s->fields, i5);
+    uint64_t mask;
+
+    /* If this is a test-only form, arrange to discard the result.  */
+    if (i3 & 0x80) {
+        o->out = tcg_temp_new_i64();
+        o->g_out = false;
+    }
+
+    i3 &= 63;
+    i4 &= 63;
+    i5 &= 63;
+
+    /* MASK is the set of bits to be operated on from R2.
+       Take care for I3/I4 wraparound.  */
+    mask = ~0ull >> i3;
+    if (i3 <= i4) {
+        mask ^= ~0ull >> i4 >> 1;
+    } else {
+        mask |= ~(~0ull >> i4 >> 1);
+    }
+
+    /* Rotate the input as necessary.  */
+    tcg_gen_rotli_i64(o->in2, o->in2, i5);
+
+    /* Operate.  */
+    switch (s->fields->op2) {
+    case 0x55: /* AND */
+        tcg_gen_ori_i64(o->in2, o->in2, ~mask);
+        tcg_gen_and_i64(o->out, o->out, o->in2);
         break;
-    case 0xe: /* CHI     R1,I2     [RI] */
-        tmp32_1 = load_reg32(r1);
-        cmp_s32c(s, tmp32_1, i2);
-        tcg_temp_free_i32(tmp32_1);
+    case 0x56: /* OR */
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
+        tcg_gen_or_i64(o->out, o->out, o->in2);
         break;
-    case 0xf: /* CGHI     R1,I2     [RI] */
-        tmp = load_reg(r1);
-        cmp_s64c(s, tmp, i2);
-        tcg_temp_free_i64(tmp);
+    case 0x57: /* XOR */
+        tcg_gen_andi_i64(o->in2, o->in2, mask);
+        tcg_gen_xor_i64(o->out, o->out, o->in2);
         break;
     default:
-        LOG_DISAS("illegal a7 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 2);
-        return;
+        abort();
     }
+
+    /* Set the CC.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rev16(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_bswap16_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rev32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_bswap32_i64(o->out, o->in2);
+    return NO_EXIT;
 }
 
-static void disas_b2(CPUS390XState *env, DisasContext *s, int op,
-                     uint32_t insn)
+static ExitStatus op_rev64(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
-    int r1, r2;
-    int ilc = 2;
+    tcg_gen_bswap64_i64(o->out, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rll32(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 t1 = tcg_temp_new_i32();
+    TCGv_i32 t2 = tcg_temp_new_i32();
+    TCGv_i32 to = tcg_temp_new_i32();
+    tcg_gen_trunc_i64_i32(t1, o->in1);
+    tcg_gen_trunc_i64_i32(t2, o->in2);
+    tcg_gen_rotl_i32(to, t1, t2);
+    tcg_gen_extu_i32_i64(o->out, to);
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(t2);
+    tcg_temp_free_i32(to);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rll64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_rotl_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
 #ifndef CONFIG_USER_ONLY
-    int r3, d2, b2;
+static ExitStatus op_rrbe(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_rrbe(cc_op, cpu_env, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sacf(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_sacf(cpu_env, o->in2);
+    /* Addressing mode has changed, so end the block.  */
+    return EXIT_PC_STALE;
+}
 #endif
 
-    r1 = (insn >> 4) & 0xf;
-    r2 = insn & 0xf;
+static ExitStatus op_sar(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    tcg_gen_st32_i64(o->in2, cpu_env, offsetof(CPUS390XState, aregs[r1]));
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_b2: op 0x%x r1 %d r2 %d\n", op, r1, r2);
+static ExitStatus op_seb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_seb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
 
-    switch (op) {
-    case 0x22: /* IPM    R1               [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_op_calc_cc(s);
-        gen_helper_ipm(cpu_env, cc_op, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x41: /* CKSM    R1,R2     [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_cksm(cpu_env, tmp32_1, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        gen_op_movi_cc(s, 0);
-        break;
-    case 0x4e: /* SAR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, aregs[r1]));
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x4f: /* EAR     R1,R2     [RRE] */
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, aregs[r2]));
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x52: /* MSR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tcg_gen_mul_i32(tmp32_1, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x54: /* MVPG     R1,R2     [RRE] */
-        tmp = load_reg(0);
-        tmp2 = load_reg(r1);
-        tmp3 = load_reg(r2);
-        potential_page_fault(s);
-        gen_helper_mvpg(cpu_env, tmp, tmp2, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        /* XXX check CCO bit and set CC accordingly */
-        gen_op_movi_cc(s, 0);
-        break;
-    case 0x55: /* MVST     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = tcg_const_i32(r1);
-        tmp32_3 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_mvst(cpu_env, tmp32_1, tmp32_2, tmp32_3);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        gen_op_movi_cc(s, 1);
-        break;
-    case 0x5d: /* CLST     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = tcg_const_i32(r1);
-        tmp32_3 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_clst(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x5e: /* SRST     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = tcg_const_i32(r1);
-        tmp32_3 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_srst(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
+static ExitStatus op_sdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sdb(o->out, cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sxb(o->out, cpu_env, o->out, o->out2, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sqeb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sqeb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sqdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sqdb(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sqxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sqxb(o->out, cpu_env, o->in1, o->in2);
+    return_low128(o->out2);
+    return NO_EXIT;
+}
 
 #ifndef CONFIG_USER_ONLY
-    case 0x02: /* STIDP     D2(B2)     [S] */
-        /* Store CPU ID */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stidp(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x04: /* SCK       D2(B2)     [S] */
-        /* Set Clock */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_sck(cc_op, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x05: /* STCK     D2(B2)     [S] */
-        /* Store Clock */
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stck(cc_op, cpu_env, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x06: /* SCKC     D2(B2)     [S] */
-        /* Set Clock Comparator */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_sckc(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x07: /* STCKC    D2(B2)     [S] */
-        /* Store Clock Comparator */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stckc(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x08: /* SPT      D2(B2)     [S] */
-        /* Set CPU Timer */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_spt(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x09: /* STPT     D2(B2)     [S] */
-        /* Store CPU Timer */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stpt(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x0a: /* SPKA     D2(B2)     [S] */
-        /* Set PSW Key from Address */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_andi_i64(tmp2, psw_mask, ~PSW_MASK_KEY);
-        tcg_gen_shli_i64(tmp, tmp, PSW_SHIFT_KEY - 4);
-        tcg_gen_or_i64(psw_mask, tmp2, tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x0d: /* PTLB                [S] */
-        /* Purge TLB */
-        check_privileged(env, s, ilc);
-        gen_helper_ptlb(cpu_env);
-        break;
-    case 0x10: /* SPX      D2(B2)     [S] */
-        /* Set Prefix Register */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_spx(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x11: /* STPX     D2(B2)     [S] */
-        /* Store Prefix */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_ld_i64(tmp2, cpu_env, offsetof(CPUS390XState, psa));
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x12: /* STAP     D2(B2)     [S] */
-        /* Store CPU Address */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, cpu_num));
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x21: /* IPTE     R1,R2      [RRE] */
-        /* Invalidate PTE */
-        check_privileged(env, s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        gen_helper_ipte(cpu_env, tmp, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x29: /* ISKE     R1,R2      [RRE] */
-        /* Insert Storage Key Extended */
-        check_privileged(env, s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp = load_reg(r2);
-        tmp2 = tcg_temp_new_i64();
-        gen_helper_iske(tmp2, cpu_env, tmp);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x2a: /* RRBE     R1,R2      [RRE] */
-        /* Set Storage Key Extended */
-        check_privileged(env, s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = load_reg32(r1);
-        tmp = load_reg(r2);
-        gen_helper_rrbe(cc_op, cpu_env, tmp32_1, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x2b: /* SSKE     R1,R2      [RRE] */
-        /* Set Storage Key Extended */
-        check_privileged(env, s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = load_reg32(r1);
-        tmp = load_reg(r2);
-        gen_helper_sske(cpu_env, tmp32_1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x34: /* STCH ? */
-        /* Store Subchannel */
-        check_privileged(env, s, ilc);
-        gen_op_movi_cc(s, 3);
-        break;
-    case 0x46: /* STURA    R1,R2      [RRE] */
-        /* Store Using Real Address */
-        check_privileged(env, s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = load_reg32(r1);
-        tmp = load_reg(r2);
-        potential_page_fault(s);
-        gen_helper_stura(cpu_env, tmp, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x50: /* CSP      R1,R2      [RRE] */
-        /* Compare And Swap And Purge */
-        check_privileged(env, s, ilc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        gen_helper_csp(cc_op, cpu_env, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x5f: /* CHSC ? */
-        /* Channel Subsystem Call */
-        check_privileged(env, s, ilc);
-        gen_op_movi_cc(s, 3);
-        break;
-    case 0x78: /* STCKE    D2(B2)     [S] */
-        /* Store Clock Extended */
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_stcke(cc_op, cpu_env, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x79: /* SACF    D2(B2)     [S] */
-        /* Store Clock Extended */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        potential_page_fault(s);
-        gen_helper_sacf(cpu_env, tmp);
-        tcg_temp_free_i64(tmp);
-        /* addressing mode has changed, so end the block */
-        s->pc += ilc * 2;
-        update_psw_addr(s);
-        s->is_jmp = DISAS_EXCP;
-        break;
-    case 0x7d: /* STSI     D2,(B2)     [S] */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(0);
-        tmp32_2 = load_reg32(1);
-        potential_page_fault(s);
-        gen_helper_stsi(cc_op, cpu_env, tmp, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x9d: /* LFPC      D2(B2)   [S] */
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xb1: /* STFL     D2(B2)     [S] */
-        /* Store Facility List (CPU features) at 200 */
-        check_privileged(env, s, ilc);
-        tmp2 = tcg_const_i64(0xc0000000);
-        tmp = tcg_const_i64(200);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xb2: /* LPSWE    D2(B2)     [S] */
-        /* Load PSW Extended */
-        check_privileged(env, s, ilc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, tmp, get_mem_index(s));
-        tcg_gen_addi_i64(tmp, tmp, 8);
-        tcg_gen_qemu_ld64(tmp3, tmp, get_mem_index(s));
-        gen_helper_load_psw(cpu_env, tmp2, tmp3);
-        /* we need to keep cc_op intact */
-        s->is_jmp = DISAS_JUMP;
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x20: /* SERVC     R1,R2     [RRE] */
-        /* SCLP Service call (PV hypercall) */
-        check_privileged(env, s, ilc);
-        potential_page_fault(s);
-        tmp32_1 = load_reg32(r2);
-        tmp = load_reg(r1);
-        gen_helper_servc(cc_op, cpu_env, tmp32_1, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
+static ExitStatus op_servc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_servc(cc_op, cpu_env, o->in2, o->in1);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sigp(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_sigp(cc_op, cpu_env, o->in2, r1, o->in1);
+    tcg_temp_free_i32(r1);
+    return NO_EXIT;
+}
 #endif
-    default:
-        LOG_DISAS("illegal b2 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, ilc);
-        break;
+
+static ExitStatus op_soc(DisasContext *s, DisasOps *o)
+{
+    DisasCompare c;
+    TCGv_i64 a;
+    int lab, r1;
+
+    disas_jcc(s, &c, get_field(s->fields, m3));
+
+    lab = gen_new_label();
+    if (c.is_64) {
+        tcg_gen_brcond_i64(c.cond, c.u.s64.a, c.u.s64.b, lab);
+    } else {
+        tcg_gen_brcond_i32(c.cond, c.u.s32.a, c.u.s32.b, lab);
     }
+    free_compare(&c);
+
+    r1 = get_field(s->fields, r1);
+    a = get_address(s, 0, get_field(s->fields, b2), get_field(s->fields, d2));
+    if (s->insn->data) {
+        tcg_gen_qemu_st64(regs[r1], a, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_st32(regs[r1], a, get_mem_index(s));
+    }
+    tcg_temp_free_i64(a);
+
+    gen_set_label(lab);
+    return NO_EXIT;
 }
 
-static void disas_b3(CPUS390XState *env, DisasContext *s, int op, int m3,
-                     int r1, int r2)
+static ExitStatus op_sla(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
-    LOG_DISAS("disas_b3: op 0x%x m3 0x%x r1 %d r2 %d\n", op, m3, r1, r2);
-#define FP_HELPER(i) \
-    tmp32_1 = tcg_const_i32(r1); \
-    tmp32_2 = tcg_const_i32(r2); \
-    gen_helper_ ## i(cpu_env, tmp32_1, tmp32_2); \
-    tcg_temp_free_i32(tmp32_1); \
-    tcg_temp_free_i32(tmp32_2);
-
-#define FP_HELPER_CC(i) \
-    tmp32_1 = tcg_const_i32(r1); \
-    tmp32_2 = tcg_const_i32(r2); \
-    gen_helper_ ## i(cc_op, cpu_env, tmp32_1, tmp32_2); \
-    set_cc_static(s); \
-    tcg_temp_free_i32(tmp32_1); \
-    tcg_temp_free_i32(tmp32_2);
+    uint64_t sign = 1ull << s->insn->data;
+    enum cc_op cco = s->insn->data == 31 ? CC_OP_SLA_32 : CC_OP_SLA_64;
+    gen_op_update2_cc_i64(s, cco, o->in1, o->in2);
+    tcg_gen_shl_i64(o->out, o->in1, o->in2);
+    /* The arithmetic left shift is curious in that it does not affect
+       the sign bit.  Copy that over from the source unchanged.  */
+    tcg_gen_andi_i64(o->out, o->out, ~sign);
+    tcg_gen_andi_i64(o->in1, o->in1, sign);
+    tcg_gen_or_i64(o->out, o->out, o->in1);
+    return NO_EXIT;
+}
 
-    switch (op) {
-    case 0x0: /* LPEBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lpebr);
-        break;
-    case 0x2: /* LTEBR       R1,R2             [RRE] */
-        FP_HELPER_CC(ltebr);
-        break;
-    case 0x3: /* LCEBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lcebr);
-        break;
-    case 0x4: /* LDEBR       R1,R2             [RRE] */
-        FP_HELPER(ldebr);
-        break;
-    case 0x5: /* LXDBR       R1,R2             [RRE] */
-        FP_HELPER(lxdbr);
-        break;
-    case 0x9: /* CEBR        R1,R2             [RRE] */
-        FP_HELPER_CC(cebr);
-        break;
-    case 0xa: /* AEBR        R1,R2             [RRE] */
-        FP_HELPER_CC(aebr);
-        break;
-    case 0xb: /* SEBR        R1,R2             [RRE] */
-        FP_HELPER_CC(sebr);
-        break;
-    case 0xd: /* DEBR        R1,R2             [RRE] */
-        FP_HELPER(debr);
-        break;
-    case 0x10: /* LPDBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lpdbr);
-        break;
-    case 0x12: /* LTDBR       R1,R2             [RRE] */
-        FP_HELPER_CC(ltdbr);
-        break;
-    case 0x13: /* LCDBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lcdbr);
-        break;
-    case 0x15: /* SQBDR       R1,R2             [RRE] */
-        FP_HELPER(sqdbr);
-        break;
-    case 0x17: /* MEEBR       R1,R2             [RRE] */
-        FP_HELPER(meebr);
-        break;
-    case 0x19: /* CDBR        R1,R2             [RRE] */
-        FP_HELPER_CC(cdbr);
-        break;
-    case 0x1a: /* ADBR        R1,R2             [RRE] */
-        FP_HELPER_CC(adbr);
-        break;
-    case 0x1b: /* SDBR        R1,R2             [RRE] */
-        FP_HELPER_CC(sdbr);
-        break;
-    case 0x1c: /* MDBR        R1,R2             [RRE] */
-        FP_HELPER(mdbr);
-        break;
-    case 0x1d: /* DDBR        R1,R2             [RRE] */
-        FP_HELPER(ddbr);
-        break;
-    case 0xe: /* MAEBR  R1,R3,R2 [RRF] */
-    case 0x1e: /* MADBR R1,R3,R2 [RRF] */
-    case 0x1f: /* MSDBR R1,R3,R2 [RRF] */
-        /* for RRF insns, m3 is R1, r1 is R3, and r2 is R2 */
-        tmp32_1 = tcg_const_i32(m3);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(r1);
-        switch (op) {
-        case 0xe:
-            gen_helper_maebr(cpu_env, tmp32_1, tmp32_3, tmp32_2);
-            break;
-        case 0x1e:
-            gen_helper_madbr(cpu_env, tmp32_1, tmp32_3, tmp32_2);
-            break;
-        case 0x1f:
-            gen_helper_msdbr(cpu_env, tmp32_1, tmp32_3, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x40: /* LPXBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lpxbr);
-        break;
-    case 0x42: /* LTXBR       R1,R2             [RRE] */
-        FP_HELPER_CC(ltxbr);
-        break;
-    case 0x43: /* LCXBR       R1,R2             [RRE] */
-        FP_HELPER_CC(lcxbr);
-        break;
-    case 0x44: /* LEDBR       R1,R2             [RRE] */
-        FP_HELPER(ledbr);
-        break;
-    case 0x45: /* LDXBR       R1,R2             [RRE] */
-        FP_HELPER(ldxbr);
-        break;
-    case 0x46: /* LEXBR       R1,R2             [RRE] */
-        FP_HELPER(lexbr);
-        break;
-    case 0x49: /* CXBR        R1,R2             [RRE] */
-        FP_HELPER_CC(cxbr);
-        break;
-    case 0x4a: /* AXBR        R1,R2             [RRE] */
-        FP_HELPER_CC(axbr);
-        break;
-    case 0x4b: /* SXBR        R1,R2             [RRE] */
-        FP_HELPER_CC(sxbr);
-        break;
-    case 0x4c: /* MXBR        R1,R2             [RRE] */
-        FP_HELPER(mxbr);
-        break;
-    case 0x4d: /* DXBR        R1,R2             [RRE] */
-        FP_HELPER(dxbr);
-        break;
-    case 0x65: /* LXR         R1,R2             [RRE] */
-        tmp = load_freg(r2);
-        store_freg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        tmp = load_freg(r2 + 2);
-        store_freg(r1 + 2, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x74: /* LZER        R1                [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzer(cpu_env, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x75: /* LZDR        R1                [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzdr(cpu_env, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x76: /* LZXR        R1                [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzxr(cpu_env, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x84: /* SFPC        R1                [RRE] */
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x8c: /* EFPC        R1                [RRE] */
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x94: /* CEFBR       R1,R2             [RRE] */
-    case 0x95: /* CDFBR       R1,R2             [RRE] */
-    case 0x96: /* CXFBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = load_reg32(r2);
-        switch (op) {
-        case 0x94:
-            gen_helper_cefbr(cpu_env, tmp32_1, tmp32_2);
-            break;
-        case 0x95:
-            gen_helper_cdfbr(cpu_env, tmp32_1, tmp32_2);
-            break;
-        case 0x96:
-            gen_helper_cxfbr(cpu_env, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x98: /* CFEBR       R1,R2             [RRE] */
-    case 0x99: /* CFDBR              R1,R2             [RRE] */
-    case 0x9a: /* CFXBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        switch (op) {
-        case 0x98:
-            gen_helper_cfebr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x99:
-            gen_helper_cfdbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x9a:
-            gen_helper_cfxbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xa4: /* CEGBR       R1,R2             [RRE] */
-    case 0xa5: /* CDGBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp = load_reg(r2);
-        switch (op) {
-        case 0xa4:
-            gen_helper_cegbr(cpu_env, tmp32_1, tmp);
-            break;
-        case 0xa5:
-            gen_helper_cdgbr(cpu_env, tmp32_1, tmp);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xa6: /* CXGBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp = load_reg(r2);
-        gen_helper_cxgbr(cpu_env, tmp32_1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xa8: /* CGEBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgebr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xa9: /* CGDBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgdbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+static ExitStatus op_sll(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_shl_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sra(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_sar_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_srl(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_shr_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sfpc(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sfpc(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sfas(DisasContext *s, DisasOps *o)
+{
+    gen_helper_sfas(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_srnm(DisasContext *s, DisasOps *o)
+{
+    int b2 = get_field(s->fields, b2);
+    int d2 = get_field(s->fields, d2);
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    int mask, pos, len;
+
+    switch (s->fields->op2) {
+    case 0x99: /* SRNM */
+        pos = 0, len = 2;
         break;
-    case 0xaa: /* CGXBR       R1,R2             [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgxbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
+    case 0xb8: /* SRNMB */
+        pos = 0, len = 3;
         break;
+    case 0xb9: /* SRNMT */
+        pos = 4, len = 3;
     default:
-        LOG_DISAS("illegal b3 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 2);
-        break;
+        tcg_abort();
     }
+    mask = (1 << len) - 1;
 
-#undef FP_HELPER_CC
-#undef FP_HELPER
+    /* Insert the value into the appropriate field of the FPC.  */
+    if (b2 == 0) {
+        tcg_gen_movi_i64(t1, d2 & mask);
+    } else {
+        tcg_gen_addi_i64(t1, regs[b2], d2);
+        tcg_gen_andi_i64(t1, t1, mask);
+    }
+    tcg_gen_ld32u_i64(t2, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_gen_deposit_i64(t2, t2, t1, pos, len);
+    tcg_temp_free_i64(t1);
+
+    /* Then install the new FPC to set the rounding mode in fpu_status.  */
+    gen_helper_sfpc(cpu_env, t2);
+    tcg_temp_free_i64(t2);
+    return NO_EXIT;
 }
 
-static void disas_b9(CPUS390XState *env, DisasContext *s, int op, int r1,
-                     int r2)
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_spka(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
+    check_privileged(s);
+    tcg_gen_shri_i64(o->in2, o->in2, 4);
+    tcg_gen_deposit_i64(psw_mask, psw_mask, o->in2, PSW_SHIFT_KEY - 4, 4);
+    return NO_EXIT;
+}
 
-    LOG_DISAS("disas_b9: op 0x%x r1 %d r2 %d\n", op, r1, r2);
-    switch (op) {
-    case 0x0: /* LPGR     R1,R2     [RRE] */
-    case 0x1: /* LNGR     R1,R2     [RRE] */
-    case 0x2: /* LTGR R1,R2 [RRE] */
-    case 0x3: /* LCGR     R1,R2     [RRE] */
-    case 0x10: /* LPGFR R1,R2 [RRE] */
-    case 0x11: /* LNFGR     R1,R2     [RRE] */
-    case 0x12: /* LTGFR R1,R2 [RRE] */
-    case 0x13: /* LCGFR    R1,R2     [RRE] */
-        if (op & 0x10) {
-            tmp = load_reg32_i64(r2);
-        } else {
-            tmp = load_reg(r2);
-        }
-        switch (op & 0xf) {
-        case 0x0: /* LP?GR */
-            set_cc_abs64(s, tmp);
-            gen_helper_abs_i64(tmp, tmp);
-            store_reg(r1, tmp);
-            break;
-        case 0x1: /* LN?GR */
-            set_cc_nabs64(s, tmp);
-            gen_helper_nabs_i64(tmp, tmp);
-            store_reg(r1, tmp);
-            break;
-        case 0x2: /* LT?GR */
-            if (r1 != r2) {
-                store_reg(r1, tmp);
-            }
-            set_cc_s64(s, tmp);
-            break;
-        case 0x3: /* LC?GR */
-            tcg_gen_neg_i64(regs[r1], tmp);
-            set_cc_comp64(s, regs[r1]);
-            break;
-        }
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x4: /* LGR R1,R2 [RRE] */
-        store_reg(r1, regs[r2]);
-        break;
-    case 0x6: /* LGBR R1,R2 [RRE] */
-        tmp2 = load_reg(r2);
-        tcg_gen_ext8s_i64(tmp2, tmp2);
-        store_reg(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x8: /* AGR     R1,R2     [RRE] */
-    case 0xa: /* ALGR     R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_add_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x8:
-            set_cc_add64(s, tmp, tmp2, tmp3);
-            break;
-        case 0xa:
-            set_cc_addu64(s, tmp, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+static ExitStatus op_sske(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_sske(cpu_env, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ssm(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    tcg_gen_deposit_i64(psw_mask, psw_mask, o->in2, 56, 8);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stap(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    /* ??? Surely cpu address != cpu number.  In any case the previous
+       version of this stored more than the required half-word, so it
+       is unlikely this has ever been tested.  */
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, cpu_num));
+    return NO_EXIT;
+}
+
+static ExitStatus op_stck(DisasContext *s, DisasOps *o)
+{
+    gen_helper_stck(o->out, cpu_env);
+    /* ??? We don't implement clock states.  */
+    gen_op_movi_cc(s, 0);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stcke(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 c1 = tcg_temp_new_i64();
+    TCGv_i64 c2 = tcg_temp_new_i64();
+    gen_helper_stck(c1, cpu_env);
+    /* Shift the 64-bit value into its place as a zero-extended
+       104-bit value.  Note that "bit positions 64-103 are always
+       non-zero so that they compare differently to STCK"; we set
+       the least significant bit to 1.  */
+    tcg_gen_shli_i64(c2, c1, 56);
+    tcg_gen_shri_i64(c1, c1, 8);
+    tcg_gen_ori_i64(c2, c2, 0x10000);
+    tcg_gen_qemu_st64(c1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(o->in2, o->in2, 8);
+    tcg_gen_qemu_st64(c2, o->in2, get_mem_index(s));
+    tcg_temp_free_i64(c1);
+    tcg_temp_free_i64(c2);
+    /* ??? We don't implement clock states.  */
+    gen_op_movi_cc(s, 0);
+    return NO_EXIT;
+}
+
+static ExitStatus op_sckc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_sckc(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stckc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_stckc(o->out, cpu_env);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stctg(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stctg(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stctl(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stctl(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stidp(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, cpu_num));
+    return NO_EXIT;
+}
+
+static ExitStatus op_spt(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_spt(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stfl(DisasContext *s, DisasOps *o)
+{
+    TCGv_i64 f, a;
+    /* We really ought to have more complete indication of facilities
+       that we implement.  Address this when STFLE is implemented.  */
+    check_privileged(s);
+    f = tcg_const_i64(0xc0000000);
+    a = tcg_const_i64(200);
+    tcg_gen_qemu_st32(f, a, get_mem_index(s));
+    tcg_temp_free_i64(f);
+    tcg_temp_free_i64(a);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stpt(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_stpt(o->out, cpu_env);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stsi(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stsi(cc_op, cpu_env, o->in2, regs[0], regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_spx(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    gen_helper_spx(cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_subchannel(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    /* Not operational.  */
+    gen_op_movi_cc(s, 3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stpx(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    tcg_gen_ld_i64(o->out, cpu_env, offsetof(CPUS390XState, psa));
+    tcg_gen_andi_i64(o->out, o->out, 0x7fffe000);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stnosm(DisasContext *s, DisasOps *o)
+{
+    uint64_t i2 = get_field(s->fields, i2);
+    TCGv_i64 t;
+
+    check_privileged(s);
+
+    /* It is important to do what the instruction name says: STORE THEN.
+       If we let the output hook perform the store then if we fault and
+       restart, we'll have the wrong SYSTEM MASK in place.  */
+    t = tcg_temp_new_i64();
+    tcg_gen_shri_i64(t, psw_mask, 56);
+    tcg_gen_qemu_st8(t, o->addr1, get_mem_index(s));
+    tcg_temp_free_i64(t);
+
+    if (s->fields->op == 0xac) {
+        tcg_gen_andi_i64(psw_mask, psw_mask,
+                         (i2 << 56) | 0x00ffffffffffffffull);
+    } else {
+        tcg_gen_ori_i64(psw_mask, psw_mask, i2 << 56);
+    }
+    return NO_EXIT;
+}
+
+static ExitStatus op_stura(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stura(cpu_env, o->in2, o->in1);
+    return NO_EXIT;
+}
+#endif
+
+static ExitStatus op_st8(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st8(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_st16(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st16(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_st32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st32(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_st64(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_qemu_st64(o->in1, o->in2, get_mem_index(s));
+    return NO_EXIT;
+}
+
+static ExitStatus op_stam(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    potential_page_fault(s);
+    gen_helper_stam(cpu_env, r1, o->in2, r3);
+    tcg_temp_free_i32(r1);
+    tcg_temp_free_i32(r3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stcm(DisasContext *s, DisasOps *o)
+{
+    int m3 = get_field(s->fields, m3);
+    int pos, base = s->insn->data;
+    TCGv_i64 tmp = tcg_temp_new_i64();
+
+    pos = base + ctz32(m3) * 8;
+    switch (m3) {
+    case 0xf:
+        /* Effectively a 32-bit store.  */
+        tcg_gen_shri_i64(tmp, o->in1, pos);
+        tcg_gen_qemu_st32(tmp, o->in2, get_mem_index(s));
         break;
-    case 0x9: /* SGR     R1,R2     [RRE] */
-    case 0xb: /* SLGR     R1,R2     [RRE] */
-    case 0x1b: /* SLGFR     R1,R2     [RRE] */
-    case 0x19: /* SGFR     R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        switch (op) {
-        case 0x1b:
-            tmp32_1 = load_reg32(r2);
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-            tcg_temp_free_i32(tmp32_1);
-            break;
-        case 0x19:
-            tmp32_1 = load_reg32(r2);
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-            tcg_temp_free_i32(tmp32_1);
-            break;
-        default:
-            tmp2 = load_reg(r2);
-            break;
-        }
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_sub_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        switch (op) {
-        case 0x9:
-        case 0x19:
-            set_cc_sub64(s, tmp, tmp2, tmp3);
-            break;
-        case 0xb:
-        case 0x1b:
-            set_cc_subu64(s, tmp, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+
+    case 0xc:
+    case 0x6:
+    case 0x3:
+        /* Effectively a 16-bit store.  */
+        tcg_gen_shri_i64(tmp, o->in1, pos);
+        tcg_gen_qemu_st16(tmp, o->in2, get_mem_index(s));
         break;
-    case 0xc: /* MSGR      R1,R2     [RRE] */
-    case 0x1c: /* MSGFR      R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        if (op == 0x1c) {
-            tcg_gen_ext32s_i64(tmp2, tmp2);
-        }
-        tcg_gen_mul_i64(tmp, tmp, tmp2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
+
+    case 0x8:
+    case 0x4:
+    case 0x2:
+    case 0x1:
+        /* Effectively an 8-bit store.  */
+        tcg_gen_shri_i64(tmp, o->in1, pos);
+        tcg_gen_qemu_st8(tmp, o->in2, get_mem_index(s));
         break;
-    case 0xd: /* DSGR      R1,R2     [RRE] */
-    case 0x1d: /* DSGFR      R1,R2     [RRE] */
-        tmp = load_reg(r1 + 1);
-        if (op == 0xd) {
-            tmp2 = load_reg(r2);
-        } else {
-            tmp32_1 = load_reg32(r2);
-            tmp2 = tcg_temp_new_i64();
-            tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-            tcg_temp_free_i32(tmp32_1);
+
+    default:
+        /* This is going to be a sequence of shifts and stores.  */
+        pos = base + 32 - 8;
+        while (m3) {
+            if (m3 & 0x8) {
+                tcg_gen_shri_i64(tmp, o->in1, pos);
+                tcg_gen_qemu_st8(tmp, o->in2, get_mem_index(s));
+                tcg_gen_addi_i64(o->in2, o->in2, 1);
+            }
+            m3 = (m3 << 1) & 0xf;
+            pos -= 8;
         }
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_div_i64(tmp3, tmp, tmp2);
-        store_reg(r1 + 1, tmp3);
-        tcg_gen_rem_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
         break;
-    case 0x14: /* LGFR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp = tcg_temp_new_i64();
-        tcg_gen_ext_i32_i64(tmp, tmp32_1);
-        store_reg(r1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x16: /* LLGFR      R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp = tcg_temp_new_i64();
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        store_reg(r1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x17: /* LLGTR      R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp = tcg_temp_new_i64();
-        tcg_gen_andi_i32(tmp32_1, tmp32_1, 0x7fffffffUL);
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        store_reg(r1, tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x18: /* AGFR     R1,R2     [RRE] */
-    case 0x1a: /* ALGFR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp2 = tcg_temp_new_i64();
-        if (op == 0x18) {
-            tcg_gen_ext_i32_i64(tmp2, tmp32_1);
-        } else {
-            tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        }
-        tcg_temp_free_i32(tmp32_1);
-        tmp = load_reg(r1);
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_add_i64(tmp3, tmp, tmp2);
-        store_reg(r1, tmp3);
-        if (op == 0x18) {
-            set_cc_add64(s, tmp, tmp2, tmp3);
+    }
+    tcg_temp_free_i64(tmp);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stm(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    int size = s->insn->data;
+    TCGv_i64 tsize = tcg_const_i64(size);
+
+    while (1) {
+        if (size == 8) {
+            tcg_gen_qemu_st64(regs[r1], o->in2, get_mem_index(s));
         } else {
-            set_cc_addu64(s, tmp, tmp2, tmp3);
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x0f: /* LRVGR    R1,R2     [RRE] */
-        tcg_gen_bswap64_i64(regs[r1], regs[r2]);
-        break;
-    case 0x1f: /* LRVR     R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_bswap32_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x20: /* CGR      R1,R2     [RRE] */
-    case 0x30: /* CGFR     R1,R2     [RRE] */
-        tmp2 = load_reg(r2);
-        if (op == 0x30) {
-            tcg_gen_ext32s_i64(tmp2, tmp2);
+            tcg_gen_qemu_st32(regs[r1], o->in2, get_mem_index(s));
         }
-        tmp = load_reg(r1);
-        cmp_s64(s, tmp, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x21: /* CLGR     R1,R2     [RRE] */
-    case 0x31: /* CLGFR    R1,R2     [RRE] */
-        tmp2 = load_reg(r2);
-        if (op == 0x31) {
-            tcg_gen_ext32u_i64(tmp2, tmp2);
-        }
-        tmp = load_reg(r1);
-        cmp_u64(s, tmp, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x26: /* LBR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_ext8s_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x27: /* LHR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_ext16s_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x80: /* NGR R1,R2 [RRE] */
-    case 0x81: /* OGR R1,R2 [RRE] */
-    case 0x82: /* XGR R1,R2 [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        switch (op) {
-        case 0x80:
-            tcg_gen_and_i64(tmp, tmp, tmp2);
-            break;
-        case 0x81:
-            tcg_gen_or_i64(tmp, tmp, tmp2);
-            break;
-        case 0x82:
-            tcg_gen_xor_i64(tmp, tmp, tmp2);
+        if (r1 == r3) {
             break;
-        default:
-            tcg_abort();
         }
-        store_reg(r1, tmp);
-        set_cc_nz_u64(s, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x83: /* FLOGR R1,R2 [RRE] */
-        tmp = load_reg(r2);
-        tmp32_1 = tcg_const_i32(r1);
-        gen_helper_flogr(cc_op, cpu_env, tmp32_1, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x84: /* LLGCR R1,R2 [RRE] */
-        tmp = load_reg(r2);
-        tcg_gen_andi_i64(tmp, tmp, 0xff);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x85: /* LLGHR R1,R2 [RRE] */
-        tmp = load_reg(r2);
-        tcg_gen_andi_i64(tmp, tmp, 0xffff);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x87: /* DLGR      R1,R2     [RRE] */
-        tmp32_1 = tcg_const_i32(r1);
-        tmp = load_reg(r2);
-        gen_helper_dlg(cpu_env, tmp32_1, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x88: /* ALCGR     R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp3 = tcg_temp_new_i64();
-        gen_op_calc_cc(s);
-        tcg_gen_extu_i32_i64(tmp3, cc_op);
-        tcg_gen_shri_i64(tmp3, tmp3, 1);
-        tcg_gen_andi_i64(tmp3, tmp3, 1);
-        tcg_gen_add_i64(tmp3, tmp2, tmp3);
-        tcg_gen_add_i64(tmp3, tmp, tmp3);
-        store_reg(r1, tmp3);
-        set_cc_addu64(s, tmp, tmp2, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x89: /* SLBGR   R1,R2     [RRE] */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp32_1 = tcg_const_i32(r1);
-        gen_op_calc_cc(s);
-        gen_helper_slbg(cc_op, cpu_env, cc_op, tmp32_1, tmp, tmp2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x94: /* LLCR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_andi_i32(tmp32_1, tmp32_1, 0xff);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x95: /* LLHR R1,R2 [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_andi_i32(tmp32_1, tmp32_1, 0xffff);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x96: /* MLR     R1,R2     [RRE] */
-        /* reg(r1, r1+1) = reg(r1+1) * reg(r2) */
-        tmp2 = load_reg(r2);
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp2, tmp2);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x97: /* DLR     R1,R2     [RRE] */
-        /* reg(r1) = reg(r1, r1+1) % reg(r2) */
-        /* reg(r1+1) = reg(r1, r1+1) / reg(r2) */
-        tmp = load_reg(r1);
-        tmp2 = load_reg(r2);
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32u_i64(tmp2, tmp2);
-        tcg_gen_ext32u_i64(tmp3, tmp3);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp3);
-
-        tcg_gen_rem_i64(tmp3, tmp, tmp2);
-        tcg_gen_div_i64(tmp, tmp, tmp2);
-        store_reg32_i64((r1 + 1) & 15, tmp);
-        store_reg32_i64(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x98: /* ALCR    R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tmp32_3 = tcg_temp_new_i32();
-        /* XXX possible optimization point */
-        gen_op_calc_cc(s);
-        gen_helper_addc_u32(tmp32_3, cc_op, tmp32_1, tmp32_2);
-        set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-        store_reg32(r1, tmp32_3);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x99: /* SLBR    R1,R2     [RRE] */
-        tmp32_1 = load_reg32(r2);
-        tmp32_2 = tcg_const_i32(r1);
-        gen_op_calc_cc(s);
-        gen_helper_slb(cc_op, cpu_env, cc_op, tmp32_2, tmp32_1);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    default:
-        LOG_DISAS("illegal b9 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 2);
-        break;
+        tcg_gen_add_i64(o->in2, o->in2, tsize);
+        r1 = (r1 + 1) & 15;
     }
+
+    tcg_temp_free_i64(tsize);
+    return NO_EXIT;
 }
 
-static void disas_c0(CPUS390XState *env, DisasContext *s, int op, int r1, int i2)
+static ExitStatus op_stmh(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp;
-    TCGv_i32 tmp32_1, tmp32_2;
-    uint64_t target = s->pc + i2 * 2LL;
-    int l1;
-
-    LOG_DISAS("disas_c0: op 0x%x r1 %d i2 %d\n", op, r1, i2);
+    int r1 = get_field(s->fields, r1);
+    int r3 = get_field(s->fields, r3);
+    TCGv_i64 t = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_const_i64(4);
+    TCGv_i64 t32 = tcg_const_i64(32);
 
-    switch (op) {
-    case 0: /* larl r1, i2 */
-        tmp = tcg_const_i64(target);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x1: /* LGFI R1,I2 [RIL] */
-        tmp = tcg_const_i64((int64_t)i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x4: /* BRCL     M1,I2     [RIL] */
-        /* m1 & (1 << (3 - cc)) */
-        tmp32_1 = tcg_const_i32(3);
-        tmp32_2 = tcg_const_i32(1);
-        gen_op_calc_cc(s);
-        tcg_gen_sub_i32(tmp32_1, tmp32_1, cc_op);
-        tcg_gen_shl_i32(tmp32_2, tmp32_2, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tmp32_1 = tcg_const_i32(r1); /* m1 == r1 */
-        tcg_gen_and_i32(tmp32_1, tmp32_1, tmp32_2);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp32_1, 0, l1);
-        gen_goto_tb(s, 0, target);
-        gen_set_label(l1);
-        gen_goto_tb(s, 1, s->pc + 6);
-        s->is_jmp = DISAS_TB_JUMP;
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x5: /* brasl r1, i2 */
-        tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 6));
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        gen_goto_tb(s, 0, target);
-        s->is_jmp = DISAS_TB_JUMP;
-        break;
-    case 0x7: /* XILF R1,I2 [RIL] */
-    case 0xb: /* NILF R1,I2 [RIL] */
-    case 0xd: /* OILF R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        switch (op) {
-        case 0x7:
-            tcg_gen_xori_i32(tmp32_1, tmp32_1, (uint32_t)i2);
-            break;
-        case 0xb:
-            tcg_gen_andi_i32(tmp32_1, tmp32_1, (uint32_t)i2);
+    while (1) {
+        tcg_gen_shl_i64(t, regs[r1], t32);
+        tcg_gen_qemu_st32(t, o->in2, get_mem_index(s));
+        if (r1 == r3) {
             break;
-        case 0xd:
-            tcg_gen_ori_i32(tmp32_1, tmp32_1, (uint32_t)i2);
-            break;
-        default:
-            tcg_abort();
         }
-        store_reg32(r1, tmp32_1);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x9: /* IILF R1,I2 [RIL] */
-        tmp32_1 = tcg_const_i32((uint32_t)i2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xa: /* NIHF R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_andi_i64(tmp, tmp, (((uint64_t)((uint32_t)i2)) << 32)
-                                   | 0xffffffffULL);
-        store_reg(r1, tmp);
-        tcg_gen_shri_i64(tmp, tmp, 32);
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xe: /* LLIHF R1,I2 [RIL] */
-        tmp = tcg_const_i64(((uint64_t)(uint32_t)i2) << 32);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xf: /* LLILF R1,I2 [RIL] */
-        tmp = tcg_const_i64((uint32_t)i2);
-        store_reg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    default:
-        LOG_DISAS("illegal c0 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 3);
-        break;
+        tcg_gen_add_i64(o->in2, o->in2, t4);
+        r1 = (r1 + 1) & 15;
     }
+
+    tcg_temp_free_i64(t);
+    tcg_temp_free_i64(t4);
+    tcg_temp_free_i64(t32);
+    return NO_EXIT;
 }
 
-static void disas_c2(CPUS390XState *env, DisasContext *s, int op, int r1,
-                     int i2)
+static ExitStatus op_srst(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
+    potential_page_fault(s);
+    gen_helper_srst(o->in1, cpu_env, regs[0], o->in1, o->in2);
+    set_cc_static(s);
+    return_low128(o->in2);
+    return NO_EXIT;
+}
 
-    switch (op) {
-    case 0x4: /* SLGFI R1,I2 [RIL] */
-    case 0xa: /* ALGFI R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        tmp2 = tcg_const_i64((uint64_t)(uint32_t)i2);
-        tmp3 = tcg_temp_new_i64();
-        switch (op) {
-        case 0x4:
-            tcg_gen_sub_i64(tmp3, tmp, tmp2);
-            set_cc_subu64(s, tmp, tmp2, tmp3);
-            break;
-        case 0xa:
-            tcg_gen_add_i64(tmp3, tmp, tmp2);
-            set_cc_addu64(s, tmp, tmp2, tmp3);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg(r1, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x5: /* SLFI R1,I2 [RIL] */
-    case 0xb: /* ALFI R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_const_i32(i2);
-        tmp32_3 = tcg_temp_new_i32();
-        switch (op) {
-        case 0x5:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_subu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0xb:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_3);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xc: /* CGFI R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        cmp_s64c(s, tmp, (int64_t)i2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xe: /* CLGFI R1,I2 [RIL] */
-        tmp = load_reg(r1);
-        cmp_u64c(s, tmp, (uint64_t)(uint32_t)i2);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xd: /* CFI R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        cmp_s32c(s, tmp32_1, i2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xf: /* CLFI R1,I2 [RIL] */
-        tmp32_1 = load_reg32(r1);
-        cmp_u32c(s, tmp32_1, i2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    default:
-        LOG_DISAS("illegal c2 operation 0x%x\n", op);
-        gen_illegal_opcode(env, s, 3);
-        break;
-    }
+static ExitStatus op_sub(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_sub_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
 }
 
-static void gen_and_or_xor_i32(int opc, TCGv_i32 tmp, TCGv_i32 tmp2)
+static ExitStatus op_subb(DisasContext *s, DisasOps *o)
 {
-    switch (opc & 0xf) {
-    case 0x4:
-        tcg_gen_and_i32(tmp, tmp, tmp2);
-        break;
-    case 0x6:
-        tcg_gen_or_i32(tmp, tmp, tmp2);
-        break;
-    case 0x7:
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        break;
-    default:
-        tcg_abort();
+    DisasCompare cmp;
+    TCGv_i64 borrow;
+
+    tcg_gen_sub_i64(o->out, o->in1, o->in2);
+
+    /* The !borrow flag is the msb of CC.  Since we want the inverse of
+       that, we ask for a comparison of CC=0 | CC=1 -> mask of 8 | 4.  */
+    disas_jcc(s, &cmp, 8 | 4);
+    borrow = tcg_temp_new_i64();
+    if (cmp.is_64) {
+        tcg_gen_setcond_i64(cmp.cond, borrow, cmp.u.s64.a, cmp.u.s64.b);
+    } else {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_setcond_i32(cmp.cond, t, cmp.u.s32.a, cmp.u.s32.b);
+        tcg_gen_extu_i32_i64(borrow, t);
+        tcg_temp_free_i32(t);
     }
+    free_compare(&cmp);
+
+    tcg_gen_sub_i64(o->out, o->out, borrow);
+    tcg_temp_free_i64(borrow);
+    return NO_EXIT;
 }
 
-static void disas_s390_insn(CPUS390XState *env, DisasContext *s)
+static ExitStatus op_svc(DisasContext *s, DisasOps *o)
 {
-    TCGv_i64 tmp, tmp2, tmp3, tmp4;
-    TCGv_i32 tmp32_1, tmp32_2, tmp32_3, tmp32_4;
-    unsigned char opc;
-    uint64_t insn;
-    int op, r1, r2, r3, d1, d2, x2, b1, b2, i, i2, r1b;
-    TCGv_i32 vl;
-    int ilc;
-    int l1;
+    TCGv_i32 t;
 
-    opc = cpu_ldub_code(env, s->pc);
-    LOG_DISAS("opc 0x%x\n", opc);
+    update_psw_addr(s);
+    update_cc_op(s);
 
-    ilc = get_ilc(opc);
+    t = tcg_const_i32(get_field(s->fields, i1) & 0xff);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, int_svc_code));
+    tcg_temp_free_i32(t);
+
+    t = tcg_const_i32(s->next_pc - s->pc);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, int_svc_ilen));
+    tcg_temp_free_i32(t);
+
+    gen_exception(EXCP_SVC);
+    return EXIT_NORETURN;
+}
+
+static ExitStatus op_tceb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_tceb(cc_op, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_tcdb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_tcdb(cc_op, o->in1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_tcxb(DisasContext *s, DisasOps *o)
+{
+    gen_helper_tcxb(cc_op, o->out, o->out2, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 
-    switch (opc) {
 #ifndef CONFIG_USER_ONLY
-    case 0x01: /* SAM */
-        insn = ld_code2(env, s->pc);
-        /* set addressing mode, but we only do 64bit anyways */
-        break;
+static ExitStatus op_tprot(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_tprot(cc_op, o->addr1, o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
 #endif
-    case 0x6: /* BCTR     R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
-        store_reg32(r1, tmp32_1);
-
-        if (r2) {
-            gen_update_cc_op(s);
-            l1 = gen_new_label();
-            tcg_gen_brcondi_i32(TCG_COND_NE, tmp32_1, 0, l1);
-
-            /* not taking the branch, jump to after the instruction */
-            gen_goto_tb(s, 0, s->pc + 2);
-            gen_set_label(l1);
-
-            /* take the branch, move R2 into psw.addr */
-            tmp32_1 = load_reg32(r2);
-            tmp = tcg_temp_new_i64();
-            tcg_gen_extu_i32_i64(tmp, tmp32_1);
-            tcg_gen_mov_i64(psw_addr, tmp);
-            s->is_jmp = DISAS_JUMP;
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i64(tmp);
-        }
-        break;
-    case 0x7: /* BCR    M1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        if (r2) {
-            tmp = load_reg(r2);
-            gen_bcr(s, r1, tmp, s->pc);
-            tcg_temp_free_i64(tmp);
-            s->is_jmp = DISAS_TB_JUMP;
-        } else {
-            /* XXX: "serialization and checkpoint-synchronization function"? */
-        }
-        break;
-    case 0xa: /* SVC    I         [RR] */
-        insn = ld_code2(env, s->pc);
-        debug_insn(insn);
-        i = insn & 0xff;
-        update_psw_addr(s);
-        gen_op_calc_cc(s);
-        tmp32_1 = tcg_const_i32(i);
-        tmp32_2 = tcg_const_i32(ilc * 2);
-        tmp32_3 = tcg_const_i32(EXCP_SVC);
-        tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, int_svc_code));
-        tcg_gen_st_i32(tmp32_2, cpu_env, offsetof(CPUS390XState, int_svc_ilc));
-        gen_helper_exception(cpu_env, tmp32_3);
-        s->is_jmp = DISAS_EXCP;
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0xd: /* BASR   R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 2));
-        store_reg(r1, tmp);
-        if (r2) {
-            tmp2 = load_reg(r2);
-            tcg_gen_mov_i64(psw_addr, tmp2);
-            tcg_temp_free_i64(tmp2);
-            s->is_jmp = DISAS_JUMP;
-        }
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0xe: /* MVCL   R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r2);
-        potential_page_fault(s);
-        gen_helper_mvcl(cc_op, cpu_env, tmp32_1, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x10: /* LPR    R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        set_cc_abs32(s, tmp32_1);
-        gen_helper_abs_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x11: /* LNR    R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        set_cc_nabs32(s, tmp32_1);
-        gen_helper_nabs_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x12: /* LTR    R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        if (r1 != r2) {
-            store_reg32(r1, tmp32_1);
-        }
-        set_cc_s32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x13: /* LCR    R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        tcg_gen_neg_i32(tmp32_1, tmp32_1);
-        store_reg32(r1, tmp32_1);
-        set_cc_comp32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x14: /* NR     R1,R2     [RR] */
-    case 0x16: /* OR     R1,R2     [RR] */
-    case 0x17: /* XR     R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_2 = load_reg32(r2);
-        tmp32_1 = load_reg32(r1);
-        gen_and_or_xor_i32(opc, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x18: /* LR     R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x15: /* CLR    R1,R2     [RR] */
-    case 0x19: /* CR     R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        if (opc == 0x15) {
-            cmp_u32(s, tmp32_1, tmp32_2);
-        } else {
-            cmp_s32(s, tmp32_1, tmp32_2);
+
+static ExitStatus op_tr(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_tr(cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_unpk(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_unpk(cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    return NO_EXIT;
+}
+
+static ExitStatus op_xc(DisasContext *s, DisasOps *o)
+{
+    int d1 = get_field(s->fields, d1);
+    int d2 = get_field(s->fields, d2);
+    int b1 = get_field(s->fields, b1);
+    int b2 = get_field(s->fields, b2);
+    int l = get_field(s->fields, l1);
+    TCGv_i32 t32;
+
+    o->addr1 = get_address(s, 0, b1, d1);
+
+    /* If the addresses are identical, this is a store/memset of zero.  */
+    if (b1 == b2 && d1 == d2 && (l + 1) <= 32) {
+        o->in2 = tcg_const_i64(0);
+
+        l++;
+        while (l >= 8) {
+            tcg_gen_qemu_st64(o->in2, o->addr1, get_mem_index(s));
+            l -= 8;
+            if (l > 0) {
+                tcg_gen_addi_i64(o->addr1, o->addr1, 8);
+            }
         }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x1a: /* AR     R1,R2     [RR] */
-    case 0x1e: /* ALR    R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_3);
-        if (opc == 0x1a) {
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-        } else {
-            set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
+        if (l >= 4) {
+            tcg_gen_qemu_st32(o->in2, o->addr1, get_mem_index(s));
+            l -= 4;
+            if (l > 0) {
+                tcg_gen_addi_i64(o->addr1, o->addr1, 4);
+            }
         }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x1b: /* SR     R1,R2     [RR] */
-    case 0x1f: /* SLR    R1,R2     [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r2);
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_3);
-        if (opc == 0x1b) {
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-        } else {
-            set_cc_subu32(s, tmp32_1, tmp32_2, tmp32_3);
+        if (l >= 2) {
+            tcg_gen_qemu_st16(o->in2, o->addr1, get_mem_index(s));
+            l -= 2;
+            if (l > 0) {
+                tcg_gen_addi_i64(o->addr1, o->addr1, 2);
+            }
         }
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x1c: /* MR     R1,R2     [RR] */
-        /* reg(r1, r1+1) = reg(r1+1) * reg(r2) */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp2 = load_reg(r2);
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32s_i64(tmp2, tmp2);
-        tcg_gen_ext32s_i64(tmp3, tmp3);
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x1d: /* DR     R1,R2               [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r1 + 1);
-        tmp32_3 = load_reg32(r2);
-
-        tmp = tcg_temp_new_i64(); /* dividend */
-        tmp2 = tcg_temp_new_i64(); /* divisor */
-        tmp3 = tcg_temp_new_i64();
-
-        /* dividend is r(r1 << 32) | r(r1 + 1) */
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_2);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp2);
-
-        /* divisor is r(r2) */
-        tcg_gen_ext_i32_i64(tmp2, tmp32_3);
-
-        tcg_gen_div_i64(tmp3, tmp, tmp2);
-        tcg_gen_rem_i64(tmp, tmp, tmp2);
-
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp3);
-
-        store_reg32(r1, tmp32_1); /* remainder */
-        store_reg32(r1 + 1, tmp32_2); /* quotient */
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x28: /* LDR    R1,R2               [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp = load_freg(r2);
-        store_freg(r1, tmp);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x38: /* LER    R1,R2               [RR] */
-        insn = ld_code2(env, s->pc);
-        decode_rr(s, insn, &r1, &r2);
-        tmp32_1 = load_freg32(r2);
-        store_freg32(r1, tmp32_1);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x40: /* STH    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tcg_gen_qemu_st16(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x41:        /* la */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        store_reg(r1, tmp); /* FIXME: 31/24-bit addressing */
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x42: /* STC    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x43: /* IC     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        store_reg8(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x44: /* EX     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tmp3 = tcg_const_i64(s->pc + 4);
-        update_psw_addr(s);
-        gen_op_calc_cc(s);
-        gen_helper_ex(cc_op, cpu_env, cc_op, tmp2, tmp, tmp3);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x46: /* BCT    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tcg_temp_free_i64(tmp);
-
-        tmp32_1 = load_reg32(r1);
-        tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
-        store_reg32(r1, tmp32_1);
-
-        gen_update_cc_op(s);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp32_1, 0, l1);
-
-        /* not taking the branch, jump to after the instruction */
-        gen_goto_tb(s, 0, s->pc + 4);
-        gen_set_label(l1);
-
-        /* take the branch, move R2 into psw.addr */
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tcg_gen_mov_i64(psw_addr, tmp);
-        s->is_jmp = DISAS_JUMP;
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp);
-        break;
-    case 0x47: /* BC     M1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        gen_bcr(s, r1, tmp, s->pc + 4);
-        tcg_temp_free_i64(tmp);
-        s->is_jmp = DISAS_TB_JUMP;
-        break;
-    case 0x48: /* LH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x49: /* CH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        cmp_s32(s, tmp32_1, tmp32_2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x4a: /* AH     R1,D2(X2,B2)     [RX] */
-    case 0x4b: /* SH     R1,D2(X2,B2)     [RX] */
-    case 0x4c: /* MH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-
-        tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        switch (opc) {
-        case 0x4a:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x4b:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x4c:
-            tcg_gen_mul_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
+        if (l) {
+            tcg_gen_qemu_st8(o->in2, o->addr1, get_mem_index(s));
         }
-        store_reg32(r1, tmp32_3);
+        gen_op_movi_cc(s, 0);
+        return NO_EXIT;
+    }
 
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x4d: /* BAS    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_const_i64(pc_to_link_info(s, s->pc + 4));
-        store_reg(r1, tmp2);
-        tcg_gen_mov_i64(psw_addr, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        s->is_jmp = DISAS_JUMP;
-        break;
-    case 0x4e: /* CVD    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp32_1, regs[r1]);
-        gen_helper_cvd(tmp2, tmp32_1);
-        tcg_gen_qemu_st64(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x50: /* st r1, d2(x2, b2) */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_reg(r1);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x55: /* CL     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tmp32_2 = load_reg32(r1);
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        cmp_u32(s, tmp32_2, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x54: /* N      R1,D2(X2,B2)     [RX] */
-    case 0x56: /* O      R1,D2(X2,B2)     [RX] */
-    case 0x57: /* X      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        gen_and_or_xor_i32(opc, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        set_cc_nz_u32(s, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x58: /* l r1, d2(x2, b2) */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0x59: /* C      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tmp32_2 = load_reg32(r1);
-        tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        cmp_s32(s, tmp32_2, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x5a: /* A      R1,D2(X2,B2)     [RX] */
-    case 0x5b: /* S      R1,D2(X2,B2)     [RX] */
-    case 0x5e: /* AL     R1,D2(X2,B2)     [RX] */
-    case 0x5f: /* SL     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tmp32_3 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32s(tmp, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp);
-        switch (opc) {
-        case 0x5a:
-        case 0x5e:
-            tcg_gen_add_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        case 0x5b:
-        case 0x5f:
-            tcg_gen_sub_i32(tmp32_3, tmp32_1, tmp32_2);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_3);
-        switch (opc) {
-        case 0x5a:
-            set_cc_add32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5e:
-            set_cc_addu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5b:
-            set_cc_sub32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        case 0x5f:
-            set_cc_subu32(s, tmp32_1, tmp32_2, tmp32_3);
-            break;
-        default:
-            tcg_abort();
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i32(tmp32_3);
-        break;
-    case 0x5c: /* M      R1,D2(X2,B2)        [RX] */
-        /* reg(r1, r1+1) = reg(r1+1) * *(s32*)addr */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
-        tmp3 = load_reg((r1 + 1) & 15);
-        tcg_gen_ext32s_i64(tmp2, tmp2);
-        tcg_gen_ext32s_i64(tmp3, tmp3);
-        tcg_gen_mul_i64(tmp2, tmp2, tmp3);
-        store_reg32_i64((r1 + 1) & 15, tmp2);
-        tcg_gen_shri_i64(tmp2, tmp2, 32);
-        store_reg32_i64(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x5d: /* D      R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp3 = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r1 + 1);
-
-        tmp = tcg_temp_new_i64();
-        tmp2 = tcg_temp_new_i64();
-
-        /* dividend is r(r1 << 32) | r(r1 + 1) */
-        tcg_gen_extu_i32_i64(tmp, tmp32_1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_2);
-        tcg_gen_shli_i64(tmp, tmp, 32);
-        tcg_gen_or_i64(tmp, tmp, tmp2);
-
-        /* divisor is in memory */
-        tcg_gen_qemu_ld32s(tmp2, tmp3, get_mem_index(s));
-
-        /* XXX divisor == 0 -> FixP divide exception */
-
-        tcg_gen_div_i64(tmp3, tmp, tmp2);
-        tcg_gen_rem_i64(tmp, tmp, tmp2);
-
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp3);
-
-        store_reg32(r1, tmp32_1); /* remainder */
-        store_reg32(r1 + 1, tmp32_2); /* quotient */
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-    case 0x60: /* STD    R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = load_freg(r1);
-        tcg_gen_qemu_st64(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x68: /* LD    R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld64(tmp2, tmp, get_mem_index(s));
-        store_freg(r1, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
+    /* But in general we'll defer to a helper.  */
+    o->in2 = get_address(s, 0, b2, d2);
+    t32 = tcg_const_i32(l);
+    potential_page_fault(s);
+    gen_helper_xc(cc_op, cpu_env, t32, o->addr1, o->in2);
+    tcg_temp_free_i32(t32);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_xor(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_xor_i64(o->out, o->in1, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_xori(DisasContext *s, DisasOps *o)
+{
+    int shift = s->insn->data & 0xff;
+    int size = s->insn->data >> 8;
+    uint64_t mask = ((1ull << size) - 1) << shift;
+
+    assert(!o->g_in2);
+    tcg_gen_shli_i64(o->in2, o->in2, shift);
+    tcg_gen_xor_i64(o->out, o->in1, o->in2);
+
+    /* Produce the CC from only the bits manipulated.  */
+    tcg_gen_andi_i64(cc_dst, o->out, mask);
+    set_cc_nz_u64(s, cc_dst);
+    return NO_EXIT;
+}
+
+static ExitStatus op_zero(DisasContext *s, DisasOps *o)
+{
+    o->out = tcg_const_i64(0);
+    return NO_EXIT;
+}
+
+static ExitStatus op_zero2(DisasContext *s, DisasOps *o)
+{
+    o->out = tcg_const_i64(0);
+    o->out2 = o->out;
+    o->g_out2 = true;
+    return NO_EXIT;
+}
+
+/* ====================================================================== */
+/* The "Cc OUTput" generators.  Given the generated output (and in some cases
+   the original inputs), update the various cc data structures in order to
+   be able to compute the new condition code.  */
+
+static void cout_abs32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_ABS_32, o->out);
+}
+
+static void cout_abs64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_ABS_64, o->out);
+}
+
+static void cout_adds32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADD_32, o->in1, o->in2, o->out);
+}
+
+static void cout_adds64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADD_64, o->in1, o->in2, o->out);
+}
+
+static void cout_addu32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDU_32, o->in1, o->in2, o->out);
+}
+
+static void cout_addu64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDU_64, o->in1, o->in2, o->out);
+}
+
+static void cout_addc32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDC_32, o->in1, o->in2, o->out);
+}
+
+static void cout_addc64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_ADDC_64, o->in1, o->in2, o->out);
+}
+
+static void cout_cmps32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTGT_32, o->in1, o->in2);
+}
+
+static void cout_cmps64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTGT_64, o->in1, o->in2);
+}
+
+static void cout_cmpu32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTUGTU_32, o->in1, o->in2);
+}
+
+static void cout_cmpu64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_LTUGTU_64, o->in1, o->in2);
+}
+
+static void cout_f32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F32, o->out);
+}
+
+static void cout_f64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NZ_F64, o->out);
+}
+
+static void cout_f128(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_NZ_F128, o->out, o->out2);
+}
+
+static void cout_nabs32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NABS_32, o->out);
+}
+
+static void cout_nabs64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NABS_64, o->out);
+}
+
+static void cout_neg32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_COMP_32, o->out);
+}
+
+static void cout_neg64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_COMP_64, o->out);
+}
+
+static void cout_nz32(DisasContext *s, DisasOps *o)
+{
+    tcg_gen_ext32u_i64(cc_dst, o->out);
+    gen_op_update1_cc_i64(s, CC_OP_NZ, cc_dst);
+}
+
+static void cout_nz64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_NZ, o->out);
+}
+
+static void cout_s32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_LTGT0_32, o->out);
+}
+
+static void cout_s64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_LTGT0_64, o->out);
+}
+
+static void cout_subs32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUB_32, o->in1, o->in2, o->out);
+}
+
+static void cout_subs64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUB_64, o->in1, o->in2, o->out);
+}
+
+static void cout_subu32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBU_32, o->in1, o->in2, o->out);
+}
+
+static void cout_subu64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBU_64, o->in1, o->in2, o->out);
+}
+
+static void cout_subb32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBB_32, o->in1, o->in2, o->out);
+}
+
+static void cout_subb64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update3_cc_i64(s, CC_OP_SUBB_64, o->in1, o->in2, o->out);
+}
+
+static void cout_tm32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_TM_32, o->in1, o->in2);
+}
+
+static void cout_tm64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_TM_64, o->in1, o->in2);
+}
+
+/* ====================================================================== */
+/* The "PREPeration" generators.  These initialize the DisasOps.OUT fields
+   with the TCG register to which we will write.  Used in combination with
+   the "wout" generators, in some cases we need a new temporary, and in
+   some cases we can write to a TCG global.  */
+
+static void prep_new(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = tcg_temp_new_i64();
+}
+#define SPEC_prep_new 0
+
+static void prep_new_P(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = tcg_temp_new_i64();
+    o->out2 = tcg_temp_new_i64();
+}
+#define SPEC_prep_new_P 0
+
+static void prep_r1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = regs[get_field(f, r1)];
+    o->g_out = true;
+}
+#define SPEC_prep_r1 0
+
+static void prep_r1_P(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->out = regs[r1];
+    o->out2 = regs[r1 + 1];
+    o->g_out = o->g_out2 = true;
+}
+#define SPEC_prep_r1_P SPEC_r1_even
+
+static void prep_f1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->out = fregs[get_field(f, r1)];
+    o->g_out = true;
+}
+#define SPEC_prep_f1 0
+
+static void prep_x1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->out = fregs[r1];
+    o->out2 = fregs[r1 + 2];
+    o->g_out = o->g_out2 = true;
+}
+#define SPEC_prep_x1 SPEC_r1_f128
+
+/* ====================================================================== */
+/* The "Write OUTput" generators.  These generally perform some non-trivial
+   copy of data to TCG globals, or to main memory.  The trivial cases are
+   generally handled by having a "prep" generator install the TCG global
+   as the destination of the operation.  */
+
+static void wout_r1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_reg(get_field(f, r1), o->out);
+}
+#define SPEC_wout_r1 0
+
+static void wout_r1_8(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    tcg_gen_deposit_i64(regs[r1], regs[r1], o->out, 0, 8);
+}
+#define SPEC_wout_r1_8 0
+
+static void wout_r1_16(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    tcg_gen_deposit_i64(regs[r1], regs[r1], o->out, 0, 16);
+}
+#define SPEC_wout_r1_16 0
+
+static void wout_r1_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_reg32_i64(get_field(f, r1), o->out);
+}
+#define SPEC_wout_r1_32 0
+
+static void wout_r1_P32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    store_reg32_i64(r1, o->out);
+    store_reg32_i64(r1 + 1, o->out2);
+}
+#define SPEC_wout_r1_P32 SPEC_r1_even
+
+static void wout_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    store_reg32_i64(r1 + 1, o->out);
+    tcg_gen_shri_i64(o->out, o->out, 32);
+    store_reg32_i64(r1, o->out);
+}
+#define SPEC_wout_r1_D32 SPEC_r1_even
+
+static void wout_e1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_freg32_i64(get_field(f, r1), o->out);
+}
+#define SPEC_wout_e1 0
+
+static void wout_f1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_freg(get_field(f, r1), o->out);
+}
+#define SPEC_wout_f1 0
+
+static void wout_x1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int f1 = get_field(s->fields, r1);
+    store_freg(f1, o->out);
+    store_freg(f1 + 2, o->out2);
+}
+#define SPEC_wout_x1 SPEC_r1_f128
+
+static void wout_cond_r1r2_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    if (get_field(f, r1) != get_field(f, r2)) {
+        store_reg32_i64(get_field(f, r1), o->out);
+    }
+}
+#define SPEC_wout_cond_r1r2_32 0
+
+static void wout_cond_e1e2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    if (get_field(f, r1) != get_field(f, r2)) {
+        store_freg32_i64(get_field(f, r1), o->out);
+    }
+}
+#define SPEC_wout_cond_e1e2 0
+
+static void wout_m1_8(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st8(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_8 0
+
+static void wout_m1_16(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st16(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_16 0
+
+static void wout_m1_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st32(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_32 0
+
+static void wout_m1_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st64(o->out, o->addr1, get_mem_index(s));
+}
+#define SPEC_wout_m1_64 0
+
+static void wout_m2_32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    tcg_gen_qemu_st32(o->out, o->in2, get_mem_index(s));
+}
+#define SPEC_wout_m2_32 0
+
+/* ====================================================================== */
+/* The "INput 1" generators.  These load the first operand to an insn.  */
+
+static void in1_r1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r1));
+}
+#define SPEC_in1_r1 0
+
+static void in1_r1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = regs[get_field(f, r1)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_r1_o 0
+
+static void in1_r1_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in1, regs[get_field(f, r1)]);
+}
+#define SPEC_in1_r1_32s 0
+
+static void in1_r1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in1, regs[get_field(f, r1)]);
+}
+#define SPEC_in1_r1_32u 0
+
+static void in1_r1_sr32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(o->in1, regs[get_field(f, r1)], 32);
+}
+#define SPEC_in1_r1_sr32 0
+
+static void in1_r1p1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r1) + 1);
+}
+#define SPEC_in1_r1p1 SPEC_r1_even
+
+static void in1_r1p1_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in1, regs[get_field(f, r1) + 1]);
+}
+#define SPEC_in1_r1p1_32s SPEC_r1_even
+
+static void in1_r1p1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in1, regs[get_field(f, r1) + 1]);
+}
+#define SPEC_in1_r1p1_32u SPEC_r1_even
+
+static void in1_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in1, regs[r1 + 1], regs[r1]);
+}
+#define SPEC_in1_r1_D32 SPEC_r1_even
+
+static void in1_r2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r2));
+}
+#define SPEC_in1_r2 0
+
+static void in1_r3(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_reg(get_field(f, r3));
+}
+#define SPEC_in1_r3 0
+
+static void in1_r3_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = regs[get_field(f, r3)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_r3_o 0
+
+static void in1_r3_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in1, regs[get_field(f, r3)]);
+}
+#define SPEC_in1_r3_32s 0
+
+static void in1_r3_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in1, regs[get_field(f, r3)]);
+}
+#define SPEC_in1_r3_32u 0
+
+static void in1_r3_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r3 = get_field(f, r3);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in1, regs[r3 + 1], regs[r3]);
+}
+#define SPEC_in1_r3_D32 SPEC_r3_even
+
+static void in1_e1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = load_freg32_i64(get_field(f, r1));
+}
+#define SPEC_in1_e1 0
+
+static void in1_f1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = fregs[get_field(f, r1)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_f1_o 0
+
+static void in1_x1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->out = fregs[r1];
+    o->out2 = fregs[r1 + 2];
+    o->g_out = o->g_out2 = true;
+}
+#define SPEC_in1_x1_o SPEC_r1_f128
+
+static void in1_f3_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = fregs[get_field(f, r3)];
+    o->g_in1 = true;
+}
+#define SPEC_in1_f3_o 0
+
+static void in1_la1(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->addr1 = get_address(s, 0, get_field(f, b1), get_field(f, d1));
+}
+#define SPEC_in1_la1 0
+
+static void in1_la2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int x2 = have_field(f, x2) ? get_field(f, x2) : 0;
+    o->addr1 = get_address(s, x2, get_field(f, b2), get_field(f, d2));
+}
+#define SPEC_in1_la2 0
+
+static void in1_m1_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld8u(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_8u 0
+
+static void in1_m1_16s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld16s(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_16s 0
+
+static void in1_m1_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld16u(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_16u 0
+
+static void in1_m1_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32s(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_32s 0
+
+static void in1_m1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32u(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_32u 0
+
+static void in1_m1_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in1_la1(s, f, o);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld64(o->in1, o->addr1, get_mem_index(s));
+}
+#define SPEC_in1_m1_64 0
+
+/* ====================================================================== */
+/* The "INput 2" generators.  These load the second operand to an insn.  */
+
+static void in2_r1_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = regs[get_field(f, r1)];
+    o->g_in2 = true;
+}
+#define SPEC_in2_r1_o 0
+
+static void in2_r1_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext16u_i64(o->in2, regs[get_field(f, r1)]);
+}
+#define SPEC_in2_r1_16u 0
+
+static void in2_r1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in2, regs[get_field(f, r1)]);
+}
+#define SPEC_in2_r1_32u 0
+
+static void in2_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in2, regs[r1 + 1], regs[r1]);
+}
+#define SPEC_in2_r1_D32 SPEC_r1_even
+
+static void in2_r2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = load_reg(get_field(f, r2));
+}
+#define SPEC_in2_r2 0
+
+static void in2_r2_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = regs[get_field(f, r2)];
+    o->g_in2 = true;
+}
+#define SPEC_in2_r2_o 0
+
+static void in2_r2_nz(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r2 = get_field(f, r2);
+    if (r2 != 0) {
+        o->in2 = load_reg(r2);
+    }
+}
+#define SPEC_in2_r2_nz 0
+
+static void in2_r2_8s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext8s_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_8s 0
+
+static void in2_r2_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext8u_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_8u 0
+
+static void in2_r2_16s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext16s_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_16s 0
+
+static void in2_r2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext16u_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_16u 0
+
+static void in2_r3(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = load_reg(get_field(f, r3));
+}
+#define SPEC_in2_r3 0
+
+static void in2_r2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext32s_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_32s 0
+
+static void in2_r2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_ext32u_i64(o->in2, regs[get_field(f, r2)]);
+}
+#define SPEC_in2_r2_32u 0
+
+static void in2_e2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = load_freg32_i64(get_field(f, r2));
+}
+#define SPEC_in2_e2 0
+
+static void in2_f2_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = fregs[get_field(f, r2)];
+    o->g_in2 = true;
+}
+#define SPEC_in2_f2_o 0
+
+static void in2_x2_o(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r2 = get_field(f, r2);
+    o->in1 = fregs[r2];
+    o->in2 = fregs[r2 + 2];
+    o->g_in1 = o->g_in2 = true;
+}
+#define SPEC_in2_x2_o SPEC_r2_f128
+
+static void in2_ra2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = get_address(s, 0, get_field(f, r2), 0);
+}
+#define SPEC_in2_ra2 0
+
+static void in2_a2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int x2 = have_field(f, x2) ? get_field(f, x2) : 0;
+    o->in2 = get_address(s, x2, get_field(f, b2), get_field(f, d2));
+}
+#define SPEC_in2_a2 0
+
+static void in2_ri2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64(s->pc + (int64_t)get_field(f, i2) * 2);
+}
+#define SPEC_in2_ri2 0
+
+static void in2_sh32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    help_l2_shift(s, f, o, 31);
+}
+#define SPEC_in2_sh32 0
+
+static void in2_sh64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    help_l2_shift(s, f, o, 63);
+}
+#define SPEC_in2_sh64 0
+
+static void in2_m2_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld8u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_8u 0
+
+static void in2_m2_16s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld16s(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_16s 0
+
+static void in2_m2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld16u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_16u 0
+
+static void in2_m2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld32s(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_32s 0
+
+static void in2_m2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld32u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_32u 0
+
+static void in2_m2_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_a2(s, f, o);
+    tcg_gen_qemu_ld64(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_m2_64 0
+
+static void in2_mri2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld16u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_16u 0
+
+static void in2_mri2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld32s(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_32s 0
+
+static void in2_mri2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld32u(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_32u 0
+
+static void in2_mri2_64(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    in2_ri2(s, f, o);
+    tcg_gen_qemu_ld64(o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_64 0
+
+static void in2_i2(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64(get_field(f, i2));
+}
+#define SPEC_in2_i2 0
+
+static void in2_i2_8u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64((uint8_t)get_field(f, i2));
+}
+#define SPEC_in2_i2_8u 0
+
+static void in2_i2_16u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64((uint16_t)get_field(f, i2));
+}
+#define SPEC_in2_i2_16u 0
+
+static void in2_i2_32u(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64((uint32_t)get_field(f, i2));
+}
+#define SPEC_in2_i2_32u 0
+
+static void in2_i2_16u_shl(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    uint64_t i2 = (uint16_t)get_field(f, i2);
+    o->in2 = tcg_const_i64(i2 << s->insn->data);
+}
+#define SPEC_in2_i2_16u_shl 0
+
+static void in2_i2_32u_shl(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    uint64_t i2 = (uint32_t)get_field(f, i2);
+    o->in2 = tcg_const_i64(i2 << s->insn->data);
+}
+#define SPEC_in2_i2_32u_shl 0
+
+/* ====================================================================== */
+
+/* Find opc within the table of insns.  This is formulated as a switch
+   statement so that (1) we get compile-time notice of cut-paste errors
+   for duplicated opcodes, and (2) the compiler generates the binary
+   search tree, rather than us having to post-process the table.  */
+
+#define C(OPC, NM, FT, FC, I1, I2, P, W, OP, CC) \
+    D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, 0)
+
+#define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) insn_ ## NM,
+
+enum DisasInsnEnum {
+#include "insn-data.def"
+};
+
+#undef D
+#define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) {                       \
+    .opc = OPC,                                                             \
+    .fmt = FMT_##FT,                                                        \
+    .fac = FAC_##FC,                                                        \
+    .spec = SPEC_in1_##I1 | SPEC_in2_##I2 | SPEC_prep_##P | SPEC_wout_##W,  \
+    .name = #NM,                                                            \
+    .help_in1 = in1_##I1,                                                   \
+    .help_in2 = in2_##I2,                                                   \
+    .help_prep = prep_##P,                                                  \
+    .help_wout = wout_##W,                                                  \
+    .help_cout = cout_##CC,                                                 \
+    .help_op = op_##OP,                                                     \
+    .data = D                                                               \
+ },
+
+/* Allow 0 to be used for NULL in the table below.  */
+#define in1_0  NULL
+#define in2_0  NULL
+#define prep_0  NULL
+#define wout_0  NULL
+#define cout_0  NULL
+#define op_0  NULL
+
+#define SPEC_in1_0 0
+#define SPEC_in2_0 0
+#define SPEC_prep_0 0
+#define SPEC_wout_0 0
+
+static const DisasInsn insn_info[] = {
+#include "insn-data.def"
+};
+
+#undef D
+#define D(OPC, NM, FT, FC, I1, I2, P, W, OP, CC, D) \
+    case OPC: return &insn_info[insn_ ## NM];
+
+static const DisasInsn *lookup_opc(uint16_t opc)
+{
+    switch (opc) {
+#include "insn-data.def"
+    default:
+        return NULL;
+    }
+}
+
+#undef D
+#undef C
+
+/* Extract a field from the insn.  The INSN should be left-aligned in
+   the uint64_t so that we can more easily utilize the big-bit-endian
+   definitions we extract from the Principals of Operation.  */
+
+static void extract_field(DisasFields *o, const DisasField *f, uint64_t insn)
+{
+    uint32_t r, m;
+
+    if (f->size == 0) {
+        return;
+    }
+
+    /* Zero extract the field from the insn.  */
+    r = (insn << f->beg) >> (64 - f->size);
+
+    /* Sign-extend, or un-swap the field as necessary.  */
+    switch (f->type) {
+    case 0: /* unsigned */
         break;
-    case 0x70: /* STE R1,D2(X2,B2) [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_freg32(r1);
-        tcg_gen_extu_i32_i64(tmp2, tmp32_1);
-        tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
+    case 1: /* signed */
+        assert(f->size <= 32);
+        m = 1u << (f->size - 1);
+        r = (r ^ m) - m;
         break;
-    case 0x71: /* MS      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        tcg_gen_mul_i32(tmp32_1, tmp32_1, tmp32_2);
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+    case 2: /* dl+dh split, signed 20 bit. */
+        r = ((int8_t)r << 12) | (r >> 8);
         break;
-    case 0x78: /* LE     R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = tcg_temp_new_i32();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp2);
-        store_freg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
+    default:
+        abort();
+    }
+
+    /* Validate that the "compressed" encoding we selected above is valid.
+       I.e. we havn't make two different original fields overlap.  */
+    assert(((o->presentC >> f->indexC) & 1) == 0);
+    o->presentC |= 1 << f->indexC;
+    o->presentO |= 1 << f->indexO;
+
+    o->c[f->indexC] = r;
+}
+
+/* Lookup the insn at the current PC, extracting the operands into O and
+   returning the info struct for the insn.  Returns NULL for invalid insn.  */
+
+static const DisasInsn *extract_insn(CPUS390XState *env, DisasContext *s,
+                                     DisasFields *f)
+{
+    uint64_t insn, pc = s->pc;
+    int op, op2, ilen;
+    const DisasInsn *info;
+
+    insn = ld_code2(env, pc);
+    op = (insn >> 8) & 0xff;
+    ilen = get_ilen(op);
+    s->next_pc = s->pc + ilen;
+
+    switch (ilen) {
+    case 2:
+        insn = insn << 48;
         break;
-#ifndef CONFIG_USER_ONLY
-    case 0x80: /* SSM      D2(B2)       [S] */
-        /* Set System Mask */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_andi_i64(tmp3, psw_mask, ~0xff00000000000000ULL);
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_shli_i64(tmp2, tmp2, 56);
-        tcg_gen_or_i64(psw_mask, tmp3, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    case 4:
+        insn = ld_code4(env, pc) << 32;
         break;
-    case 0x82: /* LPSW     D2(B2)       [S] */
-        /* Load PSW */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-        tcg_gen_addi_i64(tmp, tmp, 4);
-        tcg_gen_qemu_ld32u(tmp3, tmp, get_mem_index(s));
-        gen_helper_load_psw(cpu_env, tmp2, tmp3);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        /* we need to keep cc_op intact */
-        s->is_jmp = DISAS_JUMP;
+    case 6:
+        insn = (insn << 48) | (ld_code4(env, pc + 2) << 16);
         break;
-    case 0x83: /* DIAG     R1,R3,D2     [RS] */
-        /* Diagnose call (KVM hypercall) */
-        check_privileged(env, s, ilc);
-        potential_page_fault(s);
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp32_1 = tcg_const_i32(insn & 0xfff);
-        tmp2 = load_reg(2);
-        tmp3 = load_reg(1);
-        gen_helper_diag(tmp2, cpu_env, tmp32_1, tmp2, tmp3);
-        store_reg(2, tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
+    default:
+        abort();
+    }
+
+    /* We can't actually determine the insn format until we've looked up
+       the full insn opcode.  Which we can't do without locating the
+       secondary opcode.  Assume by default that OP2 is at bit 40; for
+       those smaller insns that don't actually have a secondary opcode
+       this will correctly result in OP2 = 0. */
+    switch (op) {
+    case 0x01: /* E */
+    case 0x80: /* S */
+    case 0x82: /* S */
+    case 0x93: /* S */
+    case 0xb2: /* S, RRF, RRE */
+    case 0xb3: /* RRE, RRD, RRF */
+    case 0xb9: /* RRE, RRF */
+    case 0xe5: /* SSE, SIL */
+        op2 = (insn << 8) >> 56;
+        break;
+    case 0xa5: /* RI */
+    case 0xa7: /* RI */
+    case 0xc0: /* RIL */
+    case 0xc2: /* RIL */
+    case 0xc4: /* RIL */
+    case 0xc6: /* RIL */
+    case 0xc8: /* SSF */
+    case 0xcc: /* RIL */
+        op2 = (insn << 12) >> 60;
+        break;
+    case 0xd0 ... 0xdf: /* SS */
+    case 0xe1: /* SS */
+    case 0xe2: /* SS */
+    case 0xe8: /* SS */
+    case 0xe9: /* SS */
+    case 0xea: /* SS */
+    case 0xee ... 0xf3: /* SS */
+    case 0xf8 ... 0xfd: /* SS */
+        op2 = 0;
         break;
-#endif
-    case 0x88: /* SRL    R1,D2(B2)        [RS] */
-    case 0x89: /* SLL    R1,D2(B2)        [RS] */
-    case 0x8a: /* SRA    R1,D2(B2)        [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp);
-        tcg_gen_andi_i32(tmp32_2, tmp32_2, 0x3f);
-        switch (opc) {
-        case 0x88:
-            tcg_gen_shr_i32(tmp32_1, tmp32_1, tmp32_2);
-            break;
-        case 0x89:
-            tcg_gen_shl_i32(tmp32_1, tmp32_1, tmp32_2);
-            break;
-        case 0x8a:
-            tcg_gen_sar_i32(tmp32_1, tmp32_1, tmp32_2);
-            set_cc_s32(s, tmp32_1);
-            break;
-        default:
-            tcg_abort();
-        }
-        store_reg32(r1, tmp32_1);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
+    default:
+        op2 = (insn << 40) >> 56;
         break;
-    case 0x8c: /* SRDL   R1,D2(B2)        [RS] */
-    case 0x8d: /* SLDL   R1,D2(B2)        [RS] */
-    case 0x8e: /* SRDA   R1,D2(B2)        [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2); /* shift */
-        tmp2 = tcg_temp_new_i64();
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = load_reg32(r1 + 1);
-        tcg_gen_concat_i32_i64(tmp2, tmp32_2, tmp32_1); /* operand */
-        switch (opc) {
-        case 0x8c:
-            tcg_gen_shr_i64(tmp2, tmp2, tmp);
-            break;
-        case 0x8d:
-            tcg_gen_shl_i64(tmp2, tmp2, tmp);
-            break;
-        case 0x8e:
-            tcg_gen_sar_i64(tmp2, tmp2, tmp);
-            set_cc_s64(s, tmp2);
-            break;
+    }
+
+    memset(f, 0, sizeof(*f));
+    f->op = op;
+    f->op2 = op2;
+
+    /* Lookup the instruction.  */
+    info = lookup_opc(op << 8 | op2);
+
+    /* If we found it, extract the operands.  */
+    if (info != NULL) {
+        DisasFormat fmt = info->fmt;
+        int i;
+
+        for (i = 0; i < NUM_C_FIELD; ++i) {
+            extract_field(f, &format_info[fmt].op[i], insn);
         }
-        tcg_gen_shri_i64(tmp, tmp2, 32);
-        tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-        store_reg32(r1, tmp32_1);
-        tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-        store_reg32(r1 + 1, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x98: /* LM     R1,R3,D2(B2)     [RS] */
-    case 0x90: /* STM    R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = tcg_temp_new_i64();
-        tmp3 = tcg_const_i64(4);
-        tmp4 = tcg_const_i64(0xffffffff00000000ULL);
-        for (i = r1;; i = (i + 1) % 16) {
-            if (opc == 0x98) {
-                tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
-                tcg_gen_and_i64(regs[i], regs[i], tmp4);
-                tcg_gen_or_i64(regs[i], regs[i], tmp2);
-            } else {
-                tcg_gen_qemu_st32(regs[i], tmp, get_mem_index(s));
-            }
-            if (i == r3) {
-                break;
+    }
+    return info;
+}
+
+static ExitStatus translate_one(CPUS390XState *env, DisasContext *s)
+{
+    const DisasInsn *insn;
+    ExitStatus ret = NO_EXIT;
+    DisasFields f;
+    DisasOps o;
+
+    /* Search for the insn in the table.  */
+    insn = extract_insn(env, s, &f);
+
+    /* Not found means unimplemented/illegal opcode.  */
+    if (insn == NULL) {
+        qemu_log_mask(LOG_UNIMP, "unimplemented opcode 0x%02x%02x\n",
+                      f.op, f.op2);
+        gen_illegal_opcode(s);
+        return EXIT_NORETURN;
+    }
+
+    /* Check for insn specification exceptions.  */
+    if (insn->spec) {
+        int spec = insn->spec, excp = 0, r;
+
+        if (spec & SPEC_r1_even) {
+            r = get_field(&f, r1);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
             }
-            tcg_gen_add_i64(tmp, tmp, tmp3);
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        tcg_temp_free_i64(tmp4);
-        break;
-    case 0x91: /* TM     D1(B1),I2        [SI] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_const_i64(i2);
-        tcg_gen_qemu_ld8u(tmp, tmp, get_mem_index(s));
-        cmp_64(s, tmp, tmp2, CC_OP_TM_32);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x92: /* MVI    D1(B1),I2        [SI] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_const_i64(i2);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x94: /* NI     D1(B1),I2        [SI] */
-    case 0x96: /* OI     D1(B1),I2        [SI] */
-    case 0x97: /* XI     D1(B1),I2        [SI] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        switch (opc) {
-        case 0x94:
-            tcg_gen_andi_i64(tmp2, tmp2, i2);
-            break;
-        case 0x96:
-            tcg_gen_ori_i64(tmp2, tmp2, i2);
-            break;
-        case 0x97:
-            tcg_gen_xori_i64(tmp2, tmp2, i2);
-            break;
-        default:
-            tcg_abort();
         }
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        set_cc_nz_u64(s, tmp2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x95: /* CLI    D1(B1),I2        [SI] */
-        insn = ld_code4(env, s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-        cmp_u64c(s, tmp2, i2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0x9a: /* LAM      R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_lam(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0x9b: /* STAM     R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stam(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xa5:
-        insn = ld_code4(env, s->pc);
-        r1 = (insn >> 20) & 0xf;
-        op = (insn >> 16) & 0xf;
-        i2 = insn & 0xffff;
-        disas_a5(env, s, op, r1, i2);
-        break;
-    case 0xa7:
-        insn = ld_code4(env, s->pc);
-        r1 = (insn >> 20) & 0xf;
-        op = (insn >> 16) & 0xf;
-        i2 = (short)insn;
-        disas_a7(env, s, op, r1, i2);
-        break;
-    case 0xa8: /* MVCLE   R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_mvcle(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xa9: /* CLCLE   R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_clcle(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xac: /* STNSM   D1(B1),I2     [SI] */
-    case 0xad: /* STOSM   D1(B1),I2     [SI] */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        tmp = decode_si(s, insn, &i2, &b1, &d1);
-        tmp2 = tcg_temp_new_i64();
-        tcg_gen_shri_i64(tmp2, psw_mask, 56);
-        tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
-        if (opc == 0xac) {
-            tcg_gen_andi_i64(psw_mask, psw_mask,
-                    ((uint64_t)i2 << 56) | 0x00ffffffffffffffULL);
-        } else {
-            tcg_gen_ori_i64(psw_mask, psw_mask, (uint64_t)i2 << 56);
-        }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-    case 0xae: /* SIGP   R1,R3,D2(B2)     [RS] */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp2 = load_reg(r3);
-        tmp32_1 = tcg_const_i32(r1);
-        potential_page_fault(s);
-        gen_helper_sigp(cc_op, cpu_env, tmp, tmp32_1, tmp2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-    case 0xb1: /* LRA    R1,D2(X2, B2)     [RX] */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
-        tmp32_1 = tcg_const_i32(r1);
-        potential_page_fault(s);
-        gen_helper_lra(cc_op, cpu_env, tmp, tmp32_1);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        break;
-#endif
-    case 0xb2:
-        insn = ld_code4(env, s->pc);
-        op = (insn >> 16) & 0xff;
-        switch (op) {
-        case 0x9c: /* STFPC    D2(B2) [S] */
-            d2 = insn & 0xfff;
-            b2 = (insn >> 12) & 0xf;
-            tmp32_1 = tcg_temp_new_i32();
-            tmp = tcg_temp_new_i64();
-            tmp2 = get_address(s, 0, b2, d2);
-            tcg_gen_ld_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, fpc));
-            tcg_gen_extu_i32_i64(tmp, tmp32_1);
-            tcg_gen_qemu_st32(tmp, tmp2, get_mem_index(s));
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i64(tmp);
-            tcg_temp_free_i64(tmp2);
-            break;
-        default:
-            disas_b2(env, s, op, insn);
-            break;
+        if (spec & SPEC_r2_even) {
+            r = get_field(&f, r2);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
+            }
         }
-        break;
-    case 0xb3:
-        insn = ld_code4(env, s->pc);
-        op = (insn >> 16) & 0xff;
-        r3 = (insn >> 12) & 0xf; /* aka m3 */
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        disas_b3(env, s, op, r3, r1, r2);
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xb6: /* STCTL     R1,R3,D2(B2)     [RS] */
-        /* Store Control */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stctl(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xb7: /* LCTL      R1,R3,D2(B2)     [RS] */
-        /* Load Control */
-        check_privileged(env, s, ilc);
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_lctl(cpu_env, tmp32_1, tmp, tmp32_2);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-#endif
-    case 0xb9:
-        insn = ld_code4(env, s->pc);
-        r1 = (insn >> 4) & 0xf;
-        r2 = insn & 0xf;
-        op = (insn >> 16) & 0xff;
-        disas_b9(env, s, op, r1, r2);
-        break;
-    case 0xba: /* CS     R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = tcg_const_i32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_cs(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xbd: /* CLM    R1,M3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_clm(cc_op, cpu_env, tmp32_1, tmp32_2, tmp);
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xbe: /* STCM R1,M3,D2(B2) [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        tmp = get_address(s, 0, b2, d2);
-        tmp32_1 = load_reg32(r1);
-        tmp32_2 = tcg_const_i32(r3);
-        potential_page_fault(s);
-        gen_helper_stcm(cpu_env, tmp32_1, tmp32_2, tmp);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i32(tmp32_1);
-        tcg_temp_free_i32(tmp32_2);
-        break;
-    case 0xbf: /* ICM    R1,M3,D2(B2)     [RS] */
-        insn = ld_code4(env, s->pc);
-        decode_rs(s, insn, &r1, &r3, &b2, &d2);
-        if (r3 == 15) {
-            /* effectively a 32-bit load */
-            tmp = get_address(s, 0, b2, d2);
-            tmp32_1 = tcg_temp_new_i32();
-            tmp32_2 = tcg_const_i32(r3);
-            tcg_gen_qemu_ld32u(tmp, tmp, get_mem_index(s));
-            store_reg32_i64(r1, tmp);
-            tcg_gen_trunc_i64_i32(tmp32_1, tmp);
-            set_cc_icm(s, tmp32_2, tmp32_1);
-            tcg_temp_free_i64(tmp);
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i32(tmp32_2);
-        } else if (r3) {
-            uint32_t mask = 0x00ffffffUL;
-            uint32_t shift = 24;
-            int m3 = r3;
-            tmp = get_address(s, 0, b2, d2);
-            tmp2 = tcg_temp_new_i64();
-            tmp32_1 = load_reg32(r1);
-            tmp32_2 = tcg_temp_new_i32();
-            tmp32_3 = tcg_const_i32(r3);
-            tmp32_4 = tcg_const_i32(0);
-            while (m3) {
-                if (m3 & 8) {
-                    tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
-                    tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
-                    if (shift) {
-                        tcg_gen_shli_i32(tmp32_2, tmp32_2, shift);
-                    }
-                    tcg_gen_andi_i32(tmp32_1, tmp32_1, mask);
-                    tcg_gen_or_i32(tmp32_1, tmp32_1, tmp32_2);
-                    tcg_gen_or_i32(tmp32_4, tmp32_4, tmp32_2);
-                    tcg_gen_addi_i64(tmp, tmp, 1);
-                }
-                m3 = (m3 << 1) & 0xf;
-                mask = (mask >> 8) | 0xff000000UL;
-                shift -= 8;
+        if (spec & SPEC_r3_even) {
+            r = get_field(&f, r3);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
             }
-            store_reg32(r1, tmp32_1);
-            set_cc_icm(s, tmp32_3, tmp32_4);
-            tcg_temp_free_i64(tmp);
-            tcg_temp_free_i64(tmp2);
-            tcg_temp_free_i32(tmp32_1);
-            tcg_temp_free_i32(tmp32_2);
-            tcg_temp_free_i32(tmp32_3);
-            tcg_temp_free_i32(tmp32_4);
-        } else {
-            /* i.e. env->cc = 0 */
-            gen_op_movi_cc(s, 0);
         }
-        break;
-    case 0xc0:
-    case 0xc2:
-        insn = ld_code6(env, s->pc);
-        r1 = (insn >> 36) & 0xf;
-        op = (insn >> 32) & 0xf;
-        i2 = (int)insn;
-        switch (opc) {
-        case 0xc0:
-            disas_c0(env, s, op, r1, i2);
-            break;
-        case 0xc2:
-            disas_c2(env, s, op, r1, i2);
-            break;
-        default:
-            tcg_abort();
+        if (spec & SPEC_r1_f128) {
+            r = get_field(&f, r1);
+            if (r > 13) {
+                excp = PGM_SPECIFICATION;
+            }
         }
-        break;
-    case 0xd2: /* MVC    D1(L,B1),D2(B2)         [SS] */
-    case 0xd4: /* NC     D1(L,B1),D2(B2)         [SS] */
-    case 0xd5: /* CLC    D1(L,B1),D2(B2)         [SS] */
-    case 0xd6: /* OC     D1(L,B1),D2(B2)         [SS] */
-    case 0xd7: /* XC     D1(L,B1),D2(B2)         [SS] */
-    case 0xdc: /* TR     D1(L,B1),D2(B2)         [SS] */
-    case 0xf3: /* UNPK   D1(L1,B1),D2(L2,B2)     [SS] */
-        insn = ld_code6(env, s->pc);
-        vl = tcg_const_i32((insn >> 32) & 0xff);
-        b1 = (insn >> 28) & 0xf;
-        b2 = (insn >> 12) & 0xf;
-        d1 = (insn >> 16) & 0xfff;
-        d2 = insn & 0xfff;
-        tmp = get_address(s, 0, b1, d1);
-        tmp2 = get_address(s, 0, b2, d2);
-        switch (opc) {
-        case 0xd2:
-            gen_op_mvc(s, (insn >> 32) & 0xff, tmp, tmp2);
-            break;
-        case 0xd4:
-            potential_page_fault(s);
-            gen_helper_nc(cc_op, cpu_env, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xd5:
-            gen_op_clc(s, (insn >> 32) & 0xff, tmp, tmp2);
-            break;
-        case 0xd6:
-            potential_page_fault(s);
-            gen_helper_oc(cc_op, cpu_env, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xd7:
-            potential_page_fault(s);
-            gen_helper_xc(cc_op, cpu_env, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xdc:
-            potential_page_fault(s);
-            gen_helper_tr(cpu_env, vl, tmp, tmp2);
-            set_cc_static(s);
-            break;
-        case 0xf3:
-            potential_page_fault(s);
-            gen_helper_unpk(cpu_env, vl, tmp, tmp2);
-            break;
-        default:
-            tcg_abort();
+        if (spec & SPEC_r2_f128) {
+            r = get_field(&f, r2);
+            if (r > 13) {
+                excp = PGM_SPECIFICATION;
+            }
         }
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xda: /* MVCP     D1(R1,B1),D2(B2),R3   [SS] */
-    case 0xdb: /* MVCS     D1(R1,B1),D2(B2),R3   [SS] */
-        check_privileged(env, s, ilc);
-        potential_page_fault(s);
-        insn = ld_code6(env, s->pc);
-        r1 = (insn >> 36) & 0xf;
-        r3 = (insn >> 32) & 0xf;
-        b1 = (insn >> 28) & 0xf;
-        d1 = (insn >> 16) & 0xfff;
-        b2 = (insn >> 12) & 0xf;
-        d2 = insn & 0xfff;
-        tmp = load_reg(r1);
-        /* XXX key in r3 */
-        tmp2 = get_address(s, 0, b1, d1);
-        tmp3 = get_address(s, 0, b2, d2);
-        if (opc == 0xda) {
-            gen_helper_mvcp(cc_op, cpu_env, tmp, tmp2, tmp3);
-        } else {
-            gen_helper_mvcs(cc_op, cpu_env, tmp, tmp2, tmp3);
+        if (excp) {
+            gen_program_exception(s, excp);
+            return EXIT_NORETURN;
         }
-        set_cc_static(s);
-        tcg_temp_free_i64(tmp);
-        tcg_temp_free_i64(tmp2);
-        tcg_temp_free_i64(tmp3);
-        break;
-#endif
-    case 0xe3:
-        insn = ld_code6(env, s->pc);
-        debug_insn(insn);
-        op = insn & 0xff;
-        r1 = (insn >> 36) & 0xf;
-        x2 = (insn >> 32) & 0xf;
-        b2 = (insn >> 28) & 0xf;
-        d2 = ((int)((((insn >> 16) & 0xfff)
-           | ((insn << 4) & 0xff000)) << 12)) >> 12;
-        disas_e3(env, s, op,  r1, x2, b2, d2 );
-        break;
-#ifndef CONFIG_USER_ONLY
-    case 0xe5:
-        /* Test Protection */
-        check_privileged(env, s, ilc);
-        insn = ld_code6(env, s->pc);
-        debug_insn(insn);
-        disas_e5(env, s, insn);
-        break;
-#endif
-    case 0xeb:
-        insn = ld_code6(env, s->pc);
-        debug_insn(insn);
-        op = insn & 0xff;
-        r1 = (insn >> 36) & 0xf;
-        r3 = (insn >> 32) & 0xf;
-        b2 = (insn >> 28) & 0xf;
-        d2 = ((int)((((insn >> 16) & 0xfff)
-           | ((insn << 4) & 0xff000)) << 12)) >> 12;
-        disas_eb(env, s, op, r1, r3, b2, d2);
-        break;
-    case 0xed:
-        insn = ld_code6(env, s->pc);
-        debug_insn(insn);
-        op = insn & 0xff;
-        r1 = (insn >> 36) & 0xf;
-        x2 = (insn >> 32) & 0xf;
-        b2 = (insn >> 28) & 0xf;
-        d2 = (short)((insn >> 16) & 0xfff);
-        r1b = (insn >> 12) & 0xf;
-        disas_ed(env, s, op, r1, x2, b2, d2, r1b);
-        break;
-    default:
-        qemu_log_mask(LOG_UNIMP, "unimplemented opcode 0x%x\n", opc);
-        gen_illegal_opcode(env, s, ilc);
-        break;
     }
 
-    /* Instruction length is encoded in the opcode */
-    s->pc += (ilc * 2);
+    /* Set up the strutures we use to communicate with the helpers. */
+    s->insn = insn;
+    s->fields = &f;
+    o.g_out = o.g_out2 = o.g_in1 = o.g_in2 = false;
+    TCGV_UNUSED_I64(o.out);
+    TCGV_UNUSED_I64(o.out2);
+    TCGV_UNUSED_I64(o.in1);
+    TCGV_UNUSED_I64(o.in2);
+    TCGV_UNUSED_I64(o.addr1);
+
+    /* Implement the instruction.  */
+    if (insn->help_in1) {
+        insn->help_in1(s, &f, &o);
+    }
+    if (insn->help_in2) {
+        insn->help_in2(s, &f, &o);
+    }
+    if (insn->help_prep) {
+        insn->help_prep(s, &f, &o);
+    }
+    if (insn->help_op) {
+        ret = insn->help_op(s, &o);
+    }
+    if (insn->help_wout) {
+        insn->help_wout(s, &f, &o);
+    }
+    if (insn->help_cout) {
+        insn->help_cout(s, &o);
+    }
+
+    /* Free any temporaries created by the helpers.  */
+    if (!TCGV_IS_UNUSED_I64(o.out) && !o.g_out) {
+        tcg_temp_free_i64(o.out);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.out2) && !o.g_out2) {
+        tcg_temp_free_i64(o.out2);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.in1) && !o.g_in1) {
+        tcg_temp_free_i64(o.in1);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.in2) && !o.g_in2) {
+        tcg_temp_free_i64(o.in2);
+    }
+    if (!TCGV_IS_UNUSED_I64(o.addr1)) {
+        tcg_temp_free_i64(o.addr1);
+    }
+
+    /* Advance to the next instruction.  */
+    s->pc = s->next_pc;
+    return ret;
 }
 
 static inline void gen_intermediate_code_internal(CPUS390XState *env,
@@ -5121,6 +4745,8 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
     int j, lj = -1;
     int num_insns, max_insns;
     CPUBreakpoint *bp;
+    ExitStatus status;
+    bool do_debug;
 
     pc_start = tb->pc;
 
@@ -5129,10 +4755,10 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
         pc_start &= 0x7fffffff;
     }
 
-    dc.pc = pc_start;
-    dc.is_jmp = DISAS_NEXT;
     dc.tb = tb;
+    dc.pc = pc_start;
     dc.cc_op = CC_OP_DYNAMIC;
+    do_debug = dc.singlestep_enabled = env->singlestep_enabled;
 
     gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
@@ -5147,28 +4773,20 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
     gen_icount_start();
 
     do {
-        if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
-            QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
-                if (bp->pc == dc.pc) {
-                    gen_debug(&dc);
-                    break;
-                }
-            }
-        }
         if (search_pc) {
             j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc.pc;
+            tcg_ctx.gen_opc_pc[lj] = dc.pc;
             gen_opc_cc_op[lj] = dc.cc_op;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (++num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
@@ -5176,48 +4794,71 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
             tcg_gen_debug_insn_start(dc.pc);
         }
 
-        disas_s390_insn(env, &dc);
-
-        num_insns++;
-        if (env->singlestep_enabled) {
-            gen_debug(&dc);
+        status = NO_EXIT;
+        if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
+            QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
+                if (bp->pc == dc.pc) {
+                    status = EXIT_PC_STALE;
+                    do_debug = true;
+                    break;
+                }
+            }
+        }
+        if (status == NO_EXIT) {
+            status = translate_one(env, &dc);
         }
-    } while (!dc.is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end
-             && dc.pc < next_page_start
-             && num_insns < max_insns && !env->singlestep_enabled
-             && !singlestep);
-
-    if (!dc.is_jmp) {
-        update_psw_addr(&dc);
-    }
 
-    if (singlestep && dc.cc_op != CC_OP_DYNAMIC) {
-        gen_op_calc_cc(&dc);
-    } else {
-        /* next TB starts off with CC_OP_DYNAMIC, so make sure the cc op type
-           is in env */
-        gen_op_set_cc_op(&dc);
-    }
+        /* If we reach a page boundary, are single stepping,
+           or exhaust instruction count, stop generation.  */
+        if (status == NO_EXIT
+            && (dc.pc >= next_page_start
+                || tcg_ctx.gen_opc_ptr >= gen_opc_end
+                || num_insns >= max_insns
+                || singlestep
+                || env->singlestep_enabled)) {
+            status = EXIT_PC_STALE;
+        }
+    } while (status == NO_EXIT);
 
     if (tb->cflags & CF_LAST_IO) {
         gen_io_end();
     }
-    /* Generate the return instruction */
-    if (dc.is_jmp != DISAS_TB_JUMP) {
-        tcg_gen_exit_tb(0);
+
+    switch (status) {
+    case EXIT_GOTO_TB:
+    case EXIT_NORETURN:
+        break;
+    case EXIT_PC_STALE:
+        update_psw_addr(&dc);
+        /* FALLTHRU */
+    case EXIT_PC_UPDATED:
+        /* Next TB starts off with CC_OP_DYNAMIC, so make sure the
+           cc op type is in env */
+        update_cc_op(&dc);
+        /* Exit the TB, either by raising a debug exception or by return.  */
+        if (do_debug) {
+            gen_exception(EXCP_DEBUG);
+        } else {
+            tcg_gen_exit_tb(0);
+        }
+        break;
+    default:
+        abort();
     }
+
     gen_icount_end(tb, num_insns);
     *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc.pc - pc_start;
         tb->icount = num_insns;
     }
+
 #if defined(S390X_DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
@@ -5240,7 +4881,7 @@ void gen_intermediate_code_pc (CPUS390XState *env, struct TranslationBlock *tb)
 void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, int pc_pos)
 {
     int cc_op;
-    env->psw.addr = gen_opc_pc[pc_pos];
+    env->psw.addr = tcg_ctx.gen_opc_pc[pc_pos];
     cc_op = gen_opc_cc_op[pc_pos];
     if ((cc_op != CC_OP_DYNAMIC) && (cc_op != CC_OP_STATIC)) {
         env->cc_op = cc_op;
diff --git a/target-sh4/Makefile.objs b/target-sh4/Makefile.objs
index ca20f21..cb448a8 100644
--- a/target-sh4/Makefile.objs
+++ b/target-sh4/Makefile.objs
@@ -1,2 +1 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
-obj-$(CONFIG_SOFTMMU) += machine.o
diff --git a/target-sh4/cpu-qom.h b/target-sh4/cpu-qom.h
index c41164a..09573c9 100644
--- a/target-sh4/cpu-qom.h
+++ b/target-sh4/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_SUPERH_CPU_QOM_H
 #define QEMU_SUPERH_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 
 #define TYPE_SUPERH_CPU "superh-cpu"
 
diff --git a/target-sh4/cpu.c b/target-sh4/cpu.c
index a1a177f..d283122 100644
--- a/target-sh4/cpu.c
+++ b/target-sh4/cpu.c
@@ -21,6 +21,7 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
+#include "migration/vmstate.h"
 
 
 /* CPUClass::reset() */
@@ -31,7 +32,7 @@ static void superh_cpu_reset(CPUState *s)
     CPUSH4State *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -63,13 +64,21 @@ static void superh_cpu_initfn(Object *obj)
     env->movcal_backup_tail = &(env->movcal_backup);
 }
 
+static const VMStateDescription vmstate_sh_cpu = {
+    .name = "cpu",
+    .unmigratable = 1,
+};
+
 static void superh_cpu_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     CPUClass *cc = CPU_CLASS(oc);
     SuperHCPUClass *scc = SUPERH_CPU_CLASS(oc);
 
     scc->parent_reset = cc->reset;
     cc->reset = superh_cpu_reset;
+
+    dc->vmsd = &vmstate_sh_cpu;
 }
 
 static const TypeInfo superh_cpu_type_info = {
diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h
index 9a0e72b..34e9b0a 100644
--- a/target-sh4/cpu.h
+++ b/target-sh4/cpu.h
@@ -39,9 +39,9 @@
 
 #define CPUArchState struct CPUSH4State
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_PAGE_BITS 12	/* 4k XXXXX */
 
@@ -230,8 +230,6 @@ static inline void cpu_set_tls(CPUSH4State *env, target_ulong newtls)
 
 void cpu_load_tlb(CPUSH4State * env);
 
-#include "softfloat.h"
-
 static inline CPUSH4State *cpu_init(const char *cpu_model)
 {
     SuperHCPU *cpu = cpu_sh4_init(cpu_model);
@@ -264,7 +262,7 @@ static inline void cpu_clone_regs(CPUSH4State *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 /* Memory access type */
 enum {
@@ -378,7 +376,7 @@ static inline bool cpu_has_work(CPUState *cpu)
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUSH4State *env, TranslationBlock *tb)
 {
diff --git a/target-sh4/helper.h b/target-sh4/helper.h
index 304b77b..7162448 100644
--- a/target-sh4/helper.h
+++ b/target-sh4/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_1(ldtlb, void, env)
 DEF_HELPER_1(raise_illegal_instruction, noreturn, env)
@@ -47,4 +47,4 @@ DEF_HELPER_2(ftrc_DT, i32, env, f64)
 DEF_HELPER_3(fipr, void, env, i32, i32)
 DEF_HELPER_2(ftrv, void, env, i32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-sh4/machine.c b/target-sh4/machine.c
deleted file mode 100644
index e69de29..0000000
--- a/target-sh4/machine.c
+++ /dev/null
diff --git a/target-sh4/op_helper.c b/target-sh4/op_helper.c
index 60ec4cb..09e3d23 100644
--- a/target-sh4/op_helper.c
+++ b/target-sh4/op_helper.c
@@ -21,37 +21,22 @@
 #include "cpu.h"
 #include "helper.h"
 
-static inline void cpu_restore_state_from_retaddr(CPUSH4State *env,
-                                                  uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-
-    if (retaddr) {
-        tb = tb_find_pc(retaddr);
-        if (tb) {
-            /* the PC is inside the translated code. It means that we have
-               a virtual CPU fault */
-            cpu_restore_state(tb, env, retaddr);
-        }
-    }
-}
-
 #ifndef CONFIG_USER_ONLY
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 void tlb_fill(CPUSH4State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
@@ -61,7 +46,9 @@ void tlb_fill(CPUSH4State *env, target_ulong addr, int is_write, int mmu_idx,
     ret = cpu_sh4_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
         /* now we have a real cpu fault */
-        cpu_restore_state_from_retaddr(env, retaddr);
+        if (retaddr) {
+            cpu_restore_state(env, retaddr);
+        }
         cpu_loop_exit(env);
     }
 }
@@ -82,7 +69,9 @@ static inline void QEMU_NORETURN raise_exception(CPUSH4State *env, int index,
                                                  uintptr_t retaddr)
 {
     env->exception_index = index;
-    cpu_restore_state_from_retaddr(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
     cpu_loop_exit(env);
 }
 
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 5497ded..260aaab 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -21,7 +21,7 @@
 //#define SH4_SINGLE_STEP
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
 
 #include "helper.h"
@@ -69,7 +69,7 @@ static TCGv cpu_flags, cpu_delayed_pc;
 
 static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 static void sh4_translate_init(void)
 {
@@ -2003,12 +2003,12 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
             if (ii < i) {
                 ii++;
                 while (ii < i)
-                    gen_opc_instr_start[ii++] = 0;
+                    tcg_ctx.gen_opc_instr_start[ii++] = 0;
             }
-            gen_opc_pc[ii] = ctx.pc;
+            tcg_ctx.gen_opc_pc[ii] = ctx.pc;
             gen_opc_hflags[ii] = ctx.flags;
-            gen_opc_instr_start[ii] = 1;
-            gen_opc_icount[ii] = num_insns;
+            tcg_ctx.gen_opc_instr_start[ii] = 1;
+            tcg_ctx.gen_opc_icount[ii] = num_insns;
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
@@ -2061,7 +2061,7 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
         i = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         ii++;
         while (ii <= i)
-            gen_opc_instr_start[ii++] = 0;
+            tcg_ctx.gen_opc_instr_start[ii++] = 0;
     } else {
         tb->size = ctx.pc - pc_start;
         tb->icount = num_insns;
@@ -2088,6 +2088,6 @@ void gen_intermediate_code_pc(CPUSH4State * env, struct TranslationBlock *tb)
 
 void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
     env->flags = gen_opc_hflags[pc_pos];
 }
diff --git a/target-sparc/cpu-qom.h b/target-sparc/cpu-qom.h
index 3d3ac0f..2a738ae 100644
--- a/target-sparc/cpu-qom.h
+++ b/target-sparc/cpu-qom.h
@@ -20,7 +20,7 @@
 #ifndef QEMU_SPARC_CPU_QOM_H
 #define QEMU_SPARC_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #ifdef TARGET_SPARC64
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index 882d306..4bc1afc 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -31,7 +31,7 @@ static void sparc_cpu_reset(CPUState *s)
     CPUSPARCState *env = &cpu->env;
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
-        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        qemu_log("CPU Reset (CPU %d)\n", s->cpu_index);
         log_cpu_state(env, 0);
     }
 
@@ -119,7 +119,7 @@ SPARCCPU *cpu_sparc_init(const char *cpu_model)
     }
 
     if (cpu_sparc_register(env, cpu_model) < 0) {
-        object_delete(OBJECT(cpu));
+        object_unref(OBJECT(cpu));
         return NULL;
     }
     qemu_init_vcpu(env);
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index 042d52a..7389b03 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -3,7 +3,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "bswap.h"
+#include "qemu/bswap.h"
 
 #if !defined(TARGET_SPARC64)
 #define TARGET_LONG_BITS 32
@@ -25,9 +25,9 @@
 
 #define CPUArchState struct CPUSPARCState
 
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 
-#include "softfloat.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -392,7 +392,6 @@ struct CPUSPARCState {
     target_ulong cc_dst;
     uint32_t cc_op;
 
-    target_ulong t0, t1; /* temporaries live across basic blocks */
     target_ulong cond; /* conditional branch result (XXX: save it in a
                           temporary register when possible) */
 
@@ -702,7 +701,7 @@ static inline void cpu_clone_regs(CPUSPARCState *env, target_ulong newsp)
 }
 #endif
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 
 #ifdef TARGET_SPARC64
 /* sun4u.c */
@@ -711,7 +710,6 @@ uint64_t cpu_tick_get_count(CPUTimer *timer);
 void cpu_tick_set_limit(CPUTimer *timer, uint64_t limit);
 trap_state* cpu_tsptr(CPUSPARCState* env);
 #endif
-void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr);
 
 #define TB_FLAG_FPU_ENABLED (1 << 4)
 #define TB_FLAG_AM_ENABLED (1 << 5)
@@ -769,7 +767,7 @@ static inline bool cpu_has_work(CPUState *cpu)
            cpu_interrupts_enabled(env1);
 }
 
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUSPARCState *env, TranslationBlock *tb)
 {
diff --git a/target-sparc/helper.c b/target-sparc/helper.c
index 556ac28..91ecfc7 100644
--- a/target-sparc/helper.c
+++ b/target-sparc/helper.c
@@ -18,9 +18,9 @@
  */
 
 #include "cpu.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include "helper.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 void helper_raise_exception(CPUSPARCState *env, int tt)
 {
@@ -75,7 +75,7 @@ static target_ulong helper_udiv_common(CPUSPARCState *env, target_ulong a,
     x1 = (b & 0xffffffff);
 
     if (x1 == 0) {
-        cpu_restore_state2(env, GETPC());
+        cpu_restore_state(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
 
@@ -114,7 +114,7 @@ static target_ulong helper_sdiv_common(CPUSPARCState *env, target_ulong a,
     x1 = (b & 0xffffffff);
 
     if (x1 == 0) {
-        cpu_restore_state2(env, GETPC());
+        cpu_restore_state(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
 
@@ -147,7 +147,7 @@ int64_t helper_sdivx(CPUSPARCState *env, int64_t a, int64_t b)
 {
     if (b == 0) {
         /* Raise divide by zero trap.  */
-        cpu_restore_state2(env, GETPC());
+        cpu_restore_state(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     } else if (b == -1) {
         /* Avoid overflow trap with i386 divide insn.  */
@@ -161,7 +161,7 @@ uint64_t helper_udivx(CPUSPARCState *env, uint64_t a, uint64_t b)
 {
     if (b == 0) {
         /* Raise divide by zero trap.  */
-        cpu_restore_state2(env, GETPC());
+        cpu_restore_state(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
     return a / b;
@@ -193,7 +193,7 @@ target_ulong helper_taddcctv(CPUSPARCState *env, target_ulong src1,
     return dst;
 
  tag_overflow:
-    cpu_restore_state2(env, GETPC());
+    cpu_restore_state(env, GETPC());
     helper_raise_exception(env, TT_TOVF);
 }
 
@@ -222,6 +222,6 @@ target_ulong helper_tsubcctv(CPUSPARCState *env, target_ulong src1,
     return dst;
 
  tag_overflow:
-    cpu_restore_state2(env, GETPC());
+    cpu_restore_state(env, GETPC());
     helper_raise_exception(env, TT_TOVF);
 }
diff --git a/target-sparc/helper.h b/target-sparc/helper.h
index 098c482..cfcdab1 100644
--- a/target-sparc/helper.h
+++ b/target-sparc/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 #ifndef TARGET_SPARC64
 DEF_HELPER_1(rett, void, env)
@@ -173,4 +173,4 @@ VIS_CMPHELPER(cmpne);
 DEF_HELPER_1(compute_psr, void, env);
 DEF_HELPER_1(compute_C_icc, i32, env);
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-sparc/int32_helper.c b/target-sparc/int32_helper.c
index 507c355..c35f522 100644
--- a/target-sparc/int32_helper.c
+++ b/target-sparc/int32_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "trace.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #define DEBUG_PCALL
 
diff --git a/target-sparc/ldst_helper.c b/target-sparc/ldst_helper.c
index f3e08fd..cf1bddf 100644
--- a/target-sparc/ldst_helper.c
+++ b/target-sparc/ldst_helper.c
@@ -68,21 +68,21 @@
 static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
                                               target_ulong addr, int is_write,
                                               int is_user, uintptr_t retaddr);
-#include "softmmu_exec.h"
+#include "exec/softmmu_exec.h"
 #define MMUSUFFIX _mmu
 #define ALIGNED_ONLY
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 #endif
 
 #if defined(TARGET_SPARC64) && !defined(CONFIG_USER_ONLY)
@@ -2393,22 +2393,6 @@ void cpu_unassigned_access(CPUSPARCState *env, hwaddr addr,
 #endif
 #endif
 
-/* XXX: make it generic ? */
-void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-
-    if (retaddr) {
-        /* now we have a real cpu fault */
-        tb = tb_find_pc(retaddr);
-        if (tb) {
-            /* the PC is inside the translated code. It means that we have
-               a virtual CPU fault */
-            cpu_restore_state(tb, env, retaddr);
-        }
-    }
-}
-
 #if !defined(CONFIG_USER_ONLY)
 static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
                                               target_ulong addr, int is_write,
@@ -2418,7 +2402,9 @@ static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
     printf("Unaligned access to 0x" TARGET_FMT_lx " from 0x" TARGET_FMT_lx
            "\n", addr, env->pc);
 #endif
-    cpu_restore_state2(env, retaddr);
+    if (retaddr) {
+        cpu_restore_state(env, retaddr);
+    }
     helper_raise_exception(env, TT_UNALIGNED);
 }
 
@@ -2433,7 +2419,9 @@ void tlb_fill(CPUSPARCState *env, target_ulong addr, int is_write, int mmu_idx,
 
     ret = cpu_sparc_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
-        cpu_restore_state2(env, retaddr);
+        if (retaddr) {
+            cpu_restore_state(env, retaddr);
+        }
         cpu_loop_exit(env);
     }
 }
diff --git a/target-sparc/machine.c b/target-sparc/machine.c
index eb4d87f..a353dab 100644
--- a/target-sparc/machine.c
+++ b/target-sparc/machine.c
@@ -1,6 +1,6 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 #include "cpu.h"
 
diff --git a/target-sparc/mmu_helper.c b/target-sparc/mmu_helper.c
index 2c89b20..a9649ae 100644
--- a/target-sparc/mmu_helper.c
+++ b/target-sparc/mmu_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 /* Sparc MMU emulation */
 
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index 2ae8036..ca75e1a 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -25,7 +25,7 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "helper.h"
 #include "tcg-op.h"
 
@@ -64,7 +64,7 @@ static TCGv_i64 cpu_fpr[TARGET_DPREGS];
 static target_ulong gen_opc_npc[OPC_BUF_SIZE];
 static target_ulong gen_opc_jump_pc[2];
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 typedef struct DisasContext {
     target_ulong pc;    /* current Program Counter: integer or DYNAMIC_PC */
@@ -5283,11 +5283,11 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
             if (lj < j) {
                 lj++;
                 while (lj < j)
-                    gen_opc_instr_start[lj++] = 0;
-                gen_opc_pc[lj] = dc->pc;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
+                tcg_ctx.gen_opc_pc[lj] = dc->pc;
                 gen_opc_npc[lj] = dc->npc;
-                gen_opc_instr_start[lj] = 1;
-                gen_opc_icount[lj] = num_insns;
+                tcg_ctx.gen_opc_instr_start[lj] = 1;
+                tcg_ctx.gen_opc_icount[lj] = num_insns;
             }
         }
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
@@ -5339,7 +5339,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
 #if 0
         log_page_dump();
 #endif
@@ -5478,7 +5478,7 @@ void gen_intermediate_code_init(CPUSPARCState *env)
 void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
 {
     target_ulong npc;
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
     npc = gen_opc_npc[pc_pos];
     if (npc == 1) {
         /* dynamic NPC: already stored */
diff --git a/target-unicore32/Makefile.objs b/target-unicore32/Makefile.objs
index 8e143da..6b41b1e 100644
--- a/target-unicore32/Makefile.objs
+++ b/target-unicore32/Makefile.objs
@@ -1,4 +1,4 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += ucf64_helper.o
 
-obj-$(CONFIG_SOFTMMU) += machine.o softmmu.o
+obj-$(CONFIG_SOFTMMU) += softmmu.o
diff --git a/target-unicore32/cpu-qom.h b/target-unicore32/cpu-qom.h
index 342d85e..fe40b2d 100644
--- a/target-unicore32/cpu-qom.h
+++ b/target-unicore32/cpu-qom.h
@@ -11,7 +11,7 @@
 #ifndef QEMU_UC32_CPU_QOM_H
 #define QEMU_UC32_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_UNICORE32_CPU "unicore32-cpu"
diff --git a/target-unicore32/cpu.c b/target-unicore32/cpu.c
index 884c101..4e4177f 100644
--- a/target-unicore32/cpu.c
+++ b/target-unicore32/cpu.c
@@ -14,6 +14,7 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
+#include "migration/vmstate.h"
 
 static inline void set_feature(CPUUniCore32State *env, int feature)
 {
@@ -22,6 +23,25 @@ static inline void set_feature(CPUUniCore32State *env, int feature)
 
 /* CPU models */
 
+static ObjectClass *uc32_cpu_class_by_name(const char *cpu_model)
+{
+    ObjectClass *oc;
+    char *typename;
+
+    if (cpu_model == NULL) {
+        return NULL;
+    }
+
+    typename = g_strdup_printf("%s-" TYPE_UNICORE32_CPU, cpu_model);
+    oc = object_class_by_name(typename);
+    g_free(typename);
+    if (oc != NULL && (!object_class_dynamic_cast(oc, TYPE_UNICORE32_CPU) ||
+                       object_class_is_abstract(oc))) {
+        oc = NULL;
+    }
+    return oc;
+}
+
 typedef struct UniCore32CPUInfo {
     const char *name;
     void (*instance_init)(Object *obj);
@@ -67,7 +87,6 @@ static void uc32_cpu_initfn(Object *obj)
     CPUUniCore32State *env = &cpu->env;
 
     cpu_exec_init(env);
-    env->cpu_model_str = object_get_typename(obj);
 
 #ifdef CONFIG_USER_ONLY
     env->uncached_asr = ASR_MODE_USER;
@@ -80,15 +99,30 @@ static void uc32_cpu_initfn(Object *obj)
     tlb_flush(env, 1);
 }
 
+static const VMStateDescription vmstate_uc32_cpu = {
+    .name = "cpu",
+    .unmigratable = 1,
+};
+
+static void uc32_cpu_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    CPUClass *cc = CPU_CLASS(oc);
+
+    cc->class_by_name = uc32_cpu_class_by_name;
+    dc->vmsd = &vmstate_uc32_cpu;
+}
+
 static void uc32_register_cpu_type(const UniCore32CPUInfo *info)
 {
     TypeInfo type_info = {
-        .name = info->name,
         .parent = TYPE_UNICORE32_CPU,
         .instance_init = info->instance_init,
     };
 
-    type_register_static(&type_info);
+    type_info.name = g_strdup_printf("%s-" TYPE_UNICORE32_CPU, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
 }
 
 static const TypeInfo uc32_cpu_type_info = {
@@ -98,6 +132,7 @@ static const TypeInfo uc32_cpu_type_info = {
     .instance_init = uc32_cpu_initfn,
     .abstract = true,
     .class_size = sizeof(UniCore32CPUClass),
+    .class_init = uc32_cpu_class_init,
 };
 
 static void uc32_cpu_register_types(void)
diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h
index 676c5d9..ae9a9d6 100644
--- a/target-unicore32/cpu.h
+++ b/target-unicore32/cpu.h
@@ -23,8 +23,8 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
-#include "softfloat.h"
+#include "exec/cpu-defs.h"
+#include "fpu/softfloat.h"
 
 #define NB_MMU_MODES            2
 
@@ -133,8 +133,6 @@ int uc32_cpu_signal_handler(int host_signum, void *pinfo, void *puc);
 int uc32_cpu_handle_mmu_fault(CPUUniCore32State *env, target_ulong address, int rw,
                               int mmu_idx);
 
-#define CPU_SAVE_VERSION 2
-
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
@@ -157,9 +155,9 @@ static inline void cpu_set_tls(CPUUniCore32State *env, target_ulong newtls)
     env->regs[16] = newtls;
 }
 
-#include "cpu-all.h"
+#include "exec/cpu-all.h"
 #include "cpu-qom.h"
-#include "exec-all.h"
+#include "exec/exec-all.h"
 
 static inline void cpu_pc_from_tb(CPUUniCore32State *env, TranslationBlock *tb)
 {
diff --git a/target-unicore32/helper.c b/target-unicore32/helper.c
index a9e226b..3a92232 100644
--- a/target-unicore32/helper.c
+++ b/target-unicore32/helper.c
@@ -10,10 +10,12 @@
  */
 
 #include "cpu.h"
-#include "gdbstub.h"
+#include "exec/gdbstub.h"
 #include "helper.h"
-#include "host-utils.h"
-#include "console.h"
+#include "qemu/host-utils.h"
+#ifndef CONFIG_USER_ONLY
+#include "ui/console.h"
+#endif
 
 #undef DEBUG_UC32
 
@@ -27,13 +29,16 @@ CPUUniCore32State *uc32_cpu_init(const char *cpu_model)
 {
     UniCore32CPU *cpu;
     CPUUniCore32State *env;
+    ObjectClass *oc;
     static int inited = 1;
 
-    if (object_class_by_name(cpu_model) == NULL) {
+    oc = cpu_class_by_name(TYPE_UNICORE32_CPU, cpu_model);
+    if (oc == NULL) {
         return NULL;
     }
-    cpu = UNICORE32_CPU(object_new(cpu_model));
+    cpu = UNICORE32_CPU(object_new(object_class_get_name(oc)));
     env = &cpu->env;
+    env->cpu_model_str = cpu_model;
 
     if (inited) {
         inited = 0;
diff --git a/target-unicore32/helper.h b/target-unicore32/helper.h
index a4b8149..e85ce6c 100644
--- a/target-unicore32/helper.h
+++ b/target-unicore32/helper.h
@@ -6,7 +6,7 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_4(cp0_set, void, env, i32, i32, i32)
@@ -65,4 +65,4 @@ DEF_HELPER_2(ucf64_si2df, f64, f32, env)
 DEF_HELPER_2(ucf64_sf2si, f32, f32, env)
 DEF_HELPER_2(ucf64_df2si, f32, f64, env)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-unicore32/machine.c b/target-unicore32/machine.c
deleted file mode 100644
index 60b2ec1..0000000
--- a/target-unicore32/machine.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Generic machine functions for UniCore32 ISA
- *
- * Copyright (C) 2010-2012 Guan Xuetao
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation, or any later version.
- * See the COPYING file in the top-level directory.
- */
-#include "hw/hw.h"
-
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    hw_error("%s not supported yet.\n", __func__);
-}
-
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-    hw_error("%s not supported yet.\n", __func__);
-
-    return 0;
-}
diff --git a/target-unicore32/op_helper.c b/target-unicore32/op_helper.c
index f474d1b..6443ffe 100644
--- a/target-unicore32/op_helper.c
+++ b/target-unicore32/op_helper.c
@@ -242,34 +242,27 @@ uint32_t HELPER(ror_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 void tlb_fill(CPUUniCore32State *env, target_ulong addr, int is_write,
               int mmu_idx, uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    unsigned long pc;
     int ret;
 
     ret = uc32_cpu_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
-            pc = (unsigned long)retaddr;
-            tb = tb_find_pc(pc);
-            if (tb) {/* the PC is inside the translated code.
-                        It means that we have a virtual CPU fault */
-                cpu_restore_state(tb, env, pc);
-            }
+            cpu_restore_state(env, retaddr);
         }
         cpu_loop_exit(env);
     }
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index 052bb45..f4498bc 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -15,9 +15,9 @@
 #include <inttypes.h>
 
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -55,7 +55,7 @@ static TCGv_i32 cpu_R[32];
 static TCGv cpu_F0s, cpu_F1s;
 static TCGv_i64 cpu_F0d, cpu_F1d;
 
-#include "gen-icount.h"
+#include "exec/gen-icount.h"
 
 static const char *regnames[] = {
       "r00", "r01", "r02", "r03", "r04", "r05", "r06", "r07",
@@ -2003,12 +2003,12 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc->pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = num_insns;
+            tcg_ctx.gen_opc_pc[lj] = dc->pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
@@ -2117,7 +2117,7 @@ done_generating:
         j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
-            gen_opc_instr_start[lj++] = 0;
+            tcg_ctx.gen_opc_instr_start[lj++] = 0;
         }
     } else {
         tb->size = dc->pc - pc_start;
@@ -2203,5 +2203,5 @@ void cpu_dump_state(CPUUniCore32State *env, FILE *f,
 
 void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb, int pc_pos)
 {
-    env->regs[31] = gen_opc_pc[pc_pos];
+    env->regs[31] = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-xtensa/Makefile.objs b/target-xtensa/Makefile.objs
index b30e5a8..644b7f9 100644
--- a/target-xtensa/Makefile.objs
+++ b/target-xtensa/Makefile.objs
@@ -3,4 +3,3 @@ obj-y += core-dc232b.o
 obj-y += core-dc233c.o
 obj-y += core-fsf.o
 obj-y += translate.o op_helper.o helper.o cpu.o
-obj-$(CONFIG_SOFTMMU) += machine.o
diff --git a/target-xtensa/core-dc232b.c b/target-xtensa/core-dc232b.c
index 804fdef..0bfcf24 100644
--- a/target-xtensa/core-dc232b.c
+++ b/target-xtensa/core-dc232b.c
@@ -26,9 +26,9 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 
 #include "core-dc232b/core-isa.h"
 #include "overlay_tool.h"
diff --git a/target-xtensa/core-dc233c.c b/target-xtensa/core-dc233c.c
index d643f41..11acbf3 100644
--- a/target-xtensa/core-dc233c.c
+++ b/target-xtensa/core-dc233c.c
@@ -26,10 +26,10 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
 #include "qemu-common.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 #include "core-dc233c/core-isa.h"
 #include "overlay_tool.h"
diff --git a/target-xtensa/core-fsf.c b/target-xtensa/core-fsf.c
index e36b0de..d4660ed 100644
--- a/target-xtensa/core-fsf.c
+++ b/target-xtensa/core-fsf.c
@@ -26,9 +26,9 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 
 #include "core-fsf/core-isa.h"
 #include "overlay_tool.h"
diff --git a/target-xtensa/cpu-qom.h b/target-xtensa/cpu-qom.h
index 1fd2f27..e344a9a 100644
--- a/target-xtensa/cpu-qom.h
+++ b/target-xtensa/cpu-qom.h
@@ -29,7 +29,7 @@
 #ifndef QEMU_XTENSA_CPU_QOM_H
 #define QEMU_XTENSA_CPU_QOM_H
 
-#include "qemu/cpu.h"
+#include "qom/cpu.h"
 #include "cpu.h"
 
 #define TYPE_XTENSA_CPU "xtensa-cpu"
diff --git a/target-xtensa/cpu.c b/target-xtensa/cpu.c
index 9d01983..ebc7e99 100644
--- a/target-xtensa/cpu.c
+++ b/target-xtensa/cpu.c
@@ -30,6 +30,7 @@
 
 #include "cpu.h"
 #include "qemu-common.h"
+#include "migration/vmstate.h"
 
 
 /* CPUClass::reset() */
@@ -48,6 +49,9 @@ static void xtensa_cpu_reset(CPUState *s)
             XTENSA_OPTION_INTERRUPT) ? 0x1f : 0x10;
     env->sregs[VECBASE] = env->config->vecbase;
     env->sregs[IBREAKENABLE] = 0;
+    env->sregs[CACHEATTR] = 0x22222222;
+    env->sregs[ATOMCTL] = xtensa_option_enabled(env->config,
+            XTENSA_OPTION_ATOMCTL) ? 0x28 : 0x15;
 
     env->pending_irq_level = 0;
     reset_mmu(env);
@@ -61,13 +65,21 @@ static void xtensa_cpu_initfn(Object *obj)
     cpu_exec_init(env);
 }
 
+static const VMStateDescription vmstate_xtensa_cpu = {
+    .name = "cpu",
+    .unmigratable = 1,
+};
+
 static void xtensa_cpu_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     CPUClass *cc = CPU_CLASS(oc);
     XtensaCPUClass *xcc = XTENSA_CPU_CLASS(cc);
 
     xcc->parent_reset = cc->reset;
     cc->reset = xtensa_cpu_reset;
+
+    dc->vmsd = &vmstate_xtensa_cpu;
 }
 
 static const TypeInfo xtensa_cpu_type_info = {
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index 74e9888..5acf78c 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -35,7 +35,7 @@
 
 #include "config.h"
 #include "qemu-common.h"
-#include "cpu-defs.h"
+#include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
@@ -65,6 +65,7 @@ enum {
     XTENSA_OPTION_FP_COPROCESSOR,
     XTENSA_OPTION_MP_SYNCHRO,
     XTENSA_OPTION_CONDITIONAL_STORE,
+    XTENSA_OPTION_ATOMCTL,
 
     /* Interrupts and exceptions */
     XTENSA_OPTION_EXCEPTION,
@@ -93,6 +94,7 @@ enum {
     XTENSA_OPTION_REGION_PROTECTION,
     XTENSA_OPTION_REGION_TRANSLATION,
     XTENSA_OPTION_MMU,
+    XTENSA_OPTION_CACHEATTR,
 
     /* Other */
     XTENSA_OPTION_WINDOWED_REGISTER,
@@ -128,6 +130,8 @@ enum {
     ITLBCFG = 91,
     DTLBCFG = 92,
     IBREAKENABLE = 96,
+    CACHEATTR = 98,
+    ATOMCTL = 99,
     IBREAKA = 128,
     DBREAKA = 144,
     DBREAKC = 160,
@@ -149,6 +153,7 @@ enum {
     ICOUNTLEVEL = 237,
     EXCVADDR = 238,
     CCOMPARE = 240,
+    MISC = 244,
 };
 
 #define PS_INTLEVEL 0xf
@@ -193,6 +198,14 @@ enum {
 
 #define REGION_PAGE_MASK 0xe0000000
 
+#define PAGE_CACHE_MASK    0x700
+#define PAGE_CACHE_SHIFT   8
+#define PAGE_CACHE_INVALID 0x000
+#define PAGE_CACHE_BYPASS  0x100
+#define PAGE_CACHE_WT      0x200
+#define PAGE_CACHE_WB      0x400
+#define PAGE_CACHE_ISOLATE 0x600
+
 enum {
     /* Static vectors */
     EXC_RESET,
@@ -404,6 +417,7 @@ void debug_exception_env(CPUXtensaState *new_env, uint32_t cause);
 
 
 #define XTENSA_OPTION_BIT(opt) (((uint64_t)1) << (opt))
+#define XTENSA_OPTION_ALL (~(uint64_t)0)
 
 static inline bool xtensa_option_bits_enabled(const XtensaConfig *config,
         uint64_t opt)
@@ -498,8 +512,8 @@ static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
     }
 }
 
-#include "cpu-all.h"
-#include "exec-all.h"
+#include "exec/cpu-all.h"
+#include "exec/exec-all.h"
 
 static inline int cpu_has_work(CPUState *cpu)
 {
diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c
index d94bae2..94c03a1 100644
--- a/target-xtensa/helper.c
+++ b/target-xtensa/helper.c
@@ -26,9 +26,9 @@
  */
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "gdbstub.h"
-#include "host-utils.h"
+#include "exec/exec-all.h"
+#include "exec/gdbstub.h"
+#include "qemu/host-utils.h"
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/loader.h"
 #endif
@@ -390,6 +390,7 @@ int xtensa_tlb_lookup(const CPUXtensaState *env, uint32_t addr, bool dtlb,
 static unsigned mmu_attr_to_access(uint32_t attr)
 {
     unsigned access = 0;
+
     if (attr < 12) {
         access |= PAGE_READ;
         if (attr & 0x1) {
@@ -398,8 +399,22 @@ static unsigned mmu_attr_to_access(uint32_t attr)
         if (attr & 0x2) {
             access |= PAGE_WRITE;
         }
+
+        switch (attr & 0xc) {
+        case 0:
+            access |= PAGE_CACHE_BYPASS;
+            break;
+
+        case 4:
+            access |= PAGE_CACHE_WB;
+            break;
+
+        case 8:
+            access |= PAGE_CACHE_WT;
+            break;
+        }
     } else if (attr == 13) {
-        access |= PAGE_READ | PAGE_WRITE;
+        access |= PAGE_READ | PAGE_WRITE | PAGE_CACHE_ISOLATE;
     }
     return access;
 }
@@ -410,14 +425,35 @@ static unsigned mmu_attr_to_access(uint32_t attr)
  */
 static unsigned region_attr_to_access(uint32_t attr)
 {
-    unsigned access = 0;
-    if ((attr < 6 && attr != 3) || attr == 14) {
-        access |= PAGE_READ | PAGE_WRITE;
-    }
-    if (attr > 0 && attr < 6) {
-        access |= PAGE_EXEC;
-    }
-    return access;
+    static const unsigned access[16] = {
+         [0] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_WT,
+         [1] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WT,
+         [2] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_BYPASS,
+         [3] =                          PAGE_EXEC | PAGE_CACHE_WB,
+         [4] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WB,
+         [5] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WB,
+        [14] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_ISOLATE,
+    };
+
+    return access[attr & 0xf];
+}
+
+/*!
+ * Convert cacheattr to PAGE_{READ,WRITE,EXEC} mask.
+ * See ISA, A.2.14 The Cache Attribute Register
+ */
+static unsigned cacheattr_attr_to_access(uint32_t attr)
+{
+    static const unsigned access[16] = {
+         [0] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_WT,
+         [1] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WT,
+         [2] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_BYPASS,
+         [3] =                          PAGE_EXEC | PAGE_CACHE_WB,
+         [4] = PAGE_READ | PAGE_WRITE | PAGE_EXEC | PAGE_CACHE_WB,
+        [14] = PAGE_READ | PAGE_WRITE             | PAGE_CACHE_ISOLATE,
+    };
+
+    return access[attr & 0xf];
 }
 
 static bool is_access_granted(unsigned access, int is_write)
@@ -486,7 +522,8 @@ static int get_physical_addr_mmu(CPUXtensaState *env, bool update_tlb,
             INST_FETCH_PRIVILEGE_CAUSE;
     }
 
-    *access = mmu_attr_to_access(entry->attr);
+    *access = mmu_attr_to_access(entry->attr) &
+        ~(dtlb ? PAGE_EXEC : PAGE_READ | PAGE_WRITE);
     if (!is_access_granted(*access, is_write)) {
         return dtlb ?
             (is_write ?
@@ -566,7 +603,8 @@ int xtensa_get_physical_addr(CPUXtensaState *env, bool update_tlb,
     } else {
         *paddr = vaddr;
         *page_size = TARGET_PAGE_SIZE;
-        *access = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+        *access = cacheattr_attr_to_access(
+                env->sregs[CACHEATTR] >> ((vaddr & 0xe0000000) >> 27));
         return 0;
     }
 }
@@ -599,24 +637,34 @@ static void dump_tlb(FILE *f, fprintf_function cpu_fprintf,
                 xtensa_tlb_get_entry(env, dtlb, wi, ei);
 
             if (entry->asid) {
+                static const char * const cache_text[8] = {
+                    [PAGE_CACHE_BYPASS >> PAGE_CACHE_SHIFT] = "Bypass",
+                    [PAGE_CACHE_WT >> PAGE_CACHE_SHIFT] = "WT",
+                    [PAGE_CACHE_WB >> PAGE_CACHE_SHIFT] = "WB",
+                    [PAGE_CACHE_ISOLATE >> PAGE_CACHE_SHIFT] = "Isolate",
+                };
                 unsigned access = attr_to_access(entry->attr);
+                unsigned cache_idx = (access & PAGE_CACHE_MASK) >>
+                    PAGE_CACHE_SHIFT;
 
                 if (print_header) {
                     print_header = false;
                     cpu_fprintf(f, "Way %u (%d %s)\n", wi, sz, sz_text);
                     cpu_fprintf(f,
-                            "\tVaddr       Paddr       ASID  Attr RWX\n"
-                            "\t----------  ----------  ----  ---- ---\n");
+                            "\tVaddr       Paddr       ASID  Attr RWX Cache\n"
+                            "\t----------  ----------  ----  ---- --- -------\n");
                 }
                 cpu_fprintf(f,
-                        "\t0x%08x  0x%08x  0x%02x  0x%02x %c%c%c\n",
+                        "\t0x%08x  0x%08x  0x%02x  0x%02x %c%c%c %-7s\n",
                         entry->vaddr,
                         entry->paddr,
                         entry->asid,
                         entry->attr,
                         (access & PAGE_READ) ? 'R' : '-',
                         (access & PAGE_WRITE) ? 'W' : '-',
-                        (access & PAGE_EXEC) ? 'X' : '-');
+                        (access & PAGE_EXEC) ? 'X' : '-',
+                        cache_text[cache_idx] ? cache_text[cache_idx] :
+                            "Invalid");
             }
         }
     }
diff --git a/target-xtensa/helper.h b/target-xtensa/helper.h
index 1163c09..38d7157 100644
--- a/target-xtensa/helper.h
+++ b/target-xtensa/helper.h
@@ -1,4 +1,4 @@
-#include "def-helper.h"
+#include "exec/def-helper.h"
 
 DEF_HELPER_2(exception, noreturn, env, i32)
 DEF_HELPER_3(exception_cause, noreturn, env, i32, i32)
@@ -23,6 +23,7 @@ DEF_HELPER_3(waiti, void, env, i32, i32)
 DEF_HELPER_3(timer_irq, void, env, i32, i32)
 DEF_HELPER_2(advance_ccount, void, env, i32)
 DEF_HELPER_1(check_interrupts, void, env)
+DEF_HELPER_3(check_atomctl, void, env, i32, i32)
 
 DEF_HELPER_2(wsr_rasid, void, env, i32)
 DEF_HELPER_FLAGS_3(rtlb0, TCG_CALL_NO_RWG_SE, i32, env, i32, i32)
@@ -57,4 +58,4 @@ DEF_HELPER_4(ult_s, void, env, i32, f32, f32)
 DEF_HELPER_4(ole_s, void, env, i32, f32, f32)
 DEF_HELPER_4(ule_s, void, env, i32, f32, f32)
 
-#include "def-helper.h"
+#include "exec/def-helper.h"
diff --git a/target-xtensa/machine.c b/target-xtensa/machine.c
deleted file mode 100644
index ddeffb2..0000000
--- a/target-xtensa/machine.c
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2011, Max Filippov, Open Source and Linux Lab.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the Open Source and Linux Lab nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "hw/hw.h"
-#include "hw/boards.h"
-
-void cpu_save(QEMUFile *f, void *opaque)
-{
-}
-
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-    return 0;
-}
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index ae0c099..3813a72 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -27,7 +27,7 @@
 
 #include "cpu.h"
 #include "helper.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 static void do_unaligned_access(CPUXtensaState *env,
         target_ulong addr, int is_write, int is_user, uintptr_t retaddr);
@@ -36,33 +36,23 @@ static void do_unaligned_access(CPUXtensaState *env,
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 1
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 2
-#include "softmmu_template.h"
+#include "exec/softmmu_template.h"
 
 #define SHIFT 3
-#include "softmmu_template.h"
-
-static void do_restore_state(CPUXtensaState *env, uintptr_t pc)
-{
-    TranslationBlock *tb;
-
-    tb = tb_find_pc(pc);
-    if (tb) {
-        cpu_restore_state(tb, env, pc);
-    }
-}
+#include "exec/softmmu_template.h"
 
 static void do_unaligned_access(CPUXtensaState *env,
         target_ulong addr, int is_write, int is_user, uintptr_t retaddr)
 {
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_UNALIGNED_EXCEPTION) &&
             !xtensa_option_enabled(env->config, XTENSA_OPTION_HW_ALIGNMENT)) {
-        do_restore_state(env, retaddr);
+        cpu_restore_state(env, retaddr);
         HELPER(exception_cause_vaddr)(env,
                 env->pc, LOAD_STORE_ALIGNMENT_CAUSE, addr);
     }
@@ -86,7 +76,7 @@ void tlb_fill(CPUXtensaState *env,
                 paddr & TARGET_PAGE_MASK,
                 access, mmu_idx, page_size);
     } else {
-        do_restore_state(env, retaddr);
+        cpu_restore_state(env, retaddr);
         HELPER(exception_cause_vaddr)(env, env->pc, ret, vaddr);
     }
 }
@@ -415,6 +405,63 @@ void HELPER(check_interrupts)(CPUXtensaState *env)
     check_interrupts(env);
 }
 
+/*!
+ * Check vaddr accessibility/cache attributes and raise an exception if
+ * specified by the ATOMCTL SR.
+ *
+ * Note: local memory exclusion is not implemented
+ */
+void HELPER(check_atomctl)(CPUXtensaState *env, uint32_t pc, uint32_t vaddr)
+{
+    uint32_t paddr, page_size, access;
+    uint32_t atomctl = env->sregs[ATOMCTL];
+    int rc = xtensa_get_physical_addr(env, true, vaddr, 1,
+            xtensa_get_cring(env), &paddr, &page_size, &access);
+
+    /*
+     * s32c1i never causes LOAD_PROHIBITED_CAUSE exceptions,
+     * see opcode description in the ISA
+     */
+    if (rc == 0 &&
+            (access & (PAGE_READ | PAGE_WRITE)) != (PAGE_READ | PAGE_WRITE)) {
+        rc = STORE_PROHIBITED_CAUSE;
+    }
+
+    if (rc) {
+        HELPER(exception_cause_vaddr)(env, pc, rc, vaddr);
+    }
+
+    /*
+     * When data cache is not configured use ATOMCTL bypass field.
+     * See ISA, 4.3.12.4 The Atomic Operation Control Register (ATOMCTL)
+     * under the Conditional Store Option.
+     */
+    if (!xtensa_option_enabled(env->config, XTENSA_OPTION_DCACHE)) {
+        access = PAGE_CACHE_BYPASS;
+    }
+
+    switch (access & PAGE_CACHE_MASK) {
+    case PAGE_CACHE_WB:
+        atomctl >>= 2;
+    case PAGE_CACHE_WT:
+        atomctl >>= 2;
+    case PAGE_CACHE_BYPASS:
+        if ((atomctl & 0x3) == 0) {
+            HELPER(exception_cause_vaddr)(env, pc,
+                    LOAD_STORE_ERROR_CAUSE, vaddr);
+        }
+        break;
+
+    case PAGE_CACHE_ISOLATE:
+        HELPER(exception_cause_vaddr)(env, pc,
+                LOAD_STORE_ERROR_CAUSE, vaddr);
+        break;
+
+    default:
+        break;
+    }
+}
+
 void HELPER(wsr_rasid)(CPUXtensaState *env, uint32_t v)
 {
     v = (v & 0xffffff00) | 0x1;
diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h
index e395053..dd4f51a 100644
--- a/target-xtensa/overlay_tool.h
+++ b/target-xtensa/overlay_tool.h
@@ -42,6 +42,10 @@
 #define XCHAL_VECBASE_RESET_VADDR 0
 #endif
 
+#ifndef XCHAL_HW_MIN_VERSION
+#define XCHAL_HW_MIN_VERSION 0
+#endif
+
 #define XCHAL_OPTION(xchal, qemu) ((xchal) ? XTENSA_OPTION_BIT(qemu) : 0)
 
 #define XTENSA_OPTIONS ( \
@@ -62,6 +66,8 @@
     XCHAL_OPTION(XCHAL_HAVE_FP, XTENSA_OPTION_FP_COPROCESSOR) | \
     XCHAL_OPTION(XCHAL_HAVE_RELEASE_SYNC, XTENSA_OPTION_MP_SYNCHRO) | \
     XCHAL_OPTION(XCHAL_HAVE_S32C1I, XTENSA_OPTION_CONDITIONAL_STORE) | \
+    XCHAL_OPTION(XCHAL_HAVE_S32C1I && XCHAL_HW_MIN_VERSION >= 230000, \
+        XTENSA_OPTION_ATOMCTL) | \
     /* Interrupts and exceptions */ \
     XCHAL_OPTION(XCHAL_HAVE_EXCEPTIONS, XTENSA_OPTION_EXCEPTION) | \
     XCHAL_OPTION(XCHAL_HAVE_VECBASE, XTENSA_OPTION_RELOCATABLE_VECTOR) | \
@@ -85,9 +91,13 @@
     XCHAL_OPTION(XCHAL_HAVE_XLT_CACHEATTR, \
             XTENSA_OPTION_REGION_TRANSLATION) | \
     XCHAL_OPTION(XCHAL_HAVE_PTP_MMU, XTENSA_OPTION_MMU) | \
+    XCHAL_OPTION(XCHAL_HAVE_CACHEATTR, XTENSA_OPTION_CACHEATTR) | \
     /* Other, TODO */ \
     XCHAL_OPTION(XCHAL_HAVE_WINDOWED, XTENSA_OPTION_WINDOWED_REGISTER) | \
-    XCHAL_OPTION(XCHAL_HAVE_DEBUG, XTENSA_OPTION_DEBUG))
+    XCHAL_OPTION(XCHAL_HAVE_DEBUG, XTENSA_OPTION_DEBUG) |\
+    XCHAL_OPTION(XCHAL_NUM_MISC_REGS > 0, XTENSA_OPTION_MISC_SR) | \
+    XCHAL_OPTION(XCHAL_HAVE_THREADPTR, XTENSA_OPTION_THREAD_POINTER) | \
+    XCHAL_OPTION(XCHAL_HAVE_PRID, XTENSA_OPTION_PROCESSOR_ID))
 
 #ifndef XCHAL_WINDOW_OF4_VECOFS
 #define XCHAL_WINDOW_OF4_VECOFS         0x00000000
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index e5a3f49..7029ac4 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -31,11 +31,11 @@
 #include <stdio.h>
 
 #include "cpu.h"
-#include "exec-all.h"
-#include "disas.h"
+#include "exec/exec-all.h"
+#include "disas/disas.h"
 #include "tcg-op.h"
-#include "qemu-log.h"
-#include "sysemu.h"
+#include "qemu/log.h"
+#include "sysemu/sysemu.h"
 
 #include "helper.h"
 #define GEN_HELPER 1
@@ -76,78 +76,119 @@ static TCGv_i32 cpu_FR[16];
 static TCGv_i32 cpu_SR[256];
 static TCGv_i32 cpu_UR[256];
 
-#include "gen-icount.h"
-
-static const char * const sregnames[256] = {
-    [LBEG] = "LBEG",
-    [LEND] = "LEND",
-    [LCOUNT] = "LCOUNT",
-    [SAR] = "SAR",
-    [BR] = "BR",
-    [LITBASE] = "LITBASE",
-    [SCOMPARE1] = "SCOMPARE1",
-    [ACCLO] = "ACCLO",
-    [ACCHI] = "ACCHI",
-    [MR] = "MR0",
-    [MR + 1] = "MR1",
-    [MR + 2] = "MR2",
-    [MR + 3] = "MR3",
-    [WINDOW_BASE] = "WINDOW_BASE",
-    [WINDOW_START] = "WINDOW_START",
-    [PTEVADDR] = "PTEVADDR",
-    [RASID] = "RASID",
-    [ITLBCFG] = "ITLBCFG",
-    [DTLBCFG] = "DTLBCFG",
-    [IBREAKENABLE] = "IBREAKENABLE",
-    [IBREAKA] = "IBREAKA0",
-    [IBREAKA + 1] = "IBREAKA1",
-    [DBREAKA] = "DBREAKA0",
-    [DBREAKA + 1] = "DBREAKA1",
-    [DBREAKC] = "DBREAKC0",
-    [DBREAKC + 1] = "DBREAKC1",
-    [EPC1] = "EPC1",
-    [EPC1 + 1] = "EPC2",
-    [EPC1 + 2] = "EPC3",
-    [EPC1 + 3] = "EPC4",
-    [EPC1 + 4] = "EPC5",
-    [EPC1 + 5] = "EPC6",
-    [EPC1 + 6] = "EPC7",
-    [DEPC] = "DEPC",
-    [EPS2] = "EPS2",
-    [EPS2 + 1] = "EPS3",
-    [EPS2 + 2] = "EPS4",
-    [EPS2 + 3] = "EPS5",
-    [EPS2 + 4] = "EPS6",
-    [EPS2 + 5] = "EPS7",
-    [EXCSAVE1] = "EXCSAVE1",
-    [EXCSAVE1 + 1] = "EXCSAVE2",
-    [EXCSAVE1 + 2] = "EXCSAVE3",
-    [EXCSAVE1 + 3] = "EXCSAVE4",
-    [EXCSAVE1 + 4] = "EXCSAVE5",
-    [EXCSAVE1 + 5] = "EXCSAVE6",
-    [EXCSAVE1 + 6] = "EXCSAVE7",
-    [CPENABLE] = "CPENABLE",
-    [INTSET] = "INTSET",
-    [INTCLEAR] = "INTCLEAR",
-    [INTENABLE] = "INTENABLE",
-    [PS] = "PS",
-    [VECBASE] = "VECBASE",
-    [EXCCAUSE] = "EXCCAUSE",
-    [DEBUGCAUSE] = "DEBUGCAUSE",
-    [CCOUNT] = "CCOUNT",
-    [PRID] = "PRID",
-    [ICOUNT] = "ICOUNT",
-    [ICOUNTLEVEL] = "ICOUNTLEVEL",
-    [EXCVADDR] = "EXCVADDR",
-    [CCOMPARE] = "CCOMPARE0",
-    [CCOMPARE + 1] = "CCOMPARE1",
-    [CCOMPARE + 2] = "CCOMPARE2",
+#include "exec/gen-icount.h"
+
+typedef struct XtensaReg {
+    const char *name;
+    uint64_t opt_bits;
+    enum {
+        SR_R = 1,
+        SR_W = 2,
+        SR_X = 4,
+        SR_RW = 3,
+        SR_RWX = 7,
+    } access;
+} XtensaReg;
+
+#define XTENSA_REG_ACCESS(regname, opt, acc) { \
+        .name = (regname), \
+        .opt_bits = XTENSA_OPTION_BIT(opt), \
+        .access = (acc), \
+    }
+
+#define XTENSA_REG(regname, opt) XTENSA_REG_ACCESS(regname, opt, SR_RWX)
+
+#define XTENSA_REG_BITS(regname, opt) { \
+        .name = (regname), \
+        .opt_bits = (opt), \
+        .access = SR_RWX, \
+    }
+
+static const XtensaReg sregnames[256] = {
+    [LBEG] = XTENSA_REG("LBEG", XTENSA_OPTION_LOOP),
+    [LEND] = XTENSA_REG("LEND", XTENSA_OPTION_LOOP),
+    [LCOUNT] = XTENSA_REG("LCOUNT", XTENSA_OPTION_LOOP),
+    [SAR] = XTENSA_REG_BITS("SAR", XTENSA_OPTION_ALL),
+    [BR] = XTENSA_REG("BR", XTENSA_OPTION_BOOLEAN),
+    [LITBASE] = XTENSA_REG("LITBASE", XTENSA_OPTION_EXTENDED_L32R),
+    [SCOMPARE1] = XTENSA_REG("SCOMPARE1", XTENSA_OPTION_CONDITIONAL_STORE),
+    [ACCLO] = XTENSA_REG("ACCLO", XTENSA_OPTION_MAC16),
+    [ACCHI] = XTENSA_REG("ACCHI", XTENSA_OPTION_MAC16),
+    [MR] = XTENSA_REG("MR0", XTENSA_OPTION_MAC16),
+    [MR + 1] = XTENSA_REG("MR1", XTENSA_OPTION_MAC16),
+    [MR + 2] = XTENSA_REG("MR2", XTENSA_OPTION_MAC16),
+    [MR + 3] = XTENSA_REG("MR3", XTENSA_OPTION_MAC16),
+    [WINDOW_BASE] = XTENSA_REG("WINDOW_BASE", XTENSA_OPTION_WINDOWED_REGISTER),
+    [WINDOW_START] = XTENSA_REG("WINDOW_START",
+            XTENSA_OPTION_WINDOWED_REGISTER),
+    [PTEVADDR] = XTENSA_REG("PTEVADDR", XTENSA_OPTION_MMU),
+    [RASID] = XTENSA_REG("RASID", XTENSA_OPTION_MMU),
+    [ITLBCFG] = XTENSA_REG("ITLBCFG", XTENSA_OPTION_MMU),
+    [DTLBCFG] = XTENSA_REG("DTLBCFG", XTENSA_OPTION_MMU),
+    [IBREAKENABLE] = XTENSA_REG("IBREAKENABLE", XTENSA_OPTION_DEBUG),
+    [CACHEATTR] = XTENSA_REG("CACHEATTR", XTENSA_OPTION_CACHEATTR),
+    [ATOMCTL] = XTENSA_REG("ATOMCTL", XTENSA_OPTION_ATOMCTL),
+    [IBREAKA] = XTENSA_REG("IBREAKA0", XTENSA_OPTION_DEBUG),
+    [IBREAKA + 1] = XTENSA_REG("IBREAKA1", XTENSA_OPTION_DEBUG),
+    [DBREAKA] = XTENSA_REG("DBREAKA0", XTENSA_OPTION_DEBUG),
+    [DBREAKA + 1] = XTENSA_REG("DBREAKA1", XTENSA_OPTION_DEBUG),
+    [DBREAKC] = XTENSA_REG("DBREAKC0", XTENSA_OPTION_DEBUG),
+    [DBREAKC + 1] = XTENSA_REG("DBREAKC1", XTENSA_OPTION_DEBUG),
+    [EPC1] = XTENSA_REG("EPC1", XTENSA_OPTION_EXCEPTION),
+    [EPC1 + 1] = XTENSA_REG("EPC2", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 2] = XTENSA_REG("EPC3", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 3] = XTENSA_REG("EPC4", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 4] = XTENSA_REG("EPC5", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 5] = XTENSA_REG("EPC6", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPC1 + 6] = XTENSA_REG("EPC7", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [DEPC] = XTENSA_REG("DEPC", XTENSA_OPTION_EXCEPTION),
+    [EPS2] = XTENSA_REG("EPS2", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 1] = XTENSA_REG("EPS3", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 2] = XTENSA_REG("EPS4", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 3] = XTENSA_REG("EPS5", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 4] = XTENSA_REG("EPS6", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EPS2 + 5] = XTENSA_REG("EPS7", XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1] = XTENSA_REG("EXCSAVE1", XTENSA_OPTION_EXCEPTION),
+    [EXCSAVE1 + 1] = XTENSA_REG("EXCSAVE2",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 2] = XTENSA_REG("EXCSAVE3",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 3] = XTENSA_REG("EXCSAVE4",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 4] = XTENSA_REG("EXCSAVE5",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 5] = XTENSA_REG("EXCSAVE6",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [EXCSAVE1 + 6] = XTENSA_REG("EXCSAVE7",
+            XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT),
+    [CPENABLE] = XTENSA_REG("CPENABLE", XTENSA_OPTION_COPROCESSOR),
+    [INTSET] = XTENSA_REG_ACCESS("INTSET", XTENSA_OPTION_INTERRUPT, SR_RW),
+    [INTCLEAR] = XTENSA_REG_ACCESS("INTCLEAR", XTENSA_OPTION_INTERRUPT, SR_W),
+    [INTENABLE] = XTENSA_REG("INTENABLE", XTENSA_OPTION_INTERRUPT),
+    [PS] = XTENSA_REG_BITS("PS", XTENSA_OPTION_ALL),
+    [VECBASE] = XTENSA_REG("VECBASE", XTENSA_OPTION_RELOCATABLE_VECTOR),
+    [EXCCAUSE] = XTENSA_REG("EXCCAUSE", XTENSA_OPTION_EXCEPTION),
+    [DEBUGCAUSE] = XTENSA_REG_ACCESS("DEBUGCAUSE", XTENSA_OPTION_DEBUG, SR_R),
+    [CCOUNT] = XTENSA_REG("CCOUNT", XTENSA_OPTION_TIMER_INTERRUPT),
+    [PRID] = XTENSA_REG_ACCESS("PRID", XTENSA_OPTION_PROCESSOR_ID, SR_R),
+    [ICOUNT] = XTENSA_REG("ICOUNT", XTENSA_OPTION_DEBUG),
+    [ICOUNTLEVEL] = XTENSA_REG("ICOUNTLEVEL", XTENSA_OPTION_DEBUG),
+    [EXCVADDR] = XTENSA_REG("EXCVADDR", XTENSA_OPTION_EXCEPTION),
+    [CCOMPARE] = XTENSA_REG("CCOMPARE0", XTENSA_OPTION_TIMER_INTERRUPT),
+    [CCOMPARE + 1] = XTENSA_REG("CCOMPARE1",
+            XTENSA_OPTION_TIMER_INTERRUPT),
+    [CCOMPARE + 2] = XTENSA_REG("CCOMPARE2",
+            XTENSA_OPTION_TIMER_INTERRUPT),
+    [MISC] = XTENSA_REG("MISC0", XTENSA_OPTION_MISC_SR),
+    [MISC + 1] = XTENSA_REG("MISC1", XTENSA_OPTION_MISC_SR),
+    [MISC + 2] = XTENSA_REG("MISC2", XTENSA_OPTION_MISC_SR),
+    [MISC + 3] = XTENSA_REG("MISC3", XTENSA_OPTION_MISC_SR),
 };
 
-static const char * const uregnames[256] = {
-    [THREADPTR] = "THREADPTR",
-    [FCR] = "FCR",
-    [FSR] = "FSR",
+static const XtensaReg uregnames[256] = {
+    [THREADPTR] = XTENSA_REG("THREADPTR", XTENSA_OPTION_THREAD_POINTER),
+    [FCR] = XTENSA_REG("FCR", XTENSA_OPTION_FP_COPROCESSOR),
+    [FSR] = XTENSA_REG("FSR", XTENSA_OPTION_FP_COPROCESSOR),
 };
 
 void xtensa_translate_init(void)
@@ -183,18 +224,18 @@ void xtensa_translate_init(void)
     }
 
     for (i = 0; i < 256; ++i) {
-        if (sregnames[i]) {
+        if (sregnames[i].name) {
             cpu_SR[i] = tcg_global_mem_new_i32(TCG_AREG0,
                     offsetof(CPUXtensaState, sregs[i]),
-                    sregnames[i]);
+                    sregnames[i].name);
         }
     }
 
     for (i = 0; i < 256; ++i) {
-        if (uregnames[i]) {
+        if (uregnames[i].name) {
             cpu_UR[i] = tcg_global_mem_new_i32(TCG_AREG0,
                     offsetof(CPUXtensaState, uregs[i]),
-                    uregnames[i]);
+                    uregnames[i].name);
         }
     }
 #define GEN_HELPER 2
@@ -450,6 +491,28 @@ static void gen_brcondi(DisasContext *dc, TCGCond cond,
     tcg_temp_free(tmp);
 }
 
+static void gen_check_sr(DisasContext *dc, uint32_t sr, unsigned access)
+{
+    if (!xtensa_option_bits_enabled(dc->config, sregnames[sr].opt_bits)) {
+        if (sregnames[sr].name) {
+            qemu_log("SR %s is not configured\n", sregnames[sr].name);
+        } else {
+            qemu_log("SR %d is not implemented\n", sr);
+        }
+        gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
+    } else if (!(sregnames[sr].access & access)) {
+        static const char * const access_text[] = {
+            [SR_R] = "rsr",
+            [SR_W] = "wsr",
+            [SR_X] = "xsr",
+        };
+        assert(access < ARRAY_SIZE(access_text) && access_text[access]);
+        qemu_log("SR %s is not available for %s\n", sregnames[sr].name,
+                access_text[access]);
+        gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
+    }
+}
+
 static void gen_rsr_ccount(DisasContext *dc, TCGv_i32 d, uint32_t sr)
 {
     gen_advance_ccount(dc);
@@ -471,14 +534,10 @@ static void gen_rsr(DisasContext *dc, TCGv_i32 d, uint32_t sr)
         [PTEVADDR] = gen_rsr_ptevaddr,
     };
 
-    if (sregnames[sr]) {
-        if (rsr_handler[sr]) {
-            rsr_handler[sr](dc, d, sr);
-        } else {
-            tcg_gen_mov_i32(d, cpu_SR[sr]);
-        }
+    if (rsr_handler[sr]) {
+        rsr_handler[sr](dc, d, sr);
     } else {
-        qemu_log("RSR %d not implemented, ", sr);
+        tcg_gen_mov_i32(d, cpu_SR[sr]);
     }
 }
 
@@ -556,6 +615,11 @@ static void gen_wsr_ibreakenable(DisasContext *dc, uint32_t sr, TCGv_i32 v)
     gen_jumpi_check_loop_end(dc, 0);
 }
 
+static void gen_wsr_atomctl(DisasContext *dc, uint32_t sr, TCGv_i32 v)
+{
+    tcg_gen_andi_i32(cpu_SR[sr], v, 0x3f);
+}
+
 static void gen_wsr_ibreaka(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     unsigned id = sr - IBREAKA;
@@ -640,14 +704,6 @@ static void gen_wsr_ps(DisasContext *dc, uint32_t sr, TCGv_i32 v)
     gen_jumpi_check_loop_end(dc, -1);
 }
 
-static void gen_wsr_debugcause(DisasContext *dc, uint32_t sr, TCGv_i32 v)
-{
-}
-
-static void gen_wsr_prid(DisasContext *dc, uint32_t sr, TCGv_i32 v)
-{
-}
-
 static void gen_wsr_icount(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     if (dc->icount) {
@@ -693,6 +749,7 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
         [ITLBCFG] = gen_wsr_tlbcfg,
         [DTLBCFG] = gen_wsr_tlbcfg,
         [IBREAKENABLE] = gen_wsr_ibreakenable,
+        [ATOMCTL] = gen_wsr_atomctl,
         [IBREAKA] = gen_wsr_ibreaka,
         [IBREAKA + 1] = gen_wsr_ibreaka,
         [DBREAKA] = gen_wsr_dbreaka,
@@ -704,8 +761,6 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
         [INTCLEAR] = gen_wsr_intclear,
         [INTENABLE] = gen_wsr_intenable,
         [PS] = gen_wsr_ps,
-        [DEBUGCAUSE] = gen_wsr_debugcause,
-        [PRID] = gen_wsr_prid,
         [ICOUNT] = gen_wsr_icount,
         [ICOUNTLEVEL] = gen_wsr_icountlevel,
         [CCOMPARE] = gen_wsr_ccompare,
@@ -713,14 +768,10 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
         [CCOMPARE + 2] = gen_wsr_ccompare,
     };
 
-    if (sregnames[sr]) {
-        if (wsr_handler[sr]) {
-            wsr_handler[sr](dc, sr, s);
-        } else {
-            tcg_gen_mov_i32(cpu_SR[sr], s);
-        }
+    if (wsr_handler[sr]) {
+        wsr_handler[sr](dc, sr, s);
     } else {
-        qemu_log("WSR %d not implemented, ", sr);
+        tcg_gen_mov_i32(cpu_SR[sr], s);
     }
 }
 
@@ -1352,12 +1403,14 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 
                 case 1: /*ABS*/
                     {
-                        int label = gen_new_label();
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
-                        tcg_gen_brcondi_i32(
-                                TCG_COND_GE, cpu_R[RRR_R], 0, label);
-                        tcg_gen_neg_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
-                        gen_set_label(label);
+                        TCGv_i32 zero = tcg_const_i32(0);
+                        TCGv_i32 neg = tcg_temp_new_i32();
+
+                        tcg_gen_neg_i32(neg, cpu_R[RRR_T]);
+                        tcg_gen_movcond_i32(TCG_COND_GE, cpu_R[RRR_R],
+                                cpu_R[RRR_T], zero, cpu_R[RRR_T], neg);
+                        tcg_temp_free(neg);
+                        tcg_temp_free(zero);
                     }
                     break;
 
@@ -1431,6 +1484,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
             case 6: /*XSR*/
                 {
                     TCGv_i32 tmp = tcg_temp_new_i32();
+                    gen_check_sr(dc, RSR_SR, SR_X);
                     if (RSR_SR >= 64) {
                         gen_check_privilege(dc);
                     }
@@ -1439,9 +1493,6 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                     gen_rsr(dc, cpu_R[RRR_T], RSR_SR);
                     gen_wsr(dc, RSR_SR, tmp);
                     tcg_temp_free(tmp);
-                    if (!sregnames[RSR_SR]) {
-                        TBD();
-                    }
                 }
                 break;
 
@@ -1664,25 +1715,21 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
         case 3: /*RST3*/
             switch (OP2) {
             case 0: /*RSR*/
+                gen_check_sr(dc, RSR_SR, SR_R);
                 if (RSR_SR >= 64) {
                     gen_check_privilege(dc);
                 }
                 gen_window_check1(dc, RRR_T);
                 gen_rsr(dc, cpu_R[RRR_T], RSR_SR);
-                if (!sregnames[RSR_SR]) {
-                    TBD();
-                }
                 break;
 
             case 1: /*WSR*/
+                gen_check_sr(dc, RSR_SR, SR_W);
                 if (RSR_SR >= 64) {
                     gen_check_privilege(dc);
                 }
                 gen_window_check1(dc, RRR_T);
                 gen_wsr(dc, RSR_SR, cpu_R[RRR_T]);
-                if (!sregnames[RSR_SR]) {
-                    TBD();
-                }
                 break;
 
             case 2: /*SEXTu*/
@@ -1710,22 +1757,20 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 {
                     TCGv_i32 tmp1 = tcg_temp_new_i32();
                     TCGv_i32 tmp2 = tcg_temp_new_i32();
-                    int label = gen_new_label();
+                    TCGv_i32 zero = tcg_const_i32(0);
 
                     tcg_gen_sari_i32(tmp1, cpu_R[RRR_S], 24 - RRR_T);
                     tcg_gen_xor_i32(tmp2, tmp1, cpu_R[RRR_S]);
                     tcg_gen_andi_i32(tmp2, tmp2, 0xffffffff << (RRR_T + 7));
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    tcg_gen_brcondi_i32(TCG_COND_EQ, tmp2, 0, label);
 
                     tcg_gen_sari_i32(tmp1, cpu_R[RRR_S], 31);
-                    tcg_gen_xori_i32(cpu_R[RRR_R], tmp1,
-                            0xffffffff >> (25 - RRR_T));
-
-                    gen_set_label(label);
+                    tcg_gen_xori_i32(tmp1, tmp1, 0xffffffff >> (25 - RRR_T));
 
+                    tcg_gen_movcond_i32(TCG_COND_EQ, cpu_R[RRR_R], tmp2, zero,
+                            cpu_R[RRR_S], tmp1);
                     tcg_temp_free(tmp1);
                     tcg_temp_free(tmp2);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -1742,19 +1787,9 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                         TCG_COND_LEU,
                         TCG_COND_GEU
                     };
-                    int label = gen_new_label();
-
-                    if (RRR_R != RRR_T) {
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                        tcg_gen_brcond_i32(cond[OP2 - 4],
-                                cpu_R[RRR_S], cpu_R[RRR_T], label);
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
-                    } else {
-                        tcg_gen_brcond_i32(cond[OP2 - 4],
-                                cpu_R[RRR_T], cpu_R[RRR_S], label);
-                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    }
-                    gen_set_label(label);
+                    tcg_gen_movcond_i32(cond[OP2 - 4], cpu_R[RRR_R],
+                            cpu_R[RRR_S], cpu_R[RRR_T],
+                            cpu_R[RRR_S], cpu_R[RRR_T]);
                 }
                 break;
 
@@ -1765,15 +1800,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
                 {
                     static const TCGCond cond[] = {
-                        TCG_COND_NE,
                         TCG_COND_EQ,
+                        TCG_COND_NE,
+                        TCG_COND_LT,
                         TCG_COND_GE,
-                        TCG_COND_LT
                     };
-                    int label = gen_new_label();
-                    tcg_gen_brcondi_i32(cond[OP2 - 8], cpu_R[RRR_T], 0, label);
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    gen_set_label(label);
+                    TCGv_i32 zero = tcg_const_i32(0);
+
+                    tcg_gen_movcond_i32(cond[OP2 - 8], cpu_R[RRR_R],
+                            cpu_R[RRR_T], zero, cpu_R[RRR_S], cpu_R[RRR_R]);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -1782,16 +1818,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 HAS_OPTION(XTENSA_OPTION_BOOLEAN);
                 gen_window_check2(dc, RRR_R, RRR_S);
                 {
-                    int label = gen_new_label();
+                    TCGv_i32 zero = tcg_const_i32(0);
                     TCGv_i32 tmp = tcg_temp_new_i32();
 
                     tcg_gen_andi_i32(tmp, cpu_SR[BR], 1 << RRR_T);
-                    tcg_gen_brcondi_i32(
-                            OP2 & 1 ? TCG_COND_EQ : TCG_COND_NE,
-                            tmp, 0, label);
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_R[RRR_S]);
-                    gen_set_label(label);
+                    tcg_gen_movcond_i32(OP2 & 1 ? TCG_COND_NE : TCG_COND_EQ,
+                            cpu_R[RRR_R], tmp, zero,
+                            cpu_R[RRR_S], cpu_R[RRR_R]);
+
                     tcg_temp_free(tmp);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -1799,7 +1835,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 gen_window_check1(dc, RRR_R);
                 {
                     int st = (RRR_S << 4) + RRR_T;
-                    if (uregnames[st]) {
+                    if (uregnames[st].name) {
                         tcg_gen_mov_i32(cpu_R[RRR_R], cpu_UR[st]);
                     } else {
                         qemu_log("RUR %d not implemented, ", st);
@@ -1810,7 +1846,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 
             case 15: /*WUR*/
                 gen_window_check1(dc, RRR_T);
-                if (uregnames[RSR_SR]) {
+                if (uregnames[RSR_SR].name) {
                     gen_wur(RSR_SR, cpu_R[RRR_T]);
                 } else {
                     qemu_log("WUR %d not implemented, ", RSR_SR);
@@ -2082,15 +2118,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 gen_check_cpenable(dc, 0);
                 {
                     static const TCGCond cond[] = {
-                        TCG_COND_NE,
                         TCG_COND_EQ,
+                        TCG_COND_NE,
+                        TCG_COND_LT,
                         TCG_COND_GE,
-                        TCG_COND_LT
                     };
-                    int label = gen_new_label();
-                    tcg_gen_brcondi_i32(cond[OP2 - 8], cpu_R[RRR_T], 0, label);
-                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
-                    gen_set_label(label);
+                    TCGv_i32 zero = tcg_const_i32(0);
+
+                    tcg_gen_movcond_i32(cond[OP2 - 8], cpu_FR[RRR_R],
+                            cpu_R[RRR_T], zero, cpu_FR[RRR_S], cpu_FR[RRR_R]);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -2099,16 +2136,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 HAS_OPTION(XTENSA_OPTION_BOOLEAN);
                 gen_check_cpenable(dc, 0);
                 {
-                    int label = gen_new_label();
+                    TCGv_i32 zero = tcg_const_i32(0);
                     TCGv_i32 tmp = tcg_temp_new_i32();
 
                     tcg_gen_andi_i32(tmp, cpu_SR[BR], 1 << RRR_T);
-                    tcg_gen_brcondi_i32(
-                            OP2 & 1 ? TCG_COND_EQ : TCG_COND_NE,
-                            tmp, 0, label);
-                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
-                    gen_set_label(label);
+                    tcg_gen_movcond_i32(OP2 & 1 ? TCG_COND_NE : TCG_COND_EQ,
+                            cpu_FR[RRR_R], tmp, zero,
+                            cpu_FR[RRR_S], cpu_FR[RRR_R]);
+
                     tcg_temp_free(tmp);
+                    tcg_temp_free(zero);
                 }
                 break;
 
@@ -2317,10 +2354,15 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 int label = gen_new_label();
                 TCGv_i32 tmp = tcg_temp_local_new_i32();
                 TCGv_i32 addr = tcg_temp_local_new_i32();
+                TCGv_i32 tpc;
 
                 tcg_gen_mov_i32(tmp, cpu_R[RRI8_T]);
                 tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2);
                 gen_load_store_alignment(dc, 2, addr, true);
+
+                gen_advance_ccount(dc);
+                tpc = tcg_const_i32(dc->pc);
+                gen_helper_check_atomctl(cpu_env, tpc, addr);
                 tcg_gen_qemu_ld32u(cpu_R[RRI8_T], addr, dc->cring);
                 tcg_gen_brcond_i32(TCG_COND_NE, cpu_R[RRI8_T],
                         cpu_SR[SCOMPARE1], label);
@@ -2328,6 +2370,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 tcg_gen_qemu_st32(tmp, addr, dc->cring);
 
                 gen_set_label(label);
+                tcg_temp_free(tpc);
                 tcg_temp_free(addr);
                 tcg_temp_free(tmp);
             }
@@ -2897,12 +2940,12 @@ static void gen_intermediate_code_internal(
             if (lj < j) {
                 lj++;
                 while (lj < j) {
-                    gen_opc_instr_start[lj++] = 0;
+                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
                 }
             }
-            gen_opc_pc[lj] = dc.pc;
-            gen_opc_instr_start[lj] = 1;
-            gen_opc_icount[lj] = insn_count;
+            tcg_ctx.gen_opc_pc[lj] = dc.pc;
+            tcg_ctx.gen_opc_instr_start[lj] = 1;
+            tcg_ctx.gen_opc_icount[lj] = insn_count;
         }
 
         if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
@@ -2962,7 +3005,11 @@ static void gen_intermediate_code_internal(
     gen_icount_end(tb, insn_count);
     *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
-    if (!search_pc) {
+    if (search_pc) {
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
+        memset(tcg_ctx.gen_opc_instr_start + lj + 1, 0,
+                (j - lj) * sizeof(tcg_ctx.gen_opc_instr_start[0]));
+    } else {
         tb->size = dc.pc - pc_start;
         tb->icount = insn_count;
     }
@@ -2986,8 +3033,8 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "PC=%08x\n\n", env->pc);
 
     for (i = j = 0; i < 256; ++i) {
-        if (sregnames[i]) {
-            cpu_fprintf(f, "%s=%08x%c", sregnames[i], env->sregs[i],
+        if (xtensa_option_bits_enabled(env->config, sregnames[i].opt_bits)) {
+            cpu_fprintf(f, "%12s=%08x%c", sregnames[i].name, env->sregs[i],
                     (j++ % 4) == 3 ? '\n' : ' ');
         }
     }
@@ -2995,8 +3042,8 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, (j % 4) == 0 ? "\n" : "\n\n");
 
     for (i = j = 0; i < 256; ++i) {
-        if (uregnames[i]) {
-            cpu_fprintf(f, "%s=%08x%c", uregnames[i], env->uregs[i],
+        if (xtensa_option_bits_enabled(env->config, uregnames[i].opt_bits)) {
+            cpu_fprintf(f, "%s=%08x%c", uregnames[i].name, env->uregs[i],
                     (j++ % 4) == 3 ? '\n' : ' ');
         }
     }
@@ -3004,7 +3051,7 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, (j % 4) == 0 ? "\n" : "\n\n");
 
     for (i = 0; i < 16; ++i) {
-        cpu_fprintf(f, "A%02d=%08x%c", i, env->regs[i],
+        cpu_fprintf(f, " A%02d=%08x%c", i, env->regs[i],
                 (i % 4) == 3 ? '\n' : ' ');
     }
 
@@ -3028,5 +3075,5 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
 
 void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb, int pc_pos)
 {
-    env->pc = gen_opc_pc[pc_pos];
+    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
 }
diff --git a/target-xtensa/xtensa-semi.c b/target-xtensa/xtensa-semi.c
index 851ff54..5fe0361 100644
--- a/target-xtensa/xtensa-semi.c
+++ b/target-xtensa/xtensa-semi.c
@@ -31,7 +31,7 @@
 #include <stddef.h>
 #include "cpu.h"
 #include "helper.h"
-#include "qemu-log.h"
+#include "qemu/log.h"
 
 enum {
     TARGET_SYS_exit = 1,
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index 47612fe..d9c33d8 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -992,7 +992,7 @@ static inline void tcg_out_goto_label(TCGContext *s, int cond, int label_index)
 
 #ifdef CONFIG_SOFTMMU
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
@@ -1145,7 +1145,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                     TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
     /* We assume that the offset is contained within 20 bits.  */
     tlb_offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
-    assert(tlb_offset & ~0xfffff == 0);
+    assert((tlb_offset & ~0xfffff) == 0);
     if (tlb_offset > 0xfff) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_REG_R0,
                         0xa00 | (tlb_offset >> 12));
@@ -1354,7 +1354,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
                     TCG_AREG0, TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
     /* We assume that the offset is contained within 20 bits.  */
     tlb_offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
-    assert(tlb_offset & ~0xfffff == 0);
+    assert((tlb_offset & ~0xfffff) == 0);
     if (tlb_offset > 0xfff) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_REG_R0,
                         0xa00 | (tlb_offset >> 12));
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index 98fa11b..7083f3a 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_ARM 
 #define TCG_TARGET_ARM 1
 
 #undef TCG_TARGET_WORDS_BIGENDIAN
@@ -91,3 +92,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
 #endif
 }
+
+#endif
diff --git a/tcg/hppa/tcg-target.c b/tcg/hppa/tcg-target.c
index de500ae..656e736 100644
--- a/tcg/hppa/tcg-target.c
+++ b/tcg/hppa/tcg-target.c
@@ -824,7 +824,7 @@ static void tcg_out_brcond2(TCGContext *s, int cond, TCGArg al, TCGArg ah,
         tcg_out_brcond(s, TCG_COND_EQ, ah, bh, bhconst, label_index);
         break;
     case TCG_COND_NE:
-        tcg_out_brcond(s, TCG_COND_NE, al, bl, bhconst, label_index);
+        tcg_out_brcond(s, TCG_COND_NE, al, bl, blconst, label_index);
         tcg_out_brcond(s, TCG_COND_NE, ah, bh, bhconst, label_index);
         break;
     default:
@@ -906,7 +906,7 @@ static void tcg_out_movcond(TCGContext *s, int cond, TCGArg ret,
 }
 
 #if defined(CONFIG_SOFTMMU)
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
diff --git a/tcg/hppa/tcg-target.h b/tcg/hppa/tcg-target.h
index f43fb41..e2754fe 100644
--- a/tcg/hppa/tcg-target.h
+++ b/tcg/hppa/tcg-target.h
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#ifndef TCG_TARGET_HPPA
 #define TCG_TARGET_HPPA 1
 
 #if TCG_TARGET_REG_BITS != 32
@@ -119,3 +120,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
         start += 32;
     }
 }
+
+#endif
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index 6f3ad3c..7aec304 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -97,6 +97,18 @@ static const int tcg_target_call_oarg_regs[] = {
 # define TCG_REG_L1 TCG_REG_EDX
 #endif
 
+/* For 32-bit, we are going to attempt to determine at runtime whether cmov
+   is available.  However, the host compiler must supply <cpuid.h>, as we're
+   not going to go so far as our own inline assembly.  */
+#if TCG_TARGET_REG_BITS == 64
+# define have_cmov 1
+#elif defined(CONFIG_CPUID_H)
+#include <cpuid.h>
+static bool have_cmov;
+#else
+# define have_cmov 0
+#endif
+
 static uint8_t *tb_ret_addr;
 
 static void patch_reloc(uint8_t *code_ptr, int type,
@@ -943,7 +955,14 @@ static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGArg dest,
                               TCGArg v1)
 {
     tcg_out_cmp(s, c1, c2, const_c2, 0);
-    tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond], dest, v1);
+    if (have_cmov) {
+        tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond], dest, v1);
+    } else {
+        int over = gen_new_label();
+        tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
+        tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
+        tcg_out_label(s, over, s->code_ptr);
+    }
 }
 
 #if TCG_TARGET_REG_BITS == 64
@@ -982,7 +1001,7 @@ static void tcg_out_jmp(TCGContext *s, tcg_target_long dest)
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
@@ -2080,7 +2099,7 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_st32_i64, { "ri", "r" } },
     { INDEX_op_st_i64, { "re", "r" } },
 
-    { INDEX_op_add_i64, { "r", "0", "re" } },
+    { INDEX_op_add_i64, { "r", "r", "re" } },
     { INDEX_op_mul_i64, { "r", "0", "re" } },
     { INDEX_op_div2_i64, { "a", "d", "0", "1", "r" } },
     { INDEX_op_divu2_i64, { "a", "d", "0", "1", "r" } },
@@ -2243,6 +2262,16 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 
 static void tcg_target_init(TCGContext *s)
 {
+    /* For 32-bit, 99% certainty that we're running on hardware that supports
+       cmov, but we still need to check.  In case cmov is not available, we'll
+       use a small forward branch.  */
+#ifndef have_cmov
+    {
+        unsigned a, b, c, d;
+        have_cmov = (__get_cpuid(1, &a, &b, &c, &d) && (d & bit_CMOV));
+    }
+#endif
+
 #if !defined(CONFIG_USER_ONLY)
     /* fail safe */
     if ((1 << CPU_TLB_ENTRY_BITS) != sizeof(CPUTLBEntry))
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index dbc6756..e63db9c 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_I386 
 #define TCG_TARGET_I386 1
 
 //#define TCG_TARGET_WORDS_BIGENDIAN
@@ -90,12 +91,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      1
-#if defined(__x86_64__) || defined(__i686__)
-/* Use cmov only if the compiler is already doing so.  */
 #define TCG_TARGET_HAS_movcond_i32      1
-#else
-#define TCG_TARGET_HAS_movcond_i32      0
-#endif
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div2_i64         1
@@ -135,3 +131,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
 {
 }
+
+#endif
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
index 06570be..2373d9e 100644
--- a/tcg/ia64/tcg-target.c
+++ b/tcg/ia64/tcg-target.c
@@ -1491,7 +1491,7 @@ static inline void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGArg ret,
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* Load and compare a TLB entry, and return the result in (p6, p7).
    R2 is loaded with the address of the addend TLB entry.
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index 91fe7a3..7f3401e 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_IA64 
 #define TCG_TARGET_IA64 1
 
 /* We only map the first 64 registers */
@@ -158,3 +159,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
     }
     asm volatile (";;sync.i;;srlz.i;;");
 }
+
+#endif
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index ae2b274..bd8c858 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -920,7 +920,7 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index 65b5c59..78af664 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_MIPS 
 #define TCG_TARGET_MIPS 1
 
 #ifdef __MIPSEB__
@@ -127,3 +128,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
 {
     cacheflush ((void *)start, stop-start, ICACHE);
 }
+
+#endif
diff --git a/tcg/optimize.c b/tcg/optimize.c
index 9109b81..973d2d6 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -46,6 +46,7 @@ struct tcg_temp_info {
     uint16_t prev_copy;
     uint16_t next_copy;
     tcg_target_ulong val;
+    tcg_target_ulong mask;
 };
 
 static struct tcg_temp_info temps[TCG_MAX_TEMPS];
@@ -63,6 +64,17 @@ static void reset_temp(TCGArg temp)
         }
     }
     temps[temp].state = TCG_TEMP_UNDEF;
+    temps[temp].mask = -1;
+}
+
+/* Reset all temporaries, given that there are NB_TEMPS of them.  */
+static void reset_all_temps(int nb_temps)
+{
+    int i;
+    for (i = 0; i < nb_temps; i++) {
+        temps[i].state = TCG_TEMP_UNDEF;
+        temps[i].mask = -1;
+    }
 }
 
 static int op_bits(TCGOpcode op)
@@ -139,33 +151,35 @@ static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
 static void tcg_opt_gen_mov(TCGContext *s, TCGArg *gen_args,
                             TCGArg dst, TCGArg src)
 {
-        reset_temp(dst);
-        assert(temps[src].state != TCG_TEMP_CONST);
-
-        if (s->temps[src].type == s->temps[dst].type) {
-            if (temps[src].state != TCG_TEMP_COPY) {
-                temps[src].state = TCG_TEMP_COPY;
-                temps[src].next_copy = src;
-                temps[src].prev_copy = src;
-            }
-            temps[dst].state = TCG_TEMP_COPY;
-            temps[dst].next_copy = temps[src].next_copy;
-            temps[dst].prev_copy = src;
-            temps[temps[dst].next_copy].prev_copy = dst;
-            temps[src].next_copy = dst;
+    reset_temp(dst);
+    temps[dst].mask = temps[src].mask;
+    assert(temps[src].state != TCG_TEMP_CONST);
+
+    if (s->temps[src].type == s->temps[dst].type) {
+        if (temps[src].state != TCG_TEMP_COPY) {
+            temps[src].state = TCG_TEMP_COPY;
+            temps[src].next_copy = src;
+            temps[src].prev_copy = src;
         }
+        temps[dst].state = TCG_TEMP_COPY;
+        temps[dst].next_copy = temps[src].next_copy;
+        temps[dst].prev_copy = src;
+        temps[temps[dst].next_copy].prev_copy = dst;
+        temps[src].next_copy = dst;
+    }
 
-        gen_args[0] = dst;
-        gen_args[1] = src;
+    gen_args[0] = dst;
+    gen_args[1] = src;
 }
 
 static void tcg_opt_gen_movi(TCGArg *gen_args, TCGArg dst, TCGArg val)
 {
-        reset_temp(dst);
-        temps[dst].state = TCG_TEMP_CONST;
-        temps[dst].val = val;
-        gen_args[0] = dst;
-        gen_args[1] = val;
+    reset_temp(dst);
+    temps[dst].state = TCG_TEMP_CONST;
+    temps[dst].val = val;
+    temps[dst].mask = val;
+    gen_args[0] = dst;
+    gen_args[1] = val;
 }
 
 static TCGOpcode op_to_mov(TCGOpcode op)
@@ -470,6 +484,7 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
                                     TCGArg *args, TCGOpDef *tcg_op_defs)
 {
     int i, nb_ops, op_index, nb_temps, nb_globals, nb_call_args;
+    tcg_target_ulong mask, affected;
     TCGOpcode op;
     const TCGOpDef *def;
     TCGArg *gen_args;
@@ -482,7 +497,7 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
 
     nb_temps = s->nb_temps;
     nb_globals = s->nb_globals;
-    memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+    reset_all_temps(nb_temps);
 
     nb_ops = tcg_opc_ptr - s->gen_opc_buf;
     gen_args = args;
@@ -612,6 +627,113 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
             break;
         }
 
+        /* Simplify using known-zero bits */
+        mask = -1;
+        affected = -1;
+        switch (op) {
+        CASE_OP_32_64(ext8s):
+            if ((temps[args[1]].mask & 0x80) != 0) {
+                break;
+            }
+        CASE_OP_32_64(ext8u):
+            mask = 0xff;
+            goto and_const;
+        CASE_OP_32_64(ext16s):
+            if ((temps[args[1]].mask & 0x8000) != 0) {
+                break;
+            }
+        CASE_OP_32_64(ext16u):
+            mask = 0xffff;
+            goto and_const;
+        case INDEX_op_ext32s_i64:
+            if ((temps[args[1]].mask & 0x80000000) != 0) {
+                break;
+            }
+        case INDEX_op_ext32u_i64:
+            mask = 0xffffffffU;
+            goto and_const;
+
+        CASE_OP_32_64(and):
+            mask = temps[args[2]].mask;
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+        and_const:
+                affected = temps[args[1]].mask & ~mask;
+            }
+            mask = temps[args[1]].mask & mask;
+            break;
+
+        CASE_OP_32_64(sar):
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+                mask = ((tcg_target_long)temps[args[1]].mask
+                        >> temps[args[2]].val);
+            }
+            break;
+
+        CASE_OP_32_64(shr):
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+                mask = temps[args[1]].mask >> temps[args[2]].val;
+            }
+            break;
+
+        CASE_OP_32_64(shl):
+            if (temps[args[2]].state == TCG_TEMP_CONST) {
+                mask = temps[args[1]].mask << temps[args[2]].val;
+            }
+            break;
+
+        CASE_OP_32_64(neg):
+            /* Set to 1 all bits to the left of the rightmost.  */
+            mask = -(temps[args[1]].mask & -temps[args[1]].mask);
+            break;
+
+        CASE_OP_32_64(deposit):
+            tmp = ((1ull << args[4]) - 1);
+            mask = ((temps[args[1]].mask & ~(tmp << args[3]))
+                    | ((temps[args[2]].mask & tmp) << args[3]));
+            break;
+
+        CASE_OP_32_64(or):
+        CASE_OP_32_64(xor):
+            mask = temps[args[1]].mask | temps[args[2]].mask;
+            break;
+
+        CASE_OP_32_64(setcond):
+            mask = 1;
+            break;
+
+        CASE_OP_32_64(movcond):
+            mask = temps[args[3]].mask | temps[args[4]].mask;
+            break;
+
+        default:
+            break;
+        }
+
+        if (mask == 0) {
+            assert(def->nb_oargs == 1);
+            s->gen_opc_buf[op_index] = op_to_movi(op);
+            tcg_opt_gen_movi(gen_args, args[0], 0);
+            args += def->nb_oargs + def->nb_iargs + def->nb_cargs;
+            gen_args += 2;
+            continue;
+        }
+        if (affected == 0) {
+            assert(def->nb_oargs == 1);
+            if (temps_are_copies(args[0], args[1])) {
+                s->gen_opc_buf[op_index] = INDEX_op_nop;
+            } else if (temps[args[1]].state != TCG_TEMP_CONST) {
+                s->gen_opc_buf[op_index] = op_to_mov(op);
+                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
+                gen_args += 2;
+            } else {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], temps[args[1]].val);
+                gen_args += 2;
+            }
+            args += def->nb_iargs + 1;
+            continue;
+        }
+
         /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
         switch (op) {
         CASE_OP_32_64(and):
@@ -768,7 +890,7 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
             tmp = do_constant_folding_cond(op, args[0], args[1], args[2]);
             if (tmp != 2) {
                 if (tmp) {
-                    memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                    reset_all_temps(nb_temps);
                     s->gen_opc_buf[op_index] = INDEX_op_br;
                     gen_args[0] = args[3];
                     gen_args += 1;
@@ -861,7 +983,7 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
             tmp = do_constant_folding_cond2(&args[0], &args[2], args[4]);
             if (tmp != 2) {
                 if (tmp) {
-                    memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                    reset_all_temps(nb_temps);
                     s->gen_opc_buf[op_index] = INDEX_op_br;
                     gen_args[0] = args[5];
                     gen_args += 1;
@@ -875,7 +997,7 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
                        && temps[args[3]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
-                memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                reset_all_temps(nb_temps);
                 s->gen_opc_buf[op_index] = INDEX_op_brcond_i32;
                 gen_args[0] = args[1];
                 gen_args[1] = args[3];
@@ -938,9 +1060,10 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
             /* Default case: we know nothing about operation (or were unable
                to compute the operation result) so no propagation is done.
                We trash everything if the operation is the end of a basic
-               block, otherwise we only trash the output args.  */
+               block, otherwise we only trash the output args.  "mask" is
+               the non-zero bits mask for the first output arg.  */
             if (def->flags & TCG_OPF_BB_END) {
-                memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                reset_all_temps(nb_temps);
             } else {
                 for (i = 0; i < def->nb_oargs; i++) {
                     reset_temp(args[i]);
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index d72d396..29ca934 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -549,7 +549,7 @@ static void add_qemu_ldst_label (TCGContext *s,
     label->label_ptr[0] = label_ptr;
 }
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index ad433ae..ea26769 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_PPC 
 #define TCG_TARGET_PPC 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -99,3 +100,5 @@ typedef enum {
 #define tcg_qemu_tb_exec(env, tb_ptr) \
     ((long __attribute__ ((longcall)) \
       (*)(void *, void *))code_gen_prologue)(env, tb_ptr)
+
+#endif
diff --git a/tcg/ppc64/tcg-target.c b/tcg/ppc64/tcg-target.c
index 5403fc1..833fe0c 100644
--- a/tcg/ppc64/tcg-target.c
+++ b/tcg/ppc64/tcg-target.c
@@ -546,7 +546,7 @@ static void tcg_out_ldsta (TCGContext *s, int ret, int addr,
 
 #if defined (CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
diff --git a/tcg/ppc64/tcg-target.h b/tcg/ppc64/tcg-target.h
index 97fc5c9..9b8e9a0 100644
--- a/tcg/ppc64/tcg-target.h
+++ b/tcg/ppc64/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_PPC64 
 #define TCG_TARGET_PPC64 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -109,3 +110,5 @@ typedef enum {
 #define TCG_AREG0 TCG_REG_R27
 
 #define TCG_TARGET_EXTEND_ARGS 1
+
+#endif
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index fd9286f..e12a152 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -299,7 +299,7 @@ static const uint8_t tcg_cond_to_ltr_cond[] = {
 
 #ifdef CONFIG_SOFTMMU
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index a0181ae..c87b413 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_S390 
 #define TCG_TARGET_S390 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -103,3 +104,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
 {
 }
+
+#endif
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index f146647..03db514 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -831,7 +831,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 
 #if defined(CONFIG_SOFTMMU)
 
-#include "../../softmmu_defs.h"
+#include "exec/softmmu_defs.h"
 
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index 0e7d398..256f973 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef TCG_TARGET_SPARC 
 #define TCG_TARGET_SPARC 1
 
 #define TCG_TARGET_WORDS_BIGENDIAN
@@ -138,3 +139,5 @@ static inline void flush_icache_range(tcg_target_ulong start,
     for (; p < stop; p += 8)
         __asm__ __volatile__("flush\t%0" : : "r" (p));
 }
+
+#endif
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index 0b3cb0b..91c9d80 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -2329,6 +2329,7 @@ static inline void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret,
 #define tcg_gen_qemu_ldst_op tcg_gen_op3i_i32
 #define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i32
 #define TCGV_UNUSED(x) TCGV_UNUSED_I32(x)
+#define TCGV_IS_UNUSED(x) TCGV_IS_UNUSED_I32(x)
 #define TCGV_EQUAL(a, b) TCGV_EQUAL_I32(a, b)
 #else
 #define TCGv TCGv_i64
@@ -2340,6 +2341,7 @@ static inline void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret,
 #define tcg_gen_qemu_ldst_op tcg_gen_op3i_i64
 #define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i64
 #define TCGV_UNUSED(x) TCGV_UNUSED_I64(x)
+#define TCGV_IS_UNUSED(x) TCGV_IS_UNUSED_I64(x)
 #define TCGV_EQUAL(a, b) TCGV_EQUAL_I64(a, b)
 #endif
 
diff --git a/tcg/tcg.c b/tcg/tcg.c
index cb193f2..9275e37 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -37,9 +37,9 @@
 #endif
 
 #include "qemu-common.h"
-#include "cache-utils.h"
-#include "host-utils.h"
-#include "qemu-timer.h"
+#include "qemu/cache-utils.h"
+#include "qemu/host-utils.h"
+#include "qemu/timer.h"
 
 /* Note: the long term plan is to reduce the dependancies on the QEMU
    CPU definitions. Currently they are used for qemu_ld/st
@@ -800,7 +800,6 @@ static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size,
 
     assert(idx >= 0 && idx < s->nb_temps);
     ts = &s->temps[idx];
-    assert(ts);
     if (idx < s->nb_globals) {
         pstrcpy(buf, buf_size, ts->name);
     } else {
diff --git a/tcg/tcg.h b/tcg/tcg.h
index 9481e35..a427972 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -270,6 +270,9 @@ typedef int TCGv_i64;
 #define TCGV_UNUSED_I32(x) x = MAKE_TCGV_I32(-1)
 #define TCGV_UNUSED_I64(x) x = MAKE_TCGV_I64(-1)
 
+#define TCGV_IS_UNUSED_I32(x) (GET_TCGV_I32(x) == -1)
+#define TCGV_IS_UNUSED_I64(x) (GET_TCGV_I64(x) == -1)
+
 /* call flags */
 /* Helper does not read globals (either directly or through an exception). It
    implies TCG_CALL_NO_WRITE_GLOBALS. */
@@ -290,8 +293,8 @@ typedef int TCGv_i64;
 #define TCG_CALL_DUMMY_TCGV     MAKE_TCGV_I32(-1)
 #define TCG_CALL_DUMMY_ARG      ((TCGArg)(-1))
 
-/* Conditions.  Note that these are layed out for easy manipulation by
-   the the functions below:
+/* Conditions.  Note that these are laid out for easy manipulation by
+   the functions below:
      bit 0 is used for inverting;
      bit 1 is signed,
      bit 2 is unsigned,
@@ -455,6 +458,9 @@ struct TCGContext {
 
     uint16_t *gen_opc_ptr;
     TCGArg *gen_opparam_ptr;
+    target_ulong gen_opc_pc[OPC_BUF_SIZE];
+    uint16_t gen_opc_icount[OPC_BUF_SIZE];
+    uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
 
 #if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
     /* labels info for qemu_ld/st IRs
diff --git a/tci.c b/tci.c
index 54cf1d9..2b2c11f 100644
--- a/tci.c
+++ b/tci.c
@@ -25,7 +25,7 @@
 #endif
 
 #include "qemu-common.h"
-#include "exec-all.h"           /* MAX_OPC_PARAM_IARGS */
+#include "exec/exec-all.h"           /* MAX_OPC_PARAM_IARGS */
 #include "tcg-op.h"
 
 /* Marker for missing code. */
diff --git a/tests/.gitignore b/tests/.gitignore
index f9041f3..38c94ef 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -10,4 +10,5 @@ test-qmp-commands.h
 test-qmp-commands
 test-qmp-input-strict
 test-qmp-marshal.c
+test-x86-cpuid
 *-test
diff --git a/tests/Makefile b/tests/Makefile
index b60f0fb..a2d62b8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,33 +1,78 @@
 export SRC_PATH
 
 check-unit-y = tests/check-qdict$(EXESUF)
+gcov-files-check-qdict-y = qobject/qdict.c
 check-unit-y += tests/check-qfloat$(EXESUF)
+gcov-files-check-qfloat-y = qobject/qfloat.c
 check-unit-y += tests/check-qint$(EXESUF)
+gcov-files-check-qint-y = qobject/qint.c
 check-unit-y += tests/check-qstring$(EXESUF)
+gcov-files-check-qstring-y = qobject/qstring.c
 check-unit-y += tests/check-qlist$(EXESUF)
+gcov-files-check-qlist-y = qobject/qlist.c
 check-unit-y += tests/check-qjson$(EXESUF)
+gcov-files-check-qjson-y = qobject/qjson.c
 check-unit-y += tests/test-qmp-output-visitor$(EXESUF)
+gcov-files-test-qmp-output-visitor-y = qapi/qmp-output-visitor.c
 check-unit-y += tests/test-qmp-input-visitor$(EXESUF)
+gcov-files-test-qmp-input-visitor-y = qapi/qmp-input-visitor.c
 check-unit-y += tests/test-qmp-input-strict$(EXESUF)
 check-unit-y += tests/test-qmp-commands$(EXESUF)
+gcov-files-test-qmp-commands-y = qapi/qmp-dispatch.c
 check-unit-y += tests/test-string-input-visitor$(EXESUF)
+gcov-files-test-string-input-visitor-y = qapi/string-input-visitor.c
 check-unit-y += tests/test-string-output-visitor$(EXESUF)
+gcov-files-test-string-output-visitor-y = qapi/string-output-visitor.c
 check-unit-y += tests/test-coroutine$(EXESUF)
+ifeq ($(CONFIG_WIN32),y)
+gcov-files-test-coroutine-y = coroutine-win32.c
+else
+ifeq ($(CONFIG_UCONTEXT_COROUTINE),y)
+gcov-files-test-coroutine-y = coroutine-ucontext.c
+else
+ifeq ($(CONFIG_SIGALTSTACK_COROUTINE),y)
+gcov-files-test-coroutine-y = coroutine-sigaltstack.c
+else
+gcov-files-test-coroutine-y = coroutine-gthread.c
+endif
+endif
+endif
 check-unit-y += tests/test-visitor-serialization$(EXESUF)
 check-unit-y += tests/test-iov$(EXESUF)
+gcov-files-test-iov-y = util/iov.c
 check-unit-y += tests/test-aio$(EXESUF)
+gcov-files-test-aio-$(CONFIG_WIN32) = aio-win32.c
+gcov-files-test-aio-$(CONFIG_POSIX) = aio-posix.c
 check-unit-y += tests/test-thread-pool$(EXESUF)
+gcov-files-test-thread-pool-y = thread-pool.c
+gcov-files-test-hbitmap-y = util/hbitmap.c
+check-unit-y += tests/test-hbitmap$(EXESUF)
+check-unit-y += tests/test-x86-cpuid$(EXESUF)
+# all code tested by test-x86-cpuid is inside topology.h
+gcov-files-test-x86-cpuid-y =
+check-unit-y += tests/test-xbzrle$(EXESUF)
+gcov-files-test-xbzrle-y = xbzrle.c
+check-unit-y += tests/test-cutils$(EXESUF)
+gcov-files-test-cutils-y += util/cutils.c
 
 check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
 
 # All QTests for now are POSIX-only, but the dependencies are
 # really in libqtest, not in the testcases themselves.
 check-qtest-i386-y = tests/fdc-test$(EXESUF)
+gcov-files-i386-y = hw/fdc.c
 check-qtest-i386-y += tests/hd-geo-test$(EXESUF)
+gcov-files-i386-y += hw/hd-geometry.c
 check-qtest-i386-y += tests/rtc-test$(EXESUF)
 check-qtest-x86_64-y = $(check-qtest-i386-y)
-check-qtest-sparc-y = tests/m48t59-test$(EXESUF)
-check-qtest-sparc64-y = tests/m48t59-test$(EXESUF)
+gcov-files-i386-y += i386-softmmu/hw/mc146818rtc.c
+gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
+#check-qtest-sparc-y = tests/m48t59-test$(EXESUF)
+#check-qtest-sparc64-y = tests/m48t59-test$(EXESUF)
+gcov-files-sparc-y += hw/m48t59.c
+gcov-files-sparc64-y += hw/m48t59.c
+check-qtest-arm-y = tests/tmp105-test$(EXESUF)
+gcov-files-arm-y += hw/tmp105.c
 
 GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h tests/test-qmp-commands.h
 
@@ -36,24 +81,29 @@ test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/test-coroutine.o tests/test-string-output-visitor.o \
 	tests/test-string-input-visitor.o tests/test-qmp-output-visitor.o \
 	tests/test-qmp-input-visitor.o tests/test-qmp-input-strict.o \
-	tests/test-qmp-commands.o tests/test-visitor-serialization.o
+	tests/test-qmp-commands.o tests/test-visitor-serialization.o \
+	tests/test-x86-cpuid.o
 
-test-qapi-obj-y =  $(qobject-obj-y) $(qapi-obj-y) qemu-tool.o
-test-qapi-obj-y += tests/test-qapi-visit.o tests/test-qapi-types.o
-test-qapi-obj-y += module.o
+test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o
 
 $(test-obj-y): QEMU_INCLUDES += -Itests
 
-tests/check-qint$(EXESUF): tests/check-qint.o qint.o
-tests/check-qstring$(EXESUF): tests/check-qstring.o qstring.o
-tests/check-qdict$(EXESUF): tests/check-qdict.o qdict.o qfloat.o qint.o qstring.o qbool.o qlist.o
-tests/check-qlist$(EXESUF): tests/check-qlist.o qlist.o qint.o
-tests/check-qfloat$(EXESUF): tests/check-qfloat.o qfloat.o
-tests/check-qjson$(EXESUF): tests/check-qjson.o $(qobject-obj-y) qemu-tool.o
-tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) iov.o libqemustub.a
-tests/test-aio$(EXESUF): tests/test-aio.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) libqemustub.a
-tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) libqemustub.a
-tests/test-iov$(EXESUF): tests/test-iov.o iov.o
+tests/test-x86-cpuid.o: QEMU_INCLUDES += -I$(SRC_PATH)/target-i386
+
+tests/check-qint$(EXESUF): tests/check-qint.o libqemuutil.a
+tests/check-qstring$(EXESUF): tests/check-qstring.o libqemuutil.a
+tests/check-qdict$(EXESUF): tests/check-qdict.o libqemuutil.a
+tests/check-qlist$(EXESUF): tests/check-qlist.o libqemuutil.a
+tests/check-qfloat$(EXESUF): tests/check-qfloat.o libqemuutil.a
+tests/check-qjson$(EXESUF): tests/check-qjson.o libqemuutil.a libqemustub.a
+tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a
+tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a
+tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
+tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o xbzrle.o page_cache.o libqemuutil.a
+tests/test-cutils$(EXESUF): tests/test-cutils.o util/cutils.o
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
@@ -66,18 +116,19 @@ $(SRC_PATH)/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-commands.py
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py $(gen-out-type) -o tests -p "test-" < $<, "  GEN   $@")
 
 
-tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y)
-tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y)
-tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y)
-tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y)
-tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y)
-tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y)
-tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y)
+tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
+tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) qapi-types.o qapi-visit.o libqemuutil.a libqemustub.a
+tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
 
-tests/rtc-test$(EXESUF): tests/rtc-test.o $(trace-obj-y)
-tests/m48t59-test$(EXESUF): tests/m48t59-test.o $(trace-obj-y)
-tests/fdc-test$(EXESUF): tests/fdc-test.o tests/libqtest.o $(trace-obj-y)
-tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o tests/libqtest.o $(trace-obj-y)
+tests/rtc-test$(EXESUF): tests/rtc-test.o
+tests/m48t59-test$(EXESUF): tests/m48t59-test.o
+tests/fdc-test$(EXESUF): tests/fdc-test.o
+tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
+tests/tmp105-test$(EXESUF): tests/tmp105-test.o
 
 # QTest rules
 
@@ -85,7 +136,8 @@ TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS)))
 QTEST_TARGETS=$(foreach TARGET,$(TARGETS), $(if $(check-qtest-$(TARGET)-y), $(TARGET),))
 check-qtest-$(CONFIG_POSIX)=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y))
 
-qtest-obj-y = tests/libqtest.o $(oslib-obj-y) libqemustub.a
+qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
+qtest-obj-y += tests/libi2c.o tests/libi2c-omap.o
 $(check-qtest-y): $(qtest-obj-y)
 
 .PHONY: check-help
@@ -108,17 +160,28 @@ check-help:
 
 SPEED = quick
 GTESTER_OPTIONS = -k $(if $(V),--verbose,-q)
+GCOV_OPTIONS = -n $(if $(V),-f,)
 
 # gtester tests, possibly with verbose output
 
 .PHONY: $(patsubst %, check-qtest-%, $(QTEST_TARGETS))
 $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: $(check-qtest-y)
+	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
 	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
 		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$*-y); do \
+	  echo Gcov report for $$f:;\
+	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
+	done,)
 
 .PHONY: $(patsubst %, check-%, $(check-unit-y))
 $(patsubst %, check-%, $(check-unit-y)): check-%: %
+	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
 	$(call quiet-command,gtester $(GTESTER_OPTIONS) -m=$(SPEED) $*,"GTESTER $*")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$(subst tests/,,$*)-y); do \
+	  echo Gcov report for $$f:;\
+	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
+	done,)
 
 # gtester tests with XML output
 
diff --git a/tests/check-qdict.c b/tests/check-qdict.c
index fc0d276..dc5f05a 100644
--- a/tests/check-qdict.c
+++ b/tests/check-qdict.c
@@ -11,9 +11,9 @@
  */
 #include <glib.h>
 
-#include "qint.h"
-#include "qdict.h"
-#include "qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/check-qfloat.c b/tests/check-qfloat.c
index cdc66ea..6404ac8 100644
--- a/tests/check-qfloat.c
+++ b/tests/check-qfloat.c
@@ -12,7 +12,7 @@
  */
 #include <glib.h>
 
-#include "qfloat.h"
+#include "qapi/qmp/qfloat.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/check-qint.c b/tests/check-qint.c
index 5a27119..8686884 100644
--- a/tests/check-qint.c
+++ b/tests/check-qint.c
@@ -11,7 +11,7 @@
  */
 #include <glib.h>
 
-#include "qint.h"
+#include "qapi/qmp/qint.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/check-qjson.c b/tests/check-qjson.c
index 3b896f5..32ffb43 100644
--- a/tests/check-qjson.c
+++ b/tests/check-qjson.c
@@ -10,13 +10,13 @@
  */
 #include <glib.h>
 
-#include "qstring.h"
-#include "qint.h"
-#include "qdict.h"
-#include "qlist.h"
-#include "qfloat.h"
-#include "qbool.h"
-#include "qjson.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qjson.h"
 
 #include "qemu-common.h"
 
diff --git a/tests/check-qlist.c b/tests/check-qlist.c
index 501ba26..b9c05d4 100644
--- a/tests/check-qlist.c
+++ b/tests/check-qlist.c
@@ -11,8 +11,8 @@
  */
 #include <glib.h>
 
-#include "qint.h"
-#include "qlist.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qlist.h"
 
 /*
  * Public Interface test-cases
diff --git a/tests/check-qstring.c b/tests/check-qstring.c
index addad6c..95dc9e3 100644
--- a/tests/check-qstring.c
+++ b/tests/check-qstring.c
@@ -11,7 +11,7 @@
  */
 #include <glib.h>
 
-#include "qstring.h"
+#include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
 /*
diff --git a/tests/libi2c-omap.c b/tests/libi2c-omap.c
new file mode 100644
index 0000000..b7b10b5
--- /dev/null
+++ b/tests/libi2c-omap.c
@@ -0,0 +1,196 @@
+/*
+ * QTest I2C driver
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "libi2c.h"
+
+#include <glib.h>
+#include <string.h>
+
+#include "qemu/osdep.h"
+#include "qemu/bswap.h"
+#include "libqtest.h"
+
+enum OMAPI2CRegisters {
+    OMAP_I2C_REV  = 0x00,
+    OMAP_I2C_STAT = 0x08,
+    OMAP_I2C_CNT  = 0x18,
+    OMAP_I2C_DATA = 0x1c,
+    OMAP_I2C_CON  = 0x24,
+    OMAP_I2C_SA   = 0x2c,
+};
+
+enum OMAPI2CSTATBits {
+    OMAP_I2C_STAT_NACK = 1 << 1,
+    OMAP_I2C_STAT_ARDY = 1 << 2,
+    OMAP_I2C_STAT_RRDY = 1 << 3,
+    OMAP_I2C_STAT_XRDY = 1 << 4,
+    OMAP_I2C_STAT_ROVR = 1 << 11,
+    OMAP_I2C_STAT_SBD  = 1 << 15,
+};
+
+enum OMAPI2CCONBits {
+    OMAP_I2C_CON_STT    = 1 << 0,
+    OMAP_I2C_CON_STP    = 1 << 1,
+    OMAP_I2C_CON_TRX    = 1 << 9,
+    OMAP_I2C_CON_MST    = 1 << 10,
+    OMAP_I2C_CON_BE     = 1 << 14,
+    OMAP_I2C_CON_I2C_EN = 1 << 15,
+};
+
+typedef struct OMAPI2C {
+    I2CAdapter parent;
+
+    uint64_t addr;
+} OMAPI2C;
+
+
+/* FIXME Use TBD readw qtest API */
+static inline uint16_t readw(uint64_t addr)
+{
+    uint16_t data;
+
+    memread(addr, &data, 2);
+    return le16_to_cpu(data);
+}
+
+/* FIXME Use TBD writew qtest API */
+static inline void writew(uint64_t addr, uint16_t data)
+{
+    data = cpu_to_le16(data);
+    memwrite(addr, &data, 2);
+}
+
+#ifdef __GNUC__
+#undef memread
+#undef memwrite
+#pragma GCC poison memread
+#pragma GCC poison memwrite
+#endif
+
+static void omap_i2c_set_slave_addr(OMAPI2C *s, uint8_t addr)
+{
+    uint16_t data = addr;
+
+    writew(s->addr + OMAP_I2C_SA, data);
+    data = readw(s->addr + OMAP_I2C_SA);
+    g_assert_cmphex(data, ==, addr);
+}
+
+static void omap_i2c_send(I2CAdapter *i2c, uint8_t addr,
+                          const uint8_t *buf, uint16_t len)
+{
+    OMAPI2C *s = (OMAPI2C *)i2c;
+    uint16_t data;
+
+    omap_i2c_set_slave_addr(s, addr);
+
+    data = len;
+    writew(s->addr + OMAP_I2C_CNT, data);
+
+    data = OMAP_I2C_CON_I2C_EN |
+           OMAP_I2C_CON_TRX |
+           OMAP_I2C_CON_MST |
+           OMAP_I2C_CON_STT |
+           OMAP_I2C_CON_STP;
+    writew(s->addr + OMAP_I2C_CON, data);
+    data = readw(s->addr + OMAP_I2C_CON);
+    g_assert((data & OMAP_I2C_CON_STP) != 0);
+
+    data = readw(s->addr + OMAP_I2C_STAT);
+    g_assert((data & OMAP_I2C_STAT_NACK) == 0);
+
+    while (len > 1) {
+        data = readw(s->addr + OMAP_I2C_STAT);
+        g_assert((data & OMAP_I2C_STAT_XRDY) != 0);
+
+        data = buf[0] | ((uint16_t)buf[1] << 8);
+        writew(s->addr + OMAP_I2C_DATA, data);
+        buf = (uint8_t *)buf + 2;
+        len -= 2;
+    }
+    if (len == 1) {
+        data = readw(s->addr + OMAP_I2C_STAT);
+        g_assert((data & OMAP_I2C_STAT_XRDY) != 0);
+
+        data = buf[0];
+        writew(s->addr + OMAP_I2C_DATA, data);
+    }
+
+    data = readw(s->addr + OMAP_I2C_CON);
+    g_assert((data & OMAP_I2C_CON_STP) == 0);
+}
+
+static void omap_i2c_recv(I2CAdapter *i2c, uint8_t addr,
+                          uint8_t *buf, uint16_t len)
+{
+    OMAPI2C *s = (OMAPI2C *)i2c;
+    uint16_t data, stat;
+
+    omap_i2c_set_slave_addr(s, addr);
+
+    data = len;
+    writew(s->addr + OMAP_I2C_CNT, data);
+
+    data = OMAP_I2C_CON_I2C_EN |
+           OMAP_I2C_CON_MST |
+           OMAP_I2C_CON_STT |
+           OMAP_I2C_CON_STP;
+    writew(s->addr + OMAP_I2C_CON, data);
+    data = readw(s->addr + OMAP_I2C_CON);
+    g_assert((data & OMAP_I2C_CON_STP) == 0);
+
+    data = readw(s->addr + OMAP_I2C_STAT);
+    g_assert((data & OMAP_I2C_STAT_NACK) == 0);
+
+    data = readw(s->addr + OMAP_I2C_CNT);
+    g_assert_cmpuint(data, ==, len);
+
+    while (len > 0) {
+        data = readw(s->addr + OMAP_I2C_STAT);
+        g_assert((data & OMAP_I2C_STAT_RRDY) != 0);
+        g_assert((data & OMAP_I2C_STAT_ROVR) == 0);
+
+        data = readw(s->addr + OMAP_I2C_DATA);
+
+        stat = readw(s->addr + OMAP_I2C_STAT);
+
+        if (unlikely(len == 1)) {
+            g_assert((stat & OMAP_I2C_STAT_SBD) != 0);
+
+            buf[0] = data & 0xff;
+            buf++;
+            len--;
+        } else {
+            buf[0] = data & 0xff;
+            buf[1] = data >> 8;
+            buf += 2;
+            len -= 2;
+        }
+    }
+
+    data = readw(s->addr + OMAP_I2C_CON);
+    g_assert((data & OMAP_I2C_CON_STP) == 0);
+}
+
+I2CAdapter *omap_i2c_create(uint64_t addr)
+{
+    OMAPI2C *s = g_malloc0(sizeof(*s));
+    I2CAdapter *i2c = (I2CAdapter *)s;
+    uint16_t data;
+
+    s->addr = addr;
+
+    i2c->send = omap_i2c_send;
+    i2c->recv = omap_i2c_recv;
+
+    /* verify the mmio address by looking for a known signature */
+    data = readw(addr + OMAP_I2C_REV);
+    g_assert_cmphex(data, ==, 0x34);
+
+    return i2c;
+}
diff --git a/tests/libi2c.c b/tests/libi2c.c
new file mode 100644
index 0000000..13ec85c
--- /dev/null
+++ b/tests/libi2c.c
@@ -0,0 +1,22 @@
+/*
+ * QTest I2C driver
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "libi2c.h"
+#include "libqtest.h"
+
+void i2c_send(I2CAdapter *i2c, uint8_t addr,
+              const uint8_t *buf, uint16_t len)
+{
+    i2c->send(i2c, addr, buf, len);
+}
+
+void i2c_recv(I2CAdapter *i2c, uint8_t addr,
+              uint8_t *buf, uint16_t len)
+{
+    i2c->recv(i2c, addr, buf, len);
+}
diff --git a/tests/libi2c.h b/tests/libi2c.h
new file mode 100644
index 0000000..1ce9af4
--- /dev/null
+++ b/tests/libi2c.h
@@ -0,0 +1,30 @@
+/*
+ * I2C libqos
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef LIBQOS_I2C_H
+#define LIBQOS_I2C_H
+
+#include <stdint.h>
+
+typedef struct I2CAdapter I2CAdapter;
+struct I2CAdapter {
+    void (*send)(I2CAdapter *adapter, uint8_t addr,
+                 const uint8_t *buf, uint16_t len);
+    void (*recv)(I2CAdapter *adapter, uint8_t addr,
+                 uint8_t *buf, uint16_t len);
+};
+
+void i2c_send(I2CAdapter *i2c, uint8_t addr,
+              const uint8_t *buf, uint16_t len);
+void i2c_recv(I2CAdapter *i2c, uint8_t addr,
+              uint8_t *buf, uint16_t len);
+
+/* libi2c-omap.c */
+I2CAdapter *omap_i2c_create(uint64_t addr);
+
+#endif
diff --git a/tests/libqtest.c b/tests/libqtest.c
index 71b84c1..762dec4 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -26,8 +26,8 @@
 #include <unistd.h>
 #include <string.h>
 
-#include "compiler.h"
-#include "osdep.h"
+#include "qemu/compiler.h"
+#include "qemu/osdep.h"
 
 #define MAX_IRQ 256
 
@@ -39,7 +39,8 @@ struct QTestState
     int qmp_fd;
     bool irq_level[MAX_IRQ];
     GString *rx;
-    gchar *pid_file;
+    gchar *pid_file; /* QEMU PID file */
+    int child_pid;   /* Child process created to execute QEMU */
     char *socket_path, *qmp_socket_path;
 };
 
@@ -144,6 +145,7 @@ QTestState *qtest_init(const char *extra_args)
 
     s->rx = g_string_new("");
     s->pid_file = pid_file;
+    s->child_pid = pid;
     for (i = 0; i < MAX_IRQ; i++) {
         s->irq_level[i] = false;
     }
@@ -165,8 +167,9 @@ void qtest_quit(QTestState *s)
 
     pid_t pid = qtest_qemu_pid(s);
     if (pid != -1) {
+        /* kill QEMU, but wait for the child created by us to run system() */
         kill(pid, SIGTERM);
-        waitpid(pid, &status, 0);
+        waitpid(s->child_pid, &status, 0);
     }
 
     unlink(s->pid_file);
diff --git a/tests/libqtest.h b/tests/libqtest.h
index c8ade85..110e2ec 100644
--- a/tests/libqtest.h
+++ b/tests/libqtest.h
@@ -109,7 +109,6 @@ void qtest_outl(QTestState *s, uint16_t addr, uint32_t value);
  * qtest_inb:
  * @s: QTestState instance to operate on.
  * @addr: I/O port to read from.
- * @value: Value being written.
  *
  * Returns an 8-bit value from an I/O port.
  */
@@ -119,7 +118,6 @@ uint8_t qtest_inb(QTestState *s, uint16_t addr);
  * qtest_inw:
  * @s: QTestState instance to operate on.
  * @addr: I/O port to read from.
- * @value: Value being written.
  *
  * Returns a 16-bit value from an I/O port.
  */
@@ -129,7 +127,6 @@ uint16_t qtest_inw(QTestState *s, uint16_t addr);
  * qtest_inl:
  * @s: QTestState instance to operate on.
  * @addr: I/O port to read from.
- * @value: Value being written.
  *
  * Returns a 32-bit value from an I/O port.
  */
@@ -279,7 +276,6 @@ void qtest_add_func(const char *str, void (*fn));
 /**
  * inb:
  * @addr: I/O port to read from.
- * @value: Value being written.
  *
  * Returns an 8-bit value from an I/O port.
  */
@@ -288,7 +284,6 @@ void qtest_add_func(const char *str, void (*fn));
 /**
  * inw:
  * @addr: I/O port to read from.
- * @value: Value being written.
  *
  * Returns a 16-bit value from an I/O port.
  */
@@ -297,7 +292,6 @@ void qtest_add_func(const char *str, void (*fn));
 /**
  * inl:
  * @addr: I/O port to read from.
- * @value: Value being written.
  *
  * Returns a 32-bit value from an I/O port.
  */
diff --git a/tests/m48t59-test.c b/tests/m48t59-test.c
index 5179681..77d69b3 100644
--- a/tests/m48t59-test.c
+++ b/tests/m48t59-test.c
@@ -142,7 +142,9 @@ static void cmos_get_date_time(struct tm *date)
     date->tm_mday = mday;
     date->tm_mon = mon - 1;
     date->tm_year = base_year + year - 1900;
+#ifndef __sun__
     date->tm_gmtoff = 0;
+#endif
 
     ts = mktime(date);
 }
@@ -233,6 +235,11 @@ static void fuzz_registers(void)
         reg = (uint8_t)g_test_rand_int_range(0, 16);
         val = (uint8_t)g_test_rand_int_range(0, 256);
 
+        if (reg == 7) {
+            /* watchdog setup register, may trigger system reset, skip */
+            continue;
+        }
+
         cmos_write(reg, val);
         cmos_read(reg);
     }
diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041
index c6eb851..720eeff 100755
--- a/tests/qemu-iotests/041
+++ b/tests/qemu-iotests/041
@@ -207,6 +207,37 @@ class TestSingleDrive(ImageMirroringTestCase):
         self.assertTrue(self.compare_images(test_img, target_img),
                         'target image does not match source after mirroring')
 
+    def test_small_buffer(self):
+        self.assert_no_active_mirrors()
+
+        # A small buffer is rounded up automatically
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             buf_size=4096, target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_small_buffer2(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,size=%d'
+                        % (TestSingleDrive.image_len, TestSingleDrive.image_len), target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             buf_size=65536, mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
     def test_large_cluster(self):
         self.assert_no_active_mirrors()
 
@@ -292,6 +323,75 @@ class TestMirrorNoBacking(ImageMirroringTestCase):
         self.assertTrue(self.compare_images(test_img, target_img),
                         'target image does not match source after mirroring')
 
+    def test_large_cluster(self):
+        self.assert_no_active_mirrors()
+
+        # qemu-img create fails if the image is not there
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'size=%d'
+                        %(TestMirrorNoBacking.image_len), target_backing_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
+                        % (TestMirrorNoBacking.image_len, target_backing_img), target_img)
+        os.remove(target_backing_img)
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+class TestMirrorResized(ImageMirroringTestCase):
+    backing_len = 1 * 1024 * 1024 # MB
+    image_len = 2 * 1024 * 1024 # MB
+
+    def setUp(self):
+        self.create_image(backing_img, TestMirrorResized.backing_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        qemu_img('resize', test_img, '2M')
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def test_complete_top(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='top',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_complete_full(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
 class TestReadErrors(ImageMirroringTestCase):
     image_len = 2 * 1024 * 1024 # MB
 
@@ -330,6 +430,9 @@ new_state = "1"
                  '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
                        % (self.blkdebug_file, backing_img),
                  test_img)
+        # Write something for tests that use sync='top'
+        qemu_io('-c', 'write %d 512' % (self.MIRROR_GRANULARITY + 65536),
+                        test_img)
         self.vm = iotests.VM().add_drive(test_img)
         self.vm.launch()
 
@@ -383,6 +486,32 @@ new_state = "1"
         self.complete_and_wait()
         self.vm.shutdown()
 
+    def test_large_cluster(self):
+        self.assert_no_active_mirrors()
+
+        # Test COW into the target image.  The first half of the
+        # cluster at MIRROR_GRANULARITY has to be copied from
+        # backing_img, even though sync='top'.
+        qemu_img('create', '-f', iotests.imgfmt, '-ocluster_size=131072,backing_file=%s' %(backing_img), target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='top',
+                             on_source_error='ignore',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        event = self.vm.get_qmp_event(wait=True)
+        self.assertEquals(event['event'], 'BLOCK_JOB_ERROR')
+        self.assert_qmp(event, 'data/device', 'drive0')
+        self.assert_qmp(event, 'data/operation', 'read')
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.complete_and_wait()
+        self.vm.shutdown()
+
+        # Detach blkdebug to compare images successfully
+        qemu_img('rebase', '-f', iotests.imgfmt, '-u', '-b', backing_img, test_img)
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
     def test_stop_read(self):
         self.assert_no_active_mirrors()
 
diff --git a/tests/qemu-iotests/041.out b/tests/qemu-iotests/041.out
index 71009c2..42314e9 100644
--- a/tests/qemu-iotests/041.out
+++ b/tests/qemu-iotests/041.out
@@ -1,5 +1,5 @@
-..................
+........................
 ----------------------------------------------------------------------
-Ran 18 tests
+Ran 24 tests
 
 OK
diff --git a/tests/qemu-iotests/045 b/tests/qemu-iotests/045
new file mode 100755
index 0000000..2b6f1af
--- /dev/null
+++ b/tests/qemu-iotests/045
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+#
+# Tests for fdsets.
+#
+# Copyright (C) 2012 IBM Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+from iotests import qemu_img
+
+image0 = os.path.join(iotests.test_dir, 'image0')
+image1 = os.path.join(iotests.test_dir, 'image1')
+image2 = os.path.join(iotests.test_dir, 'image2')
+image3 = os.path.join(iotests.test_dir, 'image3')
+image4 = os.path.join(iotests.test_dir, 'image4')
+
+class TestFdSets(iotests.QMPTestCase):
+
+    def setUp(self):
+        self.vm = iotests.VM()
+        qemu_img('create', '-f', iotests.imgfmt, image0, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image1, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image2, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image3, '128K')
+        qemu_img('create', '-f', iotests.imgfmt, image4, '128K')
+        self.file0 = open(image0, 'r')
+        self.file1 = open(image1, 'w+')
+        self.file2 = open(image2, 'r')
+        self.file3 = open(image3, 'r')
+        self.file4 = open(image4, 'r')
+        self.vm.add_fd(self.file0.fileno(), 1, 'image0:r')
+        self.vm.add_fd(self.file1.fileno(), 1, 'image1:w+')
+        self.vm.add_fd(self.file2.fileno(), 0, 'image2:r')
+        self.vm.add_fd(self.file3.fileno(), 2, 'image3:r')
+        self.vm.add_fd(self.file4.fileno(), 2, 'image4:r')
+        self.vm.add_drive("/dev/fdset/1")
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        self.file0.close()
+        self.file1.close()
+        self.file2.close()
+        self.file3.close()
+        self.file4.close()
+        os.remove(image0)
+        os.remove(image1)
+        os.remove(image2)
+        os.remove(image3)
+        os.remove(image4)
+
+    def test_query_fdset(self):
+        result = self.vm.qmp('query-fdsets')
+        self.assert_qmp(result, 'return[0]/fdset-id', 2)
+        self.assert_qmp(result, 'return[1]/fdset-id', 1)
+        self.assert_qmp(result, 'return[2]/fdset-id', 0)
+        self.assert_qmp(result, 'return[0]/fds[0]/opaque', 'image3:r')
+        self.assert_qmp(result, 'return[0]/fds[1]/opaque', 'image4:r')
+        self.assert_qmp(result, 'return[1]/fds[0]/opaque', 'image0:r')
+        self.assert_qmp(result, 'return[1]/fds[1]/opaque', 'image1:w+')
+        self.assert_qmp(result, 'return[2]/fds[0]/opaque', 'image2:r')
+        self.vm.shutdown()
+
+    def test_remove_fdset(self):
+        result = self.vm.qmp('remove-fd', fdset_id=2)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('query-fdsets')
+        self.assert_qmp(result, 'return[0]/fdset-id', 1)
+        self.assert_qmp(result, 'return[1]/fdset-id', 0)
+        self.assert_qmp(result, 'return[0]/fds[0]/opaque', 'image0:r')
+        self.assert_qmp(result, 'return[0]/fds[1]/opaque', 'image1:w+')
+        self.assert_qmp(result, 'return[1]/fds[0]/opaque', 'image2:r')
+        self.vm.shutdown()
+
+    def test_remove_fd(self):
+        result = self.vm.qmp('query-fdsets')
+        fd_image3 = result['return'][0]['fds'][0]['fd']
+        result = self.vm.qmp('remove-fd', fdset_id=2, fd=fd_image3)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('query-fdsets')
+        self.assert_qmp(result, 'return[0]/fdset-id', 2)
+        self.assert_qmp(result, 'return[1]/fdset-id', 1)
+        self.assert_qmp(result, 'return[2]/fdset-id', 0)
+        self.assert_qmp(result, 'return[0]/fds[0]/opaque', 'image4:r')
+        self.assert_qmp(result, 'return[1]/fds[0]/opaque', 'image0:r')
+        self.assert_qmp(result, 'return[1]/fds[1]/opaque', 'image1:w+')
+        self.assert_qmp(result, 'return[2]/fds[0]/opaque', 'image2:r')
+        self.vm.shutdown()
+
+    def test_remove_fd_invalid_fdset(self):
+        result = self.vm.qmp('query-fdsets')
+        fd_image3 = result['return'][0]['fds'][0]['fd']
+        result = self.vm.qmp('remove-fd', fdset_id=3, fd=fd_image3)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+            'File descriptor named \'fdset-id:3, fd:%d\' not found' % fd_image3)
+        self.vm.shutdown()
+
+    def test_remove_fd_invalid_fd(self):
+        result = self.vm.qmp('query-fdsets')
+        result = self.vm.qmp('remove-fd', fdset_id=2, fd=999)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+            'File descriptor named \'fdset-id:2, fd:999\' not found')
+        self.vm.shutdown()
+
+    def test_add_fd_invalid_fd(self):
+        result = self.vm.qmp('add-fd', fdset_id=2)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc',
+                'No file descriptor supplied via SCM_RIGHTS')
+        self.vm.shutdown()
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['raw'])
diff --git a/tests/qemu-iotests/045.out b/tests/qemu-iotests/045.out
new file mode 100644
index 0000000..3f8a935
--- /dev/null
+++ b/tests/qemu-iotests/045.out
@@ -0,0 +1,5 @@
+......
+----------------------------------------------------------------------
+Ran 6 tests
+
+OK
diff --git a/tests/qemu-iotests/046 b/tests/qemu-iotests/046
new file mode 100755
index 0000000..e0176f4
--- /dev/null
+++ b/tests/qemu-iotests/046
@@ -0,0 +1,215 @@
+#!/bin/bash
+#
+# Test concurrent cluster allocations
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+CLUSTER_SIZE=64k
+size=128M
+
+echo
+echo "== creating backing file for COW tests =="
+
+_make_test_img $size
+
+function backing_io()
+{
+    local offset=$1
+    local sectors=$2
+    local op=$3
+    local pattern=0
+    local cur_sec=0
+
+    for i in $(seq 0 $((sectors - 1))); do
+        cur_sec=$((offset / 65536 + i))
+        pattern=$(( ( (cur_sec % 128) + (cur_sec / 128)) % 128 ))
+
+        echo "$op -P $pattern $((cur_sec * 64))k 64k"
+    done
+}
+
+backing_io 0 16 write | $QEMU_IO $TEST_IMG | _filter_qemu_io
+
+mv $TEST_IMG $TEST_IMG.base
+
+_make_test_img -b $TEST_IMG.base 6G
+
+echo
+echo "== Some concurrent requests touching the same cluster =="
+
+function overlay_io()
+{
+# Allocate middle of cluster 1, then write to somewhere before and after it
+cat  <<EOF
+break write_aio A
+aio_write -P 10 0x18000 0x2000
+wait_break A
+
+aio_write -P 11 0x12000 0x2000
+aio_write -P 12 0x1c000 0x2000
+
+resume A
+aio_flush
+EOF
+
+# Sequential write case: Alloc middle of cluster 2, then write overlapping
+# to next cluster
+cat  <<EOF
+break write_aio A
+aio_write -P 20 0x28000 0x2000
+wait_break A
+aio_write -P 21 0x2a000 0x10000
+resume A
+aio_flush
+EOF
+
+# The same with a gap between both requests
+cat  <<EOF
+break write_aio A
+aio_write -P 40 0x48000 0x2000
+wait_break A
+aio_write -P 41 0x4c000 0x10000
+resume A
+aio_flush
+EOF
+
+# Sequential write, but the next cluster is already allocated
+cat  <<EOF
+write -P 70 0x76000 0x8000
+aio_flush
+break write_aio A
+aio_write -P 60 0x66000 0x2000
+wait_break A
+aio_write -P 61 0x6a000 0xe000
+resume A
+aio_flush
+EOF
+
+# Sequential write, but the next cluster is already allocated
+# and phyiscally in the right position
+cat  <<EOF
+write -P 89 0x80000 0x1000
+write -P 90 0x96000 0x8000
+aio_flush
+discard 0x80000 0x10000
+aio_flush
+break write_aio A
+aio_write -P 80 0x86000 0x2000
+wait_break A
+aio_write -P 81 0x8a000 0xe000
+resume A
+aio_flush
+EOF
+
+# Sequential write, and the next cluster is compressed
+cat  <<EOF
+write    -P 109 0xa0000 0x1000
+write -c -P 110 0xb0000 0x10000
+aio_flush
+discard 0xa0000 0x10000
+aio_flush
+break write_aio A
+aio_write -P 100 0xa6000 0x2000
+wait_break A
+aio_write -P 101 0xaa000 0xe000
+resume A
+aio_flush
+EOF
+}
+
+overlay_io | $QEMU_IO blkdebug::$TEST_IMG | _filter_qemu_io |\
+	sed -e 's/bytes at offset [0-9]*/bytes at offset XXX/g'
+
+echo
+echo "== Verify image content =="
+
+function verify_io()
+{
+    echo read -P 0 0 0x10000
+
+    echo read -P 1  0x10000 0x2000
+    echo read -P 11 0x12000 0x2000
+    echo read -P 1  0x14000 0x4000
+    echo read -P 10 0x18000 0x2000
+    echo read -P 1  0x1a000 0x2000
+    echo read -P 12 0x1c000 0x2000
+    echo read -P 1  0x1e000 0x2000
+
+    echo read -P 2  0x20000 0x8000
+    echo read -P 20 0x28000 0x2000
+    echo read -P 21 0x2a000 0x10000
+    echo read -P 3  0x3a000 0x6000
+
+    echo read -P 4  0x40000 0x8000
+    echo read -P 40 0x48000 0x2000
+    echo read -P 4  0x4a000 0x2000
+    echo read -P 41 0x4c000 0x10000
+    echo read -P 5  0x5c000 0x4000
+
+    echo read -P 6  0x60000 0x6000
+    echo read -P 60 0x66000 0x2000
+    echo read -P 6  0x68000 0x2000
+    echo read -P 61 0x6a000 0xe000
+    echo read -P 70 0x78000 0x6000
+    echo read -P 7  0x7e000 0x2000
+
+    echo read -P 8  0x80000 0x6000
+    echo read -P 80 0x86000 0x2000
+    echo read -P 8  0x88000 0x2000
+    echo read -P 81 0x8a000 0xe000
+    echo read -P 90 0x98000 0x6000
+    echo read -P 9  0x9e000 0x2000
+
+    echo read -P 10  0xa0000 0x6000
+    echo read -P 100 0xa6000 0x2000
+    echo read -P 10  0xa8000 0x2000
+    echo read -P 101 0xaa000 0xe000
+    echo read -P 110 0xb8000 0x8000
+}
+
+verify_io | $QEMU_IO $TEST_IMG | _filter_qemu_io
+
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/046.out b/tests/qemu-iotests/046.out
new file mode 100644
index 0000000..565360f
--- /dev/null
+++ b/tests/qemu-iotests/046.out
@@ -0,0 +1,163 @@
+QA output created by 046
+
+== creating backing file for COW tests ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+qemu-io> wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 196608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 262144
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 327680
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 393216
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 458752
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 524288
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 589824
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 655360
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 720896
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 786432
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 851968
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 917504
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset 983040
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+
+== Some concurrent requests touching the same cluster ==
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 32768/32768 bytes at offset XXX
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 57344/57344 bytes at offset XXX
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 4096/4096 bytes at offset XXX
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 32768/32768 bytes at offset XXX
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> discard 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 57344/57344 bytes at offset XXX
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 4096/4096 bytes at offset XXX
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> discard 65536/65536 bytes at offset XXX
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> blkdebug: Suspended request 'A'
+qemu-io> qemu-io> qemu-io> blkdebug: Resuming request 'A'
+qemu-io> wrote 8192/8192 bytes at offset XXX
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 57344/57344 bytes at offset XXX
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> 
+== Verify image content ==
+qemu-io> read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 65536
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 73728
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 16384/16384 bytes at offset 81920
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 98304
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 106496
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 114688
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 122880
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 32768/32768 bytes at offset 131072
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 163840
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 65536/65536 bytes at offset 172032
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 237568
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 32768/32768 bytes at offset 262144
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 294912
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 303104
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 65536/65536 bytes at offset 311296
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 16384/16384 bytes at offset 376832
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 393216
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 417792
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 425984
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 57344/57344 bytes at offset 434176
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 491520
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 516096
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 524288
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 548864
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 557056
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 57344/57344 bytes at offset 565248
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 622592
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 647168
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 24576/24576 bytes at offset 655360
+24 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 679936
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 8192/8192 bytes at offset 688128
+8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 57344/57344 bytes at offset 696320
+56 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 32768/32768 bytes at offset 753664
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/047 b/tests/qemu-iotests/047
new file mode 100755
index 0000000..0cf36b4
--- /dev/null
+++ b/tests/qemu-iotests/047
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Regression test for commit b7ab0fea (which was a corruption fix,
+# despite the commit message claiming otherwise)
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+size=128M
+
+_make_test_img $size
+
+function qemu_io_cmds()
+{
+cat <<EOF
+write -P 0x66 0 320k
+write 320k 128k
+write -P 0x55 1M 128k
+write -P 0x88 448k 128k
+discard 320k 128k
+aio_flush
+
+write -P 0x77 0 480k
+aio_flush
+
+read -P 0x77 0 480k
+read -P 0x88 480k 96k
+read -P 0x55 1M 128k
+EOF
+}
+
+qemu_io_cmds | $QEMU_IO $TEST_IMG | _filter_qemu_io
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/047.out b/tests/qemu-iotests/047.out
new file mode 100644
index 0000000..81b2ff7
--- /dev/null
+++ b/tests/qemu-iotests/047.out
@@ -0,0 +1,22 @@
+QA output created by 047
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+qemu-io> wrote 327680/327680 bytes at offset 0
+320 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 131072/131072 bytes at offset 327680
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 131072/131072 bytes at offset 1048576
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> wrote 131072/131072 bytes at offset 458752
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> discard 131072/131072 bytes at offset 327680
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> wrote 491520/491520 bytes at offset 0
+480 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> qemu-io> qemu-io> read 491520/491520 bytes at offset 0
+480 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 98304/98304 bytes at offset 491520
+96 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> read 131072/131072 bytes at offset 1048576
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-io> No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index a4a9044..1bbd2bf 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -51,3 +51,6 @@
 042 rw auto quick
 043 rw auto backing
 044 rw auto
+045 rw auto
+046 rw auto aio
+047 rw auto
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index b2eaf20..569ca3d 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -43,7 +43,7 @@ def qemu_img(*args):
     return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
 
 def qemu_img_verbose(*args):
-    '''Run qemu-img without supressing its output and return the exit code'''
+    '''Run qemu-img without suppressing its output and return the exit code'''
     return subprocess.call(qemu_img_args + list(args))
 
 def qemu_io(*args):
@@ -79,6 +79,18 @@ class VM(object):
         self._num_drives += 1
         return self
 
+    def add_fd(self, fd, fdset, opaque, opts=''):
+        '''Pass a file descriptor to the VM'''
+        options = ['fd=%d' % fd,
+                   'set=%d' % fdset,
+                   'opaque=%s' % opaque]
+        if opts:
+            options.append(opts)
+
+        self._args.append('-add-fd')
+        self._args.append(','.join(options))
+        return self
+
     def launch(self):
         '''Launch the VM and establish a QMP connection'''
         devnull = open('/dev/null', 'rb')
diff --git a/tests/rtc-test.c b/tests/rtc-test.c
index 02edbf5..203c0fc 100644
--- a/tests/rtc-test.c
+++ b/tests/rtc-test.c
@@ -115,7 +115,9 @@ static void cmos_get_date_time(struct tm *date)
     date->tm_mday = mday;
     date->tm_mon = mon - 1;
     date->tm_year = base_year + year - 1900;
+#ifndef __sun__
     date->tm_gmtoff = 0;
+#endif
 
     ts = mktime(date);
 }
@@ -201,6 +203,10 @@ static void set_year_20xx(void)
     g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
     g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
 
+    if (sizeof(time_t) == 4) {
+        return;
+    }
+
     /* Set a date in 2080 to ensure there is no year-2038 overflow.  */
     cmos_write(RTC_REG_A, 0x76);
     cmos_write(RTC_YEAR, 0x80);
diff --git a/tests/tcg/cris/crisutils.h b/tests/tcg/cris/crisutils.h
index 29b71cd..3456b9d 100644
--- a/tests/tcg/cris/crisutils.h
+++ b/tests/tcg/cris/crisutils.h
@@ -1,3 +1,6 @@
+#ifndef CRISUTILS_H
+#define CRISUTILS_H 1
+
 static char *tst_cc_loc = NULL;
 
 #define cris_tst_cc_init() \
@@ -69,3 +72,5 @@ static inline void cris_tst_cc(const int n, const int z,
 	if (c) cris_tst_cc_c1(); else cris_tst_cc_c0();
 	asm volatile ("" : : "g" (_err));
 }
+
+#endif
diff --git a/tests/tcg/mips/mips32-dsp/extr_r_w.c b/tests/tcg/mips/mips32-dsp/extr_r_w.c
index 0beeefd..02e0224 100644
--- a/tests/tcg/mips/mips32-dsp/extr_r_w.c
+++ b/tests/tcg/mips/mips32-dsp/extr_r_w.c
@@ -44,5 +44,28 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x1F\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extr_rs_w.c b/tests/tcg/mips/mips32-dsp/extr_rs_w.c
index 24c748d..c3a22ee 100644
--- a/tests/tcg/mips/mips32-dsp/extr_rs_w.c
+++ b/tests/tcg/mips/mips32-dsp/extr_rs_w.c
@@ -44,5 +44,28 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x1F\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extr_s_h.c b/tests/tcg/mips/mips32-dsp/extr_s_h.c
index b212913..9bc2a63 100644
--- a/tests/tcg/mips/mips32-dsp/extr_s_h.c
+++ b/tests/tcg/mips/mips32-dsp/extr_s_h.c
@@ -59,5 +59,28 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x123;
+    acl = 0x87654321;
+    result = 0x1238;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 28\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extr_w.c b/tests/tcg/mips/mips32-dsp/extr_w.c
index 02ab9ec..bd6b0b9 100644
--- a/tests/tcg/mips/mips32-dsp/extr_w.c
+++ b/tests/tcg/mips/mips32-dsp/extr_w.c
@@ -44,5 +44,28 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x1F\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extrv_r_w.c b/tests/tcg/mips/mips32-dsp/extrv_r_w.c
index 005807b..2403b3a 100644
--- a/tests/tcg/mips/mips32-dsp/extrv_r_w.c
+++ b/tests/tcg/mips/mips32-dsp/extrv_r_w.c
@@ -50,5 +50,30 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 31;
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extrv_rs_w.c b/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
index c2d8513..ccceeb9 100644
--- a/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
+++ b/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
@@ -48,5 +48,30 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x1F;
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extrv_s_h.c b/tests/tcg/mips/mips32-dsp/extrv_s_h.c
index 8c13b5e..feac3e2 100644
--- a/tests/tcg/mips/mips32-dsp/extrv_s_h.c
+++ b/tests/tcg/mips/mips32-dsp/extrv_s_h.c
@@ -67,5 +67,22 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    rs = 0x1C;
+    ach = 0x123;
+    acl = 0x87654321;
+    result = 0x1238;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/extrv_w.c b/tests/tcg/mips/mips32-dsp/extrv_w.c
index 9cb493d..9e8b238 100644
--- a/tests/tcg/mips/mips32-dsp/extrv_w.c
+++ b/tests/tcg/mips/mips32-dsp/extrv_w.c
@@ -50,5 +50,31 @@ int main()
     assert(dsp == 0);
     assert(result == rt);
 
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 31;
+    ach = 0x3fffffff;
+    acl = 0x2bcdef01;
+    result = 0x7ffffffe;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/insv.c b/tests/tcg/mips/mips32-dsp/insv.c
index 7e3b047..243b007 100644
--- a/tests/tcg/mips/mips32-dsp/insv.c
+++ b/tests/tcg/mips/mips32-dsp/insv.c
@@ -10,7 +10,7 @@ int main()
     dsp    = 0x305;
     rt     = 0x12345678;
     rs     = 0x87654321;
-    result = 0x12345338;
+    result = 0x12345438;
     __asm
         ("wrdsp %2, 0x03\n\t"
          "insv  %0, %1\n\t"
diff --git a/tests/tcg/mips/mips32-dsp/mthlip.c b/tests/tcg/mips/mips32-dsp/mthlip.c
index 9549aae..85f94d8 100644
--- a/tests/tcg/mips/mips32-dsp/mthlip.c
+++ b/tests/tcg/mips/mips32-dsp/mthlip.c
@@ -30,7 +30,7 @@ int main()
     assert(ach == resulth);
     assert(acl == resultl);
 
-    dsp = 0x3f;
+    dsp = 0x1f;
     ach = 0x05;
     acl = 0xB4CB;
     rs  = 0x00FFBBAA;
diff --git a/tests/tcg/mips/mips32-dsp/rddsp.c b/tests/tcg/mips/mips32-dsp/rddsp.c
index e8948ec..2f30285 100644
--- a/tests/tcg/mips/mips32-dsp/rddsp.c
+++ b/tests/tcg/mips/mips32-dsp/rddsp.c
@@ -6,14 +6,13 @@ int main()
     int dsp_i, dsp_o;
     int ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
     int ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
-    int ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
 
-    ccond_i   = 0x000000BC;/* 4 */
-    outflag_i = 0x0000001B;/* 3 */
-    efi_i     = 0x00000001;/* 5 */
-    c_i       = 0x00000001;/* 2 */
-    scount_i  = 0x0000000F;/* 1 */
-    pos_i     = 0x0000000C;/* 0 */
+    ccond_i   = 0x0000000C;  /* 4 */
+    outflag_i = 0x0000001B;  /* 3 */
+    efi_i     = 0x00000001;  /* 5 */
+    c_i       = 0x00000001;  /* 2 */
+    scount_i  = 0x0000000F;  /* 1 */
+    pos_i     = 0x0000000C;  /* 0 */
 
     dsp_i = (ccond_i   << 24) | \
             (outflag_i << 16) | \
@@ -22,13 +21,6 @@ int main()
             (scount_i  <<  7) | \
             pos_i;
 
-    ccond_r   = ccond_i;
-    outflag_r = outflag_i;
-    efi_r     = efi_i;
-    c_r       = c_i;
-    scount_r  = scount_i;
-    pos_r     = pos_i;
-
     __asm
         ("wrdsp %1, 0x3F\n\t"
          "rddsp %0, 0x3F\n\t"
@@ -43,12 +35,12 @@ int main()
     scount_o  = (dsp_o >>  7) & 0x3F;
     pos_o     =  dsp_o & 0x1F;
 
-    assert(ccond_o   == ccond_r);
-    assert(outflag_o == outflag_r);
-    assert(efi_o     == efi_r);
-    assert(c_o       == c_r);
-    assert(scount_o  == scount_r);
-    assert(pos_o     == pos_r);
+    assert(ccond_o   == ccond_i);
+    assert(outflag_o == outflag_i);
+    assert(efi_o     == efi_i);
+    assert(c_o       == c_i);
+    assert(scount_o  == scount_i);
+    assert(pos_o     == pos_i);
 
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/shilo.c b/tests/tcg/mips/mips32-dsp/shilo.c
index b686616..ce8ebc6 100644
--- a/tests/tcg/mips/mips32-dsp/shilo.c
+++ b/tests/tcg/mips/mips32-dsp/shilo.c
@@ -23,5 +23,23 @@ int main()
     assert(ach == resulth);
     assert(acl == resultl);
 
+
+    ach = 0x1;
+    acl = 0x80000000;
+
+    resulth = 0x3;
+    resultl = 0x0;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilo $ac1, -1\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/shilov.c b/tests/tcg/mips/mips32-dsp/shilov.c
index f186032..e1d6cea 100644
--- a/tests/tcg/mips/mips32-dsp/shilov.c
+++ b/tests/tcg/mips/mips32-dsp/shilov.c
@@ -25,5 +25,25 @@ int main()
     assert(ach == resulth);
     assert(acl == resultl);
 
+
+    rs  = 0xffffffff;
+    ach = 0x1;
+    acl = 0x80000000;
+
+    resulth = 0x3;
+    resultl = 0x0;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilov $ac1, %2\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dsp/wrdsp.c b/tests/tcg/mips/mips32-dsp/wrdsp.c
index e8948ec..dc54943 100644
--- a/tests/tcg/mips/mips32-dsp/wrdsp.c
+++ b/tests/tcg/mips/mips32-dsp/wrdsp.c
@@ -6,14 +6,13 @@ int main()
     int dsp_i, dsp_o;
     int ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
     int ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
-    int ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
 
-    ccond_i   = 0x000000BC;/* 4 */
-    outflag_i = 0x0000001B;/* 3 */
-    efi_i     = 0x00000001;/* 5 */
-    c_i       = 0x00000001;/* 2 */
-    scount_i  = 0x0000000F;/* 1 */
-    pos_i     = 0x0000000C;/* 0 */
+    ccond_i   = 0x000000BC;  /* 4 */
+    outflag_i = 0x0000001B;  /* 3 */
+    efi_i     = 0x00000001;  /* 5 */
+    c_i       = 0x00000001;  /* 2 */
+    scount_i  = 0x0000000F;  /* 1 */
+    pos_i     = 0x0000000C;  /* 0 */
 
     dsp_i = (ccond_i   << 24) | \
             (outflag_i << 16) | \
@@ -22,13 +21,6 @@ int main()
             (scount_i  <<  7) | \
             pos_i;
 
-    ccond_r   = ccond_i;
-    outflag_r = outflag_i;
-    efi_r     = efi_i;
-    c_r       = c_i;
-    scount_r  = scount_i;
-    pos_r     = pos_i;
-
     __asm
         ("wrdsp %1, 0x3F\n\t"
          "rddsp %0, 0x3F\n\t"
@@ -43,12 +35,12 @@ int main()
     scount_o  = (dsp_o >>  7) & 0x3F;
     pos_o     =  dsp_o & 0x1F;
 
-    assert(ccond_o   == ccond_r);
-    assert(outflag_o == outflag_r);
-    assert(efi_o     == efi_r);
-    assert(c_o       == c_r);
-    assert(scount_o  == scount_r);
-    assert(pos_o     == pos_r);
+    assert(ccond_o   == (ccond_i & 0x0F));
+    assert(outflag_o == outflag_i);
+    assert(efi_o     == efi_i);
+    assert(c_o       == c_i);
+    assert(scount_o  == scount_i);
+    assert(pos_o     == pos_i);
 
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
index 1cfbdb0..fae49f1 100644
--- a/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
+++ b/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
@@ -26,8 +26,8 @@ int main()
     ach = 6, acl = 7;
     rs     = 0xFFFF00FF;
     rt     = 0xFFFF0002;
-    resulth = 0x05;
-    resultl = 0xfffe0206;
+    resulth = 0x06;
+    resultl = 0x206;
     __asm
         ("mthi  %0, $ac1\n\t"
          "mtlo  %1, $ac1\n\t"
diff --git a/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
index f756997..514797c 100644
--- a/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
+++ b/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
@@ -23,5 +23,22 @@ int main()
     assert(ach == resulth);
     assert(acl == resultl);
 
+    ach = 6, acl = 7;
+    rs     = 0xFFFF00FF;
+    rt     = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xFFFFFF06;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpax.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dspr2/dps_w_ph.c b/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
index 8303643..f51f9b9 100644
--- a/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
+++ b/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
@@ -23,5 +23,22 @@ int main()
     assert(ach == resulth);
     assert(acl == resultl);
 
+    ach = 6, acl = 7;
+    rs     = 0xFFFF00FF;
+    rt     = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xFFFFFE08;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dps.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
     return 0;
 }
diff --git a/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
index 6db59a4..bb49a40 100644
--- a/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
+++ b/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
@@ -9,8 +9,8 @@ int main()
 
     rs      = 0xBC0123AD;
     rt      = 0x01643721;
-    resulth = 0x04;
-    resultl = 0xD751F050;
+    resulth = 0x05;
+    resultl = 0xE72F050;
     __asm
         ("mthi  %0, $ac1\n\t"
          "mtlo  %1, $ac1\n\t"
diff --git a/tests/tcg/test-i386-fprem.c b/tests/tcg/test-i386-fprem.c
index 8c7a4d1..e91fb1a 100644
--- a/tests/tcg/test-i386-fprem.c
+++ b/tests/tcg/test-i386-fprem.c
@@ -22,8 +22,8 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "compiler.h"
-#include "osdep.h"
+#include "qemu/compiler.h"
+#include "qemu/osdep.h"
 #include <stdio.h>
 #include <inttypes.h>
 
diff --git a/tests/tcg/test-i386.c b/tests/tcg/test-i386.c
index 40392ac..6dc730d 100644
--- a/tests/tcg/test-i386.c
+++ b/tests/tcg/test-i386.c
@@ -17,7 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #define _GNU_SOURCE
-#include "compiler.h"
+#include "qemu/compiler.h"
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
diff --git a/tests/tcg/xtensa/Makefile b/tests/tcg/xtensa/Makefile
index 0ff0ccf..002fd87 100644
--- a/tests/tcg/xtensa/Makefile
+++ b/tests/tcg/xtensa/Makefile
@@ -42,9 +42,11 @@ endif
 TESTCASES += test_quo.tst
 TESTCASES += test_rem.tst
 TESTCASES += test_rst0.tst
+TESTCASES += test_s32c1i.tst
 TESTCASES += test_sar.tst
 TESTCASES += test_sext.tst
 TESTCASES += test_shift.tst
+TESTCASES += test_sr.tst
 TESTCASES += test_timer.tst
 TESTCASES += test_windowed.tst
 
diff --git a/tests/tcg/xtensa/macros.inc b/tests/tcg/xtensa/macros.inc
index 23bf3e9..c9be1ce 100644
--- a/tests/tcg/xtensa/macros.inc
+++ b/tests/tcg/xtensa/macros.inc
@@ -1,7 +1,7 @@
 .macro test_suite name
 .data
 status: .word result
-result: .space 20
+result: .space 256
 .text
 .global main
 .align 4
diff --git a/tests/tcg/xtensa/test_s32c1i.S b/tests/tcg/xtensa/test_s32c1i.S
new file mode 100644
index 0000000..4536015
--- /dev/null
+++ b/tests/tcg/xtensa/test_s32c1i.S
@@ -0,0 +1,39 @@
+.include "macros.inc"
+
+test_suite s32c1i
+
+test s32c1i_nowrite
+    movi    a2, 1f
+    movi    a3, 1
+    wsr     a3, scompare1
+    movi    a1, 2
+    s32c1i  a1, a2, 0
+    assert  ne, a1, a3
+    l32i    a1, a2, 0
+    assert  eqi, a1, 3
+
+.data
+.align 4
+1:
+    .word   3
+.text
+test_end
+
+test s32c1i_write
+    movi    a2, 1f
+    movi    a3, 3
+    wsr     a3, scompare1
+    movi    a1, 2
+    s32c1i  a1, a2, 0
+    assert  eq, a1, a3
+    l32i    a1, a2, 0
+    assert  eqi, a1, 2
+
+.data
+.align 4
+1:
+    .word   3
+.text
+test_end
+
+test_suite_end
diff --git a/tests/tcg/xtensa/test_sr.S b/tests/tcg/xtensa/test_sr.S
new file mode 100644
index 0000000..470c03d
--- /dev/null
+++ b/tests/tcg/xtensa/test_sr.S
@@ -0,0 +1,90 @@
+.include "macros.inc"
+
+test_suite sr
+
+.macro  sr_op sym, op_sym, op_byte, sr
+    .if \sym
+    \op_sym a4, \sr
+    .else
+    .byte 0x40, \sr, \op_byte
+    .endif
+.endm
+
+.macro 	test_sr_op sym, mask, op, op_byte, sr
+    movi    a4, 0
+    .if (\mask)
+    set_vector kernel, 0
+    sr_op   \sym, \op, \op_byte, \sr
+    .else
+    set_vector kernel, 2f
+1:
+    sr_op   \sym, \op, \op_byte, \sr
+    test_fail
+2:
+    reset_ps
+    rsr     a2, exccause
+    assert  eqi, a2, 0
+    rsr     a2, epc1
+    movi    a3, 1b
+    assert  eq, a2, a3
+    .endif
+.endm
+
+.macro  test_sr_mask sr, sym, mask
+test \sr
+    test_sr_op \sym, \mask & 1, rsr, 0x03, \sr
+    test_sr_op \sym, \mask & 2, wsr, 0x13, \sr
+    test_sr_op \sym, \mask & 4, xsr, 0x61, \sr
+test_end
+.endm
+
+.macro  test_sr sr, conf
+    test_sr_mask    \sr, \conf, 7
+.endm
+
+test_sr acchi, 1
+test_sr acclo, 1
+test_sr_mask /*atomctl*/99, 0, 0
+test_sr_mask /*br*/4, 0, 0
+test_sr_mask /*cacheattr*/98, 0, 0
+test_sr ccompare0, 1
+test_sr ccount, 1
+test_sr cpenable, 1
+test_sr dbreaka0, 1
+test_sr dbreakc0, 1
+test_sr_mask debugcause, 1, 1
+test_sr depc, 1
+test_sr dtlbcfg, 1
+test_sr epc1, 1
+test_sr epc2, 1
+test_sr eps2, 1
+test_sr exccause, 1
+test_sr excsave1, 1
+test_sr excsave2, 1
+test_sr excvaddr, 1
+test_sr ibreaka0, 1
+test_sr ibreakenable, 1
+test_sr icount, 1
+test_sr icountlevel, 1
+test_sr_mask /*intclear*/227, 0, 2
+test_sr_mask /*interrupt*/226, 0, 3
+test_sr intenable, 1
+test_sr itlbcfg, 1
+test_sr lbeg, 1
+test_sr lcount, 1
+test_sr lend, 1
+test_sr litbase, 1
+test_sr m0, 1
+test_sr misc0, 1
+test_sr_mask /*prefctl*/40, 0, 0
+test_sr_mask /*prid*/235, 0, 1
+test_sr ps, 1
+test_sr ptevaddr, 1
+test_sr rasid, 1
+test_sr sar, 1
+test_sr scompare1, 1
+test_sr vecbase, 1
+test_sr windowbase, 1
+test_sr windowstart, 1
+
+test_suite_end
diff --git a/tests/test-aio.c b/tests/test-aio.c
index f53c908..c173870 100644
--- a/tests/test-aio.c
+++ b/tests/test-aio.c
@@ -11,10 +11,18 @@
  */
 
 #include <glib.h>
-#include "qemu-aio.h"
+#include "block/aio.h"
 
 AioContext *ctx;
 
+/* Wait until there are no more BHs or AIO requests */
+static void wait_for_aio(void)
+{
+    while (aio_poll(ctx, true)) {
+        /* Do nothing */
+    }
+}
+
 /* Simple callbacks for testing.  */
 
 typedef struct {
@@ -78,14 +86,6 @@ static void test_notify(void)
     g_assert(!aio_poll(ctx, false));
 }
 
-static void test_flush(void)
-{
-    g_assert(!aio_poll(ctx, false));
-    aio_notify(ctx);
-    aio_flush(ctx);
-    g_assert(!aio_poll(ctx, false));
-}
-
 static void test_bh_schedule(void)
 {
     BHTestData data = { .n = 0 };
@@ -116,7 +116,7 @@ static void test_bh_schedule10(void)
     g_assert(aio_poll(ctx, true));
     g_assert_cmpint(data.n, ==, 2);
 
-    aio_flush(ctx);
+    wait_for_aio();
     g_assert_cmpint(data.n, ==, 10);
 
     g_assert(!aio_poll(ctx, false));
@@ -164,7 +164,7 @@ static void test_bh_delete_from_cb(void)
     qemu_bh_schedule(data1.bh);
     g_assert_cmpint(data1.n, ==, 0);
 
-    aio_flush(ctx);
+    wait_for_aio();
     g_assert_cmpint(data1.n, ==, data1.max);
     g_assert(data1.bh == NULL);
 
@@ -200,7 +200,7 @@ static void test_bh_delete_from_cb_many(void)
     g_assert_cmpint(data4.n, ==, 1);
     g_assert(data1.bh == NULL);
 
-    aio_flush(ctx);
+    wait_for_aio();
     g_assert_cmpint(data1.n, ==, data1.max);
     g_assert_cmpint(data2.n, ==, data2.max);
     g_assert_cmpint(data3.n, ==, data3.max);
@@ -219,7 +219,7 @@ static void test_bh_flush(void)
     qemu_bh_schedule(data.bh);
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_flush(ctx);
+    wait_for_aio();
     g_assert_cmpint(data.n, ==, 1);
 
     g_assert(!aio_poll(ctx, false));
@@ -281,7 +281,7 @@ static void test_flush_event_notifier(void)
     g_assert_cmpint(data.active, ==, 9);
     g_assert(aio_poll(ctx, false));
 
-    aio_flush(ctx);
+    wait_for_aio();
     g_assert_cmpint(data.n, ==, 10);
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!aio_poll(ctx, false));
@@ -315,17 +315,17 @@ static void test_wait_event_notifier_noflush(void)
     event_notifier_set(&data.e);
     g_assert(aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 1);
-    g_assert(!aio_poll(ctx, false));
+    g_assert(aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 1);
 
     event_notifier_set(&data.e);
     g_assert(aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
-    g_assert(!aio_poll(ctx, false));
+    g_assert(aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
     event_notifier_set(&dummy.e);
-    aio_flush(ctx);
+    wait_for_aio();
     g_assert_cmpint(data.n, ==, 2);
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
@@ -346,7 +346,7 @@ static void test_wait_event_notifier_noflush(void)
  * - sometimes both the AioContext and the glib main loop wake
  *   themselves up.  Hence, some "g_assert(!aio_poll(ctx, false));"
  *   are replaced by "while (g_main_context_iteration(NULL, false));".
- * - there is no exact replacement for aio_flush's blocking wait.
+ * - there is no exact replacement for a blocking wait.
  *   "while (g_main_context_iteration(NULL, true)" seems to work,
  *   but it is not documented _why_ it works.  For these tests a
  *   non-blocking loop like "while (g_main_context_iteration(NULL, false)"
@@ -637,7 +637,6 @@ int main(int argc, char **argv)
 
     g_test_init(&argc, &argv, NULL);
     g_test_add_func("/aio/notify",                  test_notify);
-    g_test_add_func("/aio/flush",                   test_flush);
     g_test_add_func("/aio/bh/schedule",             test_bh_schedule);
     g_test_add_func("/aio/bh/schedule10",           test_bh_schedule10);
     g_test_add_func("/aio/bh/cancel",               test_bh_cancel);
diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index e5d14eb..4c6cc81 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -12,7 +12,7 @@
  */
 
 #include <glib.h>
-#include "qemu-coroutine.h"
+#include "block/coroutine.h"
 
 /*
  * Check that qemu_in_coroutine() works
diff --git a/tests/test-cutils.c b/tests/test-cutils.c
new file mode 100644
index 0000000..2a4556d
--- /dev/null
+++ b/tests/test-cutils.c
@@ -0,0 +1,251 @@
+/*
+ * cutils.c unit-tests
+ *
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * Authors:
+ *  Eduardo Habkost <ehabkost@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <glib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "qemu-common.h"
+
+
+static void test_parse_uint_null(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    int r;
+
+    r = parse_uint(NULL, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, -EINVAL);
+    g_assert_cmpint(i, ==, 0);
+    g_assert(endptr == NULL);
+}
+
+static void test_parse_uint_empty(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, -EINVAL);
+    g_assert_cmpint(i, ==, 0);
+    g_assert(endptr == str);
+}
+
+static void test_parse_uint_whitespace(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "   \t   ";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, -EINVAL);
+    g_assert_cmpint(i, ==, 0);
+    g_assert(endptr == str);
+}
+
+
+static void test_parse_uint_invalid(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = " \t xxx";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, -EINVAL);
+    g_assert_cmpint(i, ==, 0);
+    g_assert(endptr == str);
+}
+
+
+static void test_parse_uint_trailing(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "123xxx";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, 0);
+    g_assert_cmpint(i, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_parse_uint_correct(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "123";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, 0);
+    g_assert_cmpint(i, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_parse_uint_octal(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "0123";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, 0);
+    g_assert_cmpint(i, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_parse_uint_decimal(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "0123";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 10);
+
+    g_assert_cmpint(r, ==, 0);
+    g_assert_cmpint(i, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+
+static void test_parse_uint_llong_max(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    char *str = g_strdup_printf("%llu", (unsigned long long)LLONG_MAX + 1);
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, 0);
+    g_assert_cmpint(i, ==, (unsigned long long)LLONG_MAX + 1);
+    g_assert(endptr == str + strlen(str));
+
+    g_free(str);
+}
+
+static void test_parse_uint_overflow(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = "99999999999999999999999999999999999999";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, -ERANGE);
+    g_assert_cmpint(i, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_parse_uint_negative(void)
+{
+    unsigned long long i = 999;
+    char f = 'X';
+    char *endptr = &f;
+    const char *str = " \t -321";
+    int r;
+
+    r = parse_uint(str, &i, &endptr, 0);
+
+    g_assert_cmpint(r, ==, -ERANGE);
+    g_assert_cmpint(i, ==, 0);
+    g_assert(endptr == str + strlen(str));
+}
+
+
+static void test_parse_uint_full_trailing(void)
+{
+    unsigned long long i = 999;
+    const char *str = "123xxx";
+    int r;
+
+    r = parse_uint_full(str, &i, 0);
+
+    g_assert_cmpint(r, ==, -EINVAL);
+    g_assert_cmpint(i, ==, 0);
+}
+
+static void test_parse_uint_full_correct(void)
+{
+    unsigned long long i = 999;
+    const char *str = "123";
+    int r;
+
+    r = parse_uint_full(str, &i, 0);
+
+    g_assert_cmpint(r, ==, 0);
+    g_assert_cmpint(i, ==, 123);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/cutils/parse_uint/null", test_parse_uint_null);
+    g_test_add_func("/cutils/parse_uint/empty", test_parse_uint_empty);
+    g_test_add_func("/cutils/parse_uint/whitespace",
+                    test_parse_uint_whitespace);
+    g_test_add_func("/cutils/parse_uint/invalid", test_parse_uint_invalid);
+    g_test_add_func("/cutils/parse_uint/trailing", test_parse_uint_trailing);
+    g_test_add_func("/cutils/parse_uint/correct", test_parse_uint_correct);
+    g_test_add_func("/cutils/parse_uint/octal", test_parse_uint_octal);
+    g_test_add_func("/cutils/parse_uint/decimal", test_parse_uint_decimal);
+    g_test_add_func("/cutils/parse_uint/llong_max", test_parse_uint_llong_max);
+    g_test_add_func("/cutils/parse_uint/overflow", test_parse_uint_overflow);
+    g_test_add_func("/cutils/parse_uint/negative", test_parse_uint_negative);
+    g_test_add_func("/cutils/parse_uint_full/trailing",
+                    test_parse_uint_full_trailing);
+    g_test_add_func("/cutils/parse_uint_full/correct",
+                    test_parse_uint_full_correct);
+
+    return g_test_run();
+}
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
new file mode 100644
index 0000000..8c902f2
--- /dev/null
+++ b/tests/test-hbitmap.c
@@ -0,0 +1,401 @@
+/*
+ * Hierarchical bitmap unit-tests.
+ *
+ * Copyright (C) 2012 Red Hat Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <glib.h>
+#include <stdarg.h>
+#include "qemu/hbitmap.h"
+
+#define LOG_BITS_PER_LONG          (BITS_PER_LONG == 32 ? 5 : 6)
+
+#define L1                         BITS_PER_LONG
+#define L2                         (BITS_PER_LONG * L1)
+#define L3                         (BITS_PER_LONG * L2)
+
+typedef struct TestHBitmapData {
+    HBitmap       *hb;
+    unsigned long *bits;
+    size_t         size;
+    int            granularity;
+} TestHBitmapData;
+
+
+/* Check that the HBitmap and the shadow bitmap contain the same data,
+ * ignoring the same "first" bits.
+ */
+static void hbitmap_test_check(TestHBitmapData *data,
+                               uint64_t first)
+{
+    uint64_t count = 0;
+    size_t pos;
+    int bit;
+    HBitmapIter hbi;
+    int64_t i, next;
+
+    hbitmap_iter_init(&hbi, data->hb, first);
+
+    i = first;
+    for (;;) {
+        next = hbitmap_iter_next(&hbi);
+        if (next < 0) {
+            next = data->size;
+        }
+
+        while (i < next) {
+            pos = i >> LOG_BITS_PER_LONG;
+            bit = i & (BITS_PER_LONG - 1);
+            i++;
+            g_assert_cmpint(data->bits[pos] & (1UL << bit), ==, 0);
+        }
+
+        if (next == data->size) {
+            break;
+        }
+
+        pos = i >> LOG_BITS_PER_LONG;
+        bit = i & (BITS_PER_LONG - 1);
+        i++;
+        count++;
+        g_assert_cmpint(data->bits[pos] & (1UL << bit), !=, 0);
+    }
+
+    if (first == 0) {
+        g_assert_cmpint(count << data->granularity, ==, hbitmap_count(data->hb));
+    }
+}
+
+/* This is provided instead of a test setup function so that the sizes
+   are kept in the test functions (and not in main()) */
+static void hbitmap_test_init(TestHBitmapData *data,
+                              uint64_t size, int granularity)
+{
+    size_t n;
+    data->hb = hbitmap_alloc(size, granularity);
+
+    n = (size + BITS_PER_LONG - 1) / BITS_PER_LONG;
+    if (n == 0) {
+        n = 1;
+    }
+    data->bits = g_new0(unsigned long, n);
+    data->size = size;
+    data->granularity = granularity;
+    if (size) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
+static void hbitmap_test_teardown(TestHBitmapData *data,
+                                  const void *unused)
+{
+    if (data->hb) {
+        hbitmap_free(data->hb);
+        data->hb = NULL;
+    }
+    if (data->bits) {
+        g_free(data->bits);
+        data->bits = NULL;
+    }
+}
+
+/* Set a range in the HBitmap and in the shadow "simple" bitmap.
+ * The two bitmaps are then tested against each other.
+ */
+static void hbitmap_test_set(TestHBitmapData *data,
+                             uint64_t first, uint64_t count)
+{
+    hbitmap_set(data->hb, first, count);
+    while (count-- != 0) {
+        size_t pos = first >> LOG_BITS_PER_LONG;
+        int bit = first & (BITS_PER_LONG - 1);
+        first++;
+
+        data->bits[pos] |= 1UL << bit;
+    }
+
+    if (data->granularity == 0) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
+/* Reset a range in the HBitmap and in the shadow "simple" bitmap.
+ */
+static void hbitmap_test_reset(TestHBitmapData *data,
+                               uint64_t first, uint64_t count)
+{
+    hbitmap_reset(data->hb, first, count);
+    while (count-- != 0) {
+        size_t pos = first >> LOG_BITS_PER_LONG;
+        int bit = first & (BITS_PER_LONG - 1);
+        first++;
+
+        data->bits[pos] &= ~(1UL << bit);
+    }
+
+    if (data->granularity == 0) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
+static void hbitmap_test_check_get(TestHBitmapData *data)
+{
+    uint64_t count = 0;
+    uint64_t i;
+
+    for (i = 0; i < data->size; i++) {
+        size_t pos = i >> LOG_BITS_PER_LONG;
+        int bit = i & (BITS_PER_LONG - 1);
+        unsigned long val = data->bits[pos] & (1UL << bit);
+        count += hbitmap_get(data->hb, i);
+        g_assert_cmpint(hbitmap_get(data->hb, i), ==, val != 0);
+    }
+    g_assert_cmpint(count, ==, hbitmap_count(data->hb));
+}
+
+static void test_hbitmap_zero(TestHBitmapData *data,
+                               const void *unused)
+{
+    hbitmap_test_init(data, 0, 0);
+}
+
+static void test_hbitmap_unaligned(TestHBitmapData *data,
+                                   const void *unused)
+{
+    hbitmap_test_init(data, L3 + 23, 0);
+    hbitmap_test_set(data, 0, 1);
+    hbitmap_test_set(data, L3 + 22, 1);
+}
+
+static void test_hbitmap_iter_empty(TestHBitmapData *data,
+                                    const void *unused)
+{
+    hbitmap_test_init(data, L1, 0);
+}
+
+static void test_hbitmap_iter_partial(TestHBitmapData *data,
+                                      const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_set(data, 0, L3);
+    hbitmap_test_check(data, 1);
+    hbitmap_test_check(data, L1 - 1);
+    hbitmap_test_check(data, L1);
+    hbitmap_test_check(data, L1 * 2 - 1);
+    hbitmap_test_check(data, L2 - 1);
+    hbitmap_test_check(data, L2);
+    hbitmap_test_check(data, L2 + 1);
+    hbitmap_test_check(data, L2 + L1);
+    hbitmap_test_check(data, L2 + L1 * 2 - 1);
+    hbitmap_test_check(data, L2 * 2 - 1);
+    hbitmap_test_check(data, L2 * 2);
+    hbitmap_test_check(data, L2 * 2 + 1);
+    hbitmap_test_check(data, L2 * 2 + L1);
+    hbitmap_test_check(data, L2 * 2 + L1 * 2 - 1);
+    hbitmap_test_check(data, L3 / 2);
+}
+
+static void test_hbitmap_set_all(TestHBitmapData *data,
+                                 const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_set(data, 0, L3);
+}
+
+static void test_hbitmap_get_all(TestHBitmapData *data,
+                                 const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_set(data, 0, L3);
+    hbitmap_test_check_get(data);
+}
+
+static void test_hbitmap_get_some(TestHBitmapData *data,
+                                  const void *unused)
+{
+    hbitmap_test_init(data, 2 * L2, 0);
+    hbitmap_test_set(data, 10, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L1 - 1, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L1, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L2 - 1, 1);
+    hbitmap_test_check_get(data);
+    hbitmap_test_set(data, L2, 1);
+    hbitmap_test_check_get(data);
+}
+
+static void test_hbitmap_set_one(TestHBitmapData *data,
+                                 const void *unused)
+{
+    hbitmap_test_init(data, 2 * L2, 0);
+    hbitmap_test_set(data, 10, 1);
+    hbitmap_test_set(data, L1 - 1, 1);
+    hbitmap_test_set(data, L1, 1);
+    hbitmap_test_set(data, L2 - 1, 1);
+    hbitmap_test_set(data, L2, 1);
+}
+
+static void test_hbitmap_set_two_elem(TestHBitmapData *data,
+                                      const void *unused)
+{
+    hbitmap_test_init(data, 2 * L2, 0);
+    hbitmap_test_set(data, L1 - 1, 2);
+    hbitmap_test_set(data, L1 * 2 - 1, 4);
+    hbitmap_test_set(data, L1 * 4, L1 + 1);
+    hbitmap_test_set(data, L1 * 8 - 1, L1 + 1);
+    hbitmap_test_set(data, L2 - 1, 2);
+    hbitmap_test_set(data, L2 + L1 - 1, 8);
+    hbitmap_test_set(data, L2 + L1 * 4, L1 + 1);
+    hbitmap_test_set(data, L2 + L1 * 8 - 1, L1 + 1);
+}
+
+static void test_hbitmap_set(TestHBitmapData *data,
+                             const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_set(data, L1 * 3 - 1, L1 + 2);
+    hbitmap_test_set(data, L1 * 5, L1 * 2 + 1);
+    hbitmap_test_set(data, L1 * 8 - 1, L1 * 2 + 1);
+    hbitmap_test_set(data, L2 - 1, L1 + 2);
+    hbitmap_test_set(data, L2 + L1 * 2 - 1, L1 + 2);
+    hbitmap_test_set(data, L2 + L1 * 4, L1 * 2 + 1);
+    hbitmap_test_set(data, L2 + L1 * 7 - 1, L1 * 2 + 1);
+    hbitmap_test_set(data, L2 * 2 - 1, L3 * 2 - L2 * 2);
+}
+
+static void test_hbitmap_set_twice(TestHBitmapData *data,
+                                   const void *unused)
+{
+    hbitmap_test_init(data, L1 * 3, 0);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_set(data, L1, 1);
+}
+
+static void test_hbitmap_set_overlap(TestHBitmapData *data,
+                                     const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_set(data, L1 * 2 - 1, L1 * 2 + 2);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_set(data, L1 * 8 - 1, L2);
+    hbitmap_test_set(data, L2, L1);
+    hbitmap_test_set(data, L2 - L1 - 1, L1 * 8 + 2);
+    hbitmap_test_set(data, L2, L3 - L2 + 1);
+    hbitmap_test_set(data, L3 - L1, L1 * 3);
+    hbitmap_test_set(data, L3 - 1, 3);
+    hbitmap_test_set(data, L3 - 1, L2);
+}
+
+static void test_hbitmap_reset_empty(TestHBitmapData *data,
+                                     const void *unused)
+{
+    hbitmap_test_init(data, L3, 0);
+    hbitmap_test_reset(data, 0, L3);
+}
+
+static void test_hbitmap_reset(TestHBitmapData *data,
+                               const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_reset(data, L1 * 2 - 1, L1 * 2 + 2);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_reset(data, L1 * 8 - 1, L2);
+    hbitmap_test_set(data, L2, L1);
+    hbitmap_test_reset(data, L2 - L1 - 1, L1 * 8 + 2);
+    hbitmap_test_set(data, L2, L3 - L2 + 1);
+    hbitmap_test_reset(data, L3 - L1, L1 * 3);
+    hbitmap_test_set(data, L3 - 1, 3);
+    hbitmap_test_reset(data, L3 - 1, L2);
+    hbitmap_test_set(data, 0, L3 * 2);
+    hbitmap_test_reset(data, 0, L1);
+    hbitmap_test_reset(data, 0, L2);
+    hbitmap_test_reset(data, L3, L3);
+    hbitmap_test_set(data, L3 / 2, L3);
+}
+
+static void test_hbitmap_granularity(TestHBitmapData *data,
+                                     const void *unused)
+{
+    /* Note that hbitmap_test_check has to be invoked manually in this test.  */
+    hbitmap_test_init(data, L1, 1);
+    hbitmap_test_set(data, 0, 1);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 2);
+    hbitmap_test_check(data, 0);
+    hbitmap_test_set(data, 2, 1);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 4);
+    hbitmap_test_check(data, 0);
+    hbitmap_test_set(data, 0, 3);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 4);
+    hbitmap_test_reset(data, 0, 1);
+    g_assert_cmpint(hbitmap_count(data->hb), ==, 2);
+}
+
+static void test_hbitmap_iter_granularity(TestHBitmapData *data,
+                                          const void *unused)
+{
+    HBitmapIter hbi;
+
+    /* Note that hbitmap_test_check has to be invoked manually in this test.  */
+    hbitmap_test_init(data, 131072 << 7, 7);
+    hbitmap_iter_init(&hbi, data->hb, 0);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_test_set(data, ((L2 + L1 + 1) << 7) + 8, 8);
+    hbitmap_iter_init(&hbi, data->hb, 0);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, (L2 + L1 + 1) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_test_set(data, (131072 << 7) - 8, 8);
+    hbitmap_iter_init(&hbi, data->hb, 0);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, (L2 + L1 + 1) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, 131071 << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+
+    hbitmap_iter_init(&hbi, data->hb, (L2 + L1 + 2) << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), ==, 131071 << 7);
+    g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
+}
+
+static void hbitmap_test_add(const char *testpath,
+                                   void (*test_func)(TestHBitmapData *data, const void *user_data))
+{
+    g_test_add(testpath, TestHBitmapData, NULL, NULL, test_func,
+               hbitmap_test_teardown);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    hbitmap_test_add("/hbitmap/size/0", test_hbitmap_zero);
+    hbitmap_test_add("/hbitmap/size/unaligned", test_hbitmap_unaligned);
+    hbitmap_test_add("/hbitmap/iter/empty", test_hbitmap_iter_empty);
+    hbitmap_test_add("/hbitmap/iter/partial", test_hbitmap_iter_partial);
+    hbitmap_test_add("/hbitmap/iter/granularity", test_hbitmap_iter_granularity);
+    hbitmap_test_add("/hbitmap/get/all", test_hbitmap_get_all);
+    hbitmap_test_add("/hbitmap/get/some", test_hbitmap_get_some);
+    hbitmap_test_add("/hbitmap/set/all", test_hbitmap_set_all);
+    hbitmap_test_add("/hbitmap/set/one", test_hbitmap_set_one);
+    hbitmap_test_add("/hbitmap/set/two-elem", test_hbitmap_set_two_elem);
+    hbitmap_test_add("/hbitmap/set/general", test_hbitmap_set);
+    hbitmap_test_add("/hbitmap/set/twice", test_hbitmap_set_twice);
+    hbitmap_test_add("/hbitmap/set/overlap", test_hbitmap_set_overlap);
+    hbitmap_test_add("/hbitmap/reset/empty", test_hbitmap_reset_empty);
+    hbitmap_test_add("/hbitmap/reset/general", test_hbitmap_reset);
+    hbitmap_test_add("/hbitmap/granularity", test_hbitmap_granularity);
+    g_test_run();
+
+    return 0;
+}
diff --git a/tests/test-iov.c b/tests/test-iov.c
index cbe7a89..46e4ddd 100644
--- a/tests/test-iov.c
+++ b/tests/test-iov.c
@@ -1,7 +1,7 @@
 #include <glib.h>
 #include "qemu-common.h"
-#include "iov.h"
-#include "qemu_socket.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
 
 /* create a randomly-sized iovec with random vectors */
 static void iov_random(struct iovec **iovp, unsigned *iov_cntp)
@@ -250,11 +250,161 @@ static void test_io(void)
 #endif
 }
 
+static void test_discard_front(void)
+{
+    struct iovec *iov;
+    struct iovec *iov_tmp;
+    unsigned int iov_cnt;
+    unsigned int iov_cnt_tmp;
+    void *old_base;
+    size_t size;
+    size_t ret;
+
+    /* Discard zero bytes */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, 0);
+    g_assert(ret == 0);
+    g_assert(iov_tmp == iov);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    iov_free(iov, iov_cnt);
+
+    /* Discard more bytes than vector size */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size + 1);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire vector */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within first element */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov->iov_base;
+    size = g_test_rand_int_range(1, iov->iov_len);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_tmp == iov);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    g_assert(iov_tmp->iov_base == old_base + size);
+    iov_tmp->iov_base = old_base; /* undo before g_free() */
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire first element */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, iov->iov_len);
+    g_assert(ret == iov->iov_len);
+    g_assert(iov_tmp == iov + 1);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within second element */
+    iov_random(&iov, &iov_cnt);
+    iov_tmp = iov;
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[1].iov_base;
+    size = iov->iov_len + g_test_rand_int_range(1, iov[1].iov_len);
+    ret = iov_discard_front(&iov_tmp, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_tmp == iov + 1);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    g_assert(iov_tmp->iov_base == old_base + (size - iov->iov_len));
+    iov_tmp->iov_base = old_base; /* undo before g_free() */
+    iov_free(iov, iov_cnt);
+}
+
+static void test_discard_back(void)
+{
+    struct iovec *iov;
+    unsigned int iov_cnt;
+    unsigned int iov_cnt_tmp;
+    void *old_base;
+    size_t size;
+    size_t ret;
+
+    /* Discard zero bytes */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    ret = iov_discard_back(iov, &iov_cnt_tmp, 0);
+    g_assert(ret == 0);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    iov_free(iov, iov_cnt);
+
+    /* Discard more bytes than vector size */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size + 1);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire vector */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    size = iov_size(iov, iov_cnt);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == 0);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within last element */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[iov_cnt - 1].iov_base;
+    size = g_test_rand_int_range(1, iov[iov_cnt - 1].iov_len);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == iov_cnt);
+    g_assert(iov[iov_cnt - 1].iov_base == old_base);
+    iov_free(iov, iov_cnt);
+
+    /* Discard entire last element */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[iov_cnt - 1].iov_base;
+    size = iov[iov_cnt - 1].iov_len;
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    iov_free(iov, iov_cnt);
+
+    /* Discard within second-to-last element */
+    iov_random(&iov, &iov_cnt);
+    iov_cnt_tmp = iov_cnt;
+    old_base = iov[iov_cnt - 2].iov_base;
+    size = iov[iov_cnt - 1].iov_len +
+           g_test_rand_int_range(1, iov[iov_cnt - 2].iov_len);
+    ret = iov_discard_back(iov, &iov_cnt_tmp, size);
+    g_assert(ret == size);
+    g_assert(iov_cnt_tmp == iov_cnt - 1);
+    g_assert(iov[iov_cnt - 2].iov_base == old_base);
+    iov_free(iov, iov_cnt);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
     g_test_rand_int();
     g_test_add_func("/basic/iov/from-to-buf", test_to_from_buf);
     g_test_add_func("/basic/iov/io", test_io);
+    g_test_add_func("/basic/iov/discard-front", test_discard_front);
+    g_test_add_func("/basic/iov/discard-back", test_discard_back);
     return g_test_run();
 }
diff --git a/tests/test-qmp-commands.c b/tests/test-qmp-commands.c
index dc3c507..5a3e82a 100644
--- a/tests/test-qmp-commands.c
+++ b/tests/test-qmp-commands.c
@@ -1,8 +1,9 @@
 #include <glib.h>
-#include "qemu-objects.h"
+#include "qemu-common.h"
+#include "qapi/qmp/types.h"
 #include "test-qmp-commands.h"
-#include "qapi/qmp-core.h"
-#include "module.h"
+#include "qapi/qmp/dispatch.h"
+#include "qemu/module.h"
 #include "qapi/qmp-input-visitor.h"
 #include "tests/test-qapi-types.h"
 #include "tests/test-qapi-visit.h"
diff --git a/tests/test-qmp-input-strict.c b/tests/test-qmp-input-strict.c
index f6df8cb..6f68963 100644
--- a/tests/test-qmp-input-strict.c
+++ b/tests/test-qmp-input-strict.c
@@ -14,10 +14,11 @@
 #include <glib.h>
 #include <stdarg.h>
 
+#include "qemu-common.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
diff --git a/tests/test-qmp-input-visitor.c b/tests/test-qmp-input-visitor.c
index 8f5a509..955a4c0 100644
--- a/tests/test-qmp-input-visitor.c
+++ b/tests/test-qmp-input-visitor.c
@@ -13,10 +13,11 @@
 #include <glib.h>
 #include <stdarg.h>
 
+#include "qemu-common.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
diff --git a/tests/test-qmp-output-visitor.c b/tests/test-qmp-output-visitor.c
index 24a6359..71367e6 100644
--- a/tests/test-qmp-output-visitor.c
+++ b/tests/test-qmp-output-visitor.c
@@ -12,10 +12,11 @@
 
 #include <glib.h>
 
+#include "qemu-common.h"
 #include "qapi/qmp-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestOutputVisitorData {
     QmpOutputVisitor *qov;
diff --git a/tests/test-string-input-visitor.c b/tests/test-string-input-visitor.c
index 5370e32..5989f81 100644
--- a/tests/test-string-input-visitor.c
+++ b/tests/test-string-input-visitor.c
@@ -13,10 +13,11 @@
 #include <glib.h>
 #include <stdarg.h>
 
+#include "qemu-common.h"
 #include "qapi/string-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
     StringInputVisitor *siv;
@@ -164,6 +165,53 @@ static void test_visitor_in_enum(TestInputVisitorData *data,
     data->siv = NULL;
 }
 
+/* Try to crash the visitors */
+static void test_visitor_in_fuzz(TestInputVisitorData *data,
+                                 const void *unused)
+{
+    int64_t ires;
+    bool bres;
+    double nres;
+    char *sres;
+    EnumOne eres;
+    Visitor *v;
+    unsigned int i;
+    char buf[10000];
+
+    for (i = 0; i < 100; i++) {
+        unsigned int j;
+
+        j = g_test_rand_int_range(0, sizeof(buf) - 1);
+
+        buf[j] = '\0';
+
+        if (j != 0) {
+            for (j--; j != 0; j--) {
+                buf[j - 1] = (char)g_test_rand_int_range(0, 256);
+            }
+        }
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_int(v, &ires, NULL, NULL);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_bool(v, &bres, NULL, NULL);
+        visitor_input_teardown(data, NULL);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_number(v, &nres, NULL, NULL);
+
+        v = visitor_input_test_init(data, buf);
+        sres = NULL;
+        visit_type_str(v, &sres, NULL, NULL);
+        g_free(sres);
+
+        v = visitor_input_test_init(data, buf);
+        visit_type_EnumOne(v, &eres, NULL, NULL);
+        visitor_input_teardown(data, NULL);
+    }
+}
+
 static void input_visitor_test_add(const char *testpath,
                                    TestInputVisitorData *data,
                                    void (*test_func)(TestInputVisitorData *data, const void *user_data))
@@ -188,6 +236,8 @@ int main(int argc, char **argv)
                             &in_visitor_data, test_visitor_in_string);
     input_visitor_test_add("/string-visitor/input/enum",
                             &in_visitor_data, test_visitor_in_enum);
+    input_visitor_test_add("/string-visitor/input/fuzz",
+                            &in_visitor_data, test_visitor_in_fuzz);
 
     g_test_run();
 
diff --git a/tests/test-string-output-visitor.c b/tests/test-string-output-visitor.c
index 608f14a..79d815f 100644
--- a/tests/test-string-output-visitor.c
+++ b/tests/test-string-output-visitor.c
@@ -12,10 +12,11 @@
 
 #include <glib.h>
 
+#include "qemu-common.h"
 #include "qapi/string-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 
 typedef struct TestOutputVisitorData {
     StringOutputVisitor *sov;
diff --git a/tests/test-thread-pool.c b/tests/test-thread-pool.c
index fea0445..9998e03 100644
--- a/tests/test-thread-pool.c
+++ b/tests/test-thread-pool.c
@@ -1,8 +1,8 @@
 #include <glib.h>
 #include "qemu-common.h"
-#include "qemu-aio.h"
-#include "thread-pool.h"
-#include "block.h"
+#include "block/aio.h"
+#include "block/thread-pool.h"
+#include "block/block.h"
 
 static int active;
 
@@ -47,11 +47,19 @@ static void qemu_aio_wait_nonblocking(void)
     qemu_aio_wait();
 }
 
+/* Wait until all aio and bh activity has finished */
+static void qemu_aio_wait_all(void)
+{
+    while (qemu_aio_wait()) {
+        /* Do nothing */
+    }
+}
+
 static void test_submit(void)
 {
     WorkerTestData data = { .n = 0 };
     thread_pool_submit(worker_cb, &data);
-    qemu_aio_flush();
+    qemu_aio_wait_all();
     g_assert_cmpint(data.n, ==, 1);
 }
 
@@ -63,7 +71,7 @@ static void test_submit_aio(void)
     /* The callbacks are not called until after the first wait.  */
     active = 1;
     g_assert_cmpint(data.ret, ==, -EINPROGRESS);
-    qemu_aio_flush();
+    qemu_aio_wait_all();
     g_assert_cmpint(active, ==, 0);
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.ret, ==, 0);
@@ -84,7 +92,7 @@ static void co_test_cb(void *opaque)
     data->ret = 0;
     active--;
 
-    /* The test continues in test_submit_co, after qemu_aio_flush... */
+    /* The test continues in test_submit_co, after qemu_aio_wait_all... */
 }
 
 static void test_submit_co(void)
@@ -99,9 +107,9 @@ static void test_submit_co(void)
     g_assert_cmpint(active, ==, 1);
     g_assert_cmpint(data.ret, ==, -EINPROGRESS);
 
-    /* qemu_aio_flush will execute the rest of the coroutine.  */
+    /* qemu_aio_wait_all will execute the rest of the coroutine.  */
 
-    qemu_aio_flush();
+    qemu_aio_wait_all();
 
     /* Back here after the coroutine has finished.  */
 
@@ -184,7 +192,7 @@ static void test_cancel(void)
     }
 
     /* Finish execution and execute any remaining callbacks.  */
-    qemu_aio_flush();
+    qemu_aio_wait_all();
     g_assert_cmpint(active, ==, 0);
     for (i = 0; i < 100; i++) {
         if (data[i].n == 3) {
diff --git a/tests/test-visitor-serialization.c b/tests/test-visitor-serialization.c
index b8ad16f..3c6b8df 100644
--- a/tests/test-visitor-serialization.c
+++ b/tests/test-visitor-serialization.c
@@ -14,9 +14,11 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <float.h>
+
+#include "qemu-common.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
-#include "qemu-objects.h"
+#include "qapi/qmp/types.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/string-input-visitor.h"
diff --git a/tests/test-x86-cpuid.c b/tests/test-x86-cpuid.c
new file mode 100644
index 0000000..8d9f96a
--- /dev/null
+++ b/tests/test-x86-cpuid.c
@@ -0,0 +1,110 @@
+/*
+ *  Test code for x86 CPUID and Topology functions
+ *
+ *  Copyright (c) 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <glib.h>
+
+#include "topology.h"
+
+static void test_topo_bits(void)
+{
+    /* simple tests for 1 thread per core, 1 core per socket */
+    g_assert_cmpuint(apicid_smt_width(1, 1), ==, 0);
+    g_assert_cmpuint(apicid_core_width(1, 1), ==, 0);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 0), ==, 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 1), ==, 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 2), ==, 2);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(1, 1, 3), ==, 3);
+
+
+    /* Test field width calculation for multiple values
+     */
+    g_assert_cmpuint(apicid_smt_width(1, 2), ==, 1);
+    g_assert_cmpuint(apicid_smt_width(1, 3), ==, 2);
+    g_assert_cmpuint(apicid_smt_width(1, 4), ==, 2);
+
+    g_assert_cmpuint(apicid_smt_width(1, 14), ==, 4);
+    g_assert_cmpuint(apicid_smt_width(1, 15), ==, 4);
+    g_assert_cmpuint(apicid_smt_width(1, 16), ==, 4);
+    g_assert_cmpuint(apicid_smt_width(1, 17), ==, 5);
+
+
+    g_assert_cmpuint(apicid_core_width(30, 2), ==, 5);
+    g_assert_cmpuint(apicid_core_width(31, 2), ==, 5);
+    g_assert_cmpuint(apicid_core_width(32, 2), ==, 5);
+    g_assert_cmpuint(apicid_core_width(33, 2), ==, 6);
+
+
+    /* build a weird topology and see if IDs are calculated correctly
+     */
+
+    /* This will use 2 bits for thread ID and 3 bits for core ID
+     */
+    g_assert_cmpuint(apicid_smt_width(6, 3), ==, 2);
+    g_assert_cmpuint(apicid_core_width(6, 3), ==, 3);
+    g_assert_cmpuint(apicid_pkg_offset(6, 3), ==, 5);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 0), ==, 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1), ==, 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2), ==, 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 3 + 0), ==,
+                     (1 << 2) | 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 3 + 1), ==,
+                     (1 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 3 + 2), ==,
+                     (1 << 2) | 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2 * 3 + 0), ==,
+                     (2 << 2) | 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2 * 3 + 1), ==,
+                     (2 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 2 * 3 + 2), ==,
+                     (2 << 2) | 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 5 * 3 + 0), ==,
+                     (5 << 2) | 0);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 5 * 3 + 1), ==,
+                     (5 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 5 * 3 + 2), ==,
+                     (5 << 2) | 2);
+
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 6 * 3 + 0 * 3 + 0), ==,
+                     (1 << 5));
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 1 * 6 * 3 + 1 * 3 + 1), ==,
+                     (1 << 5) | (1 << 2) | 1);
+    g_assert_cmpuint(x86_apicid_from_cpu_idx(6, 3, 3 * 6 * 3 + 5 * 3 + 2), ==,
+                     (3 << 5) | (5 << 2) | 2);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/cpuid/topology/basic", test_topo_bits);
+
+    g_test_run();
+
+    return 0;
+}
diff --git a/tests/test-xbzrle.c b/tests/test-xbzrle.c
new file mode 100644
index 0000000..db93b0a
--- /dev/null
+++ b/tests/test-xbzrle.c
@@ -0,0 +1,196 @@
+/*
+ * Xor Based Zero Run Length Encoding unit tests.
+ *
+ * Copyright 2013 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Orit Wasserman  <owasserm@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <string.h>
+#include <sys/time.h>
+#include <assert.h>
+#include "qemu-common.h"
+#include "include/migration/migration.h"
+
+#define PAGE_SIZE 4096
+
+static void test_uleb(void)
+{
+    uint32_t i, val;
+    uint8_t buf[2];
+    int encode_ret, decode_ret;
+
+    for (i = 0; i <= 0x3fff; i++) {
+        encode_ret = uleb128_encode_small(&buf[0], i);
+        decode_ret = uleb128_decode_small(&buf[0], &val);
+        g_assert(encode_ret == decode_ret);
+        g_assert(i == val);
+    }
+
+    /* decode invalid value */
+    buf[0] = 0x80;
+    buf[1] = 0x80;
+
+    decode_ret = uleb128_decode_small(&buf[0], &val);
+    g_assert(decode_ret == -1);
+    g_assert(val == 0);
+}
+
+static void test_encode_decode_zero(void)
+{
+    uint8_t *buffer = g_malloc0(PAGE_SIZE);
+    uint8_t *compressed = g_malloc0(PAGE_SIZE);
+    int i = 0;
+    int dlen = 0;
+    int diff_len = g_test_rand_int_range(0, PAGE_SIZE - 1006);
+
+    for (i = diff_len; i > 0; i--) {
+        buffer[1000 + i] = i;
+    }
+
+    buffer[1000 + diff_len + 3] = 103;
+    buffer[1000 + diff_len + 5] = 105;
+
+    /* encode zero page */
+    dlen = xbzrle_encode_buffer(buffer, buffer, PAGE_SIZE, compressed,
+                       PAGE_SIZE);
+    g_assert(dlen == 0);
+
+    g_free(buffer);
+    g_free(compressed);
+}
+
+static void test_encode_decode_unchanged(void)
+{
+    uint8_t *compressed = g_malloc0(PAGE_SIZE);
+    uint8_t *test = g_malloc0(PAGE_SIZE);
+    int i = 0;
+    int dlen = 0;
+    int diff_len = g_test_rand_int_range(0, PAGE_SIZE - 1006);
+
+    for (i = diff_len; i > 0; i--) {
+        test[1000 + i] = i + 4;
+    }
+
+    test[1000 + diff_len + 3] = 107;
+    test[1000 + diff_len + 5] = 109;
+
+    /* test unchanged buffer */
+    dlen = xbzrle_encode_buffer(test, test, PAGE_SIZE, compressed,
+                                PAGE_SIZE);
+    g_assert(dlen == 0);
+
+    g_free(test);
+    g_free(compressed);
+}
+
+static void test_encode_decode_1_byte(void)
+{
+    uint8_t *buffer = g_malloc0(PAGE_SIZE);
+    uint8_t *test = g_malloc0(PAGE_SIZE);
+    uint8_t *compressed = g_malloc(PAGE_SIZE);
+    int dlen = 0, rc = 0;
+    uint8_t buf[2];
+
+    test[PAGE_SIZE - 1] = 1;
+
+    dlen = xbzrle_encode_buffer(buffer, test, PAGE_SIZE, compressed,
+                       PAGE_SIZE);
+    g_assert(dlen == (uleb128_encode_small(&buf[0], 4095) + 2));
+
+    rc = xbzrle_decode_buffer(compressed, dlen, buffer, PAGE_SIZE);
+    g_assert(rc == PAGE_SIZE);
+    g_assert(memcmp(test, buffer, PAGE_SIZE) == 0);
+
+    g_free(buffer);
+    g_free(compressed);
+    g_free(test);
+}
+
+static void test_encode_decode_overflow(void)
+{
+    uint8_t *compressed = g_malloc0(PAGE_SIZE);
+    uint8_t *test = g_malloc0(PAGE_SIZE);
+    uint8_t *buffer = g_malloc0(PAGE_SIZE);
+    int i = 0, rc = 0;
+
+    for (i = 0; i < PAGE_SIZE / 2 - 1; i++) {
+        test[i * 2] = 1;
+    }
+
+    /* encode overflow */
+    rc = xbzrle_encode_buffer(buffer, test, PAGE_SIZE, compressed,
+                              PAGE_SIZE);
+    g_assert(rc == -1);
+
+    g_free(buffer);
+    g_free(compressed);
+    g_free(test);
+}
+
+static void encode_decode_range(void)
+{
+    uint8_t *buffer = g_malloc0(PAGE_SIZE);
+    uint8_t *compressed = g_malloc(PAGE_SIZE);
+    uint8_t *test = g_malloc0(PAGE_SIZE);
+    int i = 0, rc = 0;
+    int dlen = 0;
+
+    int diff_len = g_test_rand_int_range(0, PAGE_SIZE - 1006);
+
+    for (i = diff_len; i > 0; i--) {
+        buffer[1000 + i] = i;
+        test[1000 + i] = i + 4;
+    }
+
+    buffer[1000 + diff_len + 3] = 103;
+    test[1000 + diff_len + 3] = 107;
+
+    buffer[1000 + diff_len + 5] = 105;
+    test[1000 + diff_len + 5] = 109;
+
+    /* test encode/decode */
+    dlen = xbzrle_encode_buffer(test, buffer, PAGE_SIZE, compressed,
+                                PAGE_SIZE);
+
+    rc = xbzrle_decode_buffer(compressed, dlen, test, PAGE_SIZE);
+    g_assert(rc < PAGE_SIZE);
+    g_assert(memcmp(test, buffer, PAGE_SIZE) == 0);
+
+    g_free(buffer);
+    g_free(compressed);
+    g_free(test);
+}
+
+static void test_encode_decode(void)
+{
+    int i;
+
+    for (i = 0; i < 10000; i++) {
+        encode_decode_range();
+    }
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    g_test_rand_int();
+    g_test_add_func("/xbzrle/uleb", test_uleb);
+    g_test_add_func("/xbzrle/encode_decode_zero", test_encode_decode_zero);
+    g_test_add_func("/xbzrle/encode_decode_unchanged",
+                    test_encode_decode_unchanged);
+    g_test_add_func("/xbzrle/encode_decode_1_byte", test_encode_decode_1_byte);
+    g_test_add_func("/xbzrle/encode_decode_overflow",
+                    test_encode_decode_overflow);
+    g_test_add_func("/xbzrle/encode_decode", test_encode_decode);
+
+    return g_test_run();
+}
diff --git a/tests/tmp105-test.c b/tests/tmp105-test.c
new file mode 100644
index 0000000..a6ad213
--- /dev/null
+++ b/tests/tmp105-test.c
@@ -0,0 +1,76 @@
+/*
+ * QTest testcase for the TMP105 temperature sensor
+ *
+ * Copyright (c) 2012 Andreas Färber
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "libqtest.h"
+#include "libi2c.h"
+#include "hw/tmp105_regs.h"
+
+#include <glib.h>
+
+#define OMAP2_I2C_1_BASE 0x48070000
+
+#define N8X0_ADDR 0x48
+
+static I2CAdapter *i2c;
+static uint8_t addr;
+
+static void send_and_receive(void)
+{
+    uint8_t cmd[3];
+    uint8_t resp[2];
+
+    cmd[0] = TMP105_REG_TEMPERATURE;
+    i2c_send(i2c, addr, cmd, 1);
+    i2c_recv(i2c, addr, resp, 2);
+    g_assert_cmpuint(((uint16_t)resp[0] << 8) | resp[1], ==, 0);
+
+    cmd[0] = TMP105_REG_CONFIG;
+    cmd[1] = 0x0; /* matches the reset value */
+    i2c_send(i2c, addr, cmd, 2);
+    i2c_recv(i2c, addr, resp, 1);
+    g_assert_cmphex(resp[0], ==, cmd[1]);
+
+    cmd[0] = TMP105_REG_T_LOW;
+    cmd[1] = 0x12;
+    cmd[2] = 0x34;
+    i2c_send(i2c, addr, cmd, 3);
+    i2c_recv(i2c, addr, resp, 2);
+    g_assert_cmphex(resp[0], ==, cmd[1]);
+    g_assert_cmphex(resp[1], ==, cmd[2]);
+
+    cmd[0] = TMP105_REG_T_HIGH;
+    cmd[1] = 0x42;
+    cmd[2] = 0x31;
+    i2c_send(i2c, addr, cmd, 3);
+    i2c_recv(i2c, addr, resp, 2);
+    g_assert_cmphex(resp[0], ==, cmd[1]);
+    g_assert_cmphex(resp[1], ==, cmd[2]);
+}
+
+int main(int argc, char **argv)
+{
+    QTestState *s = NULL;
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    s = qtest_start("-display none -machine n800");
+    i2c = omap_i2c_create(OMAP2_I2C_1_BASE);
+    addr = N8X0_ADDR;
+
+    qtest_add_func("/tmp105/tx-rx", send_and_receive);
+
+    ret = g_test_run();
+
+    if (s) {
+        qtest_quit(s);
+    }
+    g_free(i2c);
+
+    return ret;
+}
diff --git a/thread-pool.c b/thread-pool.c
index 204f70b..e3ca64d 100644
--- a/thread-pool.c
+++ b/thread-pool.c
@@ -15,14 +15,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "qemu-thread.h"
-#include "osdep.h"
-#include "qemu-coroutine.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "qemu/osdep.h"
+#include "block/coroutine.h"
 #include "trace.h"
-#include "block_int.h"
-#include "event_notifier.h"
-#include "thread-pool.h"
+#include "block/block_int.h"
+#include "qemu/event_notifier.h"
+#include "block/thread-pool.h"
 
 static void do_spawn_thread(void);
 
diff --git a/thunk.c b/thunk.c
index 8ebbbb4..3cca047 100644
--- a/thunk.c
+++ b/thunk.c
@@ -21,7 +21,7 @@
 #include <stdarg.h>
 
 #include "qemu.h"
-#include "thunk.h"
+#include "exec/user/thunk.h"
 
 //#define DEBUG
 
diff --git a/trace-events b/trace-events
index 6c6cbf1..1011f27 100644
--- a/trace-events
+++ b/trace-events
@@ -79,10 +79,17 @@ commit_start(void *bs, void *base, void *top, void *s, void *co, void *opaque) "
 
 # block/mirror.c
 mirror_start(void *bs, void *s, void *co, void *opaque) "bs %p s %p co %p opaque %p"
+mirror_restart_iter(void *s, int64_t cnt) "s %p dirty count %"PRId64
 mirror_before_flush(void *s) "s %p"
 mirror_before_drain(void *s, int64_t cnt) "s %p dirty count %"PRId64
 mirror_before_sleep(void *s, int64_t cnt, int synced) "s %p dirty count %"PRId64" synced %d"
 mirror_one_iteration(void *s, int64_t sector_num, int nb_sectors) "s %p sector_num %"PRId64" nb_sectors %d"
+mirror_cow(void *s, int64_t sector_num) "s %p sector_num %"PRId64
+mirror_iteration_done(void *s, int64_t sector_num, int nb_sectors, int ret) "s %p sector_num %"PRId64" nb_sectors %d ret %d"
+mirror_yield(void *s, int64_t cnt, int buf_free_count, int in_flight) "s %p dirty count %"PRId64" free buffers %d in_flight %d"
+mirror_yield_in_flight(void *s, int64_t sector_num, int in_flight) "s %p sector_num %"PRId64" in_flight %d"
+mirror_yield_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
+mirror_break_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
 
 # blockdev.c
 qmp_block_job_cancel(void *job) "job %p"
@@ -98,6 +105,15 @@ virtio_blk_rw_complete(void *req, int ret) "req %p ret %d"
 virtio_blk_handle_write(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
 virtio_blk_handle_read(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
 
+# hw/dataplane/virtio-blk.c
+virtio_blk_data_plane_start(void *s) "dataplane %p"
+virtio_blk_data_plane_stop(void *s) "dataplane %p"
+virtio_blk_data_plane_process_request(void *s, unsigned int out_num, unsigned int in_num, unsigned int head) "dataplane %p out_num %u in_num %u head %u"
+virtio_blk_data_plane_complete_request(void *s, unsigned int head, int ret) "dataplane %p head %u ret %d"
+
+# hw/dataplane/vring.c
+vring_setup(uint64_t physical, void *desc, void *avail, void *used) "vring physical %#"PRIx64" desc %p avail %p used %p"
+
 # thread-pool.c
 thread_pool_submit(void *req, void *opaque) "req %p opaque %p"
 thread_pool_complete(void *req, void *opaque, int ret) "req %p opaque %p ret %d"
@@ -158,6 +174,13 @@ ecc_mem_readl_ecr1(uint32_t ret) "Read event count 2 %08x"
 ecc_diag_mem_writeb(uint64_t addr, uint32_t val) "Write diagnostic %"PRId64" = %02x"
 ecc_diag_mem_readb(uint64_t addr, uint32_t ret) "Read diagnostic %"PRId64"= %02x"
 
+# hw/fw_cfg.c
+fw_cfg_write(void *s, uint8_t value) "%p %d"
+fw_cfg_select(void *s, uint16_t key, int ret) "%p key %d = %d"
+fw_cfg_read(void *s, uint8_t ret) "%p = %d"
+fw_cfg_add_file_dupe(void *s, char *name) "%p %s"
+fw_cfg_add_file(void *s, int index, char *name, size_t len) "%p #%d: %s (%zd bytes)"
+
 # hw/hd-geometry.c
 hd_geometry_lchs_guess(void *bs, int cyls, int heads, int secs) "bs %p LCHS %d %d %d"
 hd_geometry_guess(void *bs, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "bs %p CHS %u %u %u trans %d"
@@ -298,8 +321,6 @@ usb_uhci_frame_loop_stop_idle(void) ""
 usb_uhci_frame_loop_continue(void) ""
 usb_uhci_mmio_readw(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%04x"
 usb_uhci_mmio_writew(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%04x"
-usb_uhci_mmio_readl(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%08x"
-usb_uhci_mmio_writel(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%08x"
 usb_uhci_queue_add(uint32_t token) "token 0x%x"
 usb_uhci_queue_del(uint32_t token, const char *reason) "token 0x%x: %s"
 usb_uhci_packet_add(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
@@ -535,6 +556,7 @@ spice_vmc_write(ssize_t out, int len) "spice wrottn %zd of requested %d"
 spice_vmc_read(int bytes, int len) "spice read %d of requested %d"
 spice_vmc_register_interface(void *scd) "spice vmc registered interface %p"
 spice_vmc_unregister_interface(void *scd) "spice vmc unregistered interface %p"
+spice_vmc_event(int event) "spice vmc event %d"
 
 # hw/lm32_pic.c
 lm32_pic_raise_irq(void) "Raise CPU interrupt"
@@ -725,6 +747,14 @@ mipsnet_read(uint64_t addr, uint32_t val) "read addr=0x%" PRIx64 " val=0x%x"
 mipsnet_write(uint64_t addr, uint64_t val) "write addr=0x%" PRIx64 " val=0x%" PRIx64 ""
 mipsnet_irq(uint32_t isr, uint32_t intctl) "set irq to %d (%02x)"
 
+# hw/pc87312.c
+pc87312_io_read(uint32_t addr, uint32_t val) "read addr=%x val=%x"
+pc87312_io_write(uint32_t addr, uint32_t val) "write addr=%x val=%x"
+pc87312_info_floppy(uint32_t base) "base 0x%x"
+pc87312_info_ide(uint32_t base) "base 0x%x"
+pc87312_info_parallel(uint32_t base, uint32_t irq) "base 0x%x, irq %u"
+pc87312_info_serial(int n, uint32_t base, uint32_t irq) "id=%d, base 0x%x, irq %u"
+
 # xen-all.c
 xen_ram_alloc(unsigned long ram_addr, unsigned long size) "requested: %#lx, size %#lx"
 xen_client_set_memory(uint64_t start_addr, unsigned long size, bool log_dirty) "%#"PRIx64" size %#lx, log_dirty %i"
@@ -1006,8 +1036,10 @@ qxl_send_events_vm_stopped(int qid, uint32_t events) "%d %d"
 qxl_set_guest_bug(int qid) "%d"
 qxl_interrupt_client_monitors_config(int qid, int num_heads, void *heads) "%d %d %p"
 qxl_client_monitors_config_unsupported_by_guest(int qid, uint32_t int_mask, void *client_monitors_config) "%d %X %p"
+qxl_client_monitors_config_unsupported_by_device(int qid, int revision) "%d revision=%d"
 qxl_client_monitors_config_capped(int qid, int requested, int limit) "%d %d %d"
 qxl_client_monitors_config_crc(int qid, unsigned size, uint32_t crc32) "%d %u %u"
+qxl_set_client_capabilities_unsupported_by_revision(int qid, int revision) "%d revision=%d"
 
 # hw/qxl-render.c
 qxl_render_blit_guest_primary_initialized(void) ""
@@ -1022,3 +1054,39 @@ spapr_pci_rtas_ibm_change_msi(unsigned func, unsigned req) "func %u, requested %
 spapr_pci_rtas_ibm_query_interrupt_source_number(unsigned ioa, unsigned intr) "queries for #%u, IRQ%u"
 spapr_pci_msi_write(uint64_t addr, uint64_t data, uint32_t dt_irq) "@%"PRIx64"<=%"PRIx64" IRQ %u"
 spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
+
+# hw/xics.c
+xics_icp_check_ipi(int server, uint8_t mfrr) "CPU %d can take IPI mfrr=%#x"
+xics_icp_accept(uint32_t old_xirr, uint32_t new_xirr) "icp_accept: XIRR %#"PRIx32"->%#"PRIx32
+xics_icp_eoi(int server, uint32_t xirr, uint32_t new_xirr) "icp_eoi: server %d given XIRR %#"PRIx32" new XIRR %#"PRIx32
+xics_icp_irq(int server, int nr, uint8_t priority) "cpu %d trying to deliver irq %#"PRIx32" priority %#x"
+xics_icp_raise(uint32_t xirr, uint8_t pending_priority) "raising IRQ new XIRR=%#x new pending priority=%#x"
+xics_set_irq_msi(int srcno, int nr) "set_irq_msi: srcno %d [irq %#x]"
+xics_masked_pending(void) "set_irq_msi: masked pending"
+xics_set_irq_lsi(int srcno, int nr) "set_irq_lsi: srcno %d [irq %#x]"
+xics_ics_write_xive(int nr, int srcno, int server, uint8_t priority) "ics_write_xive: irq %#x [src %d] server %#x prio %#x"
+xics_ics_reject(int nr, int srcno) "reject irq %#x [src %d]"
+xics_ics_eoi(int nr) "ics_eoi: irq %#x"
+
+# hbitmap.c
+hbitmap_iter_skip_words(const void *hb, void *hbi, uint64_t pos, unsigned long cur) "hb %p hbi %p pos %"PRId64" cur 0x%lx"
+hbitmap_reset(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
+hbitmap_set(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
+
+# target-s390x/ioinst.c
+ioinst(const char *insn) "IOINST: %s"
+ioinst_sch_id(const char *insn, int cssid, int ssid, int schid) "IOINST: %s (%x.%x.%04x)"
+ioinst_chp_id(const char *insn, int cssid, int chpid) "IOINST: %s (%x.%02x)"
+ioinst_chsc_cmd(uint16_t cmd, uint16_t len) "IOINST: chsc command %04x, len %04x"
+
+# hw/s390x/css.c
+css_enable_facility(const char *facility) "CSS: enable %s"
+css_crw(uint8_t rsc, uint8_t erc, uint16_t rsid, const char *chained) "CSS: queueing crw: rsc=%x, erc=%x, rsid=%x %s"
+css_chpid_add(uint8_t cssid, uint8_t chpid, uint8_t type) "CSS: add chpid %x.%02x (type %02x)"
+css_new_image(uint8_t cssid, const char *default_cssid) "CSS: add css image %02x %s"
+css_assign_subch(const char *do_assign, uint8_t cssid, uint8_t ssid, uint16_t schid, uint16_t devno) "CSS: %s %x.%x.%04x (devno %04x)"
+css_io_interrupt(int cssid, int ssid, int schid, uint32_t intparm, uint8_t isc, const char *conditional) "CSS: I/O interrupt on sch %x.%x.%04x (intparm %08x, isc %x) %s"
+
+# hw/s390x/virtio-ccw.c
+virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code) "VIRTIO-CCW: %x.%x.%04x: interpret command %x"
+virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *devno_mode) "VIRTIO-CCW: add subchannel %x.%x.%04x, devno %04x (%s)"
diff --git a/trace/Makefile.objs b/trace/Makefile.objs
new file mode 100644
index 0000000..dde9d57
--- /dev/null
+++ b/trace/Makefile.objs
@@ -0,0 +1,58 @@
+# -*- mode: makefile -*-
+
+######################################################################
+# Auto-generated header for tracing routines
+
+$(obj)/generated-tracers.h: $(obj)/generated-tracers.h-timestamp
+	@cmp -s $< $@ || cp $< $@
+$(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=h \
+		--backend=$(TRACE_BACKEND) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+
+######################################################################
+# Auto-generated tracing routines (non-DTrace)
+
+ifneq ($(TRACE_BACKEND),dtrace)
+$(obj)/generated-tracers.c: $(obj)/generated-tracers.c-timestamp
+	@cmp -s $< $@ || cp $< $@
+$(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=c \
+		--backend=$(TRACE_BACKEND) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+
+$(obj)/generated-tracers.o: $(obj)/generated-tracers.c $(obj)/generated-tracers.h
+endif
+
+
+######################################################################
+# Auto-generated DTrace code
+
+# Normal practice is to name DTrace probe file with a '.d' extension
+# but that gets picked up by QEMU's Makefile as an external dependency
+# rule file. So we use '.dtrace' instead
+ifeq ($(TRACE_BACKEND),dtrace)
+$(obj)/generated-tracers.dtrace: $(obj)/generated-tracers.dtrace-timestamp
+$(obj)/generated-tracers.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	$(call quiet-command,$(TRACETOOL) \
+		--format=d \
+		--backend=$(TRACE_BACKEND) \
+		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
+	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
+
+$(obj)/generated-tracers-dtrace.h: $(obj)/generated-tracers.dtrace
+	$(call quiet-command,dtrace -o $@ -h -s $<, "  GEN   $@")
+
+$(obj)/generated-tracers.o: $(obj)/generated-tracers.dtrace
+endif
+
+######################################################################
+# Backend code
+
+util-obj-$(CONFIG_TRACE_DEFAULT) += default.o
+util-obj-$(CONFIG_TRACE_SIMPLE) += simple.o
+util-obj-$(CONFIG_TRACE_STDERR) += stderr.o
+util-obj-y += control.o
+util-obj-y += generated-tracers.o
diff --git a/trace/simple.c b/trace/simple.c
index d83681b..375d98f 100644
--- a/trace/simple.c
+++ b/trace/simple.c
@@ -16,7 +16,7 @@
 #include <signal.h>
 #include <pthread.h>
 #endif
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 #include "trace.h"
 #include "trace/control.h"
 
@@ -40,8 +40,18 @@
  * records to become available, writes them out, and then waits again.
  */
 static GStaticMutex trace_lock = G_STATIC_MUTEX_INIT;
+
+/* g_cond_new() was deprecated in glib 2.31 but we still need to support it */
+#if GLIB_CHECK_VERSION(2, 31, 0)
+static GCond the_trace_available_cond;
+static GCond the_trace_empty_cond;
+static GCond *trace_available_cond = &the_trace_available_cond;
+static GCond *trace_empty_cond = &the_trace_empty_cond;
+#else
 static GCond *trace_available_cond;
 static GCond *trace_empty_cond;
+#endif
+
 static bool trace_available;
 static bool trace_writeout_enabled;
 
@@ -51,9 +61,9 @@ enum {
 };
 
 uint8_t trace_buf[TRACE_BUF_LEN];
-static unsigned int trace_idx;
+static volatile gint trace_idx;
 static unsigned int writeout_idx;
-static uint64_t dropped_events;
+static volatile gint dropped_events;
 static FILE *trace_fp;
 static char *trace_file_name;
 
@@ -63,7 +73,7 @@ typedef struct {
     uint64_t timestamp_ns;
     uint32_t length;   /*    in bytes */
     uint32_t reserved; /*    unused */
-    uint8_t arguments[];
+    uint64_t arguments[];
 } TraceRecord;
 
 typedef struct {
@@ -160,25 +170,22 @@ static gpointer writeout_thread(gpointer opaque)
         uint8_t bytes[sizeof(TraceRecord) + sizeof(uint64_t)];
     } dropped;
     unsigned int idx = 0;
-    uint64_t dropped_count;
+    int dropped_count;
     size_t unused __attribute__ ((unused));
 
     for (;;) {
         wait_for_trace_records_available();
 
-        if (dropped_events) {
+        if (g_atomic_int_get(&dropped_events)) {
             dropped.rec.event = DROPPED_EVENT_ID,
             dropped.rec.timestamp_ns = get_clock();
-            dropped.rec.length = sizeof(TraceRecord) + sizeof(dropped_events),
+            dropped.rec.length = sizeof(TraceRecord) + sizeof(uint64_t),
             dropped.rec.reserved = 0;
-            while (1) {
-                dropped_count = dropped_events;
-                if (g_atomic_int_compare_and_exchange((gint *)&dropped_events,
-                                                      dropped_count, 0)) {
-                    break;
-                }
-            }
-            memcpy(dropped.rec.arguments, &dropped_count, sizeof(uint64_t));
+            do {
+                dropped_count = g_atomic_int_get(&dropped_events);
+            } while (!g_atomic_int_compare_and_exchange(&dropped_events,
+                                                        dropped_count, 0));
+            dropped.rec.arguments[0] = dropped_count;
             unused = fwrite(&dropped.rec, dropped.rec.length, 1, trace_fp);
         }
 
@@ -213,22 +220,17 @@ int trace_record_start(TraceBufferRecord *rec, TraceEventID event, size_t datasi
     uint32_t rec_len = sizeof(TraceRecord) + datasize;
     uint64_t timestamp_ns = get_clock();
 
-    while (1) {
-        old_idx = trace_idx;
+    do {
+        old_idx = g_atomic_int_get(&trace_idx);
         smp_rmb();
         new_idx = old_idx + rec_len;
 
         if (new_idx - writeout_idx > TRACE_BUF_LEN) {
             /* Trace Buffer Full, Event dropped ! */
-            g_atomic_int_inc((gint *)&dropped_events);
+            g_atomic_int_inc(&dropped_events);
             return -ENOSPC;
         }
-
-        if (g_atomic_int_compare_and_exchange((gint *)&trace_idx,
-                                              old_idx, new_idx)) {
-            break;
-        }
-    }
+    } while (!g_atomic_int_compare_and_exchange(&trace_idx, old_idx, new_idx));
 
     idx = old_idx % TRACE_BUF_LEN;
 
@@ -275,7 +277,8 @@ void trace_record_finish(TraceBufferRecord *rec)
     record.event |= TRACE_RECORD_VALID;
     write_to_buffer(rec->tbuf_idx, &record, sizeof(TraceRecord));
 
-    if ((trace_idx - writeout_idx) > TRACE_BUF_FLUSH_THRESHOLD) {
+    if (((unsigned int)g_atomic_int_get(&trace_idx) - writeout_idx)
+        > TRACE_BUF_FLUSH_THRESHOLD) {
         flush_trace_file(false);
     }
 }
@@ -404,7 +407,13 @@ static GThread *trace_thread_create(GThreadFunc fn)
     sigfillset(&set);
     pthread_sigmask(SIG_SETMASK, &set, &oldset);
 #endif
+
+#if GLIB_CHECK_VERSION(2, 31, 0)
+    thread = g_thread_new("trace-thread", fn, NULL);
+#else
     thread = g_thread_create(fn, NULL, FALSE, NULL);
+#endif
+
 #ifndef _WIN32
     pthread_sigmask(SIG_SETMASK, &oldset, NULL);
 #endif
@@ -425,8 +434,10 @@ bool trace_backend_init(const char *events, const char *file)
 #endif
     }
 
+#if !GLIB_CHECK_VERSION(2, 31, 0)
     trace_available_cond = g_cond_new();
     trace_empty_cond = g_cond_new();
+#endif
 
     thread = trace_thread_create(writeout_thread);
     if (!thread) {
diff --git a/translate-all.c b/translate-all.c
index d9c2e57..d367fc4 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -16,6 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/types.h>
+#include <sys/mman.h>
+#endif
 #include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -24,18 +30,119 @@
 
 #include "config.h"
 
+#include "qemu-common.h"
 #define NO_CPU_IO_DEFS
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg.h"
-#include "qemu-timer.h"
+#include "qemu/timer.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#if defined(CONFIG_USER_ONLY)
+#include "qemu.h"
+#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+#include <sys/param.h>
+#if __FreeBSD_version >= 700104
+#define HAVE_KINFO_GETVMMAP
+#define sigqueue sigqueue_freebsd  /* avoid redefinition */
+#include <sys/time.h>
+#include <sys/proc.h>
+#include <machine/profile.h>
+#define _KERNEL
+#include <sys/user.h>
+#undef _KERNEL
+#undef sigqueue
+#include <libutil.h>
+#endif
+#endif
+#endif
+
+#include "exec/cputlb.h"
+#include "translate-all.h"
+
+//#define DEBUG_TB_INVALIDATE
+//#define DEBUG_FLUSH
+/* make various TB consistency checks */
+//#define DEBUG_TB_CHECK
+
+#if !defined(CONFIG_USER_ONLY)
+/* TB consistency checks only implemented for usermode emulation.  */
+#undef DEBUG_TB_CHECK
+#endif
+
+#define SMC_BITMAP_USE_THRESHOLD 10
+
+/* Code generation and translation blocks */
+static TranslationBlock *tbs;
+static int code_gen_max_blocks;
+TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
+static int nb_tbs;
+/* any access to the tbs or the page table must use this lock */
+spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
+
+uint8_t *code_gen_prologue;
+static uint8_t *code_gen_buffer;
+static size_t code_gen_buffer_size;
+/* threshold to flush the translated code buffer */
+static size_t code_gen_buffer_max_size;
+static uint8_t *code_gen_ptr;
+
+typedef struct PageDesc {
+    /* list of TBs intersecting this ram page */
+    TranslationBlock *first_tb;
+    /* in order to optimize self modifying code, we count the number
+       of lookups we do to a given page to use a bitmap */
+    unsigned int code_write_count;
+    uint8_t *code_bitmap;
+#if defined(CONFIG_USER_ONLY)
+    unsigned long flags;
+#endif
+} PageDesc;
+
+/* In system mode we want L1_MAP to be based on ram offsets,
+   while in user mode we want it to be based on virtual addresses.  */
+#if !defined(CONFIG_USER_ONLY)
+#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
+# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
+#endif
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
+#endif
+
+/* The bits remaining after N lower levels of page tables.  */
+#define V_L1_BITS_REM \
+    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
+
+#if V_L1_BITS_REM < 4
+#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
+#else
+#define V_L1_BITS  V_L1_BITS_REM
+#endif
+
+#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
+
+#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
+
+uintptr_t qemu_real_host_page_size;
+uintptr_t qemu_host_page_size;
+uintptr_t qemu_host_page_mask;
+
+/* This is a multi-level map on the virtual address space.
+   The bottom level has pointers to PageDesc.  */
+static void *l1_map[V_L1_SIZE];
+
+/* statistics */
+static int tb_flush_count;
+static int tb_phys_invalidate_count;
 
 /* code generation context */
 TCGContext tcg_ctx;
 
-target_ulong gen_opc_pc[OPC_BUF_SIZE];
-uint16_t gen_opc_icount[OPC_BUF_SIZE];
-uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2);
+static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
 
 void cpu_gen_init(void)
 {
@@ -105,8 +212,8 @@ int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr
 
 /* The cpu state corresponding to 'searched_pc' is restored.
  */
-int cpu_restore_state(TranslationBlock *tb,
-                      CPUArchState *env, uintptr_t searched_pc)
+static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
+                                     uintptr_t searched_pc)
 {
     TCGContext *s = &tcg_ctx;
     int j;
@@ -146,9 +253,10 @@ int cpu_restore_state(TranslationBlock *tb,
     if (j < 0)
         return -1;
     /* now find start of instruction before */
-    while (gen_opc_instr_start[j] == 0)
+    while (s->gen_opc_instr_start[j] == 0) {
         j--;
-    env->icount_decr.u16.low -= gen_opc_icount[j];
+    }
+    env->icount_decr.u16.low -= s->gen_opc_icount[j];
 
     restore_state_to_opc(env, tb, j);
 
@@ -158,3 +266,1622 @@ int cpu_restore_state(TranslationBlock *tb,
 #endif
     return 0;
 }
+
+bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_pc(retaddr);
+    if (tb) {
+        cpu_restore_state_from_tb(tb, env, retaddr);
+        return true;
+    }
+    return false;
+}
+
+#ifdef _WIN32
+static inline void map_exec(void *addr, long size)
+{
+    DWORD old_protect;
+    VirtualProtect(addr, size,
+                   PAGE_EXECUTE_READWRITE, &old_protect);
+}
+#else
+static inline void map_exec(void *addr, long size)
+{
+    unsigned long start, end, page_size;
+
+    page_size = getpagesize();
+    start = (unsigned long)addr;
+    start &= ~(page_size - 1);
+
+    end = (unsigned long)addr + size;
+    end += page_size - 1;
+    end &= ~(page_size - 1);
+
+    mprotect((void *)start, end - start,
+             PROT_READ | PROT_WRITE | PROT_EXEC);
+}
+#endif
+
+static void page_init(void)
+{
+    /* NOTE: we can always suppose that qemu_host_page_size >=
+       TARGET_PAGE_SIZE */
+#ifdef _WIN32
+    {
+        SYSTEM_INFO system_info;
+
+        GetSystemInfo(&system_info);
+        qemu_real_host_page_size = system_info.dwPageSize;
+    }
+#else
+    qemu_real_host_page_size = getpagesize();
+#endif
+    if (qemu_host_page_size == 0) {
+        qemu_host_page_size = qemu_real_host_page_size;
+    }
+    if (qemu_host_page_size < TARGET_PAGE_SIZE) {
+        qemu_host_page_size = TARGET_PAGE_SIZE;
+    }
+    qemu_host_page_mask = ~(qemu_host_page_size - 1);
+
+#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
+    {
+#ifdef HAVE_KINFO_GETVMMAP
+        struct kinfo_vmentry *freep;
+        int i, cnt;
+
+        freep = kinfo_getvmmap(getpid(), &cnt);
+        if (freep) {
+            mmap_lock();
+            for (i = 0; i < cnt; i++) {
+                unsigned long startaddr, endaddr;
+
+                startaddr = freep[i].kve_start;
+                endaddr = freep[i].kve_end;
+                if (h2g_valid(startaddr)) {
+                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
+
+                    if (h2g_valid(endaddr)) {
+                        endaddr = h2g(endaddr);
+                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+                    } else {
+#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
+                        endaddr = ~0ul;
+                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+#endif
+                    }
+                }
+            }
+            free(freep);
+            mmap_unlock();
+        }
+#else
+        FILE *f;
+
+        last_brk = (unsigned long)sbrk(0);
+
+        f = fopen("/compat/linux/proc/self/maps", "r");
+        if (f) {
+            mmap_lock();
+
+            do {
+                unsigned long startaddr, endaddr;
+                int n;
+
+                n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
+
+                if (n == 2 && h2g_valid(startaddr)) {
+                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
+
+                    if (h2g_valid(endaddr)) {
+                        endaddr = h2g(endaddr);
+                    } else {
+                        endaddr = ~0ul;
+                    }
+                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
+                }
+            } while (!feof(f));
+
+            fclose(f);
+            mmap_unlock();
+        }
+#endif
+    }
+#endif
+}
+
+static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
+{
+    PageDesc *pd;
+    void **lp;
+    int i;
+
+#if defined(CONFIG_USER_ONLY)
+    /* We can't use g_malloc because it may recurse into a locked mutex. */
+# define ALLOC(P, SIZE)                                 \
+    do {                                                \
+        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
+                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
+    } while (0)
+#else
+# define ALLOC(P, SIZE) \
+    do { P = g_malloc0(SIZE); } while (0)
+#endif
+
+    /* Level 1.  Always allocated.  */
+    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
+
+    /* Level 2..N-1.  */
+    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
+        void **p = *lp;
+
+        if (p == NULL) {
+            if (!alloc) {
+                return NULL;
+            }
+            ALLOC(p, sizeof(void *) * L2_SIZE);
+            *lp = p;
+        }
+
+        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
+    }
+
+    pd = *lp;
+    if (pd == NULL) {
+        if (!alloc) {
+            return NULL;
+        }
+        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
+        *lp = pd;
+    }
+
+#undef ALLOC
+
+    return pd + (index & (L2_SIZE - 1));
+}
+
+static inline PageDesc *page_find(tb_page_addr_t index)
+{
+    return page_find_alloc(index, 0);
+}
+
+#if !defined(CONFIG_USER_ONLY)
+#define mmap_lock() do { } while (0)
+#define mmap_unlock() do { } while (0)
+#endif
+
+#if defined(CONFIG_USER_ONLY)
+/* Currently it is not recommended to allocate big chunks of data in
+   user mode. It will change when a dedicated libc will be used.  */
+/* ??? 64-bit hosts ought to have no problem mmaping data outside the
+   region in which the guest needs to run.  Revisit this.  */
+#define USE_STATIC_CODE_GEN_BUFFER
+#endif
+
+/* ??? Should configure for this, not list operating systems here.  */
+#if (defined(__linux__) \
+    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
+    || defined(__DragonFly__) || defined(__OpenBSD__) \
+    || defined(__NetBSD__))
+# define USE_MMAP
+#endif
+
+/* Minimum size of the code gen buffer.  This number is randomly chosen,
+   but not so small that we can't have a fair number of TB's live.  */
+#define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
+
+/* Maximum size of the code gen buffer we'd like to use.  Unless otherwise
+   indicated, this is constrained by the range of direct branches on the
+   host cpu, as used by the TCG implementation of goto_tb.  */
+#if defined(__x86_64__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__sparc__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__arm__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (16u * 1024 * 1024)
+#elif defined(__s390x__)
+  /* We have a +- 4GB range on the branches; leave some slop.  */
+# define MAX_CODE_GEN_BUFFER_SIZE  (3ul * 1024 * 1024 * 1024)
+#else
+# define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
+#endif
+
+#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
+
+#define DEFAULT_CODE_GEN_BUFFER_SIZE \
+  (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
+   ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
+
+static inline size_t size_code_gen_buffer(size_t tb_size)
+{
+    /* Size the buffer.  */
+    if (tb_size == 0) {
+#ifdef USE_STATIC_CODE_GEN_BUFFER
+        tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
+#else
+        /* ??? Needs adjustments.  */
+        /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
+           static buffer, we could size this on RESERVED_VA, on the text
+           segment size of the executable, or continue to use the default.  */
+        tb_size = (unsigned long)(ram_size / 4);
+#endif
+    }
+    if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
+        tb_size = MIN_CODE_GEN_BUFFER_SIZE;
+    }
+    if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
+        tb_size = MAX_CODE_GEN_BUFFER_SIZE;
+    }
+    code_gen_buffer_size = tb_size;
+    return tb_size;
+}
+
+#ifdef USE_STATIC_CODE_GEN_BUFFER
+static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
+    __attribute__((aligned(CODE_GEN_ALIGN)));
+
+static inline void *alloc_code_gen_buffer(void)
+{
+    map_exec(static_code_gen_buffer, code_gen_buffer_size);
+    return static_code_gen_buffer;
+}
+#elif defined(USE_MMAP)
+static inline void *alloc_code_gen_buffer(void)
+{
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+    uintptr_t start = 0;
+    void *buf;
+
+    /* Constrain the position of the buffer based on the host cpu.
+       Note that these addresses are chosen in concert with the
+       addresses assigned in the relevant linker script file.  */
+# if defined(__PIE__) || defined(__PIC__)
+    /* Don't bother setting a preferred location if we're building
+       a position-independent executable.  We're more likely to get
+       an address near the main executable if we let the kernel
+       choose the address.  */
+# elif defined(__x86_64__) && defined(MAP_32BIT)
+    /* Force the memory down into low memory with the executable.
+       Leave the choice of exact location with the kernel.  */
+    flags |= MAP_32BIT;
+    /* Cannot expect to map more than 800MB in low memory.  */
+    if (code_gen_buffer_size > 800u * 1024 * 1024) {
+        code_gen_buffer_size = 800u * 1024 * 1024;
+    }
+# elif defined(__sparc__)
+    start = 0x40000000ul;
+# elif defined(__s390x__)
+    start = 0x90000000ul;
+# endif
+
+    buf = mmap((void *)start, code_gen_buffer_size,
+               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
+    return buf == MAP_FAILED ? NULL : buf;
+}
+#else
+static inline void *alloc_code_gen_buffer(void)
+{
+    void *buf = g_malloc(code_gen_buffer_size);
+
+    if (buf) {
+        map_exec(buf, code_gen_buffer_size);
+    }
+    return buf;
+}
+#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
+
+static inline void code_gen_alloc(size_t tb_size)
+{
+    code_gen_buffer_size = size_code_gen_buffer(tb_size);
+    code_gen_buffer = alloc_code_gen_buffer();
+    if (code_gen_buffer == NULL) {
+        fprintf(stderr, "Could not allocate dynamic translator buffer\n");
+        exit(1);
+    }
+
+    qemu_madvise(code_gen_buffer, code_gen_buffer_size, QEMU_MADV_HUGEPAGE);
+
+    /* Steal room for the prologue at the end of the buffer.  This ensures
+       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
+       from TB's to the prologue are going to be in range.  It also means
+       that we don't need to mark (additional) portions of the data segment
+       as executable.  */
+    code_gen_prologue = code_gen_buffer + code_gen_buffer_size - 1024;
+    code_gen_buffer_size -= 1024;
+
+    code_gen_buffer_max_size = code_gen_buffer_size -
+        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
+    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
+    tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
+}
+
+/* Must be called before using the QEMU cpus. 'tb_size' is the size
+   (in bytes) allocated to the translation buffer. Zero means default
+   size. */
+void tcg_exec_init(unsigned long tb_size)
+{
+    cpu_gen_init();
+    code_gen_alloc(tb_size);
+    code_gen_ptr = code_gen_buffer;
+    tcg_register_jit(code_gen_buffer, code_gen_buffer_size);
+    page_init();
+#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+    /* There's no guest base to take into account, so go ahead and
+       initialize the prologue now.  */
+    tcg_prologue_init(&tcg_ctx);
+#endif
+}
+
+bool tcg_enabled(void)
+{
+    return code_gen_buffer != NULL;
+}
+
+/* Allocate a new translation block. Flush the translation buffer if
+   too many translation blocks or too much generated code. */
+static TranslationBlock *tb_alloc(target_ulong pc)
+{
+    TranslationBlock *tb;
+
+    if (nb_tbs >= code_gen_max_blocks ||
+        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size) {
+        return NULL;
+    }
+    tb = &tbs[nb_tbs++];
+    tb->pc = pc;
+    tb->cflags = 0;
+    return tb;
+}
+
+void tb_free(TranslationBlock *tb)
+{
+    /* In practice this is mostly used for single use temporary TB
+       Ignore the hard cases and just back up if this TB happens to
+       be the last one generated.  */
+    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
+        code_gen_ptr = tb->tc_ptr;
+        nb_tbs--;
+    }
+}
+
+static inline void invalidate_page_bitmap(PageDesc *p)
+{
+    if (p->code_bitmap) {
+        g_free(p->code_bitmap);
+        p->code_bitmap = NULL;
+    }
+    p->code_write_count = 0;
+}
+
+/* Set to NULL all the 'first_tb' fields in all PageDescs. */
+static void page_flush_tb_1(int level, void **lp)
+{
+    int i;
+
+    if (*lp == NULL) {
+        return;
+    }
+    if (level == 0) {
+        PageDesc *pd = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            pd[i].first_tb = NULL;
+            invalidate_page_bitmap(pd + i);
+        }
+    } else {
+        void **pp = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            page_flush_tb_1(level - 1, pp + i);
+        }
+    }
+}
+
+static void page_flush_tb(void)
+{
+    int i;
+
+    for (i = 0; i < V_L1_SIZE; i++) {
+        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
+    }
+}
+
+/* flush all the translation blocks */
+/* XXX: tb_flush is currently not thread safe */
+void tb_flush(CPUArchState *env1)
+{
+    CPUArchState *env;
+
+#if defined(DEBUG_FLUSH)
+    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
+           (unsigned long)(code_gen_ptr - code_gen_buffer),
+           nb_tbs, nb_tbs > 0 ?
+           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
+#endif
+    if ((unsigned long)(code_gen_ptr - code_gen_buffer)
+        > code_gen_buffer_size) {
+        cpu_abort(env1, "Internal error: code buffer overflow\n");
+    }
+    nb_tbs = 0;
+
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        memset(env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
+    }
+
+    memset(tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof(void *));
+    page_flush_tb();
+
+    code_gen_ptr = code_gen_buffer;
+    /* XXX: flush processor icache at this point if cache flush is
+       expensive */
+    tb_flush_count++;
+}
+
+#ifdef DEBUG_TB_CHECK
+
+static void tb_invalidate_check(target_ulong address)
+{
+    TranslationBlock *tb;
+    int i;
+
+    address &= TARGET_PAGE_MASK;
+    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
+        for (tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
+            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
+                  address >= tb->pc + tb->size)) {
+                printf("ERROR invalidate: address=" TARGET_FMT_lx
+                       " PC=%08lx size=%04x\n",
+                       address, (long)tb->pc, tb->size);
+            }
+        }
+    }
+}
+
+/* verify that all the pages have correct rights for code */
+static void tb_page_check(void)
+{
+    TranslationBlock *tb;
+    int i, flags1, flags2;
+
+    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
+        for (tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
+            flags1 = page_get_flags(tb->pc);
+            flags2 = page_get_flags(tb->pc + tb->size - 1);
+            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
+                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
+                       (long)tb->pc, tb->size, flags1, flags2);
+            }
+        }
+    }
+}
+
+#endif
+
+static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
+{
+    TranslationBlock *tb1;
+
+    for (;;) {
+        tb1 = *ptb;
+        if (tb1 == tb) {
+            *ptb = tb1->phys_hash_next;
+            break;
+        }
+        ptb = &tb1->phys_hash_next;
+    }
+}
+
+static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
+{
+    TranslationBlock *tb1;
+    unsigned int n1;
+
+    for (;;) {
+        tb1 = *ptb;
+        n1 = (uintptr_t)tb1 & 3;
+        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+        if (tb1 == tb) {
+            *ptb = tb1->page_next[n1];
+            break;
+        }
+        ptb = &tb1->page_next[n1];
+    }
+}
+
+static inline void tb_jmp_remove(TranslationBlock *tb, int n)
+{
+    TranslationBlock *tb1, **ptb;
+    unsigned int n1;
+
+    ptb = &tb->jmp_next[n];
+    tb1 = *ptb;
+    if (tb1) {
+        /* find tb(n) in circular list */
+        for (;;) {
+            tb1 = *ptb;
+            n1 = (uintptr_t)tb1 & 3;
+            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            if (n1 == n && tb1 == tb) {
+                break;
+            }
+            if (n1 == 2) {
+                ptb = &tb1->jmp_first;
+            } else {
+                ptb = &tb1->jmp_next[n1];
+            }
+        }
+        /* now we can suppress tb(n) from the list */
+        *ptb = tb->jmp_next[n];
+
+        tb->jmp_next[n] = NULL;
+    }
+}
+
+/* reset the jump entry 'n' of a TB so that it is not chained to
+   another TB */
+static inline void tb_reset_jump(TranslationBlock *tb, int n)
+{
+    tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
+}
+
+/* invalidate one TB */
+void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
+{
+    CPUArchState *env;
+    PageDesc *p;
+    unsigned int h, n1;
+    tb_page_addr_t phys_pc;
+    TranslationBlock *tb1, *tb2;
+
+    /* remove the TB from the hash list */
+    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
+    h = tb_phys_hash_func(phys_pc);
+    tb_hash_remove(&tb_phys_hash[h], tb);
+
+    /* remove the TB from the page list */
+    if (tb->page_addr[0] != page_addr) {
+        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
+        tb_page_remove(&p->first_tb, tb);
+        invalidate_page_bitmap(p);
+    }
+    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
+        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
+        tb_page_remove(&p->first_tb, tb);
+        invalidate_page_bitmap(p);
+    }
+
+    tb_invalidated_flag = 1;
+
+    /* remove the TB from the hash list */
+    h = tb_jmp_cache_hash_func(tb->pc);
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        if (env->tb_jmp_cache[h] == tb) {
+            env->tb_jmp_cache[h] = NULL;
+        }
+    }
+
+    /* suppress this TB from the two jump lists */
+    tb_jmp_remove(tb, 0);
+    tb_jmp_remove(tb, 1);
+
+    /* suppress any remaining jumps to this TB */
+    tb1 = tb->jmp_first;
+    for (;;) {
+        n1 = (uintptr_t)tb1 & 3;
+        if (n1 == 2) {
+            break;
+        }
+        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+        tb2 = tb1->jmp_next[n1];
+        tb_reset_jump(tb1, n1);
+        tb1->jmp_next[n1] = NULL;
+        tb1 = tb2;
+    }
+    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
+
+    tb_phys_invalidate_count++;
+}
+
+static inline void set_bits(uint8_t *tab, int start, int len)
+{
+    int end, mask, end1;
+
+    end = start + len;
+    tab += start >> 3;
+    mask = 0xff << (start & 7);
+    if ((start & ~7) == (end & ~7)) {
+        if (start < end) {
+            mask &= ~(0xff << (end & 7));
+            *tab |= mask;
+        }
+    } else {
+        *tab++ |= mask;
+        start = (start + 8) & ~7;
+        end1 = end & ~7;
+        while (start < end1) {
+            *tab++ = 0xff;
+            start += 8;
+        }
+        if (start < end) {
+            mask = ~(0xff << (end & 7));
+            *tab |= mask;
+        }
+    }
+}
+
+static void build_page_bitmap(PageDesc *p)
+{
+    int n, tb_start, tb_end;
+    TranslationBlock *tb;
+
+    p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
+
+    tb = p->first_tb;
+    while (tb != NULL) {
+        n = (uintptr_t)tb & 3;
+        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
+        /* NOTE: this is subtle as a TB may span two physical pages */
+        if (n == 0) {
+            /* NOTE: tb_end may be after the end of the page, but
+               it is not a problem */
+            tb_start = tb->pc & ~TARGET_PAGE_MASK;
+            tb_end = tb_start + tb->size;
+            if (tb_end > TARGET_PAGE_SIZE) {
+                tb_end = TARGET_PAGE_SIZE;
+            }
+        } else {
+            tb_start = 0;
+            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
+        }
+        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
+        tb = tb->page_next[n];
+    }
+}
+
+TranslationBlock *tb_gen_code(CPUArchState *env,
+                              target_ulong pc, target_ulong cs_base,
+                              int flags, int cflags)
+{
+    TranslationBlock *tb;
+    uint8_t *tc_ptr;
+    tb_page_addr_t phys_pc, phys_page2;
+    target_ulong virt_page2;
+    int code_gen_size;
+
+    phys_pc = get_page_addr_code(env, pc);
+    tb = tb_alloc(pc);
+    if (!tb) {
+        /* flush must be done */
+        tb_flush(env);
+        /* cannot fail at this point */
+        tb = tb_alloc(pc);
+        /* Don't forget to invalidate previous TB info.  */
+        tb_invalidated_flag = 1;
+    }
+    tc_ptr = code_gen_ptr;
+    tb->tc_ptr = tc_ptr;
+    tb->cs_base = cs_base;
+    tb->flags = flags;
+    tb->cflags = cflags;
+    cpu_gen_code(env, tb, &code_gen_size);
+    code_gen_ptr = (void *)(((uintptr_t)code_gen_ptr + code_gen_size +
+                             CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+
+    /* check next page if needed */
+    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
+    phys_page2 = -1;
+    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
+        phys_page2 = get_page_addr_code(env, virt_page2);
+    }
+    tb_link_page(tb, phys_pc, phys_page2);
+    return tb;
+}
+
+/*
+ * Invalidate all TBs which intersect with the target physical address range
+ * [start;end[. NOTE: start and end may refer to *different* physical pages.
+ * 'is_cpu_write_access' should be true if called from a real cpu write
+ * access: the virtual CPU will exit the current TB if code is modified inside
+ * this TB.
+ */
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
+                              int is_cpu_write_access)
+{
+    while (start < end) {
+        tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
+        start &= TARGET_PAGE_MASK;
+        start += TARGET_PAGE_SIZE;
+    }
+}
+
+/*
+ * Invalidate all TBs which intersect with the target physical address range
+ * [start;end[. NOTE: start and end must refer to the *same* physical page.
+ * 'is_cpu_write_access' should be true if called from a real cpu write
+ * access: the virtual CPU will exit the current TB if code is modified inside
+ * this TB.
+ */
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
+                                   int is_cpu_write_access)
+{
+    TranslationBlock *tb, *tb_next, *saved_tb;
+    CPUArchState *env = cpu_single_env;
+    tb_page_addr_t tb_start, tb_end;
+    PageDesc *p;
+    int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+    int current_tb_not_found = is_cpu_write_access;
+    TranslationBlock *current_tb = NULL;
+    int current_tb_modified = 0;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
+#endif /* TARGET_HAS_PRECISE_SMC */
+
+    p = page_find(start >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+    if (!p->code_bitmap &&
+        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
+        is_cpu_write_access) {
+        /* build code bitmap */
+        build_page_bitmap(p);
+    }
+
+    /* we remove all the TBs in the range [start, end[ */
+    /* XXX: see if in some cases it could be faster to invalidate all
+       the code */
+    tb = p->first_tb;
+    while (tb != NULL) {
+        n = (uintptr_t)tb & 3;
+        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
+        tb_next = tb->page_next[n];
+        /* NOTE: this is subtle as a TB may span two physical pages */
+        if (n == 0) {
+            /* NOTE: tb_end may be after the end of the page, but
+               it is not a problem */
+            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
+            tb_end = tb_start + tb->size;
+        } else {
+            tb_start = tb->page_addr[1];
+            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
+        }
+        if (!(tb_end <= start || tb_start >= end)) {
+#ifdef TARGET_HAS_PRECISE_SMC
+            if (current_tb_not_found) {
+                current_tb_not_found = 0;
+                current_tb = NULL;
+                if (env->mem_io_pc) {
+                    /* now we have a real cpu fault */
+                    current_tb = tb_find_pc(env->mem_io_pc);
+                }
+            }
+            if (current_tb == tb &&
+                (current_tb->cflags & CF_COUNT_MASK) != 1) {
+                /* If we are modifying the current TB, we must stop
+                its execution. We could be more precise by checking
+                that the modification is after the current PC, but it
+                would require a specialized function to partially
+                restore the CPU state */
+
+                current_tb_modified = 1;
+                cpu_restore_state_from_tb(current_tb, env, env->mem_io_pc);
+                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                                     &current_flags);
+            }
+#endif /* TARGET_HAS_PRECISE_SMC */
+            /* we need to do that to handle the case where a signal
+               occurs while doing tb_phys_invalidate() */
+            saved_tb = NULL;
+            if (env) {
+                saved_tb = env->current_tb;
+                env->current_tb = NULL;
+            }
+            tb_phys_invalidate(tb, -1);
+            if (env) {
+                env->current_tb = saved_tb;
+                if (env->interrupt_request && env->current_tb) {
+                    cpu_interrupt(env, env->interrupt_request);
+                }
+            }
+        }
+        tb = tb_next;
+    }
+#if !defined(CONFIG_USER_ONLY)
+    /* if no code remaining, no need to continue to use slow writes */
+    if (!p->first_tb) {
+        invalidate_page_bitmap(p);
+        if (is_cpu_write_access) {
+            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
+        }
+    }
+#endif
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (current_tb_modified) {
+        /* we generate a block containing just the instruction
+           modifying the memory. It will ensure that it cannot modify
+           itself */
+        env->current_tb = NULL;
+        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
+        cpu_resume_from_signal(env, NULL);
+    }
+#endif
+}
+
+/* len must be <= 8 and start must be a multiple of len */
+void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
+{
+    PageDesc *p;
+    int offset, b;
+
+#if 0
+    if (1) {
+        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
+                  cpu_single_env->mem_io_vaddr, len,
+                  cpu_single_env->eip,
+                  cpu_single_env->eip +
+                  (intptr_t)cpu_single_env->segs[R_CS].base);
+    }
+#endif
+    p = page_find(start >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+    if (p->code_bitmap) {
+        offset = start & ~TARGET_PAGE_MASK;
+        b = p->code_bitmap[offset >> 3] >> (offset & 7);
+        if (b & ((1 << len) - 1)) {
+            goto do_invalidate;
+        }
+    } else {
+    do_invalidate:
+        tb_invalidate_phys_page_range(start, start + len, 1);
+    }
+}
+
+#if !defined(CONFIG_SOFTMMU)
+static void tb_invalidate_phys_page(tb_page_addr_t addr,
+                                    uintptr_t pc, void *puc)
+{
+    TranslationBlock *tb;
+    PageDesc *p;
+    int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+    TranslationBlock *current_tb = NULL;
+    CPUArchState *env = cpu_single_env;
+    int current_tb_modified = 0;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
+#endif
+
+    addr &= TARGET_PAGE_MASK;
+    p = page_find(addr >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+    tb = p->first_tb;
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (tb && pc != 0) {
+        current_tb = tb_find_pc(pc);
+    }
+#endif
+    while (tb != NULL) {
+        n = (uintptr_t)tb & 3;
+        tb = (TranslationBlock *)((uintptr_t)tb & ~3);
+#ifdef TARGET_HAS_PRECISE_SMC
+        if (current_tb == tb &&
+            (current_tb->cflags & CF_COUNT_MASK) != 1) {
+                /* If we are modifying the current TB, we must stop
+                   its execution. We could be more precise by checking
+                   that the modification is after the current PC, but it
+                   would require a specialized function to partially
+                   restore the CPU state */
+
+            current_tb_modified = 1;
+            cpu_restore_state_from_tb(current_tb, env, pc);
+            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                                 &current_flags);
+        }
+#endif /* TARGET_HAS_PRECISE_SMC */
+        tb_phys_invalidate(tb, addr);
+        tb = tb->page_next[n];
+    }
+    p->first_tb = NULL;
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (current_tb_modified) {
+        /* we generate a block containing just the instruction
+           modifying the memory. It will ensure that it cannot modify
+           itself */
+        env->current_tb = NULL;
+        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
+        cpu_resume_from_signal(env, puc);
+    }
+#endif
+}
+#endif
+
+/* add the tb in the target page and protect it if necessary */
+static inline void tb_alloc_page(TranslationBlock *tb,
+                                 unsigned int n, tb_page_addr_t page_addr)
+{
+    PageDesc *p;
+#ifndef CONFIG_USER_ONLY
+    bool page_already_protected;
+#endif
+
+    tb->page_addr[n] = page_addr;
+    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
+    tb->page_next[n] = p->first_tb;
+#ifndef CONFIG_USER_ONLY
+    page_already_protected = p->first_tb != NULL;
+#endif
+    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
+    invalidate_page_bitmap(p);
+
+#if defined(TARGET_HAS_SMC) || 1
+
+#if defined(CONFIG_USER_ONLY)
+    if (p->flags & PAGE_WRITE) {
+        target_ulong addr;
+        PageDesc *p2;
+        int prot;
+
+        /* force the host page as non writable (writes will have a
+           page fault + mprotect overhead) */
+        page_addr &= qemu_host_page_mask;
+        prot = 0;
+        for (addr = page_addr; addr < page_addr + qemu_host_page_size;
+            addr += TARGET_PAGE_SIZE) {
+
+            p2 = page_find(addr >> TARGET_PAGE_BITS);
+            if (!p2) {
+                continue;
+            }
+            prot |= p2->flags;
+            p2->flags &= ~PAGE_WRITE;
+          }
+        mprotect(g2h(page_addr), qemu_host_page_size,
+                 (prot & PAGE_BITS) & ~PAGE_WRITE);
+#ifdef DEBUG_TB_INVALIDATE
+        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
+               page_addr);
+#endif
+    }
+#else
+    /* if some code is already present, then the pages are already
+       protected. So we handle the case where only the first TB is
+       allocated in a physical page */
+    if (!page_already_protected) {
+        tlb_protect_code(page_addr);
+    }
+#endif
+
+#endif /* TARGET_HAS_SMC */
+}
+
+/* add a new TB and link it to the physical page tables. phys_page2 is
+   (-1) to indicate that only one page contains the TB. */
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2)
+{
+    unsigned int h;
+    TranslationBlock **ptb;
+
+    /* Grab the mmap lock to stop another thread invalidating this TB
+       before we are done.  */
+    mmap_lock();
+    /* add in the physical hash table */
+    h = tb_phys_hash_func(phys_pc);
+    ptb = &tb_phys_hash[h];
+    tb->phys_hash_next = *ptb;
+    *ptb = tb;
+
+    /* add in the page list */
+    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
+    if (phys_page2 != -1) {
+        tb_alloc_page(tb, 1, phys_page2);
+    } else {
+        tb->page_addr[1] = -1;
+    }
+
+    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
+    tb->jmp_next[0] = NULL;
+    tb->jmp_next[1] = NULL;
+
+    /* init original jump addresses */
+    if (tb->tb_next_offset[0] != 0xffff) {
+        tb_reset_jump(tb, 0);
+    }
+    if (tb->tb_next_offset[1] != 0xffff) {
+        tb_reset_jump(tb, 1);
+    }
+
+#ifdef DEBUG_TB_CHECK
+    tb_page_check();
+#endif
+    mmap_unlock();
+}
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* check whether the given addr is in TCG generated code buffer or not */
+bool is_tcg_gen_code(uintptr_t tc_ptr)
+{
+    /* This can be called during code generation, code_gen_buffer_max_size
+       is used instead of code_gen_ptr for upper boundary checking */
+    return (tc_ptr >= (uintptr_t)code_gen_buffer &&
+            tc_ptr < (uintptr_t)(code_gen_buffer + code_gen_buffer_max_size));
+}
+#endif
+
+/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
+   tb[1].tc_ptr. Return NULL if not found */
+static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
+{
+    int m_min, m_max, m;
+    uintptr_t v;
+    TranslationBlock *tb;
+
+    if (nb_tbs <= 0) {
+        return NULL;
+    }
+    if (tc_ptr < (uintptr_t)code_gen_buffer ||
+        tc_ptr >= (uintptr_t)code_gen_ptr) {
+        return NULL;
+    }
+    /* binary search (cf Knuth) */
+    m_min = 0;
+    m_max = nb_tbs - 1;
+    while (m_min <= m_max) {
+        m = (m_min + m_max) >> 1;
+        tb = &tbs[m];
+        v = (uintptr_t)tb->tc_ptr;
+        if (v == tc_ptr) {
+            return tb;
+        } else if (tc_ptr < v) {
+            m_max = m - 1;
+        } else {
+            m_min = m + 1;
+        }
+    }
+    return &tbs[m_max];
+}
+
+static void tb_reset_jump_recursive(TranslationBlock *tb);
+
+static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
+{
+    TranslationBlock *tb1, *tb_next, **ptb;
+    unsigned int n1;
+
+    tb1 = tb->jmp_next[n];
+    if (tb1 != NULL) {
+        /* find head of list */
+        for (;;) {
+            n1 = (uintptr_t)tb1 & 3;
+            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            if (n1 == 2) {
+                break;
+            }
+            tb1 = tb1->jmp_next[n1];
+        }
+        /* we are now sure now that tb jumps to tb1 */
+        tb_next = tb1;
+
+        /* remove tb from the jmp_first list */
+        ptb = &tb_next->jmp_first;
+        for (;;) {
+            tb1 = *ptb;
+            n1 = (uintptr_t)tb1 & 3;
+            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            if (n1 == n && tb1 == tb) {
+                break;
+            }
+            ptb = &tb1->jmp_next[n1];
+        }
+        *ptb = tb->jmp_next[n];
+        tb->jmp_next[n] = NULL;
+
+        /* suppress the jump to next tb in generated code */
+        tb_reset_jump(tb, n);
+
+        /* suppress jumps in the tb on which we could have jumped */
+        tb_reset_jump_recursive(tb_next);
+    }
+}
+
+static void tb_reset_jump_recursive(TranslationBlock *tb)
+{
+    tb_reset_jump_recursive2(tb, 0);
+    tb_reset_jump_recursive2(tb, 1);
+}
+
+#if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
+void tb_invalidate_phys_addr(hwaddr addr)
+{
+    ram_addr_t ram_addr;
+    MemoryRegionSection *section;
+
+    section = phys_page_find(address_space_memory.dispatch,
+                             addr >> TARGET_PAGE_BITS);
+    if (!(memory_region_is_ram(section->mr)
+          || (section->mr->rom_device && section->mr->readable))) {
+        return;
+    }
+    ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
+        + memory_region_section_addr(section, addr);
+    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
+}
+#endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
+
+void cpu_unlink_tb(CPUArchState *env)
+{
+    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
+       problem and hope the cpu will stop of its own accord.  For userspace
+       emulation this often isn't actually as bad as it sounds.  Often
+       signals are used primarily to interrupt blocking syscalls.  */
+    TranslationBlock *tb;
+    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
+
+    spin_lock(&interrupt_lock);
+    tb = env->current_tb;
+    /* if the cpu is currently executing code, we must unlink it and
+       all the potentially executing TB */
+    if (tb) {
+        env->current_tb = NULL;
+        tb_reset_jump_recursive(tb);
+    }
+    spin_unlock(&interrupt_lock);
+}
+
+void tb_check_watchpoint(CPUArchState *env)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_pc(env->mem_io_pc);
+    if (!tb) {
+        cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
+                  (void *)env->mem_io_pc);
+    }
+    cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
+    tb_phys_invalidate(tb, -1);
+}
+
+#ifndef CONFIG_USER_ONLY
+/* mask must never be zero, except for A20 change call */
+static void tcg_handle_interrupt(CPUArchState *env, int mask)
+{
+    CPUState *cpu = ENV_GET_CPU(env);
+    int old_mask;
+
+    old_mask = env->interrupt_request;
+    env->interrupt_request |= mask;
+
+    /*
+     * If called from iothread context, wake the target cpu in
+     * case its halted.
+     */
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
+        return;
+    }
+
+    if (use_icount) {
+        env->icount_decr.u16.high = 0xffff;
+        if (!can_do_io(env)
+            && (mask & ~old_mask) != 0) {
+            cpu_abort(env, "Raised interrupt while not in I/O function");
+        }
+    } else {
+        cpu_unlink_tb(env);
+    }
+}
+
+CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
+
+/* in deterministic execution mode, instructions doing device I/Os
+   must be at the end of the TB */
+void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
+{
+    TranslationBlock *tb;
+    uint32_t n, cflags;
+    target_ulong pc, cs_base;
+    uint64_t flags;
+
+    tb = tb_find_pc(retaddr);
+    if (!tb) {
+        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
+                  (void *)retaddr);
+    }
+    n = env->icount_decr.u16.low + tb->icount;
+    cpu_restore_state_from_tb(tb, env, retaddr);
+    /* Calculate how many instructions had been executed before the fault
+       occurred.  */
+    n = n - env->icount_decr.u16.low;
+    /* Generate a new TB ending on the I/O insn.  */
+    n++;
+    /* On MIPS and SH, delay slot instructions can only be restarted if
+       they were already the first instruction in the TB.  If this is not
+       the first instruction in a TB then re-execute the preceding
+       branch.  */
+#if defined(TARGET_MIPS)
+    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
+        env->active_tc.PC -= 4;
+        env->icount_decr.u16.low++;
+        env->hflags &= ~MIPS_HFLAG_BMASK;
+    }
+#elif defined(TARGET_SH4)
+    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
+            && n > 1) {
+        env->pc -= 2;
+        env->icount_decr.u16.low++;
+        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
+    }
+#endif
+    /* This should never happen.  */
+    if (n > CF_COUNT_MASK) {
+        cpu_abort(env, "TB too big during recompile");
+    }
+
+    cflags = n | CF_LAST_IO;
+    pc = tb->pc;
+    cs_base = tb->cs_base;
+    flags = tb->flags;
+    tb_phys_invalidate(tb, -1);
+    /* FIXME: In theory this could raise an exception.  In practice
+       we have already translated the block once so it's probably ok.  */
+    tb_gen_code(env, pc, cs_base, flags, cflags);
+    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
+       the first in the TB) then we end up generating a whole new TB and
+       repeating the fault, which is horribly inefficient.
+       Better would be to execute just this insn uncached, or generate a
+       second new TB.  */
+    cpu_resume_from_signal(env, NULL);
+}
+
+void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
+{
+    unsigned int i;
+
+    /* Discard jump cache entries for any tb which might potentially
+       overlap the flushed page.  */
+    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
+    memset(&env->tb_jmp_cache[i], 0,
+           TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
+
+    i = tb_jmp_cache_hash_page(addr);
+    memset(&env->tb_jmp_cache[i], 0,
+           TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
+}
+
+void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
+{
+    int i, target_code_size, max_target_code_size;
+    int direct_jmp_count, direct_jmp2_count, cross_page;
+    TranslationBlock *tb;
+
+    target_code_size = 0;
+    max_target_code_size = 0;
+    cross_page = 0;
+    direct_jmp_count = 0;
+    direct_jmp2_count = 0;
+    for (i = 0; i < nb_tbs; i++) {
+        tb = &tbs[i];
+        target_code_size += tb->size;
+        if (tb->size > max_target_code_size) {
+            max_target_code_size = tb->size;
+        }
+        if (tb->page_addr[1] != -1) {
+            cross_page++;
+        }
+        if (tb->tb_next_offset[0] != 0xffff) {
+            direct_jmp_count++;
+            if (tb->tb_next_offset[1] != 0xffff) {
+                direct_jmp2_count++;
+            }
+        }
+    }
+    /* XXX: avoid using doubles ? */
+    cpu_fprintf(f, "Translation buffer state:\n");
+    cpu_fprintf(f, "gen code size       %td/%zd\n",
+                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
+    cpu_fprintf(f, "TB count            %d/%d\n",
+                nb_tbs, code_gen_max_blocks);
+    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
+                nb_tbs ? target_code_size / nb_tbs : 0,
+                max_target_code_size);
+    cpu_fprintf(f, "TB avg host size    %td bytes (expansion ratio: %0.1f)\n",
+                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
+                target_code_size ? (double) (code_gen_ptr - code_gen_buffer)
+                / target_code_size : 0);
+    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
+            cross_page,
+            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
+    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
+                direct_jmp_count,
+                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
+                direct_jmp2_count,
+                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
+    cpu_fprintf(f, "\nStatistics:\n");
+    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
+    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
+    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
+    tcg_dump_info(f, cpu_fprintf);
+}
+
+#else /* CONFIG_USER_ONLY */
+
+void cpu_interrupt(CPUArchState *env, int mask)
+{
+    env->interrupt_request |= mask;
+    cpu_unlink_tb(env);
+}
+
+/*
+ * Walks guest process memory "regions" one by one
+ * and calls callback function 'fn' for each region.
+ */
+struct walk_memory_regions_data {
+    walk_memory_regions_fn fn;
+    void *priv;
+    uintptr_t start;
+    int prot;
+};
+
+static int walk_memory_regions_end(struct walk_memory_regions_data *data,
+                                   abi_ulong end, int new_prot)
+{
+    if (data->start != -1ul) {
+        int rc = data->fn(data->priv, data->start, end, data->prot);
+        if (rc != 0) {
+            return rc;
+        }
+    }
+
+    data->start = (new_prot ? end : -1ul);
+    data->prot = new_prot;
+
+    return 0;
+}
+
+static int walk_memory_regions_1(struct walk_memory_regions_data *data,
+                                 abi_ulong base, int level, void **lp)
+{
+    abi_ulong pa;
+    int i, rc;
+
+    if (*lp == NULL) {
+        return walk_memory_regions_end(data, base, 0);
+    }
+
+    if (level == 0) {
+        PageDesc *pd = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            int prot = pd[i].flags;
+
+            pa = base | (i << TARGET_PAGE_BITS);
+            if (prot != data->prot) {
+                rc = walk_memory_regions_end(data, pa, prot);
+                if (rc != 0) {
+                    return rc;
+                }
+            }
+        }
+    } else {
+        void **pp = *lp;
+
+        for (i = 0; i < L2_SIZE; ++i) {
+            pa = base | ((abi_ulong)i <<
+                (TARGET_PAGE_BITS + L2_BITS * level));
+            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
+            if (rc != 0) {
+                return rc;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
+{
+    struct walk_memory_regions_data data;
+    uintptr_t i;
+
+    data.fn = fn;
+    data.priv = priv;
+    data.start = -1ul;
+    data.prot = 0;
+
+    for (i = 0; i < V_L1_SIZE; i++) {
+        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
+                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
+
+        if (rc != 0) {
+            return rc;
+        }
+    }
+
+    return walk_memory_regions_end(&data, 0, 0);
+}
+
+static int dump_region(void *priv, abi_ulong start,
+    abi_ulong end, unsigned long prot)
+{
+    FILE *f = (FILE *)priv;
+
+    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
+        " "TARGET_ABI_FMT_lx" %c%c%c\n",
+        start, end, end - start,
+        ((prot & PAGE_READ) ? 'r' : '-'),
+        ((prot & PAGE_WRITE) ? 'w' : '-'),
+        ((prot & PAGE_EXEC) ? 'x' : '-'));
+
+    return 0;
+}
+
+/* dump memory mappings */
+void page_dump(FILE *f)
+{
+    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
+            "start", "end", "size", "prot");
+    walk_memory_regions(f, dump_region);
+}
+
+int page_get_flags(target_ulong address)
+{
+    PageDesc *p;
+
+    p = page_find(address >> TARGET_PAGE_BITS);
+    if (!p) {
+        return 0;
+    }
+    return p->flags;
+}
+
+/* Modify the flags of a page and invalidate the code if necessary.
+   The flag PAGE_WRITE_ORG is positioned automatically depending
+   on PAGE_WRITE.  The mmap_lock should already be held.  */
+void page_set_flags(target_ulong start, target_ulong end, int flags)
+{
+    target_ulong addr, len;
+
+    /* This function should never be called with addresses outside the
+       guest address space.  If this assert fires, it probably indicates
+       a missing call to h2g_valid.  */
+#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
+    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+#endif
+    assert(start < end);
+
+    start = start & TARGET_PAGE_MASK;
+    end = TARGET_PAGE_ALIGN(end);
+
+    if (flags & PAGE_WRITE) {
+        flags |= PAGE_WRITE_ORG;
+    }
+
+    for (addr = start, len = end - start;
+         len != 0;
+         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
+        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
+
+        /* If the write protection bit is set, then we invalidate
+           the code inside.  */
+        if (!(p->flags & PAGE_WRITE) &&
+            (flags & PAGE_WRITE) &&
+            p->first_tb) {
+            tb_invalidate_phys_page(addr, 0, NULL);
+        }
+        p->flags = flags;
+    }
+}
+
+int page_check_range(target_ulong start, target_ulong len, int flags)
+{
+    PageDesc *p;
+    target_ulong end;
+    target_ulong addr;
+
+    /* This function should never be called with addresses outside the
+       guest address space.  If this assert fires, it probably indicates
+       a missing call to h2g_valid.  */
+#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
+    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
+#endif
+
+    if (len == 0) {
+        return 0;
+    }
+    if (start + len - 1 < start) {
+        /* We've wrapped around.  */
+        return -1;
+    }
+
+    /* must do before we loose bits in the next step */
+    end = TARGET_PAGE_ALIGN(start + len);
+    start = start & TARGET_PAGE_MASK;
+
+    for (addr = start, len = end - start;
+         len != 0;
+         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
+        p = page_find(addr >> TARGET_PAGE_BITS);
+        if (!p) {
+            return -1;
+        }
+        if (!(p->flags & PAGE_VALID)) {
+            return -1;
+        }
+
+        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
+            return -1;
+        }
+        if (flags & PAGE_WRITE) {
+            if (!(p->flags & PAGE_WRITE_ORG)) {
+                return -1;
+            }
+            /* unprotect the page if it was put read-only because it
+               contains translated code */
+            if (!(p->flags & PAGE_WRITE)) {
+                if (!page_unprotect(addr, 0, NULL)) {
+                    return -1;
+                }
+            }
+            return 0;
+        }
+    }
+    return 0;
+}
+
+/* called from signal handler: invalidate the code and unprotect the
+   page. Return TRUE if the fault was successfully handled. */
+int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
+{
+    unsigned int prot;
+    PageDesc *p;
+    target_ulong host_start, host_end, addr;
+
+    /* Technically this isn't safe inside a signal handler.  However we
+       know this only ever happens in a synchronous SEGV handler, so in
+       practice it seems to be ok.  */
+    mmap_lock();
+
+    p = page_find(address >> TARGET_PAGE_BITS);
+    if (!p) {
+        mmap_unlock();
+        return 0;
+    }
+
+    /* if the page was really writable, then we change its
+       protection back to writable */
+    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
+        host_start = address & qemu_host_page_mask;
+        host_end = host_start + qemu_host_page_size;
+
+        prot = 0;
+        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
+            p = page_find(addr >> TARGET_PAGE_BITS);
+            p->flags |= PAGE_WRITE;
+            prot |= p->flags;
+
+            /* and since the content will be modified, we must invalidate
+               the corresponding translated code. */
+            tb_invalidate_phys_page(addr, pc, puc);
+#ifdef DEBUG_TB_CHECK
+            tb_invalidate_check(addr);
+#endif
+        }
+        mprotect((void *)g2h(host_start), qemu_host_page_size,
+                 prot & PAGE_BITS);
+
+        mmap_unlock();
+        return 1;
+    }
+    mmap_unlock();
+    return 0;
+}
+#endif /* CONFIG_USER_ONLY */
diff --git a/target-s390x/machine.c b/translate-all.h
index 3e79be6..b181fb4 100644
--- a/target-s390x/machine.c
+++ b/translate-all.h
@@ -1,7 +1,7 @@
 /*
- * QEMU S390x machine definitions
+ *  Translated block handling
  *
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ *  Copyright (c) 2003 Fabrice Bellard
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -16,15 +16,19 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef TRANSLATE_ALL_H
+#define TRANSLATE_ALL_H
 
-#include "hw/hw.h"
-#include "hw/boards.h"
+/* Size of the L2 (and L3, etc) page tables.  */
+#define L2_BITS 10
+#define L2_SIZE (1 << L2_BITS)
 
-void cpu_save(QEMUFile *f, void *opaque)
-{
-}
+#define P_L2_LEVELS \
+    (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
 
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-    return 0;
-}
+/* translate-all.c */
+void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len);
+void cpu_unlink_tb(CPUArchState *env);
+void tb_check_watchpoint(CPUArchState *env);
+
+#endif /* TRANSLATE_ALL_H */
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index adc07be..d9db073 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -4,11 +4,16 @@ vnc-obj-y += vnc-enc-tight.o vnc-palette.o
 vnc-obj-y += vnc-enc-zrle.o
 vnc-obj-$(CONFIG_VNC_TLS) += vnc-tls.o vnc-auth-vencrypt.o
 vnc-obj-$(CONFIG_VNC_SASL) += vnc-auth-sasl.o
+vnc-obj-$(CONFIG_VNC_WS) += vnc-ws.o
 vnc-obj-y += vnc-jobs.o
 
-common-obj-y += keymaps.o
+common-obj-y += keymaps.o console.o cursor.o input.o qemu-pixman.o
 common-obj-$(CONFIG_SPICE) += spice-core.o spice-input.o spice-display.o
 common-obj-$(CONFIG_SDL) += sdl.o sdl_zoom.o x_keymap.o
 common-obj-$(CONFIG_COCOA) += cocoa.o
 common-obj-$(CONFIG_CURSES) += curses.o
 common-obj-$(CONFIG_VNC) += $(vnc-obj-y)
+
+$(obj)/sdl.o $(obj)/sdl_zoom.o: QEMU_CFLAGS += $(SDL_CFLAGS) 
+
+$(obj)/cocoa.o: $(SRC_PATH)/$(obj)/cocoa.m
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 87d2e44..ca42413 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -26,8 +26,8 @@
 #include <crt_externs.h>
 
 #include "qemu-common.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 #ifndef MAC_OS_X_VERSION_10_4
 #define MAC_OS_X_VERSION_10_4 1040
@@ -265,6 +265,7 @@ static int cocoa_keycode_to_qemu(int keycode)
     BOOL isTabletEnabled;
 }
 - (void) resizeContentToWidth:(int)w height:(int)h displayState:(DisplayState *)ds;
+- (void) updateDataOffset:(DisplayState *)ds;
 - (void) grabMouse;
 - (void) ungrabMouse;
 - (void) toggleFullScreen:(id)sender;
@@ -429,6 +430,20 @@ QemuCocoaView *cocoaView;
     [self setFrame:NSMakeRect(cx, cy, cw, ch)];
 }
 
+- (void) updateDataOffset:(DisplayState *)ds
+{
+    COCOA_DEBUG("QemuCocoaView: UpdateDataOffset\n");
+
+    // update screenBuffer
+    if (dataProviderRef) {
+        CGDataProviderRelease(dataProviderRef);
+    }
+
+    size_t size = ds_get_width(ds) * 4 * ds_get_height(ds);
+    dataProviderRef = CGDataProviderCreateWithData(NULL, ds_get_data(ds),
+                                                   size, NULL);
+}
+
 - (void) toggleFullScreen:(id)sender
 {
     COCOA_DEBUG("QemuCocoaView: toggleFullScreen\n");
@@ -813,9 +828,9 @@ QemuCocoaView *cocoaView;
 
         [sheet close];
 
-        asprintf(&argv[0], "%s", bin);
-        asprintf(&argv[1], "-hda");
-        asprintf(&argv[2], "%s", img);
+        argv[0] = g_strdup_printf("%s", bin);
+        argv[1] = g_strdup_printf("-hda");
+        argv[2] = g_strdup_printf("%s", img);
 
         printf("Using argc %d argv %s -hda %s\n", 3, bin, img);
 
@@ -1004,6 +1019,11 @@ static void cocoa_refresh(DisplayState *ds)
     vga_hw_update();
 }
 
+static void cocoa_setdata(DisplayState *ds)
+{
+    [cocoaView updateDataOffset:ds];
+}
+
 static void cocoa_cleanup(void)
 {
     COCOA_DEBUG("qemu_cocoa: cocoa_cleanup\n");
@@ -1020,6 +1040,7 @@ void cocoa_display_init(DisplayState *ds, int full_screen)
     dcl->dpy_gfx_update = cocoa_update;
     dcl->dpy_gfx_resize = cocoa_resize;
     dcl->dpy_refresh = cocoa_refresh;
+    dcl->dpy_gfx_setdata = cocoa_setdata;
 
 	register_displaychangelistener(ds, dcl);
 
diff --git a/console.c b/ui/console.c
index 048b48e..0a68836 100644
--- a/console.c
+++ b/ui/console.c
@@ -22,9 +22,10 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "console.h"
-#include "qemu-timer.h"
+#include "ui/console.h"
+#include "qemu/timer.h"
 #include "qmp-commands.h"
+#include "char/char.h"
 
 //#define DEBUG_CONSOLE
 #define DEFAULT_BACKSCROLL 512
@@ -193,7 +194,7 @@ void qmp_screendump(const char *filename, Error **errp)
     if (consoles[0] && consoles[0]->hw_screen_dump) {
         consoles[0]->hw_screen_dump(consoles[0]->hw, filename, cswitch, errp);
     } else {
-        error_setg(errp, "device doesn't support screendump\n");
+        error_setg(errp, "device doesn't support screendump");
     }
 
     if (cswitch) {
diff --git a/ui/curses.c b/ui/curses.c
index b40b223..d78e378 100644
--- a/ui/curses.c
+++ b/ui/curses.c
@@ -28,13 +28,9 @@
 #include <termios.h>
 #endif
 
-#ifdef __OpenBSD__
-#define resize_term resizeterm
-#endif
-
 #include "qemu-common.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 
 #define FONT_HEIGHT 16
 #define FONT_WIDTH 8
diff --git a/ui/curses_keys.h b/ui/curses_keys.h
index c0d5eb4..18ce6dc 100644
--- a/ui/curses_keys.h
+++ b/ui/curses_keys.h
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#ifndef QEMU_CURSES_KEYS_H
+#define QEMU_CURSES_KEYS_H 1
+
 #include <curses.h>
 #include "keymaps.h"
 
@@ -507,3 +510,5 @@ static const name2keysym_t name2keysym[] = {
 
     { NULL, 0 },
 };
+
+#endif
diff --git a/cursor.c b/ui/cursor.c
index 76e262c..2b8dd3f 100644
--- a/cursor.c
+++ b/ui/cursor.c
@@ -1,5 +1,5 @@
 #include "qemu-common.h"
-#include "console.h"
+#include "ui/console.h"
 
 #include "cursor_hidden.xpm"
 #include "cursor_left_ptr.xpm"
diff --git a/cursor_hidden.xpm b/ui/cursor_hidden.xpm
index 354e7a9..354e7a9 100644
--- a/cursor_hidden.xpm
+++ b/ui/cursor_hidden.xpm
diff --git a/cursor_left_ptr.xpm b/ui/cursor_left_ptr.xpm
index 6c9ada9..6c9ada9 100644
--- a/cursor_left_ptr.xpm
+++ b/ui/cursor_left_ptr.xpm
diff --git a/ui/d3des.h b/ui/d3des.h
index 78d546f..70cb6b5 100644
--- a/ui/d3des.h
+++ b/ui/d3des.h
@@ -9,6 +9,8 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  */
+#ifndef D3DES_H
+#define D3DES_H 1
 
 /* d3des.h -
  *
@@ -49,3 +51,5 @@ void des(unsigned char *, unsigned char *);
 
 /* d3des.h V5.09 rwo 9208.04 15:06 Graven Imagery
  ********************************************************************/
+
+#endif
diff --git a/input.c b/ui/input.c
index 1ffdd1b..c354852 100644
--- a/input.c
+++ b/ui/input.c
@@ -22,11 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu.h"
-#include "net.h"
-#include "monitor.h"
-#include "console.h"
-#include "error.h"
+#include "sysemu/sysemu.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "qapi/error.h"
 #include "qmp-commands.h"
 #include "qapi-types.h"
 
@@ -270,7 +269,7 @@ void qmp_send_key(KeyValueList *keys, bool has_hold_time, int64_t hold_time,
         /* key down events */
         keycode = keycode_from_keyvalue(p->value);
         if (keycode < 0x01 || keycode > 0xff) {
-            error_setg(errp, "invalid hex keycode 0x%x\n", keycode);
+            error_setg(errp, "invalid hex keycode 0x%x", keycode);
             free_keycodes();
             return;
         }
diff --git a/ui/keymaps.c b/ui/keymaps.c
index f55a2aa..f373cc5 100644
--- a/ui/keymaps.c
+++ b/ui/keymaps.c
@@ -23,7 +23,7 @@
  */
 
 #include "keymaps.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 static int get_keysym(const name2keysym_t *table,
 		      const char *name)
@@ -127,25 +127,27 @@ static kbd_layout_t *parse_keyboard_layout(const name2keysym_t *table,
                     //		    fprintf(stderr, "Warning: unknown keysym %s\n", line);
 		} else {
 		    const char *rest = end_of_keysym + 1;
-		    char *rest2;
-		    int keycode = strtol(rest, &rest2, 0);
+                    int keycode = strtol(rest, NULL, 0);
 
-		    if (rest && strstr(rest, "numlock")) {
+                    if (strstr(rest, "numlock")) {
 			add_to_key_range(&k->keypad_range, keycode);
 			add_to_key_range(&k->numlock_range, keysym);
 			//fprintf(stderr, "keypad keysym %04x keycode %d\n", keysym, keycode);
 		    }
 
-		    if (rest && strstr(rest, "shift"))
+                    if (strstr(rest, "shift")) {
 			keycode |= SCANCODE_SHIFT;
-		    if (rest && strstr(rest, "altgr"))
+                    }
+                    if (strstr(rest, "altgr")) {
 			keycode |= SCANCODE_ALTGR;
-		    if (rest && strstr(rest, "ctrl"))
+                    }
+                    if (strstr(rest, "ctrl")) {
 			keycode |= SCANCODE_CTRL;
+                    }
 
 		    add_keysym(line, keysym, keycode, k);
 
-		    if (rest && strstr(rest, "addupper")) {
+                    if (strstr(rest, "addupper")) {
 			char *c;
 			for (c = line; *c; c++)
 			    *c = qemu_toupper(*c);
diff --git a/qemu-pixman.c b/ui/qemu-pixman.c
index e46e180..6dcbe90 100644
--- a/qemu-pixman.c
+++ b/ui/qemu-pixman.c
@@ -3,7 +3,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu-pixman.h"
+#include "qemu-common.h"
+#include "ui/console.h"
 
 int qemu_pixman_get_type(int rshift, int gshift, int bshift)
 {
@@ -21,7 +22,7 @@ int qemu_pixman_get_type(int rshift, int gshift, int bshift)
         if (rshift == 0) {
             type = PIXMAN_TYPE_ABGR;
         } else {
-#if PIXMAN_VERSION >= PIXMAN_VERSION_ENCODE(0, 21, 8)
+#if PIXMAN_VERSION >= PIXMAN_VERSION_ENCODE(0, 16, 0)
             type = PIXMAN_TYPE_BGRA;
 #endif
         }
@@ -52,10 +53,10 @@ pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
 }
 
 void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
-                              int width, int y)
+                              int width, int x, int y)
 {
     pixman_image_composite(PIXMAN_OP_SRC, fb, NULL, linebuf,
-                           0, y, 0, 0, 0, 0, width, 1);
+                           x, y, 0, 0, 0, 0, width, 1);
 }
 
 pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
diff --git a/qemu-x509.h b/ui/qemu-x509.h
index 095aec1..095aec1 100644
--- a/qemu-x509.h
+++ b/ui/qemu-x509.h
diff --git a/ui/sdl.c b/ui/sdl.c
index 37f01b2..1657848 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -29,8 +29,8 @@
 #include <SDL_syswm.h>
 
 #include "qemu-common.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 #include "x_keymap.h"
 #include "sdl_zoom.h"
 
diff --git a/ui/sdl_zoom.c b/ui/sdl_zoom.c
index a986c7c..2625c45 100644
--- a/ui/sdl_zoom.c
+++ b/ui/sdl_zoom.c
@@ -12,14 +12,15 @@
  */
 
 #include "sdl_zoom.h"
-#include "osdep.h"
+#include "qemu/osdep.h"
+#include <glib.h>
 #include <stdint.h>
 #include <stdio.h>
 
-static int sdl_zoom_rgb16(SDL_Surface *src, SDL_Surface *dst, int smooth,
-                          SDL_Rect *dst_rect);
-static int sdl_zoom_rgb32(SDL_Surface *src, SDL_Surface *dst, int smooth,
-                          SDL_Rect *dst_rect);
+static void sdl_zoom_rgb16(SDL_Surface *src, SDL_Surface *dst, int smooth,
+                           SDL_Rect *dst_rect);
+static void sdl_zoom_rgb32(SDL_Surface *src, SDL_Surface *dst, int smooth,
+                           SDL_Rect *dst_rect);
 
 #define BPP 32
 #include  "sdl_zoom_template.h"
diff --git a/ui/sdl_zoom_template.h b/ui/sdl_zoom_template.h
index 64bbca8..3bb508b 100644
--- a/ui/sdl_zoom_template.h
+++ b/ui/sdl_zoom_template.h
@@ -51,7 +51,7 @@
               (((a) & (dpf->Amask >> dpf->Ashift)) << dpf->Ashift); \
 } while (0);
 
-static int glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smooth,
+static void glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smooth,
                                    SDL_Rect *dst_rect)
 {
     int x, y, sx, sy, *sax, *say, *csax, *csay, csx, csy, ex, ey, t1, t2, sstep, sstep_jump;
@@ -71,13 +71,8 @@ static int glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smoot
         sy = (int) (65536.0 * (float) src->h / (float) dst->h);
     }
 
-    if ((sax = (int *) malloc((dst->w + 1) * sizeof(Uint32))) == NULL) {
-        return (-1);
-    }
-    if ((say = (int *) malloc((dst->h + 1) * sizeof(Uint32))) == NULL) {
-        free(sax);
-        return (-1);
-    }
+    sax = g_new(int, dst->w + 1);
+    say = g_new(int, dst->h + 1);
 
     sp = csp = (SDL_TYPE *) src->pixels;
     dp = (SDL_TYPE *) (dst->pixels + dst_rect->y * dst->pitch +
@@ -216,9 +211,8 @@ static int glue(sdl_zoom_rgb, BPP)(SDL_Surface *src, SDL_Surface *dst, int smoot
         }
     }
 
-    free(sax);
-    free(say);
-    return (0);
+    g_free(sax);
+    g_free(say);
 }
 
 #undef SDL_TYPE
diff --git a/ui/spice-core.c b/ui/spice-core.c
index 261c6f2..bcc4199 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -19,25 +19,25 @@
 #include <spice-experimental.h>
 
 #include <netdb.h>
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 
 #include "qemu-common.h"
-#include "qemu-spice.h"
-#include "qemu-thread.h"
-#include "qemu-timer.h"
-#include "qemu-queue.h"
+#include "ui/qemu-spice.h"
+#include "qemu/thread.h"
+#include "qemu/timer.h"
+#include "qemu/queue.h"
 #include "qemu-x509.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 #include "qmp-commands.h"
-#include "qint.h"
-#include "qbool.h"
-#include "qstring.h"
-#include "qjson.h"
-#include "notify.h"
-#include "migration.h"
-#include "monitor.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qjson.h"
+#include "qemu/notify.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
 #include "hw/hw.h"
-#include "spice-display.h"
+#include "ui/spice-display.h"
 
 /* core bits */
 
@@ -417,6 +417,90 @@ static SpiceChannelList *qmp_query_spice_channels(void)
     return head;
 }
 
+static QemuOptsList qemu_spice_opts = {
+    .name = "spice",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_spice_opts.head),
+    .desc = {
+        {
+            .name = "port",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "tls-port",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "addr",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "password",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "disable-ticketing",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "disable-copy-paste",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "sasl",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "x509-dir",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-key-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-key-password",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-cert-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-cacert-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "x509-dh-key-file",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "tls-ciphers",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "tls-channel",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "plaintext-channel",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "image-compression",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "jpeg-wan-compression",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "zlib-glz-wan-compression",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "streaming-video",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "agent-mouse",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "playback-compression",
+            .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "seamless-migration",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
+
 SpiceInfo *qmp_query_spice(Error **errp)
 {
     QemuOpts *opts = QTAILQ_FIRST(&qemu_spice_opts.head);
@@ -709,11 +793,15 @@ void qemu_spice_init(void)
     qemu_spice_input_init();
     qemu_spice_audio_init();
 
-    qemu_add_vm_change_state_handler(vm_change_state_handler, &spice_server);
+    qemu_add_vm_change_state_handler(vm_change_state_handler, NULL);
 
     g_free(x509_key_file);
     g_free(x509_cert_file);
     g_free(x509_cacert_file);
+
+#if SPICE_SERVER_VERSION >= 0x000c02
+    qemu_spice_register_ports();
+#endif
 }
 
 int qemu_spice_add_interface(SpiceBaseInstance *sin)
@@ -732,6 +820,7 @@ int qemu_spice_add_interface(SpiceBaseInstance *sin)
          */
         spice_server = spice_server_new();
         spice_server_init(spice_server, &core_interface);
+        qemu_add_vm_change_state_handler(vm_change_state_handler, NULL);
     }
 
     return spice_server_add_interface(spice_server, sin);
@@ -759,8 +848,8 @@ static int qemu_spice_set_ticket(bool fail_if_conn, bool disconnect_if_conn)
 int qemu_spice_set_passwd(const char *passwd,
                           bool fail_if_conn, bool disconnect_if_conn)
 {
-    free(auth_passwd);
-    auth_passwd = strdup(passwd);
+    g_free(auth_passwd);
+    auth_passwd = g_strdup(passwd);
     return qemu_spice_set_ticket(fail_if_conn, disconnect_if_conn);
 }
 
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 6aff336..dc7e58d 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -16,15 +16,15 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-spice.h"
-#include "qemu-timer.h"
-#include "qemu-queue.h"
-#include "monitor.h"
-#include "console.h"
-#include "sysemu.h"
+#include "ui/qemu-spice.h"
+#include "qemu/timer.h"
+#include "qemu/queue.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 
-#include "spice-display.h"
+#include "ui/spice-display.h"
 
 static int debug = 0;
 
diff --git a/ui/spice-input.c b/ui/spice-input.c
index af4223d..3beb8de 100644
--- a/ui/spice-input.c
+++ b/ui/spice-input.c
@@ -24,8 +24,8 @@
 #include <spice/enums.h>
 
 #include "qemu-common.h"
-#include "qemu-spice.h"
-#include "console.h"
+#include "ui/qemu-spice.h"
+#include "ui/console.h"
 
 /* keyboard bits */
 
diff --git a/vgafont.h b/ui/vgafont.h
index 3606dd7..3606dd7 100644
--- a/vgafont.h
+++ b/ui/vgafont.h
diff --git a/ui/vnc-auth-sasl.h b/ui/vnc-auth-sasl.h
index ee243a9..8091d68 100644
--- a/ui/vnc-auth-sasl.h
+++ b/ui/vnc-auth-sasl.h
@@ -32,7 +32,7 @@
 typedef struct VncStateSASL VncStateSASL;
 typedef struct VncDisplaySASL VncDisplaySASL;
 
-#include "acl.h"
+#include "qemu/acl.h"
 
 struct VncStateSASL {
     sasl_conn_t *conn;
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index 9ae4cab..4ddea7d 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -44,8 +44,8 @@
 #include <jpeglib.h>
 #endif
 
-#include "bswap.h"
-#include "qint.h"
+#include "qemu/bswap.h"
+#include "qapi/qmp/qint.h"
 #include "vnc.h"
 #include "vnc-enc-tight.h"
 #include "vnc-palette.h"
@@ -1212,7 +1212,7 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
     buf = (uint8_t *)pixman_image_get_data(linebuf);
     row[0] = buf;
     for (dy = 0; dy < h; dy++) {
-        qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, dy);
+        qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
         jpeg_write_scanlines(&cinfo, row, 1);
     }
     qemu_pixman_image_unref(linebuf);
@@ -1356,7 +1356,7 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
         if (color_type == PNG_COLOR_TYPE_PALETTE) {
             memcpy(buf, vs->tight.tight.buffer + (dy * w), w);
         } else {
-            qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, dy);
+            qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
         }
         png_write_row(png_ptr, buf);
     }
diff --git a/ui/vnc-jobs.c b/ui/vnc-jobs.c
index 57c0916..0bfc0c5 100644
--- a/ui/vnc-jobs.c
+++ b/ui/vnc-jobs.c
@@ -28,7 +28,7 @@
 
 #include "vnc.h"
 #include "vnc-jobs.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 /*
  * Locking:
diff --git a/ui/vnc-palette.c b/ui/vnc-palette.c
index 63d5f64..c130dee 100644
--- a/ui/vnc-palette.c
+++ b/ui/vnc-palette.c
@@ -27,6 +27,8 @@
  */
 
 #include "vnc-palette.h"
+#include <glib.h>
+#include <string.h>
 
 static VncPaletteEntry *palette_find(const VncPalette *palette,
                                      uint32_t color, unsigned int hash)
diff --git a/ui/vnc-palette.h b/ui/vnc-palette.h
index b82dc5d..d02f023 100644
--- a/ui/vnc-palette.h
+++ b/ui/vnc-palette.h
@@ -29,8 +29,8 @@
 #ifndef VNC_PALETTE_H
 #define VNC_PALETTE_H
 
-#include "qlist.h"
-#include "qemu-queue.h"
+#include "qapi/qmp/qlist.h"
+#include "qemu/queue.h"
 #include <stdint.h>
 #include <stdbool.h>
 
diff --git a/ui/vnc-tls.c b/ui/vnc-tls.c
index a7f7d07..5629263 100644
--- a/ui/vnc-tls.c
+++ b/ui/vnc-tls.c
@@ -26,7 +26,7 @@
 
 #include "qemu-x509.h"
 #include "vnc.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 #if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
 /* Very verbose, so only enabled for _VNC_DEBUG >= 2 */
diff --git a/ui/vnc-tls.h b/ui/vnc-tls.h
index 2b93633..36a2227 100644
--- a/ui/vnc-tls.h
+++ b/ui/vnc-tls.h
@@ -31,7 +31,7 @@
 #include <gnutls/gnutls.h>
 #include <gnutls/x509.h>
 
-#include "acl.h"
+#include "qemu/acl.h"
 
 enum {
     VNC_WIREMODE_CLEAR,
diff --git a/ui/vnc-ws.c b/ui/vnc-ws.c
new file mode 100644
index 0000000..3e30209
--- /dev/null
+++ b/ui/vnc-ws.c
@@ -0,0 +1,285 @@
+/*
+ * QEMU VNC display driver: Websockets support
+ *
+ * Copyright (C) 2010 Joel Martin
+ * Copyright (C) 2012 Tim Hardeck
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnc.h"
+
+void vncws_handshake_read(void *opaque)
+{
+    VncState *vs = opaque;
+    uint8_t *handshake_end;
+    long ret;
+    buffer_reserve(&vs->ws_input, 4096);
+    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
+
+    if (!ret) {
+        if (vs->csock == -1) {
+            vnc_disconnect_finish(vs);
+        }
+        return;
+    }
+    vs->ws_input.offset += ret;
+
+    handshake_end = (uint8_t *)g_strstr_len((char *)vs->ws_input.buffer,
+            vs->ws_input.offset, WS_HANDSHAKE_END);
+    if (handshake_end) {
+        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+        vncws_process_handshake(vs, vs->ws_input.buffer, vs->ws_input.offset);
+        buffer_advance(&vs->ws_input, handshake_end - vs->ws_input.buffer +
+                strlen(WS_HANDSHAKE_END));
+    }
+}
+
+
+long vnc_client_read_ws(VncState *vs)
+{
+    int ret, err;
+    uint8_t *payload;
+    size_t payload_size, frame_size;
+    VNC_DEBUG("Read websocket %p size %zd offset %zd\n", vs->ws_input.buffer,
+            vs->ws_input.capacity, vs->ws_input.offset);
+    buffer_reserve(&vs->ws_input, 4096);
+    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
+    if (!ret) {
+        return 0;
+    }
+    vs->ws_input.offset += ret;
+
+    /* make sure that nothing is left in the ws_input buffer */
+    do {
+        err = vncws_decode_frame(&vs->ws_input, &payload,
+                              &payload_size, &frame_size);
+        if (err <= 0) {
+            return err;
+        }
+
+        buffer_reserve(&vs->input, payload_size);
+        buffer_append(&vs->input, payload, payload_size);
+
+        buffer_advance(&vs->ws_input, frame_size);
+    } while (vs->ws_input.offset > 0);
+
+    return ret;
+}
+
+long vnc_client_write_ws(VncState *vs)
+{
+    long ret;
+    VNC_DEBUG("Write WS: Pending output %p size %zd offset %zd\n",
+              vs->output.buffer, vs->output.capacity, vs->output.offset);
+    vncws_encode_frame(&vs->ws_output, vs->output.buffer, vs->output.offset);
+    buffer_reset(&vs->output);
+    ret = vnc_client_write_buf(vs, vs->ws_output.buffer, vs->ws_output.offset);
+    if (!ret) {
+        return 0;
+    }
+
+    buffer_advance(&vs->ws_output, ret);
+
+    if (vs->ws_output.offset == 0) {
+        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+    }
+
+    return ret;
+}
+
+static char *vncws_extract_handshake_entry(const char *handshake,
+        size_t handshake_len, const char *name)
+{
+    char *begin, *end, *ret = NULL;
+    char *line = g_strdup_printf("%s%s: ", WS_HANDSHAKE_DELIM, name);
+    begin = g_strstr_len(handshake, handshake_len, line);
+    if (begin != NULL) {
+        begin += strlen(line);
+        end = g_strstr_len(begin, handshake_len - (begin - handshake),
+                WS_HANDSHAKE_DELIM);
+        if (end != NULL) {
+            ret = g_strndup(begin, end - begin);
+        }
+    }
+    g_free(line);
+    return ret;
+}
+
+static void vncws_send_handshake_response(VncState *vs, const char* key)
+{
+    char combined_key[WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1];
+    unsigned char hash[SHA1_DIGEST_LEN];
+    size_t hash_size = sizeof(hash);
+    char *accept = NULL, *response = NULL;
+    gnutls_datum_t in;
+    int ret;
+
+    g_strlcpy(combined_key, key, WS_CLIENT_KEY_LEN + 1);
+    g_strlcat(combined_key, WS_GUID, WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1);
+
+    /* hash and encode it */
+    in.data = (void *)combined_key;
+    in.size = WS_CLIENT_KEY_LEN + WS_GUID_LEN;
+    ret = gnutls_fingerprint(GNUTLS_DIG_SHA1, &in, hash, &hash_size);
+    if (ret == GNUTLS_E_SUCCESS && hash_size <= SHA1_DIGEST_LEN) {
+        accept = g_base64_encode(hash, hash_size);
+    }
+    if (accept == NULL) {
+        VNC_DEBUG("Hashing Websocket combined key failed\n");
+        vnc_client_error(vs);
+        return;
+    }
+
+    response = g_strdup_printf(WS_HANDSHAKE, accept);
+    vnc_write(vs, response, strlen(response));
+    vnc_flush(vs);
+
+    g_free(accept);
+    g_free(response);
+
+    vs->encode_ws = 1;
+    vnc_init_state(vs);
+}
+
+void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size)
+{
+    char *protocols = vncws_extract_handshake_entry((const char *)line, size,
+            "Sec-WebSocket-Protocol");
+    char *version = vncws_extract_handshake_entry((const char *)line, size,
+            "Sec-WebSocket-Version");
+    char *key = vncws_extract_handshake_entry((const char *)line, size,
+            "Sec-WebSocket-Key");
+
+    if (protocols && version && key
+            && g_strrstr(protocols, "binary")
+            && !strcmp(version, WS_SUPPORTED_VERSION)
+            && strlen(key) == WS_CLIENT_KEY_LEN) {
+        vncws_send_handshake_response(vs, key);
+    } else {
+        VNC_DEBUG("Defective Websockets header or unsupported protocol\n");
+        vnc_client_error(vs);
+    }
+
+    g_free(protocols);
+    g_free(version);
+    g_free(key);
+}
+
+void vncws_encode_frame(Buffer *output, const void *payload,
+        const size_t payload_size)
+{
+    size_t header_size = 0;
+    unsigned char opcode = WS_OPCODE_BINARY_FRAME;
+    union {
+        char buf[WS_HEAD_MAX_LEN];
+        WsHeader ws;
+    } header;
+
+    if (!payload_size) {
+        return;
+    }
+
+    header.ws.b0 = 0x80 | (opcode & 0x0f);
+    if (payload_size <= 125) {
+        header.ws.b1 = (uint8_t)payload_size;
+        header_size = 2;
+    } else if (payload_size < 65536) {
+        header.ws.b1 = 0x7e;
+        header.ws.u.s16.l16 = cpu_to_be16((uint16_t)payload_size);
+        header_size = 4;
+    } else {
+        header.ws.b1 = 0x7f;
+        header.ws.u.s64.l64 = cpu_to_be64(payload_size);
+        header_size = 10;
+    }
+
+    buffer_reserve(output, header_size + payload_size);
+    buffer_append(output, header.buf, header_size);
+    buffer_append(output, payload, payload_size);
+}
+
+int vncws_decode_frame(Buffer *input, uint8_t **payload,
+                           size_t *payload_size, size_t *frame_size)
+{
+    unsigned char opcode = 0, fin = 0, has_mask = 0;
+    size_t header_size = 0;
+    uint32_t *payload32;
+    WsHeader *header = (WsHeader *)input->buffer;
+    WsMask mask;
+    int i;
+
+    if (input->offset < WS_HEAD_MIN_LEN + 4) {
+        /* header not complete */
+        return 0;
+    }
+
+    fin = (header->b0 & 0x80) >> 7;
+    opcode = header->b0 & 0x0f;
+    has_mask = (header->b1 & 0x80) >> 7;
+    *payload_size = header->b1 & 0x7f;
+
+    if (opcode == WS_OPCODE_CLOSE) {
+        /* disconnect */
+        return -1;
+    }
+
+    /* Websocket frame sanity check:
+     * * Websocket fragmentation is not supported.
+     * * All  websockets frames sent by a client have to be masked.
+     * * Only binary encoding is supported.
+     */
+    if (!fin || !has_mask || opcode != WS_OPCODE_BINARY_FRAME) {
+        VNC_DEBUG("Received faulty/unsupported Websocket frame\n");
+        return -2;
+    }
+
+    if (*payload_size < 126) {
+        header_size = 6;
+        mask = header->u.m;
+    } else if (*payload_size == 126 && input->offset >= 8) {
+        *payload_size = be16_to_cpu(header->u.s16.l16);
+        header_size = 8;
+        mask = header->u.s16.m16;
+    } else if (*payload_size == 127 && input->offset >= 14) {
+        *payload_size = be64_to_cpu(header->u.s64.l64);
+        header_size = 14;
+        mask = header->u.s64.m64;
+    } else {
+        /* header not complete */
+        return 0;
+    }
+
+    *frame_size = header_size + *payload_size;
+
+    if (input->offset < *frame_size) {
+        /* frame not complete */
+        return 0;
+    }
+
+    *payload = input->buffer + header_size;
+
+    /* unmask frame */
+    /* process 1 frame (32 bit op) */
+    payload32 = (uint32_t *)(*payload);
+    for (i = 0; i < *payload_size / 4; i++) {
+        payload32[i] ^= mask.u;
+    }
+    /* process the remaining bytes (if any) */
+    for (i *= 4; i < *payload_size; i++) {
+        (*payload)[i] ^= mask.c[i % 4];
+    }
+
+    return 1;
+}
diff --git a/ui/vnc-ws.h b/ui/vnc-ws.h
new file mode 100644
index 0000000..039a587
--- /dev/null
+++ b/ui/vnc-ws.h
@@ -0,0 +1,86 @@
+/*
+ * QEMU VNC display driver: Websockets support
+ *
+ * Copyright (C) 2010 Joel Martin
+ * Copyright (C) 2012 Tim Hardeck
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __QEMU_UI_VNC_WS_H
+#define __QEMU_UI_VNC_WS_H
+
+#include <gnutls/gnutls.h>
+
+#define B64LEN(__x) (((__x + 2) / 3) * 12 / 3)
+#define SHA1_DIGEST_LEN 20
+
+#define WS_ACCEPT_LEN (B64LEN(SHA1_DIGEST_LEN) + 1)
+#define WS_CLIENT_KEY_LEN 24
+#define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+#define WS_GUID_LEN strlen(WS_GUID)
+
+#define WS_HANDSHAKE "HTTP/1.1 101 Switching Protocols\r\n\
+Upgrade: websocket\r\n\
+Connection: Upgrade\r\n\
+Sec-WebSocket-Accept: %s\r\n\
+Sec-WebSocket-Protocol: binary\r\n\
+\r\n"
+#define WS_HANDSHAKE_DELIM "\r\n"
+#define WS_HANDSHAKE_END "\r\n\r\n"
+#define WS_SUPPORTED_VERSION "13"
+
+#define WS_HEAD_MIN_LEN sizeof(uint16_t)
+#define WS_HEAD_MAX_LEN (WS_HEAD_MIN_LEN + sizeof(uint64_t) + sizeof(uint32_t))
+
+typedef union WsMask {
+    char c[4];
+    uint32_t u;
+} WsMask;
+
+typedef struct QEMU_PACKED WsHeader {
+    unsigned char b0;
+    unsigned char b1;
+    union {
+        struct QEMU_PACKED {
+            uint16_t l16;
+            WsMask m16;
+        } s16;
+        struct QEMU_PACKED {
+            uint64_t l64;
+            WsMask m64;
+        } s64;
+        WsMask m;
+    } u;
+} WsHeader;
+
+enum {
+    WS_OPCODE_CONTINUATION = 0x0,
+    WS_OPCODE_TEXT_FRAME = 0x1,
+    WS_OPCODE_BINARY_FRAME = 0x2,
+    WS_OPCODE_CLOSE = 0x8,
+    WS_OPCODE_PING = 0x9,
+    WS_OPCODE_PONG = 0xA
+};
+
+void vncws_handshake_read(void *opaque);
+long vnc_client_write_ws(VncState *vs);
+long vnc_client_read_ws(VncState *vs);
+void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size);
+void vncws_encode_frame(Buffer *output, const void *payload,
+            const size_t payload_size);
+int vncws_decode_frame(Buffer *input, uint8_t **payload,
+                               size_t *payload_size, size_t *frame_size);
+
+#endif /* __QEMU_UI_VNC_WS_H */
diff --git a/ui/vnc.c b/ui/vnc.c
index ba30362..ff4e2ae 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -26,13 +26,13 @@
 
 #include "vnc.h"
 #include "vnc-jobs.h"
-#include "sysemu.h"
-#include "qemu_socket.h"
-#include "qemu-timer.h"
-#include "acl.h"
-#include "qemu-objects.h"
+#include "sysemu/sysemu.h"
+#include "qemu/sockets.h"
+#include "qemu/timer.h"
+#include "qemu/acl.h"
+#include "qapi/qmp/types.h"
 #include "qmp-commands.h"
-#include "osdep.h"
+#include "qemu/osdep.h"
 
 #define VNC_REFRESH_INTERVAL_BASE 30
 #define VNC_REFRESH_INTERVAL_INC  50
@@ -420,7 +420,6 @@ out_error:
 static int vnc_update_client(VncState *vs, int has_dirty);
 static int vnc_update_client_sync(VncState *vs, int has_dirty);
 static void vnc_disconnect_start(VncState *vs);
-static void vnc_disconnect_finish(VncState *vs);
 static void vnc_init_timer(VncDisplay *vd);
 static void vnc_remove_timer(VncDisplay *vd);
 
@@ -486,7 +485,7 @@ static int buffer_empty(Buffer *buffer)
     return buffer->offset == 0;
 }
 
-static uint8_t *buffer_end(Buffer *buffer)
+uint8_t *buffer_end(Buffer *buffer)
 {
     return buffer->buffer + buffer->offset;
 }
@@ -510,6 +509,13 @@ void buffer_append(Buffer *buffer, const void *data, size_t len)
     buffer->offset += len;
 }
 
+void buffer_advance(Buffer *buf, size_t len)
+{
+    memmove(buf->buffer, buf->buffer + len,
+            (buf->offset - len));
+    buf->offset -= len;
+}
+
 static void vnc_desktop_resize(VncState *vs)
 {
     DisplayState *ds = vs->ds;
@@ -1016,7 +1022,7 @@ static void vnc_disconnect_start(VncState *vs)
     vs->csock = -1;
 }
 
-static void vnc_disconnect_finish(VncState *vs)
+void vnc_disconnect_finish(VncState *vs)
 {
     int i;
 
@@ -1027,6 +1033,10 @@ static void vnc_disconnect_finish(VncState *vs)
 
     buffer_free(&vs->input);
     buffer_free(&vs->output);
+#ifdef CONFIG_VNC_WS
+    buffer_free(&vs->ws_input);
+    buffer_free(&vs->ws_output);
+#endif /* CONFIG_VNC_WS */
 
     qobject_decref(vs->info);
 
@@ -1043,20 +1053,24 @@ static void vnc_disconnect_finish(VncState *vs)
     audio_del(vs);
     vnc_release_modifiers(vs);
 
-    QTAILQ_REMOVE(&vs->vd->clients, vs, next);
+    if (vs->initialized) {
+        QTAILQ_REMOVE(&vs->vd->clients, vs, next);
+        qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
+    }
 
     if (QTAILQ_EMPTY(&vs->vd->clients)) {
         dcl->idle = 1;
     }
 
-    qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
     vnc_remove_timer(vs->vd);
     if (vs->vd->lock_key_sync)
         qemu_remove_led_event_handler(vs->led);
     vnc_unlock_output(vs);
 
     qemu_mutex_destroy(&vs->output_mutex);
-    qemu_bh_delete(vs->bh);
+    if (vs->bh != NULL) {
+        qemu_bh_delete(vs->bh);
+    }
     buffer_free(&vs->jobs_buffer);
 
     for (i = 0; i < VNC_STAT_ROWS; ++i) {
@@ -1166,8 +1180,7 @@ static long vnc_client_write_plain(VncState *vs)
     if (!ret)
         return 0;
 
-    memmove(vs->output.buffer, vs->output.buffer + ret, (vs->output.offset - ret));
-    vs->output.offset -= ret;
+    buffer_advance(&vs->output, ret);
 
     if (vs->output.offset == 0) {
         qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
@@ -1193,7 +1206,16 @@ static void vnc_client_write_locked(void *opaque)
         vnc_client_write_sasl(vs);
     } else
 #endif /* CONFIG_VNC_SASL */
-        vnc_client_write_plain(vs);
+    {
+#ifdef CONFIG_VNC_WS
+        if (vs->encode_ws) {
+            vnc_client_write_ws(vs);
+        } else
+#endif /* CONFIG_VNC_WS */
+        {
+            vnc_client_write_plain(vs);
+        }
+    }
 }
 
 void vnc_client_write(void *opaque)
@@ -1201,7 +1223,11 @@ void vnc_client_write(void *opaque)
     VncState *vs = opaque;
 
     vnc_lock_output(vs);
-    if (vs->output.offset) {
+    if (vs->output.offset
+#ifdef CONFIG_VNC_WS
+            || vs->ws_output.offset
+#endif
+            ) {
         vnc_client_write_locked(opaque);
     } else if (vs->csock != -1) {
         qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
@@ -1295,7 +1321,21 @@ void vnc_client_read(void *opaque)
         ret = vnc_client_read_sasl(vs);
     else
 #endif /* CONFIG_VNC_SASL */
+#ifdef CONFIG_VNC_WS
+        if (vs->encode_ws) {
+            ret = vnc_client_read_ws(vs);
+            if (ret == -1) {
+                vnc_disconnect_start(vs);
+                return;
+            } else if (ret == -2) {
+                vnc_client_error(vs);
+                return;
+            }
+        } else
+#endif /* CONFIG_VNC_WS */
+        {
         ret = vnc_client_read_plain(vs);
+        }
     if (!ret) {
         if (vs->csock == -1)
             vnc_disconnect_finish(vs);
@@ -1313,8 +1353,7 @@ void vnc_client_read(void *opaque)
         }
 
         if (!ret) {
-            memmove(vs->input.buffer, vs->input.buffer + len, (vs->input.offset - len));
-            vs->input.offset -= len;
+            buffer_advance(&vs->input, len);
         } else {
             vs->read_handler_expect = ret;
         }
@@ -1367,7 +1406,11 @@ void vnc_write_u8(VncState *vs, uint8_t value)
 void vnc_flush(VncState *vs)
 {
     vnc_lock_output(vs);
-    if (vs->csock != -1 && vs->output.offset) {
+    if (vs->csock != -1 && (vs->output.offset
+#ifdef CONFIG_VNC_WS
+                || vs->ws_output.offset
+#endif
+                )) {
         vnc_client_write_locked(vs);
     }
     vnc_unlock_output(vs);
@@ -2569,7 +2612,7 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
             uint8_t *server_ptr;
 
             if (vd->guest.format != VNC_SERVER_FB_FORMAT) {
-                qemu_pixman_linebuf_fill(tmpbuf, vd->guest.fb, width, y);
+                qemu_pixman_linebuf_fill(tmpbuf, vd->guest.fb, width, 0, y);
                 guest_ptr = (uint8_t *)pixman_image_get_data(tmpbuf);
             } else {
                 guest_ptr = guest_row;
@@ -2657,7 +2700,7 @@ static void vnc_remove_timer(VncDisplay *vd)
     }
 }
 
-static void vnc_connect(VncDisplay *vd, int csock, int skipauth)
+static void vnc_connect(VncDisplay *vd, int csock, int skipauth, bool websocket)
 {
     VncState *vs = g_malloc0(sizeof(VncState));
     int i;
@@ -2684,13 +2727,35 @@ static void vnc_connect(VncDisplay *vd, int csock, int skipauth)
     VNC_DEBUG("New client on socket %d\n", csock);
     dcl->idle = 0;
     socket_set_nonblock(vs->csock);
-    qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+#ifdef CONFIG_VNC_WS
+    if (websocket) {
+        vs->websocket = 1;
+        qemu_set_fd_handler2(vs->csock, NULL, vncws_handshake_read, NULL, vs);
+    } else
+#endif /* CONFIG_VNC_WS */
+    {
+        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+    }
 
     vnc_client_cache_addr(vs);
     vnc_qmp_event(vs, QEVENT_VNC_CONNECTED);
     vnc_set_share_mode(vs, VNC_SHARE_MODE_CONNECTING);
 
     vs->vd = vd;
+
+#ifdef CONFIG_VNC_WS
+    if (!vs->websocket)
+#endif
+    {
+        vnc_init_state(vs);
+    }
+}
+
+void vnc_init_state(VncState *vs)
+{
+    vs->initialized = true;
+    VncDisplay *vd = vs->vd;
+
     vs->ds = vd->ds;
     vs->last_x = -1;
     vs->last_y = -1;
@@ -2722,21 +2787,41 @@ static void vnc_connect(VncDisplay *vd, int csock, int skipauth)
     /* vs might be free()ed here */
 }
 
-static void vnc_listen_read(void *opaque)
+static void vnc_listen_read(void *opaque, bool websocket)
 {
     VncDisplay *vs = opaque;
     struct sockaddr_in addr;
     socklen_t addrlen = sizeof(addr);
+    int csock;
 
     /* Catch-up */
     vga_hw_update();
+#ifdef CONFIG_VNC_WS
+    if (websocket) {
+        csock = qemu_accept(vs->lwebsock, (struct sockaddr *)&addr, &addrlen);
+    } else
+#endif /* CONFIG_VNC_WS */
+    {
+        csock = qemu_accept(vs->lsock, (struct sockaddr *)&addr, &addrlen);
+    }
 
-    int csock = qemu_accept(vs->lsock, (struct sockaddr *)&addr, &addrlen);
     if (csock != -1) {
-        vnc_connect(vs, csock, 0);
+        vnc_connect(vs, csock, 0, websocket);
     }
 }
 
+static void vnc_listen_regular_read(void *opaque)
+{
+    vnc_listen_read(opaque, 0);
+}
+
+#ifdef CONFIG_VNC_WS
+static void vnc_listen_websocket_read(void *opaque)
+{
+    vnc_listen_read(opaque, 1);
+}
+#endif /* CONFIG_VNC_WS */
+
 void vnc_display_init(DisplayState *ds)
 {
     VncDisplay *vs = g_malloc0(sizeof(*vs));
@@ -2748,6 +2833,9 @@ void vnc_display_init(DisplayState *ds)
     vnc_display = vs;
 
     vs->lsock = -1;
+#ifdef CONFIG_VNC_WS
+    vs->lwebsock = -1;
+#endif
 
     vs->ds = ds;
     QTAILQ_INIT(&vs->clients);
@@ -2789,6 +2877,15 @@ static void vnc_display_close(DisplayState *ds)
         close(vs->lsock);
         vs->lsock = -1;
     }
+#ifdef CONFIG_VNC_WS
+    g_free(vs->ws_display);
+    vs->ws_display = NULL;
+    if (vs->lwebsock != -1) {
+        qemu_set_fd_handler2(vs->lwebsock, NULL, NULL, NULL, NULL);
+        close(vs->lwebsock);
+        vs->lwebsock = -1;
+    }
+#endif /* CONFIG_VNC_WS */
     vs->auth = VNC_AUTH_INVALID;
 #ifdef CONFIG_VNC_TLS
     vs->subauth = VNC_AUTH_INVALID;
@@ -2910,6 +3007,36 @@ void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
         } else if (strncmp(options, "sasl", 4) == 0) {
             sasl = 1; /* Require SASL auth */
 #endif
+#ifdef CONFIG_VNC_WS
+        } else if (strncmp(options, "websocket", 9) == 0) {
+            char *start, *end;
+            vs->websocket = 1;
+
+            /* Check for 'websocket=<port>' */
+            start = strchr(options, '=');
+            end = strchr(options, ',');
+            if (start && (!end || (start < end))) {
+                int len = end ? end-(start+1) : strlen(start+1);
+                if (len < 6) {
+                    /* extract the host specification from display */
+                    char  *host = NULL, *port = NULL, *host_end = NULL;
+                    port = g_strndup(start + 1, len);
+
+                    /* ipv6 hosts have colons */
+                    end = strchr(display, ',');
+                    host_end = g_strrstr_len(display, end - display, ":");
+
+                    if (host_end) {
+                        host = g_strndup(display, host_end - display + 1);
+                    } else {
+                        host = g_strndup(":", 1);
+                    }
+                    vs->ws_display = g_strconcat(host, port, NULL);
+                    g_free(host);
+                    g_free(port);
+                }
+            }
+#endif /* CONFIG_VNC_WS */
 #ifdef CONFIG_VNC_TLS
         } else if (strncmp(options, "tls", 3) == 0) {
             tls = 1; /* Require TLS */
@@ -3068,6 +3195,9 @@ void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
         /* connect to viewer */
         int csock;
         vs->lsock = -1;
+#ifdef CONFIG_VNC_WS
+        vs->lwebsock = -1;
+#endif
         if (strncmp(display, "unix:", 5) == 0) {
             csock = unix_connect(display+5, errp);
         } else {
@@ -3076,7 +3206,7 @@ void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
         if (csock < 0) {
             goto fail;
         }
-        vnc_connect(vs, csock, 0);
+        vnc_connect(vs, csock, 0, 0);
     } else {
         /* listen for connects */
         char *dpy;
@@ -3087,25 +3217,56 @@ void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
         } else {
             vs->lsock = inet_listen(display, dpy, 256,
                                     SOCK_STREAM, 5900, errp);
-        }
-        if (vs->lsock < 0) {
-            g_free(dpy);
-            goto fail;
+            if (vs->lsock < 0) {
+                g_free(dpy);
+                goto fail;
+            }
+#ifdef CONFIG_VNC_WS
+            if (vs->websocket) {
+                if (vs->ws_display) {
+                    vs->lwebsock = inet_listen(vs->ws_display, NULL, 256,
+                        SOCK_STREAM, 0, errp);
+                } else {
+                    vs->lwebsock = inet_listen(vs->display, NULL, 256,
+                        SOCK_STREAM, 5700, errp);
+                }
+
+                if (vs->lwebsock < 0) {
+                    if (vs->lsock) {
+                        close(vs->lsock);
+                        vs->lsock = -1;
+                    }
+                    g_free(dpy);
+                    goto fail;
+                }
+            }
+#endif /* CONFIG_VNC_WS */
         }
         g_free(vs->display);
         vs->display = dpy;
-        qemu_set_fd_handler2(vs->lsock, NULL, vnc_listen_read, NULL, vs);
+        qemu_set_fd_handler2(vs->lsock, NULL,
+                vnc_listen_regular_read, NULL, vs);
+#ifdef CONFIG_VNC_WS
+        if (vs->websocket) {
+            qemu_set_fd_handler2(vs->lwebsock, NULL,
+                    vnc_listen_websocket_read, NULL, vs);
+        }
+#endif /* CONFIG_VNC_WS */
     }
     return;
 
 fail:
     g_free(vs->display);
     vs->display = NULL;
+#ifdef CONFIG_VNC_WS
+    g_free(vs->ws_display);
+    vs->ws_display = NULL;
+#endif /* CONFIG_VNC_WS */
 }
 
 void vnc_display_add_client(DisplayState *ds, int csock, int skipauth)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
 
-    vnc_connect(vs, csock, skipauth);
+    vnc_connect(vs, csock, skipauth, 0);
 }
diff --git a/ui/vnc.h b/ui/vnc.h
index 6141e88..45d7686 100644
--- a/ui/vnc.h
+++ b/ui/vnc.h
@@ -28,12 +28,12 @@
 #define __QEMU_VNC_H
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "qemu-thread.h"
-#include "console.h"
-#include "monitor.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "ui/console.h"
+#include "monitor/monitor.h"
 #include "audio/audio.h"
-#include "bitmap.h"
+#include "qemu/bitmap.h"
 #include <zlib.h>
 #include <stdbool.h>
 
@@ -99,6 +99,9 @@ typedef struct VncDisplay VncDisplay;
 #ifdef CONFIG_VNC_SASL
 #include "vnc-auth-sasl.h"
 #endif
+#ifdef CONFIG_VNC_WS
+#include "vnc-ws.h"
+#endif
 
 struct VncRectStat
 {
@@ -142,6 +145,11 @@ struct VncDisplay
     QEMUTimer *timer;
     int timer_interval;
     int lsock;
+#ifdef CONFIG_VNC_WS
+    int lwebsock;
+    bool websocket;
+    char *ws_display;
+#endif
     DisplayState *ds;
     kbd_layout_t *kbd_layout;
     int lock_key_sync;
@@ -269,11 +277,19 @@ struct VncState
 #ifdef CONFIG_VNC_SASL
     VncStateSASL sasl;
 #endif
+#ifdef CONFIG_VNC_WS
+    bool encode_ws;
+    bool websocket;
+#endif
 
     QObject *info;
 
     Buffer output;
     Buffer input;
+#ifdef CONFIG_VNC_WS
+    Buffer ws_input;
+    Buffer ws_output;
+#endif
     /* current output mode information */
     VncWritePixels *write_pixels;
     PixelFormat client_pf;
@@ -290,6 +306,7 @@ struct VncState
     QEMUPutLEDEntry *led;
 
     bool abort;
+    bool initialized;
     QemuMutex output_mutex;
     QEMUBH *bh;
     Buffer jobs_buffer;
@@ -493,6 +510,8 @@ void vnc_write_u16(VncState *vs, uint16_t value);
 void vnc_write_u8(VncState *vs, uint8_t value);
 void vnc_flush(VncState *vs);
 void vnc_read_when(VncState *vs, VncReadEvent *func, size_t expecting);
+void vnc_disconnect_finish(VncState *vs);
+void vnc_init_state(VncState *vs);
 
 
 /* Buffer I/O functions */
@@ -510,6 +529,8 @@ void buffer_reserve(Buffer *buffer, size_t len);
 void buffer_reset(Buffer *buffer);
 void buffer_free(Buffer *buffer);
 void buffer_append(Buffer *buffer, const void *data, size_t len);
+void buffer_advance(Buffer *buf, size_t len);
+uint8_t *buffer_end(Buffer *buffer);
 
 
 /* Misc helpers */
diff --git a/ui/vnc_keysym.h b/ui/vnc_keysym.h
index df33cfe..6250bec 100644
--- a/ui/vnc_keysym.h
+++ b/ui/vnc_keysym.h
@@ -215,10 +215,14 @@ static const name2keysym_t name2keysym[]={
 { "Zabovedot",            0x1af},
 { "zacute",               0x1bc},
 { "Zacute",               0x1ac},
+{ "Odoubleacute",         0x1d5},
+{ "Udoubleacute",         0x1db},
 { "cacute",               0x1e6},
 { "Cacute",               0x1c6},
 { "nacute",               0x1f1},
 { "Nacute",               0x1d1},
+{ "odoubleacute",         0x1f5},
+{ "udoubleacute",         0x1fb},
 
     /* modifiers */
 {"ISO_Level3_Shift", 0xfe03}, /* XK_ISO_Level3_Shift */
diff --git a/user-exec.c b/user-exec.c
index ef9b172..c71acbc 100644
--- a/user-exec.c
+++ b/user-exec.c
@@ -18,7 +18,7 @@
  */
 #include "config.h"
 #include "cpu.h"
-#include "disas.h"
+#include "disas/disas.h"
 #include "tcg.h"
 
 #undef EAX
@@ -81,7 +81,6 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
                                     int is_write, sigset_t *old_set,
                                     void *puc)
 {
-    TranslationBlock *tb;
     int ret;
 
 #if defined(DEBUG_SIGNAL)
@@ -104,12 +103,7 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
         return 1; /* the MMU fault was handled without causing real CPU fault */
     }
     /* now we have a real cpu fault */
-    tb = tb_find_pc(pc);
-    if (tb) {
-        /* the PC is inside the translated code. It means that we have
-           a virtual CPU fault */
-        cpu_restore_state(tb, cpu_single_env, pc);
-    }
+    cpu_restore_state(cpu_single_env, pc);
 
     /* we restore the process signal mask as the sigreturn should
        do it (XXX: use sigsetjmp) */
@@ -442,7 +436,7 @@ int cpu_signal_handler(int host_signum, void *pinfo,
     unsigned long pc;
     int is_write;
 
-#if (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
+#if defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
     pc = uc->uc_mcontext.gregs[R15];
 #else
     pc = uc->uc_mcontext.arm_pc;
diff --git a/util/Makefile.objs b/util/Makefile.objs
new file mode 100644
index 0000000..495a178
--- /dev/null
+++ b/util/Makefile.objs
@@ -0,0 +1,10 @@
+util-obj-y = osdep.o cutils.o qemu-timer-common.o
+util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o event_notifier-win32.o
+util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o event_notifier-posix.o
+util-obj-y += envlist.o path.o host-utils.o cache-utils.o module.o
+util-obj-y += bitmap.o bitops.o hbitmap.o
+util-obj-y += acl.o
+util-obj-y += error.o qemu-error.o
+util-obj-$(CONFIG_POSIX) += compatfd.o
+util-obj-y += iov.o aes.o qemu-config.o qemu-sockets.o uri.o notify.o
+util-obj-y += qemu-option.o qemu-progress.o
diff --git a/acl.c b/util/acl.c
index e840b9b..a7f33ff 100644
--- a/acl.c
+++ b/util/acl.c
@@ -24,7 +24,7 @@
 
 
 #include "qemu-common.h"
-#include "acl.h"
+#include "qemu/acl.h"
 
 #ifdef CONFIG_FNMATCH
 #include <fnmatch.h>
@@ -103,8 +103,8 @@ void qemu_acl_reset(qemu_acl *acl)
     acl->defaultDeny = 1;
     QTAILQ_FOREACH_SAFE(entry, &acl->entries, next, next_entry) {
         QTAILQ_REMOVE(&acl->entries, entry, next);
-        free(entry->match);
-        free(entry);
+        g_free(entry->match);
+        g_free(entry);
     }
     acl->nentries = 0;
 }
@@ -168,6 +168,9 @@ int qemu_acl_remove(qemu_acl *acl,
         i++;
         if (strcmp(entry->match, match) == 0) {
             QTAILQ_REMOVE(&acl->entries, entry, next);
+            acl->nentries--;
+            g_free(entry->match);
+            g_free(entry);
             return i;
         }
     }
diff --git a/aes.c b/util/aes.c
index eb37adb..1da7bff 100644
--- a/aes.c
+++ b/util/aes.c
@@ -28,7 +28,7 @@
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "qemu-common.h"
-#include "aes.h"
+#include "block/aes.h"
 
 #ifndef NDEBUG
 #define NDEBUG
diff --git a/bitmap.c b/util/bitmap.c
index a62c8ba..687841d 100644
--- a/bitmap.c
+++ b/util/bitmap.c
@@ -9,8 +9,8 @@
  * Version 2.
  */
 
-#include "bitops.h"
-#include "bitmap.h"
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
 
 /*
  * bitmaps provide an array of bits, implemented using an an
diff --git a/bitops.c b/util/bitops.c
index d9de71f..7b853cf 100644
--- a/bitops.c
+++ b/util/bitops.c
@@ -11,7 +11,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "bitops.h"
+#include "qemu/bitops.h"
 
 #define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
 
@@ -60,7 +60,7 @@ found_first:
         return result + size;	/* Nope. */
     }
 found_middle:
-    return result + bitops_ffsl(tmp);
+    return result + bitops_ctzl(tmp);
 }
 
 /*
@@ -109,7 +109,7 @@ found_first:
         return result + size;	/* Nope. */
     }
 found_middle:
-    return result + ffz(tmp);
+    return result + bitops_ctol(tmp);
 }
 
 unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
diff --git a/cache-utils.c b/util/cache-utils.c
index 2db5af2..b94013a 100644
--- a/cache-utils.c
+++ b/util/cache-utils.c
@@ -1,4 +1,4 @@
-#include "cache-utils.h"
+#include "qemu/cache-utils.h"
 
 #if defined(_ARCH_PPC)
 struct qemu_cache_conf qemu_cache_conf = {
diff --git a/compatfd.c b/util/compatfd.c
index 42f81ca..9cf3f28 100644
--- a/compatfd.c
+++ b/util/compatfd.c
@@ -14,7 +14,7 @@
  */
 
 #include "qemu-common.h"
-#include "compatfd.h"
+#include "qemu/compatfd.h"
 
 #include <sys/syscall.h>
 #include <pthread.h>
diff --git a/cutils.c b/util/cutils.c
index 4f0692f..1439da4 100644
--- a/cutils.c
+++ b/util/cutils.c
@@ -22,11 +22,11 @@
  * THE SOFTWARE.
  */
 #include "qemu-common.h"
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 #include <math.h>
 
-#include "qemu_socket.h"
-#include "iov.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
 
 void strpadcpy(char *buf, int buf_size, const char *str, char pad)
 {
@@ -214,12 +214,13 @@ static int64_t suffix_mul(char suffix, int64_t unit)
 /*
  * Convert string to bytes, allowing either B/b for bytes, K/k for KB,
  * M/m for MB, G/g for GB or T/t for TB. End pointer will be returned
- * in *end, if not NULL. Return -1 on error.
+ * in *end, if not NULL. Return -ERANGE on overflow, Return -EINVAL on
+ * other error.
  */
 int64_t strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit)
 {
-    int64_t retval = -1;
+    int64_t retval = -EINVAL;
     char *endptr;
     unsigned char c;
     int mul_required = 0;
@@ -246,6 +247,7 @@ int64_t strtosz_suffix_unit(const char *nptr, char **end,
         goto fail;
     }
     if ((val * mul >= INT64_MAX) || val < 0) {
+        retval = -ERANGE;
         goto fail;
     }
     retval = val * mul;
@@ -268,6 +270,105 @@ int64_t strtosz(const char *nptr, char **end)
     return strtosz_suffix(nptr, end, STRTOSZ_DEFSUFFIX_MB);
 }
 
+/**
+ * parse_uint:
+ *
+ * @s: String to parse
+ * @value: Destination for parsed integer value
+ * @endptr: Destination for pointer to first character not consumed
+ * @base: integer base, between 2 and 36 inclusive, or 0
+ *
+ * Parse unsigned integer
+ *
+ * Parsed syntax is like strtoull()'s: arbitrary whitespace, a single optional
+ * '+' or '-', an optional "0x" if @base is 0 or 16, one or more digits.
+ *
+ * If @s is null, or @base is invalid, or @s doesn't start with an
+ * integer in the syntax above, set *@value to 0, *@endptr to @s, and
+ * return -EINVAL.
+ *
+ * Set *@endptr to point right beyond the parsed integer (even if the integer
+ * overflows or is negative, all digits will be parsed and *@endptr will
+ * point right beyond them).
+ *
+ * If the integer is negative, set *@value to 0, and return -ERANGE.
+ *
+ * If the integer overflows unsigned long long, set *@value to
+ * ULLONG_MAX, and return -ERANGE.
+ *
+ * Else, set *@value to the parsed integer, and return 0.
+ */
+int parse_uint(const char *s, unsigned long long *value, char **endptr,
+               int base)
+{
+    int r = 0;
+    char *endp = (char *)s;
+    unsigned long long val = 0;
+
+    if (!s) {
+        r = -EINVAL;
+        goto out;
+    }
+
+    errno = 0;
+    val = strtoull(s, &endp, base);
+    if (errno) {
+        r = -errno;
+        goto out;
+    }
+
+    if (endp == s) {
+        r = -EINVAL;
+        goto out;
+    }
+
+    /* make sure we reject negative numbers: */
+    while (isspace((unsigned char)*s)) {
+        s++;
+    }
+    if (*s == '-') {
+        val = 0;
+        r = -ERANGE;
+        goto out;
+    }
+
+out:
+    *value = val;
+    *endptr = endp;
+    return r;
+}
+
+/**
+ * parse_uint_full:
+ *
+ * @s: String to parse
+ * @value: Destination for parsed integer value
+ * @base: integer base, between 2 and 36 inclusive, or 0
+ *
+ * Parse unsigned integer from entire string
+ *
+ * Have the same behavior of parse_uint(), but with an additional check
+ * for additional data after the parsed number. If extra characters are present
+ * after the parsed number, the function will return -EINVAL, and *@v will
+ * be set to 0.
+ */
+int parse_uint_full(const char *s, unsigned long long *value, int base)
+{
+    char *endp;
+    int r;
+
+    r = parse_uint(s, value, &endp, base);
+    if (r < 0) {
+        return r;
+    }
+    if (*endp) {
+        *value = 0;
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
 int qemu_parse_fd(const char *param)
 {
     int fd;
diff --git a/envlist.c b/util/envlist.c
index f2303cd..ebc06cf 100644
--- a/envlist.c
+++ b/util/envlist.c
@@ -1,11 +1,6 @@
-#include <assert.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "qemu-queue.h"
-#include "envlist.h"
+#include "qemu-common.h"
+#include "qemu/queue.h"
+#include "qemu/envlist.h"
 
 struct envlist_entry {
 	const char *ev_var;			/* actual env value */
diff --git a/error.c b/util/error.c
index 128d88c..519f6b6 100644
--- a/error.c
+++ b/util/error.c
@@ -11,11 +11,11 @@
  */
 
 #include "qemu-common.h"
-#include "error.h"
-#include "qjson.h"
-#include "qdict.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qjson.h"
+#include "qapi/qmp/qdict.h"
 #include "qapi-types.h"
-#include "qerror.h"
+#include "qapi/qmp/qerror.h"
 
 struct Error
 {
diff --git a/event_notifier-posix.c b/util/event_notifier-posix.c
index 6f3239a..713d756 100644
--- a/event_notifier-posix.c
+++ b/util/event_notifier-posix.c
@@ -11,8 +11,9 @@
  */
 
 #include "qemu-common.h"
-#include "event_notifier.h"
-#include "qemu-char.h"
+#include "qemu/event_notifier.h"
+#include "char/char.h"
+#include "qemu/main-loop.h"
 
 #ifdef CONFIG_EVENTFD
 #include <sys/eventfd.h>
diff --git a/event_notifier-win32.c b/util/event_notifier-win32.c
index 4ed21c2..6dbb530 100644
--- a/event_notifier-win32.c
+++ b/util/event_notifier-win32.c
@@ -11,8 +11,8 @@
  */
 
 #include "qemu-common.h"
-#include "event_notifier.h"
-#include "main-loop.h"
+#include "qemu/event_notifier.h"
+#include "qemu/main-loop.h"
 
 int event_notifier_init(EventNotifier *e, int active)
 {
diff --git a/util/hbitmap.c b/util/hbitmap.c
new file mode 100644
index 0000000..a0df5d3
--- /dev/null
+++ b/util/hbitmap.c
@@ -0,0 +1,401 @@
+/*
+ * Hierarchical Bitmap Data Type
+ *
+ * Copyright Red Hat, Inc., 2012
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include <string.h>
+#include <glib.h>
+#include <assert.h>
+#include "qemu/osdep.h"
+#include "qemu/hbitmap.h"
+#include "qemu/host-utils.h"
+#include "trace.h"
+
+/* HBitmaps provides an array of bits.  The bits are stored as usual in an
+ * array of unsigned longs, but HBitmap is also optimized to provide fast
+ * iteration over set bits; going from one bit to the next is O(logB n)
+ * worst case, with B = sizeof(long) * CHAR_BIT: the result is low enough
+ * that the number of levels is in fact fixed.
+ *
+ * In order to do this, it stacks multiple bitmaps with progressively coarser
+ * granularity; in all levels except the last, bit N is set iff the N-th
+ * unsigned long is nonzero in the immediately next level.  When iteration
+ * completes on the last level it can examine the 2nd-last level to quickly
+ * skip entire words, and even do so recursively to skip blocks of 64 words or
+ * powers thereof (32 on 32-bit machines).
+ *
+ * Given an index in the bitmap, it can be split in group of bits like
+ * this (for the 64-bit case):
+ *
+ *   bits 0-57 => word in the last bitmap     | bits 58-63 => bit in the word
+ *   bits 0-51 => word in the 2nd-last bitmap | bits 52-57 => bit in the word
+ *   bits 0-45 => word in the 3rd-last bitmap | bits 46-51 => bit in the word
+ *
+ * So it is easy to move up simply by shifting the index right by
+ * log2(BITS_PER_LONG) bits.  To move down, you shift the index left
+ * similarly, and add the word index within the group.  Iteration uses
+ * ffs (find first set bit) to find the next word to examine; this
+ * operation can be done in constant time in most current architectures.
+ *
+ * Setting or clearing a range of m bits on all levels, the work to perform
+ * is O(m + m/W + m/W^2 + ...), which is O(m) like on a regular bitmap.
+ *
+ * When iterating on a bitmap, each bit (on any level) is only visited
+ * once.  Hence, The total cost of visiting a bitmap with m bits in it is
+ * the number of bits that are set in all bitmaps.  Unless the bitmap is
+ * extremely sparse, this is also O(m + m/W + m/W^2 + ...), so the amortized
+ * cost of advancing from one bit to the next is usually constant (worst case
+ * O(logB n) as in the non-amortized complexity).
+ */
+
+struct HBitmap {
+    /* Number of total bits in the bottom level.  */
+    uint64_t size;
+
+    /* Number of set bits in the bottom level.  */
+    uint64_t count;
+
+    /* A scaling factor.  Given a granularity of G, each bit in the bitmap will
+     * will actually represent a group of 2^G elements.  Each operation on a
+     * range of bits first rounds the bits to determine which group they land
+     * in, and then affect the entire page; iteration will only visit the first
+     * bit of each group.  Here is an example of operations in a size-16,
+     * granularity-1 HBitmap:
+     *
+     *    initial state            00000000
+     *    set(start=0, count=9)    11111000 (iter: 0, 2, 4, 6, 8)
+     *    reset(start=1, count=3)  00111000 (iter: 4, 6, 8)
+     *    set(start=9, count=2)    00111100 (iter: 4, 6, 8, 10)
+     *    reset(start=5, count=5)  00000000
+     *
+     * From an implementation point of view, when setting or resetting bits,
+     * the bitmap will scale bit numbers right by this amount of bits.  When
+     * iterating, the bitmap will scale bit numbers left by this amount of
+     * bits.
+     */
+    int granularity;
+
+    /* A number of progressively less coarse bitmaps (i.e. level 0 is the
+     * coarsest).  Each bit in level N represents a word in level N+1 that
+     * has a set bit, except the last level where each bit represents the
+     * actual bitmap.
+     *
+     * Note that all bitmaps have the same number of levels.  Even a 1-bit
+     * bitmap will still allocate HBITMAP_LEVELS arrays.
+     */
+    unsigned long *levels[HBITMAP_LEVELS];
+};
+
+static inline int popcountl(unsigned long l)
+{
+    return BITS_PER_LONG == 32 ? ctpop32(l) : ctpop64(l);
+}
+
+/* Advance hbi to the next nonzero word and return it.  hbi->pos
+ * is updated.  Returns zero if we reach the end of the bitmap.
+ */
+unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
+{
+    size_t pos = hbi->pos;
+    const HBitmap *hb = hbi->hb;
+    unsigned i = HBITMAP_LEVELS - 1;
+
+    unsigned long cur;
+    do {
+        cur = hbi->cur[--i];
+        pos >>= BITS_PER_LEVEL;
+    } while (cur == 0);
+
+    /* Check for end of iteration.  We always use fewer than BITS_PER_LONG
+     * bits in the level 0 bitmap; thus we can repurpose the most significant
+     * bit as a sentinel.  The sentinel is set in hbitmap_alloc and ensures
+     * that the above loop ends even without an explicit check on i.
+     */
+
+    if (i == 0 && cur == (1UL << (BITS_PER_LONG - 1))) {
+        return 0;
+    }
+    for (; i < HBITMAP_LEVELS - 1; i++) {
+        /* Shift back pos to the left, matching the right shifts above.
+         * The index of this word's least significant set bit provides
+         * the low-order bits.
+         */
+        pos = (pos << BITS_PER_LEVEL) + bitops_ctzl(cur);
+        hbi->cur[i] = cur & (cur - 1);
+
+        /* Set up next level for iteration.  */
+        cur = hb->levels[i + 1][pos];
+    }
+
+    hbi->pos = pos;
+    trace_hbitmap_iter_skip_words(hbi->hb, hbi, pos, cur);
+
+    assert(cur);
+    return cur;
+}
+
+void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first)
+{
+    unsigned i, bit;
+    uint64_t pos;
+
+    hbi->hb = hb;
+    pos = first >> hb->granularity;
+    assert(pos < hb->size);
+    hbi->pos = pos >> BITS_PER_LEVEL;
+    hbi->granularity = hb->granularity;
+
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        bit = pos & (BITS_PER_LONG - 1);
+        pos >>= BITS_PER_LEVEL;
+
+        /* Drop bits representing items before first.  */
+        hbi->cur[i] = hb->levels[i][pos] & ~((1UL << bit) - 1);
+
+        /* We have already added level i+1, so the lowest set bit has
+         * been processed.  Clear it.
+         */
+        if (i != HBITMAP_LEVELS - 1) {
+            hbi->cur[i] &= ~(1UL << bit);
+        }
+    }
+}
+
+bool hbitmap_empty(const HBitmap *hb)
+{
+    return hb->count == 0;
+}
+
+int hbitmap_granularity(const HBitmap *hb)
+{
+    return hb->granularity;
+}
+
+uint64_t hbitmap_count(const HBitmap *hb)
+{
+    return hb->count << hb->granularity;
+}
+
+/* Count the number of set bits between start and end, not accounting for
+ * the granularity.  Also an example of how to use hbitmap_iter_next_word.
+ */
+static uint64_t hb_count_between(HBitmap *hb, uint64_t start, uint64_t last)
+{
+    HBitmapIter hbi;
+    uint64_t count = 0;
+    uint64_t end = last + 1;
+    unsigned long cur;
+    size_t pos;
+
+    hbitmap_iter_init(&hbi, hb, start << hb->granularity);
+    for (;;) {
+        pos = hbitmap_iter_next_word(&hbi, &cur);
+        if (pos >= (end >> BITS_PER_LEVEL)) {
+            break;
+        }
+        count += popcountl(cur);
+    }
+
+    if (pos == (end >> BITS_PER_LEVEL)) {
+        /* Drop bits representing the END-th and subsequent items.  */
+        int bit = end & (BITS_PER_LONG - 1);
+        cur &= (1UL << bit) - 1;
+        count += popcountl(cur);
+    }
+
+    return count;
+}
+
+/* Setting starts at the last layer and propagates up if an element
+ * changes from zero to non-zero.
+ */
+static inline bool hb_set_elem(unsigned long *elem, uint64_t start, uint64_t last)
+{
+    unsigned long mask;
+    bool changed;
+
+    assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
+    assert(start <= last);
+
+    mask = 2UL << (last & (BITS_PER_LONG - 1));
+    mask -= 1UL << (start & (BITS_PER_LONG - 1));
+    changed = (*elem == 0);
+    *elem |= mask;
+    return changed;
+}
+
+/* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... */
+static void hb_set_between(HBitmap *hb, int level, uint64_t start, uint64_t last)
+{
+    size_t pos = start >> BITS_PER_LEVEL;
+    size_t lastpos = last >> BITS_PER_LEVEL;
+    bool changed = false;
+    size_t i;
+
+    i = pos;
+    if (i < lastpos) {
+        uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
+        changed |= hb_set_elem(&hb->levels[level][i], start, next - 1);
+        for (;;) {
+            start = next;
+            next += BITS_PER_LONG;
+            if (++i == lastpos) {
+                break;
+            }
+            changed |= (hb->levels[level][i] == 0);
+            hb->levels[level][i] = ~0UL;
+        }
+    }
+    changed |= hb_set_elem(&hb->levels[level][i], start, last);
+
+    /* If there was any change in this layer, we may have to update
+     * the one above.
+     */
+    if (level > 0 && changed) {
+        hb_set_between(hb, level - 1, pos, lastpos);
+    }
+}
+
+void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count)
+{
+    /* Compute range in the last layer.  */
+    uint64_t last = start + count - 1;
+
+    trace_hbitmap_set(hb, start, count,
+                      start >> hb->granularity, last >> hb->granularity);
+
+    start >>= hb->granularity;
+    last >>= hb->granularity;
+    count = last - start + 1;
+
+    hb->count += count - hb_count_between(hb, start, last);
+    hb_set_between(hb, HBITMAP_LEVELS - 1, start, last);
+}
+
+/* Resetting works the other way round: propagate up if the new
+ * value is zero.
+ */
+static inline bool hb_reset_elem(unsigned long *elem, uint64_t start, uint64_t last)
+{
+    unsigned long mask;
+    bool blanked;
+
+    assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
+    assert(start <= last);
+
+    mask = 2UL << (last & (BITS_PER_LONG - 1));
+    mask -= 1UL << (start & (BITS_PER_LONG - 1));
+    blanked = *elem != 0 && ((*elem & ~mask) == 0);
+    *elem &= ~mask;
+    return blanked;
+}
+
+/* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... */
+static void hb_reset_between(HBitmap *hb, int level, uint64_t start, uint64_t last)
+{
+    size_t pos = start >> BITS_PER_LEVEL;
+    size_t lastpos = last >> BITS_PER_LEVEL;
+    bool changed = false;
+    size_t i;
+
+    i = pos;
+    if (i < lastpos) {
+        uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
+
+        /* Here we need a more complex test than when setting bits.  Even if
+         * something was changed, we must not blank bits in the upper level
+         * unless the lower-level word became entirely zero.  So, remove pos
+         * from the upper-level range if bits remain set.
+         */
+        if (hb_reset_elem(&hb->levels[level][i], start, next - 1)) {
+            changed = true;
+        } else {
+            pos++;
+        }
+
+        for (;;) {
+            start = next;
+            next += BITS_PER_LONG;
+            if (++i == lastpos) {
+                break;
+            }
+            changed |= (hb->levels[level][i] != 0);
+            hb->levels[level][i] = 0UL;
+        }
+    }
+
+    /* Same as above, this time for lastpos.  */
+    if (hb_reset_elem(&hb->levels[level][i], start, last)) {
+        changed = true;
+    } else {
+        lastpos--;
+    }
+
+    if (level > 0 && changed) {
+        hb_reset_between(hb, level - 1, pos, lastpos);
+    }
+}
+
+void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count)
+{
+    /* Compute range in the last layer.  */
+    uint64_t last = start + count - 1;
+
+    trace_hbitmap_reset(hb, start, count,
+                        start >> hb->granularity, last >> hb->granularity);
+
+    start >>= hb->granularity;
+    last >>= hb->granularity;
+
+    hb->count -= hb_count_between(hb, start, last);
+    hb_reset_between(hb, HBITMAP_LEVELS - 1, start, last);
+}
+
+bool hbitmap_get(const HBitmap *hb, uint64_t item)
+{
+    /* Compute position and bit in the last layer.  */
+    uint64_t pos = item >> hb->granularity;
+    unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1));
+
+    return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0;
+}
+
+void hbitmap_free(HBitmap *hb)
+{
+    unsigned i;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        g_free(hb->levels[i]);
+    }
+    g_free(hb);
+}
+
+HBitmap *hbitmap_alloc(uint64_t size, int granularity)
+{
+    HBitmap *hb = g_malloc0(sizeof (struct HBitmap));
+    unsigned i;
+
+    assert(granularity >= 0 && granularity < 64);
+    size = (size + (1ULL << granularity) - 1) >> granularity;
+    assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
+
+    hb->size = size;
+    hb->granularity = granularity;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
+        hb->levels[i] = g_malloc0(size * sizeof(unsigned long));
+    }
+
+    /* We necessarily have free bits in level 0 due to the definition
+     * of HBITMAP_LEVELS, so use one for a sentinel.  This speeds up
+     * hbitmap_iter_skip_words.
+     */
+    assert(size == 1);
+    hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
+    return hb;
+}
diff --git a/host-utils.c b/util/host-utils.c
index dc96123..5e3915a 100644
--- a/host-utils.c
+++ b/util/host-utils.c
@@ -25,7 +25,7 @@
 
 #include <stdlib.h>
 #include <stdint.h>
-#include "host-utils.h"
+#include "qemu/host-utils.h"
 
 //#define DEBUG_MULDIV
 
diff --git a/iov.c b/util/iov.c
index a81eedc..fbe675d 100644
--- a/iov.c
+++ b/util/iov.c
@@ -16,7 +16,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "iov.h"
+#include "qemu/iov.h"
 
 #ifdef _WIN32
 # include <windows.h>
@@ -289,34 +289,53 @@ void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
 }
 
 /*
- * Concatenates (partial) iovecs from src to the end of dst.
+ * Concatenates (partial) iovecs from src_iov to the end of dst.
  * It starts copying after skipping `soffset' bytes at the
  * beginning of src and adds individual vectors from src to
  * dst copies up to `sbytes' bytes total, or up to the end
- * of src if it comes first.  This way, it is okay to specify
+ * of src_iov if it comes first.  This way, it is okay to specify
  * very large value for `sbytes' to indicate "up to the end
  * of src".
  * Only vector pointers are processed, not the actual data buffers.
  */
-void qemu_iovec_concat(QEMUIOVector *dst,
-                       QEMUIOVector *src, size_t soffset, size_t sbytes)
+void qemu_iovec_concat_iov(QEMUIOVector *dst,
+                           struct iovec *src_iov, unsigned int src_cnt,
+                           size_t soffset, size_t sbytes)
 {
     int i;
     size_t done;
-    struct iovec *siov = src->iov;
+
+    if (!sbytes) {
+        return;
+    }
     assert(dst->nalloc != -1);
-    assert(src->size >= soffset);
-    for (i = 0, done = 0; done < sbytes && i < src->niov; i++) {
-        if (soffset < siov[i].iov_len) {
-            size_t len = MIN(siov[i].iov_len - soffset, sbytes - done);
-            qemu_iovec_add(dst, siov[i].iov_base + soffset, len);
+    for (i = 0, done = 0; done < sbytes && i < src_cnt; i++) {
+        if (soffset < src_iov[i].iov_len) {
+            size_t len = MIN(src_iov[i].iov_len - soffset, sbytes - done);
+            qemu_iovec_add(dst, src_iov[i].iov_base + soffset, len);
             done += len;
             soffset = 0;
         } else {
-            soffset -= siov[i].iov_len;
+            soffset -= src_iov[i].iov_len;
         }
     }
-    /* return done; */
+    assert(soffset == 0); /* offset beyond end of src */
+}
+
+/*
+ * Concatenates (partial) iovecs from src to the end of dst.
+ * It starts copying after skipping `soffset' bytes at the
+ * beginning of src and adds individual vectors from src to
+ * dst copies up to `sbytes' bytes total, or up to the end
+ * of src if it comes first.  This way, it is okay to specify
+ * very large value for `sbytes' to indicate "up to the end
+ * of src".
+ * Only vector pointers are processed, not the actual data buffers.
+ */
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes)
+{
+    qemu_iovec_concat_iov(dst, src->iov, src->niov, soffset, sbytes);
 }
 
 void qemu_iovec_destroy(QEMUIOVector *qiov)
@@ -354,3 +373,54 @@ size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
 {
     return iov_memset(qiov->iov, qiov->niov, offset, fillc, bytes);
 }
+
+size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
+                         size_t bytes)
+{
+    size_t total = 0;
+    struct iovec *cur;
+
+    for (cur = *iov; *iov_cnt > 0; cur++) {
+        if (cur->iov_len > bytes) {
+            cur->iov_base += bytes;
+            cur->iov_len -= bytes;
+            total += bytes;
+            break;
+        }
+
+        bytes -= cur->iov_len;
+        total += cur->iov_len;
+        *iov_cnt -= 1;
+    }
+
+    *iov = cur;
+    return total;
+}
+
+size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
+                        size_t bytes)
+{
+    size_t total = 0;
+    struct iovec *cur;
+
+    if (*iov_cnt == 0) {
+        return 0;
+    }
+
+    cur = iov + (*iov_cnt - 1);
+
+    while (*iov_cnt > 0) {
+        if (cur->iov_len > bytes) {
+            cur->iov_len -= bytes;
+            total += bytes;
+            break;
+        }
+
+        bytes -= cur->iov_len;
+        total += cur->iov_len;
+        cur--;
+        *iov_cnt -= 1;
+    }
+
+    return total;
+}
diff --git a/module.c b/util/module.c
index c3a6da7..7acc33d 100644
--- a/module.c
+++ b/util/module.c
@@ -14,8 +14,8 @@
  */
 
 #include "qemu-common.h"
-#include "qemu-queue.h"
-#include "module.h"
+#include "qemu/queue.h"
+#include "qemu/module.h"
 
 typedef struct ModuleEntry
 {
diff --git a/notify.c b/util/notify.c
index 12282a6..7b7692a 100644
--- a/notify.c
+++ b/util/notify.c
@@ -14,7 +14,7 @@
  */
 
 #include "qemu-common.h"
-#include "notify.h"
+#include "qemu/notify.h"
 
 void notifier_list_init(NotifierList *list)
 {
diff --git a/osdep.c b/util/osdep.c
index 3a63d26..5b51a03 100644
--- a/osdep.c
+++ b/util/osdep.c
@@ -47,8 +47,8 @@ extern int madvise(caddr_t, size_t, int);
 
 #include "qemu-common.h"
 #include "trace.h"
-#include "qemu_socket.h"
-#include "monitor.h"
+#include "qemu/sockets.h"
+#include "monitor/monitor.h"
 
 static bool fips_enabled = false;
 
diff --git a/oslib-posix.c b/util/oslib-posix.c
index d25b52a..433dd68 100644
--- a/oslib-posix.c
+++ b/util/oslib-posix.c
@@ -49,9 +49,9 @@ extern int daemon(int, int);
 #endif
 
 #include "config-host.h"
-#include "sysemu.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 #if defined(CONFIG_VALGRIND)
 static int running_on_valgrind = -1;
@@ -105,8 +105,6 @@ void *qemu_memalign(size_t alignment, size_t size)
     return ptr;
 }
 
-/* conflicts with qemu_vmalloc in bsd-user/mmap.c */
-#if !defined(CONFIG_BSD_USER)
 /* alloc shared memory pages */
 void *qemu_vmalloc(size_t size)
 {
@@ -129,7 +127,6 @@ void *qemu_vmalloc(size_t size)
     trace_qemu_vmalloc(size, ptr);
     return ptr;
 }
-#endif
 
 void qemu_vfree(void *ptr)
 {
diff --git a/oslib-win32.c b/util/oslib-win32.c
index 51b33e8..640194c 100644
--- a/oslib-win32.c
+++ b/util/oslib-win32.c
@@ -27,10 +27,10 @@
  */
 #include <windows.h>
 #include "config-host.h"
-#include "sysemu.h"
-#include "main-loop.h"
+#include "sysemu/sysemu.h"
+#include "qemu/main-loop.h"
 #include "trace.h"
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 void *qemu_oom_check(void *ptr)
 {
@@ -71,7 +71,9 @@ void *qemu_vmalloc(size_t size)
 void qemu_vfree(void *ptr)
 {
     trace_qemu_vfree(ptr);
-    VirtualFree(ptr, 0, MEM_RELEASE);
+    if (ptr) {
+        VirtualFree(ptr, 0, MEM_RELEASE);
+    }
 }
 
 /* FIXME: add proper locking */
diff --git a/path.c b/util/path.c
index ef3f277..f0c6962 100644
--- a/path.c
+++ b/util/path.c
@@ -47,10 +47,7 @@ static struct pathelem *new_entry(const char *root,
 {
     struct pathelem *new = malloc(sizeof(*new));
     new->name = strdup(name);
-    if (asprintf(&new->pathname, "%s/%s", root, name) == -1) {
-        printf("Cannot allocate memory\n");
-        exit(1);
-    }
+    new->pathname = g_strdup_printf("%s/%s", root, name);
     new->num_entries = 0;
     return new;
 }
@@ -58,9 +55,10 @@ static struct pathelem *new_entry(const char *root,
 #define streq(a,b) (strcmp((a), (b)) == 0)
 
 /* Not all systems provide this feature */
-#if defined(DT_DIR) && defined(DT_UNKNOWN)
+#if defined(DT_DIR) && defined(DT_UNKNOWN) && defined(DT_LNK)
 # define dirent_type(dirent) ((dirent)->d_type)
-# define is_dir_maybe(type)  ((type) == DT_DIR || (type) == DT_UNKNOWN)
+# define is_dir_maybe(type) \
+    ((type) == DT_DIR || (type) == DT_UNKNOWN || (type) == DT_LNK)
 #else
 # define dirent_type(dirent) (1)
 # define is_dir_maybe(type)  (type)
diff --git a/util/qemu-config.c b/util/qemu-config.c
new file mode 100644
index 0000000..db6ec03
--- /dev/null
+++ b/util/qemu-config.c
@@ -0,0 +1,215 @@
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
+#include "hw/qdev.h"
+#include "qapi/error.h"
+
+static QemuOptsList *vm_config_groups[32];
+
+static QemuOptsList *find_list(QemuOptsList **lists, const char *group,
+                               Error **errp)
+{
+    int i;
+
+    for (i = 0; lists[i] != NULL; i++) {
+        if (strcmp(lists[i]->name, group) == 0)
+            break;
+    }
+    if (lists[i] == NULL) {
+        error_set(errp, QERR_INVALID_OPTION_GROUP, group);
+    }
+    return lists[i];
+}
+
+QemuOptsList *qemu_find_opts(const char *group)
+{
+    QemuOptsList *ret;
+    Error *local_err = NULL;
+
+    ret = find_list(vm_config_groups, group, &local_err);
+    if (error_is_set(&local_err)) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
+    }
+
+    return ret;
+}
+
+QemuOptsList *qemu_find_opts_err(const char *group, Error **errp)
+{
+    return find_list(vm_config_groups, group, errp);
+}
+
+void qemu_add_opts(QemuOptsList *list)
+{
+    int entries, i;
+
+    entries = ARRAY_SIZE(vm_config_groups);
+    entries--; /* keep list NULL terminated */
+    for (i = 0; i < entries; i++) {
+        if (vm_config_groups[i] == NULL) {
+            vm_config_groups[i] = list;
+            return;
+        }
+    }
+    fprintf(stderr, "ran out of space in vm_config_groups");
+    abort();
+}
+
+int qemu_set_option(const char *str)
+{
+    char group[64], id[64], arg[64];
+    QemuOptsList *list;
+    QemuOpts *opts;
+    int rc, offset;
+
+    rc = sscanf(str, "%63[^.].%63[^.].%63[^=]%n", group, id, arg, &offset);
+    if (rc < 3 || str[offset] != '=') {
+        error_report("can't parse: \"%s\"", str);
+        return -1;
+    }
+
+    list = qemu_find_opts(group);
+    if (list == NULL) {
+        return -1;
+    }
+
+    opts = qemu_opts_find(list, id);
+    if (!opts) {
+        error_report("there is no %s \"%s\" defined",
+                     list->name, id);
+        return -1;
+    }
+
+    if (qemu_opt_set(opts, arg, str+offset+1) == -1) {
+        return -1;
+    }
+    return 0;
+}
+
+struct ConfigWriteData {
+    QemuOptsList *list;
+    FILE *fp;
+};
+
+static int config_write_opt(const char *name, const char *value, void *opaque)
+{
+    struct ConfigWriteData *data = opaque;
+
+    fprintf(data->fp, "  %s = \"%s\"\n", name, value);
+    return 0;
+}
+
+static int config_write_opts(QemuOpts *opts, void *opaque)
+{
+    struct ConfigWriteData *data = opaque;
+    const char *id = qemu_opts_id(opts);
+
+    if (id) {
+        fprintf(data->fp, "[%s \"%s\"]\n", data->list->name, id);
+    } else {
+        fprintf(data->fp, "[%s]\n", data->list->name);
+    }
+    qemu_opt_foreach(opts, config_write_opt, data, 0);
+    fprintf(data->fp, "\n");
+    return 0;
+}
+
+void qemu_config_write(FILE *fp)
+{
+    struct ConfigWriteData data = { .fp = fp };
+    QemuOptsList **lists = vm_config_groups;
+    int i;
+
+    fprintf(fp, "# qemu config file\n\n");
+    for (i = 0; lists[i] != NULL; i++) {
+        data.list = lists[i];
+        qemu_opts_foreach(data.list, config_write_opts, &data, 0);
+    }
+}
+
+int qemu_config_parse(FILE *fp, QemuOptsList **lists, const char *fname)
+{
+    char line[1024], group[64], id[64], arg[64], value[1024];
+    Location loc;
+    QemuOptsList *list = NULL;
+    Error *local_err = NULL;
+    QemuOpts *opts = NULL;
+    int res = -1, lno = 0;
+
+    loc_push_none(&loc);
+    while (fgets(line, sizeof(line), fp) != NULL) {
+        loc_set_file(fname, ++lno);
+        if (line[0] == '\n') {
+            /* skip empty lines */
+            continue;
+        }
+        if (line[0] == '#') {
+            /* comment */
+            continue;
+        }
+        if (sscanf(line, "[%63s \"%63[^\"]\"]", group, id) == 2) {
+            /* group with id */
+            list = find_list(lists, group, &local_err);
+            if (error_is_set(&local_err)) {
+                error_report("%s", error_get_pretty(local_err));
+                error_free(local_err);
+                goto out;
+            }
+            opts = qemu_opts_create(list, id, 1, NULL);
+            continue;
+        }
+        if (sscanf(line, "[%63[^]]]", group) == 1) {
+            /* group without id */
+            list = find_list(lists, group, &local_err);
+            if (error_is_set(&local_err)) {
+                error_report("%s", error_get_pretty(local_err));
+                error_free(local_err);
+                goto out;
+            }
+            opts = qemu_opts_create_nofail(list);
+            continue;
+        }
+        if (sscanf(line, " %63s = \"%1023[^\"]\"", arg, value) == 2) {
+            /* arg = value */
+            if (opts == NULL) {
+                error_report("no group defined");
+                goto out;
+            }
+            if (qemu_opt_set(opts, arg, value) != 0) {
+                goto out;
+            }
+            continue;
+        }
+        error_report("parse error");
+        goto out;
+    }
+    if (ferror(fp)) {
+        error_report("error reading file");
+        goto out;
+    }
+    res = 0;
+out:
+    loc_pop(&loc);
+    return res;
+}
+
+int qemu_read_config_file(const char *filename)
+{
+    FILE *f = fopen(filename, "r");
+    int ret;
+
+    if (f == NULL) {
+        return -errno;
+    }
+
+    ret = qemu_config_parse(f, vm_config_groups, filename);
+    fclose(f);
+
+    if (ret == 0) {
+        return 0;
+    } else {
+        return -EINVAL;
+    }
+}
diff --git a/qemu-error.c b/util/qemu-error.c
index 7cd5ffe..08a36f4 100644
--- a/qemu-error.c
+++ b/util/qemu-error.c
@@ -11,7 +11,7 @@
  */
 
 #include <stdio.h>
-#include "monitor.h"
+#include "monitor/monitor.h"
 
 /*
  * Print to current monitor if we have one, else to stderr.
diff --git a/qemu-option.c b/util/qemu-option.c
index 27891e7..5a1d03c 100644
--- a/qemu-option.c
+++ b/util/qemu-option.c
@@ -27,11 +27,11 @@
 #include <string.h>
 
 #include "qemu-common.h"
-#include "qemu-error.h"
-#include "qemu-objects.h"
-#include "error.h"
-#include "qerror.h"
-#include "qemu-option-internal.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/types.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/option_int.h"
 
 /*
  * Extracts the name of an option from the parameter string (p points at the
@@ -231,8 +231,10 @@ static void parse_option_size(const char *name, const char *value,
             break;
         default:
             error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
+#if 0 /* conversion from qerror_report() to error_set() broke this: */
             error_printf_unless_qmp("You may use k, M, G or T suffixes for "
                     "kilobytes, megabytes, gigabytes and terabytes.\n");
+#endif
             return;
         }
     } else {
@@ -602,26 +604,36 @@ static void qemu_opt_del(QemuOpt *opt)
     g_free(opt);
 }
 
-static void opt_set(QemuOpts *opts, const char *name, const char *value,
-                    bool prepend, Error **errp)
+static bool opts_accepts_any(const QemuOpts *opts)
+{
+    return opts->list->desc[0].name == NULL;
+}
+
+static const QemuOptDesc *find_desc_by_name(const QemuOptDesc *desc,
+                                            const char *name)
 {
-    QemuOpt *opt;
-    const QemuOptDesc *desc = opts->list->desc;
-    Error *local_err = NULL;
     int i;
 
     for (i = 0; desc[i].name != NULL; i++) {
         if (strcmp(desc[i].name, name) == 0) {
-            break;
+            return &desc[i];
         }
     }
-    if (desc[i].name == NULL) {
-        if (i == 0) {
-            /* empty list -> allow any */;
-        } else {
-            error_set(errp, QERR_INVALID_PARAMETER, name);
-            return;
-        }
+
+    return NULL;
+}
+
+static void opt_set(QemuOpts *opts, const char *name, const char *value,
+                    bool prepend, Error **errp)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc;
+    Error *local_err = NULL;
+
+    desc = find_desc_by_name(opts->list->desc, name);
+    if (!desc && !opts_accepts_any(opts)) {
+        error_set(errp, QERR_INVALID_PARAMETER, name);
+        return;
     }
 
     opt = g_malloc0(sizeof(*opt));
@@ -632,12 +644,8 @@ static void opt_set(QemuOpts *opts, const char *name, const char *value,
     } else {
         QTAILQ_INSERT_TAIL(&opts->head, opt, next);
     }
-    if (desc[i].name != NULL) {
-        opt->desc = desc+i;
-    }
-    if (value) {
-        opt->str = g_strdup(value);
-    }
+    opt->desc = desc;
+    opt->str = g_strdup(value);
     qemu_opt_parse(opt, &local_err);
     if (error_is_set(&local_err)) {
         error_propagate(errp, local_err);
@@ -669,30 +677,43 @@ int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val)
 {
     QemuOpt *opt;
     const QemuOptDesc *desc = opts->list->desc;
-    int i;
 
-    for (i = 0; desc[i].name != NULL; i++) {
-        if (strcmp(desc[i].name, name) == 0) {
-            break;
-        }
-    }
-    if (desc[i].name == NULL) {
-        if (i == 0) {
-            /* empty list -> allow any */;
-        } else {
-            qerror_report(QERR_INVALID_PARAMETER, name);
-            return -1;
-        }
+    opt = g_malloc0(sizeof(*opt));
+    opt->desc = find_desc_by_name(desc, name);
+    if (!opt->desc && !opts_accepts_any(opts)) {
+        qerror_report(QERR_INVALID_PARAMETER, name);
+        g_free(opt);
+        return -1;
     }
 
-    opt = g_malloc0(sizeof(*opt));
     opt->name = g_strdup(name);
     opt->opts = opts;
+    opt->value.boolean = !!val;
+    opt->str = g_strdup(val ? "on" : "off");
     QTAILQ_INSERT_TAIL(&opts->head, opt, next);
-    if (desc[i].name != NULL) {
-        opt->desc = desc+i;
+
+    return 0;
+}
+
+int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val)
+{
+    QemuOpt *opt;
+    const QemuOptDesc *desc = opts->list->desc;
+
+    opt = g_malloc0(sizeof(*opt));
+    opt->desc = find_desc_by_name(desc, name);
+    if (!opt->desc && !opts_accepts_any(opts)) {
+        qerror_report(QERR_INVALID_PARAMETER, name);
+        g_free(opt);
+        return -1;
     }
-    opt->value.boolean = !!val;
+
+    opt->name = g_strdup(name);
+    opt->opts = opts;
+    opt->value.uint = val;
+    opt->str = g_strdup_printf("%" PRId64, val);
+    QTAILQ_INSERT_TAIL(&opts->head, opt, next);
+
     return 0;
 }
 
@@ -752,7 +773,9 @@ QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
     if (id) {
         if (!id_wellformed(id)) {
             error_set(errp,QERR_INVALID_PARAMETER_VALUE, "id", "an identifier");
+#if 0 /* conversion from qerror_report() to error_set() broke this: */
             error_printf_unless_qmp("Identifiers consist of letters, digits, '-', '.', '_', starting with a letter.\n");
+#endif
             return NULL;
         }
         opts = qemu_opts_find(list, id);
@@ -771,9 +794,7 @@ QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
         }
     }
     opts = g_malloc0(sizeof(*opts));
-    if (id) {
-        opts->id = g_strdup(id);
-    }
+    opts->id = g_strdup(id);
     opts->list = list;
     loc_save(&opts->loc);
     QTAILQ_INIT(&opts->head);
@@ -781,6 +802,15 @@ QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
     return opts;
 }
 
+QemuOpts *qemu_opts_create_nofail(QemuOptsList *list)
+{
+    QemuOpts *opts;
+    Error *errp = NULL;
+    opts = qemu_opts_create(list, NULL, 0, &errp);
+    assert_no_error(errp);
+    return opts;
+}
+
 void qemu_opts_reset(QemuOptsList *list)
 {
     QemuOpts *opts, *next_opts;
@@ -1068,23 +1098,15 @@ void qemu_opts_validate(QemuOpts *opts, const QemuOptDesc *desc, Error **errp)
     QemuOpt *opt;
     Error *local_err = NULL;
 
-    assert(opts->list->desc[0].name == NULL);
+    assert(opts_accepts_any(opts));
 
     QTAILQ_FOREACH(opt, &opts->head, next) {
-        int i;
-
-        for (i = 0; desc[i].name != NULL; i++) {
-            if (strcmp(desc[i].name, opt->name) == 0) {
-                break;
-            }
-        }
-        if (desc[i].name == NULL) {
+        opt->desc = find_desc_by_name(desc, opt->name);
+        if (!opt->desc) {
             error_set(errp, QERR_INVALID_PARAMETER, opt->name);
             return;
         }
 
-        opt->desc = &desc[i];
-
         qemu_opt_parse(opt, &local_err);
         if (error_is_set(&local_err)) {
             error_propagate(errp, local_err);
diff --git a/qemu-progress.c b/util/qemu-progress.c
index 5f1b8df..9a3f96c 100644
--- a/qemu-progress.c
+++ b/util/qemu-progress.c
@@ -23,8 +23,8 @@
  */
 
 #include "qemu-common.h"
-#include "osdep.h"
-#include "sysemu.h"
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
 #include <stdio.h>
 
 struct progress_state {
diff --git a/qemu-sockets.c b/util/qemu-sockets.c
index d314cf1..1350ccc 100644
--- a/qemu-sockets.c
+++ b/util/qemu-sockets.c
@@ -22,10 +22,10 @@
 #include <errno.h>
 #include <unistd.h>
 
-#include "monitor.h"
-#include "qemu_socket.h"
+#include "monitor/monitor.h"
+#include "qemu/sockets.h"
 #include "qemu-common.h" /* for qemu_isdigit */
-#include "main-loop.h"
+#include "qemu/main-loop.h"
 
 #ifndef AI_ADDRCONFIG
 # define AI_ADDRCONFIG 0
@@ -579,7 +579,7 @@ int inet_listen(const char *str, char *ostr, int olen,
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+        opts = qemu_opts_create_nofail(&dummy_opts);
         inet_addr_to_opts(opts, addr);
         qapi_free_InetSocketAddress(addr);
         sock = inet_listen_opts(opts, port_offset, errp);
@@ -618,7 +618,7 @@ int inet_connect(const char *str, Error **errp)
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+        opts = qemu_opts_create_nofail(&dummy_opts);
         inet_addr_to_opts(opts, addr);
         qapi_free_InetSocketAddress(addr);
         sock = inet_connect_opts(opts, errp, NULL, NULL);
@@ -652,7 +652,7 @@ int inet_nonblocking_connect(const char *str,
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+        opts = qemu_opts_create_nofail(&dummy_opts);
         inet_addr_to_opts(opts, addr);
         qapi_free_InetSocketAddress(addr);
         sock = inet_connect_opts(opts, errp, callback, opaque);
@@ -720,7 +720,7 @@ int unix_connect_opts(QemuOpts *opts, Error **errp,
     int sock, rc;
 
     if (NULL == path) {
-        error_setg(errp, "unix connect: no path specified\n");
+        error_setg(errp, "unix connect: no path specified");
         return -1;
     }
 
@@ -795,7 +795,7 @@ int unix_listen(const char *str, char *ostr, int olen, Error **errp)
     char *path, *optstr;
     int sock, len;
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    opts = qemu_opts_create_nofail(&dummy_opts);
 
     optstr = strchr(str, ',');
     if (optstr) {
@@ -823,7 +823,7 @@ int unix_connect(const char *path, Error **errp)
     QemuOpts *opts;
     int sock;
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    opts = qemu_opts_create_nofail(&dummy_opts);
     qemu_opt_set(opts, "path", path);
     sock = unix_connect_opts(opts, errp, NULL, NULL);
     qemu_opts_del(opts);
@@ -840,7 +840,7 @@ int unix_nonblocking_connect(const char *path,
 
     g_assert(callback != NULL);
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    opts = qemu_opts_create_nofail(&dummy_opts);
     qemu_opt_set(opts, "path", path);
     sock = unix_connect_opts(opts, errp, callback, opaque);
     qemu_opts_del(opts);
@@ -854,7 +854,7 @@ SocketAddress *socket_parse(const char *str, Error **errp)
     addr = g_new(SocketAddress, 1);
     if (strstart(str, "unix:", NULL)) {
         if (str[5] == '\0') {
-            error_setg(errp, "invalid Unix socket address\n");
+            error_setg(errp, "invalid Unix socket address");
             goto fail;
         } else {
             addr->kind = SOCKET_ADDRESS_KIND_UNIX;
@@ -863,7 +863,7 @@ SocketAddress *socket_parse(const char *str, Error **errp)
         }
     } else if (strstart(str, "fd:", NULL)) {
         if (str[3] == '\0') {
-            error_setg(errp, "invalid file descriptor address\n");
+            error_setg(errp, "invalid file descriptor address");
             goto fail;
         } else {
             addr->kind = SOCKET_ADDRESS_KIND_FD;
@@ -891,7 +891,7 @@ int socket_connect(SocketAddress *addr, Error **errp,
     QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    opts = qemu_opts_create_nofail(&dummy_opts);
     switch (addr->kind) {
     case SOCKET_ADDRESS_KIND_INET:
         inet_addr_to_opts(opts, addr->inet);
@@ -922,7 +922,7 @@ int socket_listen(SocketAddress *addr, Error **errp)
     QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    opts = qemu_opts_create_nofail(&dummy_opts);
     switch (addr->kind) {
     case SOCKET_ADDRESS_KIND_INET:
         inet_addr_to_opts(opts, addr->inet);
diff --git a/qemu-thread-posix.c b/util/qemu-thread-posix.c
index 4ef9c7b..4489abf 100644
--- a/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -20,7 +20,7 @@
 #include <limits.h>
 #include <unistd.h>
 #include <sys/time.h>
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 
 static void error_exit(int err, const char *msg)
 {
@@ -122,7 +122,7 @@ void qemu_sem_init(QemuSemaphore *sem, int init)
 {
     int rc;
 
-#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+#if defined(__APPLE__) || defined(__NetBSD__)
     rc = pthread_mutex_init(&sem->lock, NULL);
     if (rc != 0) {
         error_exit(rc, __func__);
@@ -147,7 +147,7 @@ void qemu_sem_destroy(QemuSemaphore *sem)
 {
     int rc;
 
-#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+#if defined(__APPLE__) || defined(__NetBSD__)
     rc = pthread_cond_destroy(&sem->cond);
     if (rc < 0) {
         error_exit(rc, __func__);
@@ -168,7 +168,7 @@ void qemu_sem_post(QemuSemaphore *sem)
 {
     int rc;
 
-#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+#if defined(__APPLE__) || defined(__NetBSD__)
     pthread_mutex_lock(&sem->lock);
     if (sem->count == INT_MAX) {
         rc = EINVAL;
@@ -206,13 +206,14 @@ int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
     int rc;
     struct timespec ts;
 
-#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+#if defined(__APPLE__) || defined(__NetBSD__)
     compute_abs_deadline(&ts, ms);
     pthread_mutex_lock(&sem->lock);
     --sem->count;
     while (sem->count < 0) {
         rc = pthread_cond_timedwait(&sem->cond, &sem->lock, &ts);
         if (rc == ETIMEDOUT) {
+            ++sem->count;
             break;
         }
         if (rc != 0) {
@@ -248,7 +249,7 @@ int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
 
 void qemu_sem_wait(QemuSemaphore *sem)
 {
-#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+#if defined(__APPLE__) || defined(__NetBSD__)
     pthread_mutex_lock(&sem->lock);
     --sem->count;
     while (sem->count < 0) {
diff --git a/qemu-thread-win32.c b/util/qemu-thread-win32.c
index 4b3db60..517878d 100644
--- a/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -11,7 +11,7 @@
  *
  */
 #include "qemu-common.h"
-#include "qemu-thread.h"
+#include "qemu/thread.h"
 #include <process.h>
 #include <assert.h>
 #include <limits.h>
@@ -239,7 +239,7 @@ struct QemuThreadData {
     CRITICAL_SECTION  cs;
 };
 
-static int qemu_thread_tls_index = TLS_OUT_OF_INDEXES;
+static __thread QemuThreadData *qemu_thread_data;
 
 static unsigned __stdcall win32_start_routine(void *arg)
 {
@@ -251,14 +251,15 @@ static unsigned __stdcall win32_start_routine(void *arg)
         g_free(data);
         data = NULL;
     }
-    TlsSetValue(qemu_thread_tls_index, data);
+    qemu_thread_data = data;
     qemu_thread_exit(start_routine(thread_arg));
     abort();
 }
 
 void qemu_thread_exit(void *arg)
 {
-    QemuThreadData *data = TlsGetValue(qemu_thread_tls_index);
+    QemuThreadData *data = qemu_thread_data;
+
     if (data) {
         assert(data->mode != QEMU_THREAD_DETACHED);
         data->ret = arg;
@@ -298,25 +299,13 @@ void *qemu_thread_join(QemuThread *thread)
     return ret;
 }
 
-static inline void qemu_thread_init(void)
-{
-    if (qemu_thread_tls_index == TLS_OUT_OF_INDEXES) {
-        qemu_thread_tls_index = TlsAlloc();
-        if (qemu_thread_tls_index == TLS_OUT_OF_INDEXES) {
-            error_exit(ERROR_NO_SYSTEM_RESOURCES, __func__);
-        }
-    }
-}
-
-
 void qemu_thread_create(QemuThread *thread,
                        void *(*start_routine)(void *),
                        void *arg, int mode)
 {
     HANDLE hThread;
-
     struct QemuThreadData *data;
-    qemu_thread_init();
+
     data = g_malloc(sizeof *data);
     data->start_routine = start_routine;
     data->arg = arg;
@@ -338,8 +327,7 @@ void qemu_thread_create(QemuThread *thread,
 
 void qemu_thread_get_self(QemuThread *thread)
 {
-    qemu_thread_init();
-    thread->data = TlsGetValue(qemu_thread_tls_index);
+    thread->data = qemu_thread_data;
     thread->tid = GetCurrentThreadId();
 }
 
diff --git a/qemu-timer-common.c b/util/qemu-timer-common.c
index 755e300..16f5e75 100644
--- a/qemu-timer-common.c
+++ b/util/qemu-timer-common.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-timer.h"
+#include "qemu/timer.h"
 
 /***********************************************************/
 /* real time host monotonic timer */
diff --git a/uri.c b/util/uri.c
index dd922de..4238729 100644
--- a/uri.c
+++ b/util/uri.c
@@ -55,7 +55,7 @@
 #include <string.h>
 #include <stdio.h>
 
-#include "uri.h"
+#include "qemu/uri.h"
 
 static void uri_clean(URI *uri);
 
@@ -432,7 +432,7 @@ rfc3986_parse_host(URI *uri, const char **str)
 
     host = cur;
     /*
-     * IPv6 and future adressing scheme are enclosed between brackets
+     * IPv6 and future addressing scheme are enclosed between brackets
      */
     if (*cur == '[') {
         cur++;
@@ -1917,7 +1917,7 @@ done:
  *     http://site1.com/docs/pic1.gif   http://site1.com/docs/pic1.gif
  *
  *
- * Note: if the URI reference is really wierd or complicated, it may be
+ * Note: if the URI reference is really weird or complicated, it may be
  *       worthwhile to first convert it into a "nice" one by calling
  *       uri_resolve (using 'base') before calling this routine,
  *       since this routine (for reasonable efficiency) assumes URI has
diff --git a/vl.c b/vl.c
index a3ab384..1355f69 100644
--- a/vl.c
+++ b/vl.c
@@ -28,7 +28,7 @@
 #include <errno.h>
 #include <sys/time.h>
 #include <zlib.h>
-#include "bitmap.h"
+#include "qemu/bitmap.h"
 
 /* Needed early for CONFIG_BSD etc. */
 #include "config-host.h"
@@ -65,7 +65,7 @@
 #endif
 
 #ifdef CONFIG_SECCOMP
-#include "qemu-seccomp.h"
+#include "sysemu/seccomp.h"
 #endif
 
 #ifdef __sun__
@@ -126,46 +126,46 @@ int main(int argc, char **argv)
 #include "hw/xen.h"
 #include "hw/qdev.h"
 #include "hw/loader.h"
-#include "bt-host.h"
-#include "net.h"
+#include "bt/bt.h"
+#include "net/net.h"
 #include "net/slirp.h"
-#include "monitor.h"
-#include "console.h"
-#include "sysemu.h"
-#include "gdbstub.h"
-#include "qemu-timer.h"
-#include "qemu-char.h"
-#include "cache-utils.h"
-#include "blockdev.h"
+#include "monitor/monitor.h"
+#include "ui/console.h"
+#include "sysemu/sysemu.h"
+#include "exec/gdbstub.h"
+#include "qemu/timer.h"
+#include "char/char.h"
+#include "qemu/cache-utils.h"
+#include "sysemu/blockdev.h"
 #include "hw/block-common.h"
-#include "block-migration.h"
-#include "dma.h"
+#include "migration/block.h"
+#include "sysemu/dma.h"
 #include "audio/audio.h"
-#include "migration.h"
-#include "kvm.h"
-#include "qjson.h"
-#include "qemu-option.h"
-#include "qemu-config.h"
+#include "migration/migration.h"
+#include "sysemu/kvm.h"
+#include "qapi/qmp/qjson.h"
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 #include "qemu-options.h"
 #include "qmp-commands.h"
-#include "main-loop.h"
+#include "qemu/main-loop.h"
 #ifdef CONFIG_VIRTFS
 #include "fsdev/qemu-fsdev.h"
 #endif
-#include "qtest.h"
+#include "sysemu/qtest.h"
 
-#include "disas.h"
+#include "disas/disas.h"
 
-#include "qemu_socket.h"
+#include "qemu/sockets.h"
 
 #include "slirp/libslirp.h"
 
 #include "trace.h"
 #include "trace/control.h"
-#include "qemu-queue.h"
-#include "cpus.h"
-#include "arch_init.h"
-#include "osdep.h"
+#include "qemu/queue.h"
+#include "sysemu/cpus.h"
+#include "sysemu/arch_init.h"
+#include "qemu/osdep.h"
 
 #include "ui/qemu-spice.h"
 #include "qapi/string-input-visitor.h"
@@ -176,6 +176,7 @@ int main(int argc, char **argv)
 #define DEFAULT_RAM_SIZE 128
 
 #define MAX_VIRTIO_CONSOLES 1
+#define MAX_SCLP_CONSOLES 1
 
 static const char *data_dir;
 const char *bios_name = NULL;
@@ -203,6 +204,7 @@ int no_quit = 0;
 CharDriverState *serial_hds[MAX_SERIAL_PORTS];
 CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 CharDriverState *virtcon_hds[MAX_VIRTIO_CONSOLES];
+CharDriverState *sclp_hds[MAX_SCLP_CONSOLES];
 int win2k_install_hack = 0;
 int singlestep = 0;
 int smp_cpus = 1;
@@ -231,7 +233,7 @@ unsigned int nb_prom_envs = 0;
 const char *prom_envs[MAX_PROM_ENVS];
 int boot_menu;
 uint8_t *boot_splash_filedata;
-int boot_splash_filedata_size;
+size_t boot_splash_filedata_size;
 uint8_t qemu_extra_params_fw[2];
 
 typedef struct FWBootEntry FWBootEntry;
@@ -261,9 +263,9 @@ static NotifierList exit_notifiers =
 static NotifierList machine_init_done_notifiers =
     NOTIFIER_LIST_INITIALIZER(machine_init_done_notifiers);
 
-static int tcg_allowed = 1;
-int kvm_allowed = 0;
-int xen_allowed = 0;
+static bool tcg_allowed = true;
+bool kvm_allowed;
+bool xen_allowed;
 uint32_t xen_domid;
 enum xen_mode xen_mode = XEN_EMULATE;
 static int tcg_tb_size;
@@ -271,6 +273,7 @@ static int tcg_tb_size;
 static int default_serial = 1;
 static int default_parallel = 1;
 static int default_virtcon = 1;
+static int default_sclp = 1;
 static int default_monitor = 1;
 static int default_floppy = 1;
 static int default_cdrom = 1;
@@ -299,6 +302,195 @@ static struct {
     { .driver = "qxl-vga",              .flag = &default_vga       },
 };
 
+static QemuOptsList qemu_rtc_opts = {
+    .name = "rtc",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_rtc_opts.head),
+    .desc = {
+        {
+            .name = "base",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "clock",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "driftfix",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_sandbox_opts = {
+    .name = "sandbox",
+    .implied_opt_name = "enable",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_sandbox_opts.head),
+    .desc = {
+        {
+            .name = "enable",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_trace_opts = {
+    .name = "trace",
+    .implied_opt_name = "trace",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
+    .desc = {
+        {
+            .name = "events",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "file",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_option_rom_opts = {
+    .name = "option-rom",
+    .implied_opt_name = "romfile",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_option_rom_opts.head),
+    .desc = {
+        {
+            .name = "bootindex",
+            .type = QEMU_OPT_NUMBER,
+        }, {
+            .name = "romfile",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_machine_opts = {
+    .name = "machine",
+    .implied_opt_name = "type",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_machine_opts.head),
+    .desc = {
+        {
+            .name = "type",
+            .type = QEMU_OPT_STRING,
+            .help = "emulated machine"
+        }, {
+            .name = "accel",
+            .type = QEMU_OPT_STRING,
+            .help = "accelerator list",
+        }, {
+            .name = "kernel_irqchip",
+            .type = QEMU_OPT_BOOL,
+            .help = "use KVM in-kernel irqchip",
+        }, {
+            .name = "kvm_shadow_mem",
+            .type = QEMU_OPT_SIZE,
+            .help = "KVM shadow MMU size",
+        }, {
+            .name = "kernel",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel image file",
+        }, {
+            .name = "initrd",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux initial ramdisk file",
+        }, {
+            .name = "append",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel command line",
+        }, {
+            .name = "dtb",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel device tree file",
+        }, {
+            .name = "dumpdtb",
+            .type = QEMU_OPT_STRING,
+            .help = "Dump current dtb to a file and quit",
+        }, {
+            .name = "phandle_start",
+            .type = QEMU_OPT_STRING,
+            .help = "The first phandle ID we may generate dynamically",
+        }, {
+            .name = "dt_compatible",
+            .type = QEMU_OPT_STRING,
+            .help = "Overrides the \"compatible\" property of the dt root node",
+        }, {
+            .name = "dump-guest-core",
+            .type = QEMU_OPT_BOOL,
+            .help = "Include guest memory in  a core dump",
+        }, {
+            .name = "mem-merge",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable memory merge support",
+        },{
+            .name = "usb",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on/off to enable/disable usb",
+        },
+        { /* End of list */ }
+    },
+};
+
+static QemuOptsList qemu_boot_opts = {
+    .name = "boot-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_boot_opts.head),
+    .desc = {
+        /* the three names below are not used now */
+        {
+            .name = "order",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "once",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "menu",
+            .type = QEMU_OPT_STRING,
+        /* following are really used */
+        }, {
+            .name = "splash",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "splash-time",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "reboot-timeout",
+            .type = QEMU_OPT_STRING,
+        },
+        { /*End of list */ }
+    },
+};
+
+static QemuOptsList qemu_add_fd_opts = {
+    .name = "add-fd",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_add_fd_opts.head),
+    .desc = {
+        {
+            .name = "fd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "file descriptor of which a duplicate is added to fd set",
+        },{
+            .name = "set",
+            .type = QEMU_OPT_NUMBER,
+            .help = "ID of the fd set to add fd to",
+        },{
+            .name = "opaque",
+            .type = QEMU_OPT_STRING,
+            .help = "free-form string used to describe fd",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
 const char *qemu_get_vm_name(void)
 {
     return qemu_name;
@@ -448,21 +640,18 @@ StatusInfo *qmp_query_status(Error **errp)
 void qemu_get_timedate(struct tm *tm, int offset)
 {
     time_t ti;
-    struct tm *ret;
 
     time(&ti);
     ti += offset;
     if (rtc_date_offset == -1) {
         if (rtc_utc)
-            ret = gmtime(&ti);
+            gmtime_r(&ti, tm);
         else
-            ret = localtime(&ti);
+            localtime_r(&ti, tm);
     } else {
         ti -= rtc_date_offset;
-        ret = gmtime(&ti);
+        gmtime_r(&ti, tm);
     }
-
-    memcpy(tm, ret, sizeof(struct tm));
 }
 
 int qemu_timedate_diff(struct tm *tm)
@@ -886,9 +1075,9 @@ static int cleanup_add_fd(QemuOpts *opts, void *opaque)
 
 static int drive_init_func(QemuOpts *opts, void *opaque)
 {
-    int *use_scsi = opaque;
+    BlockInterfaceType *block_default_type = opaque;
 
-    return drive_init(opts, *use_scsi) == NULL;
+    return drive_init(opts, *block_default_type) == NULL;
 }
 
 static int drive_enable_snapshot(QemuOpts *opts, void *opaque)
@@ -899,16 +1088,11 @@ static int drive_enable_snapshot(QemuOpts *opts, void *opaque)
     return 0;
 }
 
-static void default_drive(int enable, int snapshot, int use_scsi,
-                          BlockInterfaceType type, int index,
-                          const char *optstr)
+static void default_drive(int enable, int snapshot, BlockInterfaceType type,
+                          int index, const char *optstr)
 {
     QemuOpts *opts;
 
-    if (type == IF_DEFAULT) {
-        type = use_scsi ? IF_SCSI : IF_IDE;
-    }
-
     if (!enable || drive_get_by_index(type, index)) {
         return;
     }
@@ -917,7 +1101,7 @@ static void default_drive(int enable, int snapshot, int use_scsi,
     if (snapshot) {
         drive_enable_snapshot(opts, NULL);
     }
-    if (!drive_init(opts, use_scsi)) {
+    if (!drive_init(opts, type)) {
         exit(1);
     }
 }
@@ -1017,15 +1201,15 @@ void add_boot_device_path(int32_t bootindex, DeviceState *dev,
  * memory pointed by "size" is assigned total length of the array in bytes
  *
  */
-char *get_boot_devices_list(uint32_t *size)
+char *get_boot_devices_list(size_t *size)
 {
     FWBootEntry *i;
-    uint32_t total = 0;
+    size_t total = 0;
     char *list = NULL;
 
     QTAILQ_FOREACH(i, &fw_boot_order, link) {
         char *devpath = NULL, *bootpath;
-        int len;
+        size_t len;
 
         if (i->dev) {
             devpath = qdev_get_fw_dev_path(i->dev);
@@ -1060,21 +1244,79 @@ char *get_boot_devices_list(uint32_t *size)
     return list;
 }
 
-static void numa_add(const char *optarg)
+static void numa_node_parse_cpus(int nodenr, const char *cpus)
 {
-    char option[128];
     char *endptr;
     unsigned long long value, endvalue;
-    int nodenr;
 
-    value = endvalue = 0ULL;
+    /* Empty CPU range strings will be considered valid, they will simply
+     * not set any bit in the CPU bitmap.
+     */
+    if (!*cpus) {
+        return;
+    }
+
+    if (parse_uint(cpus, &value, &endptr, 10) < 0) {
+        goto error;
+    }
+    if (*endptr == '-') {
+        if (parse_uint_full(endptr + 1, &endvalue, 10) < 0) {
+            goto error;
+        }
+    } else if (*endptr == '\0') {
+        endvalue = value;
+    } else {
+        goto error;
+    }
+
+    if (endvalue >= MAX_CPUMASK_BITS) {
+        endvalue = MAX_CPUMASK_BITS - 1;
+        fprintf(stderr,
+            "qemu: NUMA: A max of %d VCPUs are supported\n",
+             MAX_CPUMASK_BITS);
+    }
+
+    if (endvalue < value) {
+        goto error;
+    }
+
+    bitmap_set(node_cpumask[nodenr], value, endvalue-value+1);
+    return;
 
-    optarg = get_opt_name(option, 128, optarg, ',') + 1;
+error:
+    fprintf(stderr, "qemu: Invalid NUMA CPU range: %s\n", cpus);
+    exit(1);
+}
+
+static void numa_add(const char *optarg)
+{
+    char option[128];
+    char *endptr;
+    unsigned long long nodenr;
+
+    optarg = get_opt_name(option, 128, optarg, ',');
+    if (*optarg == ',') {
+        optarg++;
+    }
     if (!strcmp(option, "node")) {
+
+        if (nb_numa_nodes >= MAX_NODES) {
+            fprintf(stderr, "qemu: too many NUMA nodes\n");
+            exit(1);
+        }
+
         if (get_param_value(option, 128, "nodeid", optarg) == 0) {
             nodenr = nb_numa_nodes;
         } else {
-            nodenr = strtoull(option, NULL, 10);
+            if (parse_uint_full(option, &nodenr, 10) < 0) {
+                fprintf(stderr, "qemu: Invalid NUMA nodeid: %s\n", option);
+                exit(1);
+            }
+        }
+
+        if (nodenr >= MAX_NODES) {
+            fprintf(stderr, "qemu: invalid NUMA nodeid: %llu\n", nodenr);
+            exit(1);
         }
 
         if (get_param_value(option, 128, "mem", optarg) == 0) {
@@ -1089,23 +1331,12 @@ static void numa_add(const char *optarg)
             node_mem[nodenr] = sval;
         }
         if (get_param_value(option, 128, "cpus", optarg) != 0) {
-            value = strtoull(option, &endptr, 10);
-            if (*endptr == '-') {
-                endvalue = strtoull(endptr+1, &endptr, 10);
-            } else {
-                endvalue = value;
-            }
-
-            if (!(endvalue < MAX_CPUMASK_BITS)) {
-                endvalue = MAX_CPUMASK_BITS - 1;
-                fprintf(stderr,
-                    "A max of %d CPUs are supported in a guest\n",
-                     MAX_CPUMASK_BITS);
-            }
-
-            bitmap_set(node_cpumask[nodenr], value, endvalue-value+1);
+            numa_node_parse_cpus(nodenr, option);
         }
         nb_numa_nodes++;
+    } else {
+        fprintf(stderr, "Invalid -numa option: %s\n", option);
+        exit(1);
     }
 }
 
@@ -1268,7 +1499,7 @@ void pcmcia_socket_unregister(PCMCIASocket *socket)
         }
 }
 
-void pcmcia_info(Monitor *mon)
+void pcmcia_info(Monitor *mon, const QDict *qdict)
 {
     struct pcmcia_socket_entry_s *iter;
 
@@ -2001,7 +2232,7 @@ static int balloon_parse(const char *arg)
                 return  -1;
         } else {
             /* create empty opts */
-            opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0, NULL);
+            opts = qemu_opts_create_nofail(qemu_find_opts("device"));
         }
         qemu_opt_set(opts, "driver", "virtio-balloon");
         return 0;
@@ -2052,16 +2283,20 @@ static int device_init_func(QemuOpts *opts, void *opaque)
     dev = qdev_device_add(opts);
     if (!dev)
         return -1;
+    object_unref(OBJECT(dev));
     return 0;
 }
 
 static int chardev_init_func(QemuOpts *opts, void *opaque)
 {
-    CharDriverState *chr;
+    Error *local_err = NULL;
 
-    chr = qemu_chr_new_from_opts(opts, NULL);
-    if (!chr)
+    qemu_chr_new_from_opts(opts, NULL, &local_err);
+    if (error_is_set(&local_err)) {
+        fprintf(stderr, "%s\n", error_get_pretty(local_err));
+        error_free(local_err);
         return -1;
+    }
     return 0;
 }
 
@@ -2156,6 +2391,7 @@ struct device_config {
         DEV_VIRTCON,   /* -virtioconsole */
         DEV_DEBUGCON,  /* -debugcon */
         DEV_GDB,       /* -gdb, -s */
+        DEV_SCLP,      /* s390 sclp */
     } type;
     const char *cmdline;
     Location loc;
@@ -2251,14 +2487,14 @@ static int virtcon_parse(const char *devname)
         exit(1);
     }
 
-    bus_opts = qemu_opts_create(device, NULL, 0, NULL);
+    bus_opts = qemu_opts_create_nofail(device);
     if (arch_type == QEMU_ARCH_S390X) {
         qemu_opt_set(bus_opts, "driver", "virtio-serial-s390");
     } else {
         qemu_opt_set(bus_opts, "driver", "virtio-serial-pci");
     } 
 
-    dev_opts = qemu_opts_create(device, NULL, 0, NULL);
+    dev_opts = qemu_opts_create_nofail(device);
     qemu_opt_set(dev_opts, "driver", "virtconsole");
 
     snprintf(label, sizeof(label), "virtcon%d", index);
@@ -2274,6 +2510,39 @@ static int virtcon_parse(const char *devname)
     return 0;
 }
 
+static int sclp_parse(const char *devname)
+{
+    QemuOptsList *device = qemu_find_opts("device");
+    static int index = 0;
+    char label[32];
+    QemuOpts *dev_opts;
+
+    if (strcmp(devname, "none") == 0) {
+        return 0;
+    }
+    if (index == MAX_SCLP_CONSOLES) {
+        fprintf(stderr, "qemu: too many sclp consoles\n");
+        exit(1);
+    }
+
+    assert(arch_type == QEMU_ARCH_S390X);
+
+    dev_opts = qemu_opts_create(device, NULL, 0, NULL);
+    qemu_opt_set(dev_opts, "driver", "sclpconsole");
+
+    snprintf(label, sizeof(label), "sclpcon%d", index);
+    sclp_hds[index] = qemu_chr_new(label, devname, NULL);
+    if (!sclp_hds[index]) {
+        fprintf(stderr, "qemu: could not connect sclp console"
+                " to character backend '%s'\n", devname);
+        return -1;
+    }
+    qemu_opt_set(dev_opts, "chardev", label);
+
+    index++;
+    return 0;
+}
+
 static int debugcon_parse(const char *devname)
 {   
     QemuOpts *opts;
@@ -2323,7 +2592,7 @@ static struct {
     const char *name;
     int (*available)(void);
     int (*init)(void);
-    int *allowed;
+    bool *allowed;
 } accel_list[] = {
     { "tcg", "tcg", tcg_available, tcg_init, &tcg_allowed },
     { "xen", "Xen", xen_available, xen_init, &xen_allowed },
@@ -2336,8 +2605,8 @@ static int configure_accelerator(void)
     const char *p = NULL;
     char buf[10];
     int i, ret;
-    bool accel_initialised = 0;
-    bool init_failed = 0;
+    bool accel_initialised = false;
+    bool init_failed = false;
 
     QemuOptsList *list = qemu_find_opts("machine");
     if (!QTAILQ_EMPTY(&list->head)) {
@@ -2356,21 +2625,21 @@ static int configure_accelerator(void)
         p = get_opt_name(buf, sizeof (buf), p, ':');
         for (i = 0; i < ARRAY_SIZE(accel_list); i++) {
             if (strcmp(accel_list[i].opt_name, buf) == 0) {
-                *(accel_list[i].allowed) = 1;
+                if (!accel_list[i].available()) {
+                    printf("%s not supported for this target\n",
+                           accel_list[i].name);
+                    continue;
+                }
+                *(accel_list[i].allowed) = true;
                 ret = accel_list[i].init();
                 if (ret < 0) {
-                    init_failed = 1;
-                    if (!accel_list[i].available()) {
-                        printf("%s not supported for this target\n",
-                               accel_list[i].name);
-                    } else {
-                        fprintf(stderr, "failed to initialize %s: %s\n",
-                                accel_list[i].name,
-                                strerror(-ret));
-                    }
-                    *(accel_list[i].allowed) = 0;
+                    init_failed = true;
+                    fprintf(stderr, "failed to initialize %s: %s\n",
+                            accel_list[i].name,
+                            strerror(-ret));
+                    *(accel_list[i].allowed) = false;
                 } else {
-                    accel_initialised = 1;
+                    accel_initialised = true;
                 }
                 break;
             }
@@ -2381,7 +2650,9 @@ static int configure_accelerator(void)
     }
 
     if (!accel_initialised) {
-        fprintf(stderr, "No accelerator found!\n");
+        if (!init_failed) {
+            fprintf(stderr, "No accelerator found!\n");
+        }
         exit(1);
     }
 
@@ -2531,7 +2802,7 @@ int main(int argc, char **argv, char **envp)
     const char *icount_option = NULL;
     const char *initrd_filename;
     const char *kernel_filename, *kernel_cmdline;
-    char boot_devices[33] = "cad"; /* default to HD->floppy->CD-ROM */
+    char boot_devices[33] = "";
     DisplayState *ds;
     int cyls, heads, secs, translation;
     QemuOpts *hda_opts = NULL, *opts, *machine_opts;
@@ -2574,6 +2845,22 @@ int main(int argc, char **argv, char **envp)
 
     module_call_init(MODULE_INIT_QOM);
 
+    qemu_add_opts(&qemu_drive_opts);
+    qemu_add_opts(&qemu_chardev_opts);
+    qemu_add_opts(&qemu_device_opts);
+    qemu_add_opts(&qemu_netdev_opts);
+    qemu_add_opts(&qemu_net_opts);
+    qemu_add_opts(&qemu_rtc_opts);
+    qemu_add_opts(&qemu_global_opts);
+    qemu_add_opts(&qemu_mon_opts);
+    qemu_add_opts(&qemu_trace_opts);
+    qemu_add_opts(&qemu_option_rom_opts);
+    qemu_add_opts(&qemu_machine_opts);
+    qemu_add_opts(&qemu_boot_opts);
+    qemu_add_opts(&qemu_sandbox_opts);
+    qemu_add_opts(&qemu_add_fd_opts);
+    qemu_add_opts(&qemu_object_opts);
+
     runstate_init();
 
     init_clocks();
@@ -2757,10 +3044,6 @@ int main(int argc, char **argv, char **envp)
                 }
                 break;
             case QEMU_OPTION_numa:
-                if (nb_numa_nodes >= MAX_NODES) {
-                    fprintf(stderr, "qemu: too many NUMA nodes\n");
-                    exit(1);
-                }
                 numa_add(optarg);
                 break;
             case QEMU_OPTION_display:
@@ -2852,8 +3135,10 @@ int main(int argc, char **argv, char **envp)
                                 exit(1);
                             }
                         }
-                        qemu_opts_parse(qemu_find_opts("boot-opts"),
-                                        optarg, 0);
+                        if (!qemu_opts_parse(qemu_find_opts("boot-opts"),
+                                             optarg, 0)) {
+                            exit(1);
+                        }
                     }
                 }
                 break;
@@ -3051,7 +3336,6 @@ int main(int argc, char **argv, char **envp)
                 }
                 opts = qemu_opts_parse(olist, optarg, 1);
                 if (!opts) {
-                    fprintf(stderr, "parse error: %s\n", optarg);
                     exit(1);
                 }
                 break;
@@ -3067,7 +3351,6 @@ int main(int argc, char **argv, char **envp)
                 }
                 opts = qemu_opts_parse(olist, optarg, 1);
                 if (!opts) {
-                    fprintf(stderr, "parse error: %s\n", optarg);
                     exit(1);
                 }
 
@@ -3110,8 +3393,7 @@ int main(int argc, char **argv, char **envp)
 
                 qemu_opt_set_bool(fsdev, "readonly",
                                 qemu_opt_get_bool(opts, "readonly", 0));
-                device = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
-                                          NULL);
+                device = qemu_opts_create_nofail(qemu_find_opts("device"));
                 qemu_opt_set(device, "driver", "virtio-9p-pci");
                 qemu_opt_set(device, "fsdev",
                              qemu_opt_get(opts, "mount_tag"));
@@ -3131,8 +3413,7 @@ int main(int argc, char **argv, char **envp)
                 }
                 qemu_opt_set(fsdev, "fsdriver", "synth");
 
-                device = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
-                                          NULL);
+                device = qemu_opts_create_nofail(qemu_find_opts("device"));
                 qemu_opt_set(device, "driver", "virtio-9p-pci");
                 qemu_opt_set(device, "fsdev", "v_synth");
                 qemu_opt_set(device, "mount_tag", "v_synth");
@@ -3240,7 +3521,6 @@ int main(int argc, char **argv, char **envp)
                 olist = qemu_find_opts("machine");
                 opts = qemu_opts_parse(olist, optarg, 1);
                 if (!opts) {
-                    fprintf(stderr, "parse error: %s\n", optarg);
                     exit(1);
                 }
                 optarg = qemu_opt_get(opts, "type");
@@ -3345,6 +3625,9 @@ int main(int argc, char **argv, char **envp)
 		    exit(1);
 		}
                 opts = qemu_opts_parse(qemu_find_opts("option-rom"), optarg, 1);
+                if (!opts) {
+                    exit(1);
+                }
                 option_rom[nb_option_roms].name = qemu_opt_get(opts, "romfile");
                 option_rom[nb_option_roms].bootindex =
                     qemu_opt_get_number(opts, "bootindex", -1);
@@ -3417,6 +3700,7 @@ int main(int argc, char **argv, char **envp)
                 default_serial = 0;
                 default_parallel = 0;
                 default_virtcon = 0;
+                default_sclp = 0;
                 default_monitor = 0;
                 default_net = 0;
                 default_floppy = 0;
@@ -3473,7 +3757,6 @@ int main(int argc, char **argv, char **envp)
                 }
                 opts = qemu_opts_parse(olist, optarg, 0);
                 if (!opts) {
-                    fprintf(stderr, "parse error: %s\n", optarg);
                     exit(1);
                 }
                 break;
@@ -3502,14 +3785,14 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_sandbox:
                 opts = qemu_opts_parse(qemu_find_opts("sandbox"), optarg, 1);
                 if (!opts) {
-                    exit(0);
+                    exit(1);
                 }
                 break;
             case QEMU_OPTION_add_fd:
 #ifndef _WIN32
                 opts = qemu_opts_parse(qemu_find_opts("add-fd"), optarg, 0);
                 if (!opts) {
-                    exit(0);
+                    exit(1);
                 }
 #else
                 error_report("File descriptor passing is disabled on this "
@@ -3519,6 +3802,9 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_object:
                 opts = qemu_opts_parse(qemu_find_opts("object"), optarg, 1);
+                if (!opts) {
+                    exit(1);
+                }
                 break;
             default:
                 os_parse_cmd_args(popt->index, optarg);
@@ -3634,6 +3920,9 @@ int main(int argc, char **argv, char **envp)
     if (!machine->use_virtcon) {
         default_virtcon = 0;
     }
+    if (!machine->use_sclp) {
+        default_sclp = 0;
+    }
     if (machine->no_floppy) {
         default_floppy = 0;
     }
@@ -3644,6 +3933,30 @@ int main(int argc, char **argv, char **envp)
         default_sdcard = 0;
     }
 
+    if (is_daemonized()) {
+        /* According to documentation and historically, -nographic redirects
+         * serial port, parallel port and monitor to stdio, which does not work
+         * with -daemonize.  We can redirect these to null instead, but since
+         * -nographic is legacy, let's just error out.
+         * We disallow -nographic only if all other ports are not redirected
+         * explicitly, to not break existing legacy setups which uses
+         * -nographic _and_ redirects all ports explicitly - this is valid
+         * usage, -nographic is just a no-op in this case.
+         */
+        if (display_type == DT_NOGRAPHIC
+            && (default_parallel || default_serial
+                || default_monitor || default_virtcon)) {
+            fprintf(stderr, "-nographic can not be used with -daemonize\n");
+            exit(1);
+        }
+#ifdef CONFIG_CURSES
+        if (display_type == DT_CURSES) {
+            fprintf(stderr, "curses display can not be used with -daemonize\n");
+            exit(1);
+        }
+#endif
+    }
+
     if (display_type == DT_NOGRAPHIC) {
         if (default_parallel)
             add_device_config(DEV_PARALLEL, "null");
@@ -3651,11 +3964,16 @@ int main(int argc, char **argv, char **envp)
             add_device_config(DEV_SERIAL, "mon:stdio");
         } else if (default_virtcon && default_monitor) {
             add_device_config(DEV_VIRTCON, "mon:stdio");
+        } else if (default_sclp && default_monitor) {
+            add_device_config(DEV_SCLP, "mon:stdio");
         } else {
             if (default_serial)
                 add_device_config(DEV_SERIAL, "stdio");
             if (default_virtcon)
                 add_device_config(DEV_VIRTCON, "stdio");
+            if (default_sclp) {
+                add_device_config(DEV_SCLP, "stdio");
+            }
             if (default_monitor)
                 monitor_parse("stdio", "readline");
         }
@@ -3668,6 +3986,9 @@ int main(int argc, char **argv, char **envp)
             monitor_parse("vc:80Cx24C", "readline");
         if (default_virtcon)
             add_device_config(DEV_VIRTCON, "vc:80Cx24C");
+        if (default_sclp) {
+            add_device_config(DEV_SCLP, "vc:80Cx24C");
+        }
     }
 
     socket_init();
@@ -3745,6 +4066,9 @@ int main(int argc, char **argv, char **envp)
     }
     configure_icount(icount_option);
 
+    /* clean up network at qemu process termination */
+    atexit(&net_cleanup);
+
     if (net_init_clients() < 0) {
         exit(1);
     }
@@ -3770,15 +4094,15 @@ int main(int argc, char **argv, char **envp)
     /* open the virtual block devices */
     if (snapshot)
         qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot, NULL, 0);
-    if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func, &machine->use_scsi, 1) != 0)
+    if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func,
+                          &machine->block_default_type, 1) != 0) {
         exit(1);
+    }
 
-    default_drive(default_cdrom, snapshot, machine->use_scsi,
-                  IF_DEFAULT, 2, CDROM_OPTS);
-    default_drive(default_floppy, snapshot, machine->use_scsi,
-                  IF_FLOPPY, 0, FD_OPTS);
-    default_drive(default_sdcard, snapshot, machine->use_scsi,
-                  IF_SD, 0, SD_OPTS);
+    default_drive(default_cdrom, snapshot, machine->block_default_type, 2,
+                  CDROM_OPTS);
+    default_drive(default_floppy, snapshot, IF_FLOPPY, 0, FD_OPTS);
+    default_drive(default_sdcard, snapshot, IF_SD, 0, SD_OPTS);
 
     register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
 
@@ -3835,6 +4159,9 @@ int main(int argc, char **argv, char **envp)
         exit(1);
     if (foreach_device_config(DEV_VIRTCON, virtcon_parse) < 0)
         exit(1);
+    if (foreach_device_config(DEV_SCLP, sclp_parse) < 0) {
+        exit(1);
+    }
     if (foreach_device_config(DEV_DEBUGCON, debugcon_parse) < 0)
         exit(1);
 
@@ -3862,7 +4189,9 @@ int main(int argc, char **argv, char **envp)
     qdev_machine_init();
 
     QEMUMachineInitArgs args = { .ram_size = ram_size,
-                                 .boot_device = boot_devices,
+                                 .boot_device = (boot_devices[0] == '\0') ?
+                                                machine->boot_order :
+                                                boot_devices,
                                  .kernel_filename = kernel_filename,
                                  .kernel_cmdline = kernel_cmdline,
                                  .initrd_filename = initrd_filename,
@@ -3910,9 +4239,7 @@ int main(int argc, char **argv, char **envp)
         break;
 #if defined(CONFIG_CURSES)
     case DT_CURSES:
-        if (!is_daemonized()) {
-            curses_display_init(ds, full_screen);
-        }
+        curses_display_init(ds, full_screen);
         break;
 #endif
 #if defined(CONFIG_SDL)
@@ -3999,7 +4326,6 @@ int main(int argc, char **argv, char **envp)
     main_loop();
     bdrv_close_all();
     pause_all_vcpus();
-    net_cleanup();
     res_free();
 
     return 0;
diff --git a/xbzrle.c b/xbzrle.c
new file mode 100644
index 0000000..fbcb35d
--- /dev/null
+++ b/xbzrle.c
@@ -0,0 +1,173 @@
+/*
+ * Xor Based Zero Run Length Encoding
+ *
+ * Copyright 2013 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Orit Wasserman  <owasserm@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include "qemu-common.h"
+#include "include/migration/migration.h"
+
+/*
+  page = zrun nzrun
+       | zrun nzrun page
+
+  zrun = length
+
+  nzrun = length byte...
+
+  length = uleb128 encoded integer
+ */
+int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
+                         uint8_t *dst, int dlen)
+{
+    uint32_t zrun_len = 0, nzrun_len = 0;
+    int d = 0, i = 0;
+    long res, xor;
+    uint8_t *nzrun_start = NULL;
+
+    g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) %
+               sizeof(long)));
+
+    while (i < slen) {
+        /* overflow */
+        if (d + 2 > dlen) {
+            return -1;
+        }
+
+        /* not aligned to sizeof(long) */
+        res = (slen - i) % sizeof(long);
+        while (res && old_buf[i] == new_buf[i]) {
+            zrun_len++;
+            i++;
+            res--;
+        }
+
+        /* word at a time for speed */
+        if (!res) {
+            while (i < slen &&
+                   (*(long *)(old_buf + i)) == (*(long *)(new_buf + i))) {
+                i += sizeof(long);
+                zrun_len += sizeof(long);
+            }
+
+            /* go over the rest */
+            while (i < slen && old_buf[i] == new_buf[i]) {
+                zrun_len++;
+                i++;
+            }
+        }
+
+        /* buffer unchanged */
+        if (zrun_len == slen) {
+            return 0;
+        }
+
+        /* skip last zero run */
+        if (i == slen) {
+            return d;
+        }
+
+        d += uleb128_encode_small(dst + d, zrun_len);
+
+        zrun_len = 0;
+        nzrun_start = new_buf + i;
+
+        /* overflow */
+        if (d + 2 > dlen) {
+            return -1;
+        }
+        /* not aligned to sizeof(long) */
+        res = (slen - i) % sizeof(long);
+        while (res && old_buf[i] != new_buf[i]) {
+            i++;
+            nzrun_len++;
+            res--;
+        }
+
+        /* word at a time for speed, use of 32-bit long okay */
+        if (!res) {
+            /* truncation to 32-bit long okay */
+            long mask = (long)0x0101010101010101ULL;
+            while (i < slen) {
+                xor = *(long *)(old_buf + i) ^ *(long *)(new_buf + i);
+                if ((xor - mask) & ~xor & (mask << 7)) {
+                    /* found the end of an nzrun within the current long */
+                    while (old_buf[i] != new_buf[i]) {
+                        nzrun_len++;
+                        i++;
+                    }
+                    break;
+                } else {
+                    i += sizeof(long);
+                    nzrun_len += sizeof(long);
+                }
+            }
+        }
+
+        d += uleb128_encode_small(dst + d, nzrun_len);
+        /* overflow */
+        if (d + nzrun_len > dlen) {
+            return -1;
+        }
+        memcpy(dst + d, nzrun_start, nzrun_len);
+        d += nzrun_len;
+        nzrun_len = 0;
+    }
+
+    return d;
+}
+
+int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen)
+{
+    int i = 0, d = 0;
+    int ret;
+    uint32_t count = 0;
+
+    while (i < slen) {
+
+        /* zrun */
+        if ((slen - i) < 2) {
+            return -1;
+        }
+
+        ret = uleb128_decode_small(src + i, &count);
+        if (ret < 0 || (i && !count)) {
+            return -1;
+        }
+        i += ret;
+        d += count;
+
+        /* overflow */
+        if (d > dlen) {
+            return -1;
+        }
+
+        /* nzrun */
+        if ((slen - i) < 2) {
+            return -1;
+        }
+
+        ret = uleb128_decode_small(src + i, &count);
+        if (ret < 0 || !count) {
+            return -1;
+        }
+        i += ret;
+
+        /* overflow */
+        if (d + count > dlen || i + count > slen) {
+            return -1;
+        }
+
+        memcpy(dst + d, src + i, count);
+        d += count;
+        i += count;
+    }
+
+    return d;
+}
diff --git a/xen-all.c b/xen-all.c
index 046cc2a..110f958 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -10,16 +10,17 @@
 
 #include <sys/mman.h>
 
-#include "hw/pci.h"
+#include "hw/pci/pci.h"
 #include "hw/pc.h"
 #include "hw/xen_common.h"
 #include "hw/xen_backend.h"
 #include "qmp-commands.h"
 
-#include "range.h"
-#include "xen-mapcache.h"
+#include "char/char.h"
+#include "qemu/range.h"
+#include "sysemu/xen-mapcache.h"
 #include "trace.h"
-#include "exec-memory.h"
+#include "exec/address-spaces.h"
 
 #include <xen/hvm/ioreq.h>
 #include <xen/hvm/params.h>
@@ -292,7 +293,8 @@ static int xen_add_to_physmap(XenIOState *state,
     return -1;
 
 go_physmap:
-    DPRINTF("mapping vram to %llx - %llx\n", start_addr, start_addr + size);
+    DPRINTF("mapping vram to %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
+            start_addr, start_addr + size);
 
     pfn = phys_offset >> TARGET_PAGE_BITS;
     start_gpfn = start_addr >> TARGET_PAGE_BITS;
@@ -365,8 +367,8 @@ static int xen_remove_from_physmap(XenIOState *state,
     phys_offset = physmap->phys_offset;
     size = physmap->size;
 
-    DPRINTF("unmapping vram to %llx - %llx, from %llx\n",
-            phys_offset, phys_offset + size, start_addr);
+    DPRINTF("unmapping vram to %"HWADDR_PRIx" - %"HWADDR_PRIx", from ",
+            "%"HWADDR_PRIx"\n", phys_offset, phys_offset + size, start_addr);
 
     size >>= TARGET_PAGE_BITS;
     start_addr >>= TARGET_PAGE_BITS;
@@ -583,9 +585,7 @@ static void xen_reset_vcpu(void *opaque)
 
 void xen_vcpu_init(void)
 {
-    CPUArchState *first_cpu;
-
-    if ((first_cpu = qemu_get_cpu(0))) {
+    if (first_cpu != NULL) {
         qemu_register_reset(xen_reset_vcpu, first_cpu);
         xen_reset_vcpu(first_cpu);
     }
@@ -682,11 +682,45 @@ static void do_outp(pio_addr_t addr,
     }
 }
 
-static void cpu_ioreq_pio(ioreq_t *req)
+/*
+ * Helper functions which read/write an object from/to physical guest
+ * memory, as part of the implementation of an ioreq.
+ *
+ * Equivalent to
+ *   cpu_physical_memory_rw(addr + (req->df ? -1 : +1) * req->size * i,
+ *                          val, req->size, 0/1)
+ * except without the integer overflow problems.
+ */
+static void rw_phys_req_item(hwaddr addr,
+                             ioreq_t *req, uint32_t i, void *val, int rw)
 {
-    int i, sign;
+    /* Do everything unsigned so overflow just results in a truncated result
+     * and accesses to undesired parts of guest memory, which is up
+     * to the guest */
+    hwaddr offset = (hwaddr)req->size * i;
+    if (req->df) {
+        addr -= offset;
+    } else {
+        addr += offset;
+    }
+    cpu_physical_memory_rw(addr, val, req->size, rw);
+}
+
+static inline void read_phys_req_item(hwaddr addr,
+                                      ioreq_t *req, uint32_t i, void *val)
+{
+    rw_phys_req_item(addr, req, i, val, 0);
+}
+static inline void write_phys_req_item(hwaddr addr,
+                                       ioreq_t *req, uint32_t i, void *val)
+{
+    rw_phys_req_item(addr, req, i, val, 1);
+}
 
-    sign = req->df ? -1 : 1;
+
+static void cpu_ioreq_pio(ioreq_t *req)
+{
+    uint32_t i;
 
     if (req->dir == IOREQ_READ) {
         if (!req->data_is_ptr) {
@@ -696,9 +730,7 @@ static void cpu_ioreq_pio(ioreq_t *req)
 
             for (i = 0; i < req->count; i++) {
                 tmp = do_inp(req->addr, req->size);
-                cpu_physical_memory_write(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t *) &tmp, req->size);
+                write_phys_req_item(req->data, req, i, &tmp);
             }
         }
     } else if (req->dir == IOREQ_WRITE) {
@@ -708,9 +740,7 @@ static void cpu_ioreq_pio(ioreq_t *req)
             for (i = 0; i < req->count; i++) {
                 uint32_t tmp = 0;
 
-                cpu_physical_memory_read(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
+                read_phys_req_item(req->data, req, i, &tmp);
                 do_outp(req->addr, req->size, tmp);
             }
         }
@@ -719,22 +749,16 @@ static void cpu_ioreq_pio(ioreq_t *req)
 
 static void cpu_ioreq_move(ioreq_t *req)
 {
-    int i, sign;
-
-    sign = req->df ? -1 : 1;
+    uint32_t i;
 
     if (!req->data_is_ptr) {
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_read(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t *) &req->data, req->size);
+                read_phys_req_item(req->addr, req, i, &req->data);
             }
         } else if (req->dir == IOREQ_WRITE) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_write(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t *) &req->data, req->size);
+                write_phys_req_item(req->addr, req, i, &req->data);
             }
         }
     } else {
@@ -742,21 +766,13 @@ static void cpu_ioreq_move(ioreq_t *req)
 
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_read(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
-                cpu_physical_memory_write(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
+                read_phys_req_item(req->addr, req, i, &tmp);
+                write_phys_req_item(req->data, req, i, &tmp);
             }
         } else if (req->dir == IOREQ_WRITE) {
             for (i = 0; i < req->count; i++) {
-                cpu_physical_memory_read(
-                        req->data + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
-                cpu_physical_memory_write(
-                        req->addr + (sign * i * (int64_t)req->size),
-                        (uint8_t*) &tmp, req->size);
+                read_phys_req_item(req->data, req, i, &tmp);
+                write_phys_req_item(req->addr, req, i, &tmp);
             }
         }
     }
diff --git a/xen-mapcache.c b/xen-mapcache.c
index 31c06dc..dc6d1fa 100644
--- a/xen-mapcache.c
+++ b/xen-mapcache.c
@@ -13,13 +13,13 @@
 #include <sys/resource.h>
 
 #include "hw/xen_backend.h"
-#include "blockdev.h"
-#include "bitmap.h"
+#include "sysemu/blockdev.h"
+#include "qemu/bitmap.h"
 
 #include <xen/hvm/params.h>
 #include <sys/mman.h>
 
-#include "xen-mapcache.h"
+#include "sysemu/xen-mapcache.h"
 #include "trace.h"
 
 
diff --git a/xen-stub.c b/xen-stub.c
index 9214392..1ee8411 100644
--- a/xen-stub.c
+++ b/xen-stub.c
@@ -10,7 +10,7 @@
 
 #include "qemu-common.h"
 #include "hw/xen.h"
-#include "memory.h"
+#include "exec/memory.h"
 #include "qmp-commands.h"
 
 void xenstore_store_pv_console_info(int i, CharDriverState *chr)