PSARC 2009/425 Additional ioctls for GEM support in i915 driver

PSARC 2009/474 Additional IOCTL Support in Agpgart Driver
6815826 GEM should be supported in drm driver
6904304 System panic in pci_get_available_prop() in busra.c

35 files changed, 8285 insertions, 1227 deletions

diff --git a/usr/src/uts/common/Makefile.files b/usr/src/uts/common/Makefile.files
index 1278f6356f..1feed57654 100644
--- a/usr/src/uts/common/Makefile.files
+++ b/usr/src/uts/common/Makefile.files
@@ -1056,7 +1056,7 @@ DRM_OBJS += drm_sunmod.o drm_kstat.o drm_agpsupport.o \
 	    drm_auth.o drm_bufs.o drm_context.o drm_dma.o \
 	    drm_drawable.o drm_drv.o drm_fops.o drm_ioctl.o drm_irq.o \
 	    drm_lock.o drm_memory.o drm_msg.o drm_pci.o drm_scatter.o \
-	    ati_pcigart.o
+	    drm_cache.o drm_gem.o drm_mm.o ati_pcigart.o 
 
 FM_OBJS += devfm.o devfm_machdep.o
 
diff --git a/usr/src/uts/common/io/busra.c b/usr/src/uts/common/io/busra.c
index 8a72954676..66fc7a8c25 100644
--- a/usr/src/uts/common/io/busra.c
+++ b/usr/src/uts/common/io/busra.c
@@ -444,7 +444,7 @@ ndi_ra_free(dev_info_t *dip, uint64_t base, uint64_t len, char *type,
 	 * Update dip's "available" property, adding this piece of
 	 * resource to the pool.
 	 */
-	(void) pci_put_available_prop(dip, base, len, type);
+	(void) pci_put_available_prop(dipmap->ra_dip, base, len, type);
 done:
 	return (NDI_SUCCESS);
 
@@ -732,7 +732,8 @@ ndi_ra_alloc(dev_info_t *dip, ndi_ra_request_t *req, uint64_t *retbasep,
 	 * resource from the pool.
 	 */
 	if ((rval == NDI_SUCCESS) || (rval == NDI_RA_PARTIAL_REQ))
-		(void) pci_get_available_prop(dip, *retbasep, *retlenp, type);
+		(void) pci_get_available_prop(dipmap->ra_dip,
+		    *retbasep, *retlenp, type);
 
 	return (rval);
 }
diff --git a/usr/src/uts/common/io/drm/drm.h b/usr/src/uts/common/io/drm/drm.h
index 13e8bcf33c..87af6eddfe 100644
--- a/usr/src/uts/common/io/drm/drm.h
+++ b/usr/src/uts/common/io/drm/drm.h
@@ -13,6 +13,7 @@
 /*
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -665,6 +666,19 @@ typedef union drm_wait_vblank {
 	struct drm_wait_vblank_reply reply;
 } drm_wait_vblank_t;
 
+#define _DRM_PRE_MODESET 1
+#define _DRM_POST_MODESET 2
+
+/**
+ * DRM_IOCTL_MODESET_CTL ioctl argument type
+ *
+ * \sa drmModesetCtl().
+ */
+typedef struct drm_modeset_ctl {
+	uint32_t crtc;
+	uint32_t cmd;
+} drm_modeset_ctl_t;
+
 /**
  * DRM_IOCTL_AGP_ENABLE ioctl argument type.
  *
@@ -737,6 +751,34 @@ typedef struct drm_set_version {
 	int drm_dd_minor;
 } drm_set_version_t;
 
+/** DRM_IOCTL_GEM_CLOSE ioctl argument type */
+typedef struct drm_gem_close {
+	/** Handle of the object to be closed. */
+	uint32_t handle;
+	uint32_t pad;
+} drm_gem_close_t;
+
+/** DRM_IOCTL_GEM_FLINK ioctl argument type */
+typedef struct drm_gem_flink {
+	/** Handle for the object being named */
+	uint32_t handle;
+
+	/** Returned global name */
+	uint32_t name;
+} drm_gem_flink_t;
+
+/** DRM_IOCTL_GEM_OPEN ioctl argument type */
+typedef struct drm_gem_open {
+	/** Name of object being opened */
+	uint32_t name;
+
+	/** Returned handle for the object */
+	uint32_t handle;
+
+	/** Returned size of the object */
+	uint64_t size;
+} drm_gem_open_t;
+
 /**
  * \name Ioctls Definitions
  */
@@ -756,6 +798,10 @@ typedef struct drm_set_version {
 #define DRM_IOCTL_GET_CLIENT            DRM_IOWR(0x05, drm_client_t)
 #define DRM_IOCTL_GET_STATS             DRM_IOR( 0x06, drm_stats_t)
 #define DRM_IOCTL_SET_VERSION		DRM_IOWR(0x07, drm_set_version_t)
+#define DRM_IOCTL_MODESET_CTL           DRM_IOW(0x08,	drm_modeset_ctl_t)
+#define DRM_IOCTL_GEM_CLOSE		DRM_IOW (0x09, drm_gem_close_t)
+#define DRM_IOCTL_GEM_FLINK		DRM_IOWR(0x0a, drm_gem_flink_t)
+#define DRM_IOCTL_GEM_OPEN		DRM_IOWR(0x0b, drm_gem_open_t)
 
 #define DRM_IOCTL_SET_UNIQUE		DRM_IOW( 0x10, drm_unique_t)
 #define DRM_IOCTL_AUTH_MAGIC		DRM_IOW( 0x11, drm_auth_t)
diff --git a/usr/src/uts/common/io/drm/drmP.h b/usr/src/uts/common/io/drm/drmP.h
index 4c6934db87..05105148b5 100644
--- a/usr/src/uts/common/io/drm/drmP.h
+++ b/usr/src/uts/common/io/drm/drmP.h
@@ -5,6 +5,7 @@
 /*
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -151,8 +152,9 @@
 #define	DRM_LOCK_OWNED()	ASSERT(mutex_owned(&dev->dev_lock))
 #define	spin_lock_irqsave(l, flag)		mutex_enter(l)
 #define	spin_unlock_irqrestore(u, flag) mutex_exit(u)
-#define	spin_lock(l)		mutex_enter(l)
-#define	spin_unlock(u)		mutex_exit(u)
+#define	spin_lock(l)	mutex_enter(l)
+#define	spin_unlock(u)	mutex_exit(u)
+
 
 #define	DRM_UDELAY(sec)  delay(drv_usectohz(sec *1000))
 #define	DRM_MEMORYBARRIER()
@@ -166,12 +168,16 @@ typedef struct drm_driver_info drm_driver_t;
 	drm_device_t *dev1, intptr_t data, drm_file_t *fpriv, int mode
 
 #define	DRM_COPYFROM_WITH_RETURN(dest, src, size)	\
-	if (ddi_copyin(src, dest, size, 0)) \
-		return (EFAULT)
+	if (ddi_copyin((src), (dest), (size), 0)) {	\
+		DRM_ERROR("%s: copy from user failed", __func__);	\
+		return (EFAULT);	\
+	}
 
 #define	DRM_COPYTO_WITH_RETURN(dest, src, size)	\
-	if (ddi_copyout((src), (dest), (size), 0)) \
-		return (EFAULT)
+	if (ddi_copyout((src), (dest), (size), 0)) {	\
+		DRM_ERROR("%s: copy to user failed", __func__);	\
+		return (EFAULT);	\
+	}
 
 #define	DRM_COPY_FROM_USER(dest, src, size) \
 	ddi_copyin((src), (dest), (size), 0) /* flag for src */
@@ -222,6 +228,8 @@ typedef struct drm_wait_queue {
 	mutex_exit(&(q)->lock);	\
 }
 
+#define	jiffies	ddi_get_lbolt()
+
 #define	DRM_WAIT_ON(ret, q, timeout, condition)  			\
 	mutex_enter(&(q)->lock);					\
 	while (!(condition)) {						\
@@ -239,6 +247,21 @@ typedef struct drm_wait_queue {
 	} 								\
 	mutex_exit(&(q)->lock);
 
+#define	DRM_WAIT(ret, q, condition)  \
+mutex_enter(&(q)->lock);	\
+if (!(condition)) {	\
+	ret = cv_timedwait_sig(&(q)->cv, &(q)->lock, jiffies + 30 * DRM_HZ); \
+	if (ret == -1) {				\
+		/* gfx maybe hang */	\
+		if (!(condition)) 	\
+			ret = -2;	\
+	} else {	\
+		ret = 0;	\
+	}	\
+} \
+mutex_exit(&(q)->lock);
+
+
 #define	DRM_GETSAREA()  					\
 {                                				\
 	drm_local_map_t *map;					\
@@ -255,8 +278,8 @@ typedef struct drm_wait_queue {
 #define	LOCK_TEST_WITH_RETURN(dev, fpriv)				\
 	if (!_DRM_LOCK_IS_HELD(dev->lock.hw_lock->lock) ||		\
 	    dev->lock.filp != fpriv) {					\
-		DRM_ERROR("called without lock held");	\
-		return (EINVAL);					\
+		DRM_DEBUG("%s called without lock held", __func__);	\
+		return (EINVAL);	\
 	}
 
 #define	DRM_IRQ_ARGS	caddr_t arg
@@ -281,6 +304,11 @@ enum {
 #define	PAGE_ALIGN(addr)	(((addr) + DRM_PAGE_SIZE - 1) & DRM_PAGE_MASK)
 #define	DRM_SUSER(p)		(crgetsgid(p) == 0 || crgetsuid(p) == 0)
 
+#define	DRM_GEM_OBJIDR_HASHNODE	1024
+#define	idr_list_for_each(entry, head) \
+	for (int key = 0; key < DRM_GEM_OBJIDR_HASHNODE; key++) \
+		list_for_each(entry, &(head)->next[key])
+
 /*
  * wait for 400 milliseconds
  */
@@ -378,6 +406,88 @@ typedef struct drm_buf_entry {
 } drm_buf_entry_t;
 
 typedef TAILQ_HEAD(drm_file_list, drm_file) drm_file_list_t;
+
+/* BEGIN CSTYLED */
+typedef struct drm_local_map {
+	unsigned long	offset;  /*  Physical address (0 for SAREA)	*/
+	unsigned long	size;	 /* Physical size (bytes)		*/
+	drm_map_type_t	type;	 /* Type of memory mapped		*/
+	drm_map_flags_t flags;	 /* Flags				*/
+	void		*handle; /* User-space: "Handle" to pass to mmap */
+				 /* Kernel-space: kernel-virtual address */
+	int		mtrr;	 /* Boolean: MTRR used 			*/
+				 /* Private data			*/
+	int		rid;	 /* PCI resource ID for bus_space 	*/
+	int		kernel_owned; /* Boolean: 1= initmapped, 0= addmapped */
+	caddr_t		dev_addr;	  /* base device address 	*/
+	ddi_acc_handle_t  dev_handle;	  /* The data access handle 	*/
+	ddi_umem_cookie_t drm_umem_cookie; /* For SAREA alloc and free  */
+	TAILQ_ENTRY(drm_local_map) link;
+} drm_local_map_t;
+/* END CSTYLED */
+
+/*
+ * This structure defines the drm_mm memory object, which will be used by the
+ * DRM for its buffer objects.
+ */
+struct drm_gem_object {
+	/* Reference count of this object */
+	atomic_t refcount;
+
+	/* Handle count of this object. Each handle also holds a reference */
+	atomic_t handlecount;
+
+	/* Related drm device */
+	struct drm_device *dev;
+
+	int flink;
+	/*
+	 * Size of the object, in bytes.  Immutable over the object's
+	 * lifetime.
+	 */
+	size_t size;
+
+	/*
+	 * Global name for this object, starts at 1. 0 means unnamed.
+	 * Access is covered by the object_name_lock in the related drm_device
+	 */
+	int name;
+
+	/*
+	 * Memory domains. These monitor which caches contain read/write data
+	 * related to the object. When transitioning from one set of domains
+	 * to another, the driver is called to ensure that caches are suitably
+	 * flushed and invalidated
+	 */
+	uint32_t read_domains;
+	uint32_t write_domain;
+
+	/*
+	 * While validating an exec operation, the
+	 * new read/write domain values are computed here.
+	 * They will be transferred to the above values
+	 * at the point that any cache flushing occurs
+	 */
+	uint32_t pending_read_domains;
+	uint32_t pending_write_domain;
+
+	void *driver_private;
+
+	drm_local_map_t *map;
+	ddi_dma_handle_t dma_hdl;
+	ddi_acc_handle_t acc_hdl;
+	caddr_t kaddr;
+	size_t real_size;	/* real size of memory */
+	pfn_t *pfnarray;
+};
+
+struct idr_list {
+	struct idr_list *next, *prev;
+	struct drm_gem_object *obj;
+	uint32_t	handle;
+	caddr_t	contain_ptr;
+};
+
 struct drm_file {
 	TAILQ_ENTRY(drm_file) link;
 	int		  authenticated;
@@ -389,6 +499,13 @@ struct drm_file {
 	drm_magic_t	  magic;
 	unsigned long	  ioctl_count;
 	void		 *driver_priv;
+	/* Mapping of mm object handles to object pointers. */
+	struct idr_list object_idr;
+	/* Lock for synchronization of access to object_idr. */
+	kmutex_t table_lock;
+
+	dev_t dev;
+	cred_t *credp;
 };
 
 typedef struct drm_lock_data {
@@ -467,26 +584,26 @@ typedef struct drm_sg_mem {
 	drm_dma_handle_t	*dmah_gart; /* Handle to PCI memory */
 } drm_sg_mem_t;
 
-typedef TAILQ_HEAD(drm_map_list, drm_local_map) drm_map_list_t;
+/*
+ * Generic memory manager structs
+ */
 
-/* BEGIN CSTYLED */
-typedef struct drm_local_map {
-	unsigned long	offset;  /*  Physical address (0 for SAREA)	*/
-	unsigned long	size;	 /* Physical size (bytes)		*/
-	drm_map_type_t	type;	 /* Type of memory mapped		*/
-	drm_map_flags_t flags;	 /* Flags				*/
-	void		*handle; /* User-space: "Handle" to pass to mmap */
-				 /* Kernel-space: kernel-virtual address */
-	int		mtrr;	 /* Boolean: MTRR used 			*/
-				 /* Private data			*/
-	int		rid;	 /* PCI resource ID for bus_space 	*/
-	int		kernel_owned; /* Boolean: 1= initmapped, 0= addmapped */
-	caddr_t		dev_addr;	  /* base device address 	*/
-	ddi_acc_handle_t  dev_handle;	  /* The data access handle 	*/
-	ddi_umem_cookie_t drm_umem_cookie; /* For SAREA alloc and free  */
-	TAILQ_ENTRY(drm_local_map) link;
-} drm_local_map_t;
-/* END CSTYLED */
+struct drm_mm_node {
+	struct list_head fl_entry;
+	struct list_head ml_entry;
+	int free;
+	unsigned long start;
+	unsigned long size;
+	struct drm_mm *mm;
+	void *private;
+};
+
+struct drm_mm {
+	struct list_head fl_entry;
+	struct list_head ml_entry;
+};
+
+typedef TAILQ_HEAD(drm_map_list, drm_local_map) drm_map_list_t;
 
 typedef TAILQ_HEAD(drm_vbl_sig_list, drm_vbl_sig) drm_vbl_sig_list_t;
 typedef struct drm_vbl_sig {
@@ -556,6 +673,16 @@ struct drm_driver_info {
 	int (*enable_vblank)(struct drm_device *dev, int crtc);
 	void (*disable_vblank)(struct drm_device *dev, int crtc);
 
+	/*
+	 * Driver-specific constructor for drm_gem_objects, to set up
+	 * obj->driver_private.
+	 *
+	 * Returns 0 on success.
+	 */
+	int (*gem_init_object) (struct drm_gem_object *obj);
+	void (*gem_free_object) (struct drm_gem_object *obj);
+
+
 	drm_ioctl_desc_t *driver_ioctls;
 	int	max_driver_ioctl;
 
@@ -577,6 +704,7 @@ struct drm_driver_info {
 	unsigned use_vbl_irq :1;
 	unsigned use_vbl_irq2 :1;
 	unsigned use_mtrr :1;
+	unsigned use_gem;
 };
 
 /*
@@ -607,11 +735,12 @@ struct drm_device {
 	int		flags;	/* Flags to open(2)		   */
 
 	/* Locks */
-	kmutex_t	  vbl_lock;	/* protects vblank operations */
-	kmutex_t	  dma_lock;	/* protects dev->dma */
-	kmutex_t	  irq_lock;	/* protects irq condition checks */
-	kmutex_t	  dev_lock;	/* protects everything else */
+	kmutex_t	vbl_lock;	/* protects vblank operations */
+	kmutex_t	dma_lock;	/* protects dev->dma */
+	kmutex_t	irq_lock;	/* protects irq condition checks */
+	kmutex_t	dev_lock;	/* protects everything else */
 	drm_lock_data_t   lock;		/* Information on hardware lock    */
+	kmutex_t	struct_mutex;	/* < For others	*/
 
 	/* Usage Counters */
 	int		  open_count;	/* Outstanding files open	   */
@@ -651,7 +780,12 @@ struct drm_device {
 
 	drm_vbl_sig_list_t vbl_sig_list;
 	drm_vbl_sig_list_t vbl_sig_list2;
-
+	/*
+	 * At load time, disabling the vblank interrupt won't be allowed since
+	 * old clients may not call the modeset ioctl and therefore misbehave.
+	 * Once the modeset ioctl *has* been called though, we can safely
+	 * disable them when unused.
+	 */
 	int vblank_disable_allowed;
 
 	wait_queue_head_t	vbl_queue;	/* vbl wait channel */
@@ -672,13 +806,13 @@ struct drm_device {
 	u32		  *last_vblank;
 	/* so we don't call enable more than */
 	atomic_t	  *vblank_enabled;
-	/* for compensation of spurious wraparounds */
-	u32		  *vblank_premodeset;
+	/* Display driver is setting mode */
+	int		*vblank_inmodeset;
 	/* Don't wait while crtc is likely disabled */
-	int		  *vblank_suspend;
+	int		*vblank_suspend;
 	/* size of vblank counter register */
-	u32		  max_vblank_count;
-	int		  num_crtcs;
+	u32		max_vblank_count;
+	int		num_crtcs;
 	kmutex_t	tasklet_lock;
 	void (*locked_tasklet_func)(struct drm_device *dev);
 
@@ -698,6 +832,22 @@ struct drm_device {
 	u32 *drw_bitfield;
 	unsigned int drw_info_length;
 	drm_drawable_info_t **drw_info;
+
+	/* \name GEM information */
+	/* @{ */
+	kmutex_t object_name_lock;
+	struct idr_list	object_name_idr;
+	atomic_t object_count;
+	atomic_t object_memory;
+	atomic_t pin_count;
+	atomic_t pin_memory;
+	atomic_t gtt_count;
+	atomic_t gtt_memory;
+	uint32_t gtt_total;
+	uint32_t invalidate_domains;	/* domains pending invalidation */
+	uint32_t flush_domains;	/* domains pending flush */
+	/* @} */
+
 	/*
 	 * Saving S3 context
 	 */
@@ -767,8 +917,8 @@ u32	drm_vblank_count(struct drm_device *dev, int crtc);
 int	drm_vblank_get(struct drm_device *dev, int crtc);
 void	drm_vblank_put(struct drm_device *dev, int crtc);
 int	drm_vblank_init(struct drm_device *dev, int num_crtcs);
-void    drm_locked_tasklet(drm_device_t *, void(*func)(drm_device_t *));
 void	drm_vblank_cleanup(struct drm_device *dev);
+int    drm_modeset_ctl(DRM_IOCTL_ARGS);
 
 /* AGP/GART support (drm_agpsupport.c) */
 int	drm_device_is_agp(drm_device_t *);
@@ -776,10 +926,21 @@ int 	drm_device_is_pcie(drm_device_t *);
 drm_agp_head_t *drm_agp_init(drm_device_t *);
 void	drm_agp_fini(drm_device_t *);
 int 	drm_agp_do_release(drm_device_t *);
-void	*drm_agp_allocate_memory(size_t, uint32_t);
-int	drm_agp_free_memory(void *);
+void	*drm_agp_allocate_memory(size_t pages,
+	    uint32_t type, drm_device_t *dev);
+int	drm_agp_free_memory(agp_allocate_t *handle, drm_device_t *dev);
 int	drm_agp_bind_memory(unsigned int, uint32_t, drm_device_t *);
 int	drm_agp_unbind_memory(unsigned long, drm_device_t *);
+int	drm_agp_bind_pages(drm_device_t *dev,
+		    pfn_t *pages,
+		    unsigned long num_pages,
+		    uint32_t gtt_offset);
+int	drm_agp_unbind_pages(drm_device_t *dev,
+		    unsigned long num_pages,
+		    uint32_t gtt_offset,
+		    uint32_t type);
+void drm_agp_chipset_flush(struct drm_device *dev);
+void drm_agp_rebind(struct drm_device *dev);
 
 /* kstat support (drm_kstats.c) */
 int	drm_init_kstats(drm_device_t *);
@@ -797,6 +958,8 @@ int	drm_lock(DRM_IOCTL_ARGS);
 int	drm_unlock(DRM_IOCTL_ARGS);
 int	drm_version(DRM_IOCTL_ARGS);
 int	drm_setversion(DRM_IOCTL_ARGS);
+/* Cache management (drm_cache.c) */
+void drm_clflush_pages(caddr_t *pages, unsigned long num_pages);
 
 /* Misc. IOCTL support (drm_ioctl.c) */
 int	drm_irq_by_busid(DRM_IOCTL_ARGS);
@@ -858,8 +1021,15 @@ int	drm_agp_bind(DRM_IOCTL_ARGS);
 int	drm_sg_alloc(DRM_IOCTL_ARGS);
 int	drm_sg_free(DRM_IOCTL_ARGS);
 
-extern int drm_debug_flag;
+/*	drm_mm.c	*/
+struct drm_mm_node *drm_mm_get_block(struct drm_mm_node *parent,
+				    unsigned long size, unsigned alignment);
+struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm,
+				    unsigned long size,
+				    unsigned alignment, int best_match);
 
+extern void drm_mm_clean_ml(const struct drm_mm *mm);
+extern int drm_debug_flag;
 
 /* We add function to support DRM_DEBUG,DRM_ERROR,DRM_INFO */
 extern void drm_debug(const char *fmt, ...);
@@ -905,4 +1075,30 @@ extern drm_cminor_t *drm_find_file_by_minor(drm_device_t *, int);
 extern int drm_open_helper(drm_device_t *, drm_cminor_t *, int, int,
     cred_t *);
 
+/* Graphics Execution Manager library functions (drm_gem.c) */
+int drm_gem_init(struct drm_device *dev);
+void drm_gem_object_free(struct drm_gem_object *obj);
+struct drm_gem_object *drm_gem_object_alloc(struct drm_device *dev,
+					    size_t size);
+void drm_gem_object_handle_free(struct drm_gem_object *obj);
+
+void drm_gem_object_reference(struct drm_gem_object *obj);
+void drm_gem_object_unreference(struct drm_gem_object *obj);
+
+int drm_gem_handle_create(struct drm_file *file_priv,
+			    struct drm_gem_object *obj,
+			    int *handlep);
+void drm_gem_object_handle_reference(struct drm_gem_object *obj);
+
+void drm_gem_object_handle_unreference(struct drm_gem_object *obj);
+
+struct drm_gem_object *drm_gem_object_lookup(struct drm_file *filp,
+					    int handle);
+int drm_gem_close_ioctl(DRM_IOCTL_ARGS);
+int drm_gem_flink_ioctl(DRM_IOCTL_ARGS);
+int drm_gem_open_ioctl(DRM_IOCTL_ARGS);
+void drm_gem_open(struct drm_file *file_private);
+void drm_gem_release(struct drm_device *dev, struct drm_file *file_private);
+
+
 #endif	/* _DRMP_H */
diff --git a/usr/src/uts/common/io/drm/drm_agpsupport.c b/usr/src/uts/common/io/drm/drm_agpsupport.c
index 48f7f5e454..ae695dabaf 100644
--- a/usr/src/uts/common/io/drm/drm_agpsupport.c
+++ b/usr/src/uts/common/io/drm/drm_agpsupport.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -10,6 +10,7 @@
 /*
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -37,8 +38,6 @@
  *
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include "drm.h"
 #include "drmP.h"
 
@@ -193,6 +192,7 @@ drm_agp_enable(DRM_IOCTL_ARGS)
 	dev->agp->mode = modes.mode;
 	setup.agps_mode = (uint32_t)modes.mode;
 
+
 	DRM_DEBUG("drm_agp_enable: dev->agp->mode=%lx", modes.mode);
 
 	ret = ldi_ioctl(dev->agp->agpgart_lh, AGPIOC_SETUP,
@@ -205,6 +205,7 @@ drm_agp_enable(DRM_IOCTL_ARGS)
 	dev->agp->base = dev->agp->agp_info.agpi_aperbase;
 	dev->agp->enabled = 1;
 
+	DRM_DEBUG("drm_agp_enable: dev->agp->base=0x%lx", dev->agp->base);
 	return (0);
 }
 
@@ -247,6 +248,8 @@ drm_agp_alloc(DRM_IOCTL_ARGS)
 		dev->agp->memory->prev = entry;
 	dev->agp->memory = entry;
 
+	DRM_DEBUG("entry->phys_addr %lx", entry->phys_addr);
+
 	/* physical is used only by i810 driver */
 	request.physical = alloc.agpa_physical;
 	request.handle = (unsigned long)entry->handle;
@@ -383,7 +386,6 @@ drm_agp_init(drm_device_t *dev)
 	drm_agp_head_t *agp   = NULL;
 	int	retval, rval;
 
-	DRM_DEBUG("drm_agp_init\n");
 	agp = kmem_zalloc(sizeof (drm_agp_head_t), KM_SLEEP);
 
 	retval = ldi_ident_from_dip(dev->dip, &agp->agpgart_li);
@@ -437,14 +439,14 @@ drm_agp_fini(drm_device_t *dev)
 
 /*ARGSUSED*/
 void *
-drm_agp_allocate_memory(size_t pages, uint32_t type)
+drm_agp_allocate_memory(size_t pages, uint32_t type, drm_device_t *dev)
 {
 	return (NULL);
 }
 
 /*ARGSUSED*/
 int
-drm_agp_free_memory(void *handle)
+drm_agp_free_memory(agp_allocate_t *handle, drm_device_t *dev)
 {
 	return (1);
 }
@@ -494,3 +496,92 @@ drm_agp_unbind_memory(unsigned long handle, drm_device_t *dev)
 	entry->bound = 0;
 	return (0);
 }
+
+/*
+ * Binds a collection of pages into AGP memory at the given offset, returning
+ * the AGP memory structure containing them.
+ *
+ * No reference is held on the pages during this time -- it is up to the
+ * caller to handle that.
+ */
+int
+drm_agp_bind_pages(drm_device_t *dev,
+		    pfn_t *pages,
+		    unsigned long num_pages,
+		    uint32_t gtt_offset)
+{
+
+	agp_bind_pages_t bind;
+	int	ret, rval;
+
+	bind.agpb_pgstart = gtt_offset / AGP_PAGE_SIZE;
+	bind.agpb_pgcount = num_pages;
+	bind.agpb_pages = pages;
+	ret = ldi_ioctl(dev->agp->agpgart_lh, AGPIOC_PAGES_BIND,
+	    (intptr_t)&bind, FKIOCTL, kcred, &rval);
+	if (ret) {
+		DRM_ERROR("AGPIOC_PAGES_BIND failed ret %d", ret);
+		return (ret);
+	}
+	return (0);
+}
+
+int
+drm_agp_unbind_pages(drm_device_t *dev,
+		    unsigned long num_pages,
+		    uint32_t gtt_offset,
+		    uint32_t type)
+{
+
+	agp_unbind_pages_t unbind;
+	int	ret, rval;
+
+	unbind.agpb_pgstart = gtt_offset / AGP_PAGE_SIZE;
+	unbind.agpb_pgcount = num_pages;
+	unbind.agpb_type = type;
+	ret = ldi_ioctl(dev->agp->agpgart_lh, AGPIOC_PAGES_UNBIND,
+	    (intptr_t)&unbind, FKIOCTL, kcred, &rval);
+	if (ret) {
+		DRM_DEBUG("drm_agp_unbind_pages AGPIOC_PAGES_UNBIND failed");
+		return (ret);
+	}
+	return (0);
+}
+
+/*
+ * Certain Intel chipsets contains a global write buffer, and this can require
+ * flushing from the drm or X.org to make sure all data has hit RAM before
+ * initiating a GPU transfer, due to a lack of coherency with the integrated
+ * graphics device and this buffer.
+ */
+void
+drm_agp_chipset_flush(struct drm_device *dev)
+{
+	int ret, rval;
+
+	DRM_DEBUG("agp_chipset_flush");
+	ret = ldi_ioctl(dev->agp->agpgart_lh, AGPIOC_FLUSHCHIPSET,
+	    (intptr_t)0, FKIOCTL, kcred, &rval);
+	if (ret != 0) {
+		DRM_ERROR("Failed to drm_agp_chipset_flush ret %d", ret);
+	}
+}
+
+/*
+ * The pages are evict on suspend, so re-bind it at resume time
+ */
+void
+drm_agp_rebind(struct drm_device *dev)
+{
+	int ret, rval;
+
+	if (!dev->agp) {
+		return;
+	}
+
+	ret = ldi_ioctl(dev->agp->agpgart_lh, AGPIOC_PAGES_REBIND,
+	    (intptr_t)0, FKIOCTL, kcred, &rval);
+	if (ret != 0) {
+		DRM_ERROR("rebind failed %d", ret);
+	}
+}
diff --git a/usr/src/uts/common/io/drm/drm_atomic.h b/usr/src/uts/common/io/drm/drm_atomic.h
index 002b974933..b8a4f56091 100644
--- a/usr/src/uts/common/io/drm/drm_atomic.h
+++ b/usr/src/uts/common/io/drm/drm_atomic.h
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2004 Eric Anholt
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -57,7 +58,7 @@ typedef uint32_t	atomic_t;
 #define	atomic_inc(p)		atomic_add_int(p, 1)
 #define	atomic_dec(p)		atomic_dec_uint(p)
 #define	atomic_add(n, p)	atomic_add_int(p, n)
-#define	atomic_sub(n, p)	atomic_dec_uint(p, n)
+#define	atomic_sub(n, p)	atomic_add_int(p, -n)
 #define	atomic_set_int(p, bits)	atomic_or_uint(p, bits)
 #define	atomic_clear_int(p, bits)	atomic_and_uint(p, ~(bits))
 #define	atomic_cmpset_int(p, c, n) \
diff --git a/usr/src/uts/common/io/drm/drm_bufs.c b/usr/src/uts/common/io/drm/drm_bufs.c
index ad1254072a..ec01d37dab 100644
--- a/usr/src/uts/common/io/drm/drm_bufs.c
+++ b/usr/src/uts/common/io/drm/drm_bufs.c
@@ -5,6 +5,7 @@
 /*
  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -170,7 +171,7 @@ int drm_addmap(drm_device_t *dev, unsigned long offset,
 		break;
 
 	case _DRM_CONSISTENT:
-		cmn_err(CE_WARN, "%d DRM_AGP_CONSISTENT", __LINE__);
+		DRM_ERROR("%d DRM_AGP_CONSISTENT", __LINE__);
 		return (ENOTSUP);
 	case _DRM_AGP:
 		map->offset += dev->agp->base;
diff --git a/usr/src/uts/common/io/drm/drm_cache.c b/usr/src/uts/common/io/drm/drm_cache.c
new file mode 100644
index 0000000000..44a19c0703
--- /dev/null
+++ b/usr/src/uts/common/io/drm/drm_cache.c
@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright(c) 2006-2007 Tungsten Graphics, Inc., Cedar Park, TX., USA
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files(the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice(including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+/*
+ * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/x86_archext.h>
+#include <vm/seg_kmem.h>
+#include "drmP.h"
+
+extern void clflush_insn(caddr_t addr);
+extern void mfence_insn(void);
+
+static void
+drm_clflush_page(caddr_t page)
+{
+	unsigned int i;
+
+	if (page == NULL)
+		return;
+
+	for (i = 0; i < PAGE_SIZE; i += x86_clflush_size)
+		clflush_insn(page + i);
+	mfence_insn();
+}
+
+void
+drm_clflush_pages(caddr_t *pages, unsigned long num_pages)
+{
+
+	if (x86_feature & X86_CLFSH) {
+		unsigned long i;
+
+		for (i = 0; i < num_pages; i++)
+			drm_clflush_page(pages[i]);
+	}
+}
diff --git a/usr/src/uts/common/io/drm/drm_drv.c b/usr/src/uts/common/io/drm/drm_drv.c
index b104843031..ae86db50d7 100644
--- a/usr/src/uts/common/io/drm/drm_drv.c
+++ b/usr/src/uts/common/io/drm/drm_drv.c
@@ -5,6 +5,7 @@
 /*
  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -61,6 +62,14 @@ drm_ioctl_desc_t drm_ioctls[DRIVER_IOCTL_COUNT] = {
 	    {drm_getstats, 0},
 	[DRM_IOCTL_NR(DRM_IOCTL_SET_VERSION)] =
 	    {drm_setversion, DRM_MASTER|DRM_ROOT_ONLY},
+	[DRM_IOCTL_NR(DRM_IOCTL_MODESET_CTL)] =
+	    {drm_modeset_ctl, 0},
+	[DRM_IOCTL_NR(DRM_IOCTL_GEM_CLOSE)] =
+	    {drm_gem_close_ioctl, 0},
+	[DRM_IOCTL_NR(DRM_IOCTL_GEM_FLINK)] =
+	    {drm_gem_flink_ioctl, DRM_AUTH},
+	[DRM_IOCTL_NR(DRM_IOCTL_GEM_OPEN)] =
+	    {drm_gem_open_ioctl, DRM_AUTH},
 	[DRM_IOCTL_NR(DRM_IOCTL_SET_UNIQUE)] =
 	    {drm_setunique, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
 	[DRM_IOCTL_NR(DRM_IOCTL_BLOCK)] =
@@ -141,6 +150,8 @@ drm_ioctl_desc_t drm_ioctls[DRIVER_IOCTL_COUNT] = {
 	    {drm_update_draw, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
 };
 
+extern void idr_list_free(struct idr_list *head);
+
 const char *
 drm_find_description(int vendor, int device, drm_pci_id_list_t *idlist)
 {
@@ -263,7 +274,7 @@ drm_lastclose(drm_device_t *dev)
 			if (entry->bound)
 				(void) drm_agp_unbind_memory(
 				    (unsigned long)entry->handle, dev);
-			(void) drm_agp_free_memory(entry->handle);
+			(void) drm_agp_free_memory(entry->handle, dev);
 			drm_free(entry, sizeof (*entry), DRM_MEM_AGPLISTS);
 		}
 		dev->agp->memory = NULL;
@@ -337,6 +348,15 @@ drm_load(drm_device_t *dev)
 		goto error;
 	}
 
+	if (dev->driver->use_gem == 1) {
+		retcode = drm_gem_init(dev);
+		if (retcode) {
+			DRM_ERROR("Cannot initialize graphics execution "
+			    "manager (GEM)\n");
+			goto error;
+		}
+	}
+
 	if (drm_init_kstats(dev)) {
 		DRM_ERROR("drm_attach => drm_load: init kstats error");
 		retcode = EFAULT;
@@ -375,6 +395,11 @@ drm_unload(drm_device_t *dev)
 
 	drm_ctxbitmap_cleanup(dev);
 
+	if (dev->driver->use_gem == 1) {
+		idr_list_free(&dev->object_name_idr);
+		mutex_destroy(&dev->object_name_lock);
+	}
+
 	DRM_LOCK();
 	(void) drm_lastclose(dev);
 	DRM_UNLOCK();
@@ -393,6 +418,10 @@ drm_unload(drm_device_t *dev)
 	mutex_destroy(&dev->dev_lock);
 	mutex_destroy(&dev->drw_lock);
 	mutex_destroy(&dev->tasklet_lock);
+
+	dev->gtt_total = 0;
+	atomic_set(&dev->pin_memory, 0);
+	DRM_ERROR("drm_unload");
 }
 
 
@@ -464,12 +493,17 @@ drm_close(drm_device_t *dev, int minor, int flag, int otyp,
 		    "retake lock not implemented yet");
 	}
 
-	if (dev->driver->use_dma)
+	if (dev->driver->use_dma) {
 		drm_reclaim_buffers(dev, fpriv);
+	}
 
+	if (dev->driver->use_gem == 1) {
+		drm_gem_release(dev, fpriv);
+	}
 
-	if (dev->driver->postclose != NULL)
+	if (dev->driver->postclose != NULL) {
 		dev->driver->postclose(dev, fpriv);
+	}
 	TAILQ_REMOVE(&dev->files, fpriv, link);
 	drm_free(fpriv, sizeof (*fpriv), DRM_MEM_FILES);
 
diff --git a/usr/src/uts/common/io/drm/drm_fops.c b/usr/src/uts/common/io/drm/drm_fops.c
index 7f2ba588bd..da61e4dd07 100644
--- a/usr/src/uts/common/io/drm/drm_fops.c
+++ b/usr/src/uts/common/io/drm/drm_fops.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -11,6 +11,7 @@
 /*-
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -41,8 +42,6 @@
 
 /* END CSTYLED */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include "drmP.h"
 
 /*ARGSUSED*/
@@ -110,6 +109,9 @@ drm_open_helper(drm_device_t *dev, drm_cminor_t *mp, int flags,
 		/* for compatibility root is always authenticated */
 		priv->authenticated	= DRM_SUSER(credp);
 
+		if (dev->driver->use_gem == 1)
+			drm_gem_open(priv);
+
 		if (dev->driver->open) {
 			retcode = dev->driver->open(dev, priv);
 			if (retcode != 0) {
diff --git a/usr/src/uts/common/io/drm/drm_gem.c b/usr/src/uts/common/io/drm/drm_gem.c
new file mode 100644
index 0000000000..9805ae7b62
--- /dev/null
+++ b/usr/src/uts/common/io/drm/drm_gem.c
@@ -0,0 +1,698 @@
+/*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <vm/anon.h>
+#include <vm/seg_kmem.h>
+#include <vm/seg_kp.h>
+#include <vm/seg_map.h>
+#include <sys/fcntl.h>
+#include <sys/vnode.h>
+#include <sys/file.h>
+#include <sys/bitmap.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <gfx_private.h>
+#include "drmP.h"
+#include "drm.h"
+
+/*
+ * @file drm_gem.c
+ *
+ * This file provides some of the base ioctls and library routines for
+ * the graphics memory manager implemented by each device driver.
+ *
+ * Because various devices have different requirements in terms of
+ * synchronization and migration strategies, implementing that is left up to
+ * the driver, and all that the general API provides should be generic --
+ * allocating objects, reading/writing data with the cpu, freeing objects.
+ * Even there, platform-dependent optimizations for reading/writing data with
+ * the CPU mean we'll likely hook those out to driver-specific calls.  However,
+ * the DRI2 implementation wants to have at least allocate/mmap be generic.
+ *
+ * The goal was to have swap-backed object allocation managed through
+ * struct file.  However, file descriptors as handles to a struct file have
+ * two major failings:
+ * - Process limits prevent more than 1024 or so being used at a time by
+ *   default.
+ * - Inability to allocate high fds will aggravate the X Server's select()
+ *   handling, and likely that of many GL client applications as well.
+ *
+ * This led to a plan of using our own integer IDs(called handles, following
+ * DRM terminology) to mimic fds, and implement the fd syscalls we need as
+ * ioctls.  The objects themselves will still include the struct file so
+ * that we can transition to fds if the required kernel infrastructure shows
+ * up at a later date, and as our interface with shmfs for memory allocation.
+ */
+
+void
+idr_list_init(struct idr_list  *head)
+{
+	struct idr_list  *entry;
+	/* HASH for accelerate */
+	entry = kmem_zalloc(DRM_GEM_OBJIDR_HASHNODE
+	    * sizeof (struct idr_list), KM_NOSLEEP);
+	head->next = entry;
+	for (int i = 0; i < DRM_GEM_OBJIDR_HASHNODE; i++) {
+		INIT_LIST_HEAD(&entry[i]);
+	}
+}
+
+int
+idr_list_get_new_above(struct idr_list	*head,
+			struct drm_gem_object *obj,
+			int *handlep)
+{
+	struct idr_list  *entry;
+	int key;
+	entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
+	key = obj->name % DRM_GEM_OBJIDR_HASHNODE;
+	list_add(entry, &head->next[key], NULL);
+	entry->obj = obj;
+	entry->handle = obj->name;
+	*handlep = obj->name;
+	return (0);
+}
+
+struct drm_gem_object *
+idr_list_find(struct idr_list  *head,
+		uint32_t	name)
+{
+	struct idr_list  *entry;
+	int key;
+	key = name % DRM_GEM_OBJIDR_HASHNODE;
+
+	list_for_each(entry, &head->next[key]) {
+		if (entry->handle == name)
+			return (entry->obj);
+	}
+	return (NULL);
+}
+
+int
+idr_list_remove(struct idr_list  *head,
+		uint32_t	name)
+{
+	struct idr_list  *entry, *temp;
+	int key;
+	key = name % DRM_GEM_OBJIDR_HASHNODE;
+	list_for_each_safe(entry, temp, &head->next[key]) {
+		if (entry->handle == name) {
+			list_del(entry);
+			kmem_free(entry, sizeof (*entry));
+			return (0);
+		}
+	}
+	DRM_ERROR("Failed to remove the object %d", name);
+	return (-1);
+}
+
+void
+idr_list_free(struct idr_list  *head)
+{
+	struct idr_list  *entry, *temp;
+	for (int key = 0; key < DRM_GEM_OBJIDR_HASHNODE; key++) {
+		list_for_each_safe(entry, temp, &head->next[key]) {
+			list_del(entry);
+			kmem_free(entry, sizeof (*entry));
+		}
+	}
+	kmem_free(head->next,
+	    DRM_GEM_OBJIDR_HASHNODE * sizeof (struct idr_list));
+	head->next = NULL;
+}
+
+int
+idr_list_empty(struct idr_list  *head)
+{
+	int empty;
+	for (int key = 0; key < DRM_GEM_OBJIDR_HASHNODE; key++) {
+		empty = list_empty(&(head)->next[key]);
+		if (!empty)
+			return (empty);
+	}
+	return (1);
+}
+
+static	uint32_t	shfile_name = 0;
+#define	SHFILE_NAME_MAX	0xffffffff
+
+/*
+ * will be set to 1 for 32 bit x86 systems only, in startup.c
+ */
+extern int	segkp_fromheap;
+extern ulong_t *segkp_bitmap;
+
+void
+drm_gem_object_reference(struct drm_gem_object *obj)
+{
+	atomic_inc(&obj->refcount);
+}
+
+void
+drm_gem_object_unreference(struct drm_gem_object *obj)
+{
+	if (obj == NULL)
+		return;
+
+	atomic_sub(1, &obj->refcount);
+	if (obj->refcount == 0)
+		drm_gem_object_free(obj);
+}
+
+void
+drm_gem_object_handle_reference(struct drm_gem_object *obj)
+{
+	drm_gem_object_reference(obj);
+	atomic_inc(&obj->handlecount);
+}
+
+void
+drm_gem_object_handle_unreference(struct drm_gem_object *obj)
+{
+	if (obj == NULL)
+		return;
+
+	/*
+	 * Must bump handle count first as this may be the last
+	 * ref, in which case the object would disappear before we
+	 * checked for a name
+	 */
+	atomic_sub(1, &obj->handlecount);
+	if (obj->handlecount == 0)
+		drm_gem_object_handle_free(obj);
+	drm_gem_object_unreference(obj);
+}
+
+/*
+ * Initialize the GEM device fields
+ */
+
+int
+drm_gem_init(struct drm_device *dev)
+{
+	mutex_init(&dev->object_name_lock, NULL, MUTEX_DRIVER, NULL);
+	idr_list_init(&dev->object_name_idr);
+
+	atomic_set(&dev->object_count, 0);
+	atomic_set(&dev->object_memory, 0);
+	atomic_set(&dev->pin_count, 0);
+	atomic_set(&dev->pin_memory, 0);
+	atomic_set(&dev->gtt_count, 0);
+	atomic_set(&dev->gtt_memory, 0);
+	return (0);
+}
+
+/*
+ * Allocate a GEM object of the specified size with shmfs backing store
+ */
+struct drm_gem_object *
+drm_gem_object_alloc(struct drm_device *dev, size_t size)
+{
+	static ddi_dma_attr_t dma_attr = {
+		DMA_ATTR_V0,
+		0U,				/* dma_attr_addr_lo */
+		0xffffffffU,			/* dma_attr_addr_hi */
+		0xffffffffU,			/* dma_attr_count_max */
+		4096,				/* dma_attr_align */
+		0x1fffU,			/* dma_attr_burstsizes */
+		1,				/* dma_attr_minxfer */
+		0xffffffffU,			/* dma_attr_maxxfer */
+		0xffffffffU,			/* dma_attr_seg */
+		1,				/* dma_attr_sgllen, variable */
+		4,				/* dma_attr_granular */
+		0				/* dma_attr_flags */
+	};
+	static ddi_device_acc_attr_t acc_attr = {
+		DDI_DEVICE_ATTR_V0,
+		DDI_NEVERSWAP_ACC,
+		DDI_MERGING_OK_ACC
+	};
+	struct drm_gem_object *obj;
+	ddi_dma_cookie_t cookie;
+	uint_t cookie_cnt;
+	drm_local_map_t *map;
+
+	pgcnt_t real_pgcnt, pgcnt = btopr(size);
+	uint32_t paddr, cookie_end;
+	int i, n;
+
+	obj = kmem_zalloc(sizeof (struct drm_gem_object), KM_NOSLEEP);
+	if (obj == NULL)
+		return (NULL);
+
+	obj->dev = dev;
+	obj->flink = 0;
+	obj->size = size;
+
+	if (shfile_name == SHFILE_NAME_MAX) {
+		DRM_ERROR("No name space for object");
+		goto err1;
+	} else {
+		obj->name = ++shfile_name;
+	}
+
+	dma_attr.dma_attr_sgllen = (int)pgcnt;
+
+	if (ddi_dma_alloc_handle(dev->dip, &dma_attr,
+	    DDI_DMA_DONTWAIT, NULL, &obj->dma_hdl)) {
+		DRM_ERROR("drm_gem_object_alloc: "
+		    "ddi_dma_alloc_handle failed");
+		goto err1;
+	}
+	if (ddi_dma_mem_alloc(obj->dma_hdl, ptob(pgcnt), &acc_attr,
+	    IOMEM_DATA_UC_WR_COMBINE, DDI_DMA_DONTWAIT, NULL,
+	    &obj->kaddr, &obj->real_size, &obj->acc_hdl)) {
+		DRM_ERROR("drm_gem_object_alloc: "
+		    "ddi_dma_mem_alloc failed");
+		goto err2;
+	}
+	if (ddi_dma_addr_bind_handle(obj->dma_hdl, NULL,
+	    obj->kaddr, obj->real_size, DDI_DMA_RDWR,
+	    DDI_DMA_DONTWAIT, NULL, &cookie, &cookie_cnt)
+	    != DDI_DMA_MAPPED) {
+		DRM_ERROR("drm_gem_object_alloc: "
+		    "ddi_dma_addr_bind_handle failed");
+		goto err3;
+	}
+
+	real_pgcnt = btopr(obj->real_size);
+
+	obj->pfnarray = kmem_zalloc(real_pgcnt * sizeof (pfn_t), KM_NOSLEEP);
+	if (obj->pfnarray == NULL) {
+		goto err4;
+	}
+	for (n = 0, i = 1; ; i++) {
+		for (paddr = cookie.dmac_address,
+		    cookie_end = cookie.dmac_address + cookie.dmac_size;
+		    paddr < cookie_end;
+		    paddr += PAGESIZE) {
+			obj->pfnarray[n++] = btop(paddr);
+			if (n >= real_pgcnt)
+				goto addmap;
+		}
+		if (i >= cookie_cnt)
+			break;
+		ddi_dma_nextcookie(obj->dma_hdl, &cookie);
+	}
+
+addmap:
+	map = drm_alloc(sizeof (struct drm_local_map), DRM_MEM_MAPS);
+	if (map == NULL) {
+		goto err5;
+	}
+
+	map->handle = obj;
+	map->offset = (uintptr_t)map->handle;
+	map->offset &= 0xffffffffUL;
+	map->dev_addr = map->handle;
+	map->size = obj->real_size;
+	map->type = _DRM_TTM;
+	map->flags = _DRM_WRITE_COMBINING | _DRM_REMOVABLE;
+	map->drm_umem_cookie =
+	    gfxp_umem_cookie_init(obj->kaddr, obj->real_size);
+	if (map->drm_umem_cookie == NULL) {
+		goto err6;
+	}
+
+	obj->map = map;
+
+	atomic_set(&obj->refcount, 1);
+	atomic_set(&obj->handlecount, 1);
+	if (dev->driver->gem_init_object != NULL &&
+	    dev->driver->gem_init_object(obj) != 0) {
+		goto err7;
+	}
+	atomic_inc(&dev->object_count);
+	atomic_add(obj->size, &dev->object_memory);
+
+	return (obj);
+
+err7:
+	gfxp_umem_cookie_destroy(map->drm_umem_cookie);
+err6:
+	drm_free(map, sizeof (struct drm_local_map), DRM_MEM_MAPS);
+err5:
+	kmem_free(obj->pfnarray, real_pgcnt * sizeof (pfn_t));
+err4:
+	(void) ddi_dma_unbind_handle(obj->dma_hdl);
+err3:
+	ddi_dma_mem_free(&obj->acc_hdl);
+err2:
+	ddi_dma_free_handle(&obj->dma_hdl);
+err1:
+	kmem_free(obj, sizeof (struct drm_gem_object));
+
+	return (NULL);
+}
+
+/*
+ * Removes the mapping from handle to filp for this object.
+ */
+static int
+drm_gem_handle_delete(struct drm_file *filp, int handle)
+{
+	struct drm_device *dev;
+	struct drm_gem_object *obj;
+	int err;
+	/*
+	 * This is gross. The idr system doesn't let us try a delete and
+	 * return an error code.  It just spews if you fail at deleting.
+	 * So, we have to grab a lock around finding the object and then
+	 * doing the delete on it and dropping the refcount, or the user
+	 * could race us to double-decrement the refcount and cause a
+	 * use-after-free later.  Given the frequency of our handle lookups,
+	 * we may want to use ida for number allocation and a hash table
+	 * for the pointers, anyway.
+	 */
+	spin_lock(&filp->table_lock);
+
+	/* Check if we currently have a reference on the object */
+	obj = idr_list_find(&filp->object_idr, handle);
+	if (obj == NULL) {
+		spin_unlock(&filp->table_lock);
+		DRM_ERROR("obj %d is not in tne list, failed to close", handle);
+		return (EINVAL);
+	}
+	dev = obj->dev;
+
+	/* Release reference and decrement refcount. */
+	err = idr_list_remove(&filp->object_idr, handle);
+	if (err == -1)
+		DRM_ERROR("%s", __func__);
+
+	spin_unlock(&filp->table_lock);
+
+	spin_lock(&dev->struct_mutex);
+	drm_gem_object_handle_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+	return (0);
+}
+
+/*
+ * Create a handle for this object. This adds a handle reference
+ * to the object, which includes a regular reference count. Callers
+ * will likely want to dereference the object afterwards.
+ */
+int
+drm_gem_handle_create(struct drm_file *file_priv,
+		    struct drm_gem_object *obj,
+		    int *handlep)
+{
+	int	ret;
+
+	/*
+	 * Get the user-visible handle using idr.
+	 */
+again:
+	/* ensure there is space available to allocate a handle */
+
+	/* do the allocation under our spinlock */
+	spin_lock(&file_priv->table_lock);
+	ret = idr_list_get_new_above(&file_priv->object_idr, obj, handlep);
+	spin_unlock(&file_priv->table_lock);
+	if (ret == -EAGAIN)
+		goto again;
+
+	if (ret != 0) {
+		DRM_ERROR("Failed to create handle");
+		return (ret);
+	}
+
+	drm_gem_object_handle_reference(obj);
+	return (0);
+}
+
+/* Returns a reference to the object named by the handle. */
+struct drm_gem_object *
+drm_gem_object_lookup(struct drm_file *filp,
+			    int handle)
+{
+	struct drm_gem_object *obj;
+
+	spin_lock(&filp->table_lock);
+
+	/* Check if we currently have a reference on the object */
+	obj = idr_list_find(&filp->object_idr, handle);
+		if (obj == NULL) {
+			spin_unlock(&filp->table_lock);
+			DRM_ERROR("object_lookup failed, handle %d", handle);
+			return (NULL);
+		}
+
+	drm_gem_object_reference(obj);
+
+	spin_unlock(&filp->table_lock);
+
+	return (obj);
+}
+
+/*
+ * Releases the handle to an mm object.
+ */
+/*ARGSUSED*/
+int
+drm_gem_close_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_gem_close args;
+	int ret;
+
+	if (!(dev->driver->use_gem == 1))
+		return (ENODEV);
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+	    (void *)data, sizeof (args));
+
+	ret = drm_gem_handle_delete(fpriv, args.handle);
+
+	return (ret);
+}
+
+/*
+ * Create a global name for an object, returning the name.
+ *
+ * Note that the name does not hold a reference; when the object
+ * is freed, the name goes away.
+ */
+/*ARGSUSED*/
+int
+drm_gem_flink_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_gem_flink args;
+	struct drm_gem_object *obj;
+	int ret, handle;
+
+	if (!(dev->driver->use_gem == 1))
+		return (ENODEV);
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+	    (void *)data, sizeof (args));
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return (EINVAL);
+	handle = args.handle;
+	spin_lock(&dev->object_name_lock);
+	if (!obj->flink) {
+		/* only creat a node in object_name_idr, no update anything */
+		ret = idr_list_get_new_above(&dev->object_name_idr,
+		    obj, &handle);
+		obj->flink = obj->name;
+		/* Allocate a reference for the name table.  */
+		drm_gem_object_reference(obj);
+	}
+	/*
+	 * Leave the reference from the lookup around as the
+	 * name table now holds one
+	 */
+	args.name = obj->name;
+
+	spin_unlock(&dev->object_name_lock);
+	ret = DRM_COPY_TO_USER((void *) data, &args, sizeof (args));
+	if (ret != 0)
+		DRM_ERROR(" gem flink error! %d", ret);
+
+	spin_lock(&dev->struct_mutex);
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	return (ret);
+}
+
+/*
+ * Open an object using the global name, returning a handle and the size.
+ *
+ * This handle (of course) holds a reference to the object, so the object
+ * will not go away until the handle is deleted.
+ */
+/*ARGSUSED*/
+int
+drm_gem_open_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_gem_open args;
+	struct drm_gem_object *obj;
+	int ret;
+	int handle;
+
+	if (!(dev->driver->use_gem == 1)) {
+		DRM_ERROR("Not support GEM");
+		return (ENODEV);
+	}
+	DRM_COPYFROM_WITH_RETURN(&args,
+	    (void *) data, sizeof (args));
+
+	spin_lock(&dev->object_name_lock);
+
+	obj = idr_list_find(&dev->object_name_idr, args.name);
+
+	if (obj)
+		drm_gem_object_reference(obj);
+	spin_unlock(&dev->object_name_lock);
+	if (!obj) {
+		DRM_ERROR("Can't find the obj %d", args.name);
+		return (ENOENT);
+	}
+
+	ret = drm_gem_handle_create(fpriv, obj, &handle);
+	spin_lock(&dev->struct_mutex);
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	args.handle = args.name;
+	args.size = obj->size;
+
+	ret = DRM_COPY_TO_USER((void *) data, &args, sizeof (args));
+	if (ret != 0)
+		DRM_ERROR(" gem open error! %d", ret);
+	return (ret);
+}
+
+/*
+ * Called at device open time, sets up the structure for handling refcounting
+ * of mm objects.
+ */
+void
+drm_gem_open(struct drm_file *file_private)
+{
+	idr_list_init(&file_private->object_idr);
+	mutex_init(&file_private->table_lock, NULL, MUTEX_DRIVER, NULL);
+}
+
+/*
+ * Called at device close to release the file's
+ * handle references on objects.
+ */
+static void
+drm_gem_object_release_handle(struct drm_gem_object *obj)
+{
+	drm_gem_object_handle_unreference(obj);
+}
+
+/*
+ * Called at close time when the filp is going away.
+ *
+ * Releases any remaining references on objects by this filp.
+ */
+void
+drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
+{
+	struct idr_list  *entry;
+	spin_lock(&dev->struct_mutex);
+
+	idr_list_for_each(entry, &file_private->object_idr)
+	    drm_gem_object_release_handle(entry->obj);
+
+	idr_list_free(&file_private->object_idr);
+	spin_unlock(&dev->struct_mutex);
+
+}
+
+/*
+ * Called after the last reference to the object has been lost.
+ *
+ * Frees the object
+ */
+void
+drm_gem_object_free(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	struct drm_local_map *map = obj->map;
+
+	if (dev->driver->gem_free_object != NULL)
+		dev->driver->gem_free_object(obj);
+
+	gfxp_umem_cookie_destroy(map->drm_umem_cookie);
+	drm_free(map, sizeof (struct drm_local_map), DRM_MEM_MAPS);
+
+	kmem_free(obj->pfnarray, btopr(obj->real_size) * sizeof (pfn_t));
+
+	(void) ddi_dma_unbind_handle(obj->dma_hdl);
+	ddi_dma_mem_free(&obj->acc_hdl);
+	ddi_dma_free_handle(&obj->dma_hdl);
+
+	atomic_dec(&dev->object_count);
+	atomic_sub(obj->size, &dev->object_memory);
+	kmem_free(obj, sizeof (struct drm_gem_object));
+}
+
+/*
+ * Called after the last handle to the object has been closed
+ *
+ * Removes any name for the object. Note that this must be
+ * called before drm_gem_object_free or we'll be touching
+ * freed memory
+ */
+void
+drm_gem_object_handle_free(struct drm_gem_object *obj)
+{
+	int err;
+	struct drm_device *dev = obj->dev;
+	/* Remove any name for this object */
+	spin_lock(&dev->object_name_lock);
+	if (obj->flink) {
+		err = idr_list_remove(&dev->object_name_idr, obj->name);
+		if (err == -1)
+			DRM_ERROR("%s", __func__);
+		obj->flink = 0;
+		spin_unlock(&dev->object_name_lock);
+		/*
+		 * The object name held a reference to this object, drop
+		 * that now.
+		 */
+		drm_gem_object_unreference(obj);
+	} else
+
+		spin_unlock(&dev->object_name_lock);
+
+}
diff --git a/usr/src/uts/common/io/drm/drm_irq.c b/usr/src/uts/common/io/drm/drm_irq.c
index 928cb63e58..3d3640a3bd 100644
--- a/usr/src/uts/common/io/drm/drm_irq.c
+++ b/usr/src/uts/common/io/drm/drm_irq.c
@@ -4,6 +4,7 @@
  */
 /*
  * Copyright 2003 Eric Anholt
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -119,6 +120,8 @@ drm_vblank_cleanup(struct drm_device *dev)
 	    dev->num_crtcs, DRM_MEM_DRIVER);
 	drm_free(dev->last_vblank, sizeof (u32) * dev->num_crtcs,
 	    DRM_MEM_DRIVER);
+	drm_free(dev->vblank_inmodeset, sizeof (*dev->vblank_inmodeset) *
+	    dev->num_crtcs, DRM_MEM_DRIVER);
 	dev->num_crtcs = 0;
 }
 
@@ -159,6 +162,12 @@ drm_vblank_init(struct drm_device *dev, int num_crtcs)
 	dev->last_vblank = drm_alloc(num_crtcs * sizeof (u32), DRM_MEM_DRIVER);
 	if (!dev->last_vblank)
 		goto err;
+
+	dev->vblank_inmodeset = drm_alloc(num_crtcs * sizeof (int),
+	    DRM_MEM_DRIVER);
+	if (!dev->vblank_inmodeset)
+		goto err;
+
 	/* Zero per-crtc vblank stuff */
 	for (i = 0; i < num_crtcs; i++) {
 		DRM_INIT_WAITQUEUE(&dev->vbl_queues[i], DRM_INTR_PRI(dev));
@@ -380,6 +389,70 @@ drm_vblank_put(struct drm_device *dev, int crtc)
 	DRM_SPINUNLOCK(&dev->vbl_lock);
 }
 
+/*
+ * drm_modeset_ctl - handle vblank event counter changes across mode switch
+ * @DRM_IOCTL_ARGS: standard ioctl arguments
+ *
+ * Applications should call the %_DRM_PRE_MODESET and %_DRM_POST_MODESET
+ * ioctls around modesetting so that any lost vblank events are accounted for.
+ *
+ * Generally the counter will reset across mode sets.  If interrupts are
+ * enabled around this call, we don't have to do anything since the counter
+ * will have already been incremented.
+ */
+/*ARGSUSED*/
+int
+drm_modeset_ctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_modeset_ctl modeset;
+	int crtc, ret = 0;
+
+	/* If drm_vblank_init() hasn't been called yet, just no-op */
+	if (!dev->num_crtcs)
+		goto out;
+
+	DRM_COPYFROM_WITH_RETURN(&modeset, (void *)data,
+	    sizeof (modeset));
+
+	crtc = modeset.crtc;
+	if (crtc >= dev->num_crtcs) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/*
+	 * To avoid all the problems that might happen if interrupts
+	 * were enabled/disabled around or between these calls, we just
+	 * have the kernel take a reference on the CRTC (just once though
+	 * to avoid corrupting the count if multiple, mismatch calls occur),
+	 * so that interrupts remain enabled in the interim.
+	 */
+	switch (modeset.cmd) {
+	case _DRM_PRE_MODESET:
+		if (!dev->vblank_inmodeset[crtc]) {
+			dev->vblank_inmodeset[crtc] = 1;
+			ret = drm_vblank_get(dev, crtc);
+		}
+		break;
+	case _DRM_POST_MODESET:
+		if (dev->vblank_inmodeset[crtc]) {
+			DRM_SPINLOCK(&dev->vbl_lock);
+			dev->vblank_disable_allowed = 1;
+			dev->vblank_inmodeset[crtc] = 0;
+			DRM_SPINUNLOCK(&dev->vbl_lock);
+			drm_vblank_put(dev, crtc);
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+out:
+	return (ret);
+}
+
 /*ARGSUSED*/
 int
 drm_wait_vblank(DRM_IOCTL_ARGS)
@@ -390,7 +463,7 @@ drm_wait_vblank(DRM_IOCTL_ARGS)
 	unsigned int	sequence;
 
 	if (!dev->irq_enabled) {
-		DRM_DEBUG("wait vblank, EINVAL");
+		DRM_ERROR("wait vblank, EINVAL");
 		return (EINVAL);
 	}
 #ifdef _MULTI_DATAMODEL
@@ -411,19 +484,20 @@ drm_wait_vblank(DRM_IOCTL_ARGS)
 
 	if (vblwait.request.type &
 	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK)) {
-		cmn_err(CE_WARN, "drm_wait_vblank: wrong request type 0x%x",
+		DRM_ERROR("drm_wait_vblank: wrong request type 0x%x",
 		    vblwait.request.type);
 		return (EINVAL);
 	}
 
 	flags = vblwait.request.type & _DRM_VBLANK_FLAGS_MASK;
 	crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
-	if (crtc >= dev->num_crtcs)
+	if (crtc >= dev->num_crtcs) {
+		DRM_ERROR("wait vblank operation not support");
 		return (ENOTSUP);
-
+	}
 	ret = drm_vblank_get(dev, crtc);
 	if (ret) {
-		DRM_DEBUG("can't get drm vblank");
+		DRM_ERROR("can't get drm vblank %d", ret);
 		return (ret);
 	}
 	sequence = drm_vblank_count(dev, crtc);
@@ -449,7 +523,7 @@ drm_wait_vblank(DRM_IOCTL_ARGS)
 		/*
 		 * Don't block process, send signal when vblank interrupt
 		 */
-		DRM_DEBUG("NOT SUPPORT YET, SHOULD BE ADDED");
+		DRM_ERROR("NOT SUPPORT YET, SHOULD BE ADDED");
 		cmn_err(CE_WARN, "NOT SUPPORT YET, SHOULD BE ADDED");
 		ret = EINVAL;
 		goto done;
@@ -457,8 +531,9 @@ drm_wait_vblank(DRM_IOCTL_ARGS)
 		/* block until vblank interupt */
 		/* shared code returns -errno */
 		DRM_WAIT_ON(ret, &dev->vbl_queues[crtc], 3 * DRM_HZ,
-		    ((drm_vblank_count(dev, crtc)
-		    - vblwait.request.sequence) <= (1 << 23)));
+		    (((drm_vblank_count(dev, crtc)
+		    - vblwait.request.sequence) <= (1 << 23)) ||
+		    !dev->irq_enabled));
 		if (ret != EINTR) {
 			struct timeval now;
 			(void) uniqtime(&now);
@@ -503,33 +578,4 @@ drm_handle_vblank(struct drm_device *dev, int crtc)
 {
 	atomic_inc(&dev->_vblank_count[crtc]);
 	DRM_WAKEUP(&dev->vbl_queues[crtc]);
-	drm_vbl_send_signals(dev);
-}
-
-/*
- * Schedule a tasklet to call back a driver hook with the HW lock held.
- *
- * \param dev DRM device.
- * \param func Driver callback.
- *
- * This is intended for triggering actions that require the HW lock from an
- * interrupt handler. The lock will be grabbed ASAP after the interrupt handler
- * completes. Note that the callback may be called from interrupt or process
- * context, it must not make any assumptions about this. Also, the HW lock will
- * be held with the kernel context or any client context.
- */
-
-void
-drm_locked_tasklet(drm_device_t *dev, void (*func)(drm_device_t *))
-{
-	mutex_enter(&dev->tasklet_lock);
-
-	if (dev->locked_tasklet_func) {
-		mutex_exit(&dev->tasklet_lock);
-		return;
-	}
-
-	dev->locked_tasklet_func = func;
-
-	mutex_exit(&dev->tasklet_lock);
 }
diff --git a/usr/src/uts/common/io/drm/drm_linux_list.h b/usr/src/uts/common/io/drm/drm_linux_list.h
index e811c9feba..02a4809bca 100644
--- a/usr/src/uts/common/io/drm/drm_linux_list.h
+++ b/usr/src/uts/common/io/drm/drm_linux_list.h
@@ -5,6 +5,7 @@
 /*
  * -
  * Copyright 2003 Eric Anholt
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -31,33 +32,48 @@
  *
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
 
 #ifndef _DRM_LINUX_LIST_H_
 #define	_DRM_LINUX_LIST_H_
 
 struct list_head {
 	struct list_head *next, *prev;
+	caddr_t contain_ptr;
 };
 
 /* Cheat, assume the list_head is at the start of the struct */
-#define	list_entry(entry, type, member)	(type *)(entry)
+#define	list_entry(entry, type, member)	(type *)(uintptr_t)(entry->contain_ptr)
 
 #define	INIT_LIST_HEAD(head) { \
 	(head)->next = head;   \
 	(head)->prev = head;   \
+	(head)->contain_ptr = (caddr_t)head;	\
 }
 
-#define	list_add_tail(entry, head) {  \
+#define	list_add(entry, head, con_ptr) {	\
+	(head)->next->prev = entry;	\
+	(entry)->next = (head)->next;	\
+	(entry)->prev = head;	\
+	(head)->next = entry;	\
+	(entry)->contain_ptr = con_ptr;	\
+}
+
+#define	list_add_tail(entry, head, con_ptr) {  \
 	(entry)->prev = (head)->prev; \
 	(entry)->next = head;         \
 	(head)->prev->next = entry;   \
 	(head)->prev = entry;         \
+	(entry)->contain_ptr = con_ptr;	\
 }
 
 #define	list_del(entry) {                         \
 	(entry)->next->prev = (entry)->prev;      \
 	(entry)->prev->next = (entry)->next;      \
+	(entry)->contain_ptr = NULL;	\
 }
 
 #define	list_for_each(entry, head)				\
@@ -65,7 +81,19 @@ struct list_head {
 
 #define	list_for_each_safe(entry, temp, head)			\
     for (entry = (head)->next, temp = (entry)->next;		\
-	temp != head; 						\
+	entry != head; 						\
 	entry = temp, temp = temp->next)
 
+#define	list_del_init(entry) {				\
+	list_del(entry);				\
+	INIT_LIST_HEAD(entry);		\
+}
+
+#define	list_move_tail(entry, head, con_ptr) {				\
+	list_del(entry);					\
+	list_add_tail(entry, head, con_ptr);		\
+}
+
+#define	list_empty(head)	((head)->next == head)
+
 #endif /* _DRM_LINUX_LIST_H_ */
diff --git a/usr/src/uts/common/io/drm/drm_memory.c b/usr/src/uts/common/io/drm/drm_memory.c
index 153841f300..cf2d5f6d9d 100644
--- a/usr/src/uts/common/io/drm/drm_memory.c
+++ b/usr/src/uts/common/io/drm/drm_memory.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -10,6 +10,7 @@
 /*
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -37,8 +38,6 @@
  *
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include "drmP.h"
 
 /* Device memory access structure */
@@ -135,6 +134,7 @@ drm_get_pci_index_reg(dev_info_t *devi, uint_t physical, uint_t size,
 			break;
 		}
 	}
+
 	kmem_free(regs, (size_t)length);
 	return (regnum);
 error:
@@ -204,8 +204,8 @@ drm_ioremap(drm_device_t *softstate, drm_local_map_t *map)
 	map->dev_addr = iomap.drm_base;
 
 	DRM_DEBUG(
-	    "map->handle is %p map->dev_addr is %lx",
-	    (void *)map->handle, (unsigned long)map->dev_addr);
+	    "map->handle is %p map->dev_addr is %lx map->size %x",
+	    (void *)map->handle, (unsigned long)map->dev_addr, map->size);
 
 	return (0);
 }
diff --git a/usr/src/uts/common/io/drm/drm_mm.c b/usr/src/uts/common/io/drm/drm_mm.c
new file mode 100644
index 0000000000..d2d70c4cba
--- /dev/null
+++ b/usr/src/uts/common/io/drm/drm_mm.c
@@ -0,0 +1,336 @@
+/*
+ *
+ * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA.
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files(the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice(including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+
+/*
+ * Generic simple memory manager implementation. Intended to be used as a base
+ * class implementation for more advanced memory managers.
+ *
+ * Note that the algorithm used is quite simple and there might be substantial
+ * performance gains if a smarter free list is implemented.
+ * Currently it is just an
+ * unordered stack of free regions. This could easily be improved if an RB-tree
+ * is used instead. At least if we expect heavy fragmentation.
+ *
+ * Aligned allocations can also see improvement.
+ *
+ * Authors:
+ * Thomas Hellström <thomas-at-tungstengraphics-dot-com>
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include "drmP.h"
+
+unsigned long
+drm_mm_tail_space(struct drm_mm *mm)
+{
+	struct list_head *tail_node;
+	struct drm_mm_node *entry;
+
+	tail_node = mm->ml_entry.prev;
+	entry = list_entry(tail_node, struct drm_mm_node, ml_entry);
+	if (!entry->free)
+		return (0);
+
+	return (entry->size);
+}
+
+int
+drm_mm_remove_space_from_tail(struct drm_mm *mm, unsigned long size)
+{
+	struct list_head *tail_node;
+	struct drm_mm_node *entry;
+
+	tail_node = mm->ml_entry.prev;
+	entry = list_entry(tail_node, struct drm_mm_node, ml_entry);
+	if (!entry->free)
+		return (ENOMEM);
+
+	if (entry->size <= size)
+		return (ENOMEM);
+
+	entry->size -= size;
+	return (0);
+}
+
+
+static int
+drm_mm_create_tail_node(struct drm_mm *mm,
+		    unsigned long start,
+		    unsigned long size)
+{
+	struct drm_mm_node *child;
+
+	child = (struct drm_mm_node *)
+	    drm_alloc(sizeof (*child), DRM_MEM_MM);
+	if (!child)
+		return (ENOMEM);
+
+	child->free = 1;
+	child->size = size;
+	child->start = start;
+	child->mm = mm;
+
+	list_add_tail(&child->ml_entry, &mm->ml_entry, (caddr_t)child);
+	list_add_tail(&child->fl_entry, &mm->fl_entry, (caddr_t)child);
+
+	return (0);
+}
+
+
+int
+drm_mm_add_space_to_tail(struct drm_mm *mm, unsigned long size)
+{
+	struct list_head *tail_node;
+	struct drm_mm_node *entry;
+
+	tail_node = mm->ml_entry.prev;
+	entry = list_entry(tail_node, struct drm_mm_node, ml_entry);
+	if (!entry->free) {
+		return (drm_mm_create_tail_node(mm,
+		    entry->start + entry->size, size));
+	}
+	entry->size += size;
+	return (0);
+}
+
+static struct drm_mm_node *
+drm_mm_split_at_start(struct drm_mm_node *parent,
+		    unsigned long size)
+{
+	struct drm_mm_node *child;
+
+	child = (struct drm_mm_node *)
+	    drm_alloc(sizeof (*child), DRM_MEM_MM);
+	if (!child)
+		return (NULL);
+
+	INIT_LIST_HEAD(&child->fl_entry);
+
+	child->free = 0;
+	child->size = size;
+	child->start = parent->start;
+	child->mm = parent->mm;
+
+	list_add_tail(&child->ml_entry, &parent->ml_entry, (caddr_t)child);
+	INIT_LIST_HEAD(&child->fl_entry);
+
+	parent->size -= size;
+	parent->start += size;
+	return (child);
+}
+
+/*
+ * Put a block. Merge with the previous and / or next block if they are free.
+ * Otherwise add to the free stack.
+ */
+
+void
+drm_mm_put_block(struct drm_mm_node *cur)
+{
+
+	struct drm_mm *mm = cur->mm;
+	struct list_head *cur_head = &cur->ml_entry;
+	struct list_head *root_head = &mm->ml_entry;
+	struct drm_mm_node *prev_node = NULL;
+	struct drm_mm_node *next_node;
+
+	int merged = 0;
+
+	if (cur_head->prev != root_head) {
+		prev_node = list_entry(cur_head->prev,
+		    struct drm_mm_node, ml_entry);
+		if (prev_node->free) {
+			prev_node->size += cur->size;
+			merged = 1;
+		}
+	}
+	if (cur_head->next != root_head) {
+		next_node = list_entry(cur_head->next,
+		    struct drm_mm_node, ml_entry);
+		if (next_node->free) {
+			if (merged) {
+				prev_node->size += next_node->size;
+				list_del(&next_node->ml_entry);
+				list_del(&next_node->fl_entry);
+				drm_free(next_node,
+				    sizeof (*next_node), DRM_MEM_MM);
+			} else {
+				next_node->size += cur->size;
+				next_node->start = cur->start;
+				merged = 1;
+			}
+		}
+	}
+	if (!merged) {
+		cur->free = 1;
+		list_add(&cur->fl_entry, &mm->fl_entry, (caddr_t)cur);
+	} else {
+		list_del(&cur->ml_entry);
+		drm_free(cur, sizeof (*cur), DRM_MEM_MM);
+	}
+}
+
+struct drm_mm_node *
+drm_mm_get_block(struct drm_mm_node *parent,
+		    unsigned long size,
+		    unsigned alignment)
+{
+
+	struct drm_mm_node *align_splitoff = NULL;
+	struct drm_mm_node *child;
+	unsigned tmp = 0;
+
+	if (alignment)
+		tmp = parent->start % alignment;
+
+	if (tmp) {
+		align_splitoff = drm_mm_split_at_start(parent, alignment - tmp);
+		if (!align_splitoff)
+			return (NULL);
+	}
+
+	if (parent->size == size) {
+		list_del_init(&parent->fl_entry);
+		parent->free = 0;
+		return (parent);
+	} else {
+		child = drm_mm_split_at_start(parent, size);
+	}
+
+	if (align_splitoff)
+		drm_mm_put_block(align_splitoff);
+
+	return (child);
+}
+
+struct drm_mm_node *
+drm_mm_search_free(const struct drm_mm *mm,
+		    unsigned long size,
+		    unsigned alignment,
+		    int best_match)
+{
+	struct list_head *list;
+	const struct list_head *free_stack = &mm->fl_entry;
+	struct drm_mm_node *entry;
+	struct drm_mm_node *best;
+	unsigned long best_size;
+	unsigned wasted;
+
+	best = NULL;
+	best_size = ~0UL;
+
+	list_for_each(list, free_stack) {
+		entry = list_entry(list, struct drm_mm_node, fl_entry);
+		wasted = 0;
+
+		if (entry->size < size)
+			continue;
+
+		if (alignment) {
+			register unsigned tmp = entry->start % alignment;
+			if (tmp)
+				wasted += alignment - tmp;
+		}
+
+
+		if (entry->size >= size + wasted) {
+			if (!best_match)
+				return (entry);
+			if (size < best_size) {
+				best = entry;
+				best_size = entry->size;
+			}
+		}
+	}
+
+	return (best);
+}
+
+int
+drm_mm_clean(struct drm_mm *mm)
+{
+	struct list_head *head = &mm->ml_entry;
+
+	return (head->next->next == head);
+}
+
+int
+drm_mm_init(struct drm_mm *mm, unsigned long start, unsigned long size)
+{
+	INIT_LIST_HEAD(&mm->ml_entry);
+	INIT_LIST_HEAD(&mm->fl_entry);
+
+	return (drm_mm_create_tail_node(mm, start, size));
+}
+
+
+void
+drm_mm_takedown(struct drm_mm *mm)
+{
+	struct list_head *bnode = mm->fl_entry.next;
+	struct drm_mm_node *entry;
+
+	entry = list_entry(bnode, struct drm_mm_node, fl_entry);
+
+	if (entry->ml_entry.next != &mm->ml_entry ||
+	    entry->fl_entry.next != &mm->fl_entry) {
+		DRM_ERROR("Memory manager not clean. Delaying takedown\n");
+		return;
+	}
+
+	list_del(&entry->fl_entry);
+	list_del(&entry->ml_entry);
+
+	drm_free(entry, sizeof (*entry), DRM_MEM_MM);
+}
+
+void
+drm_mm_clean_ml(const struct drm_mm *mm)
+{
+	const struct list_head *mlstack = &mm->ml_entry;
+	struct list_head *list, *temp;
+	struct drm_mm_node *entry;
+
+	if (mlstack->next == NULL)
+		return;
+
+	list_for_each_safe(list, temp, mlstack) {
+		entry = list_entry(list, struct drm_mm_node, ml_entry);
+		DRM_DEBUG("ml_entry 0x%x, size 0x%x, start 0x%x",
+		    entry, entry->size, entry->start);
+
+		list_del(&entry->fl_entry);
+		list_del(&entry->ml_entry);
+		drm_free(entry, sizeof (*entry), DRM_MEM_MM);
+	}
+}
diff --git a/usr/src/uts/common/io/drm/drm_sunmod.c b/usr/src/uts/common/io/drm/drm_sunmod.c
index a1ecff7f83..1cd39d29c4 100644
--- a/usr/src/uts/common/io/drm/drm_sunmod.c
+++ b/usr/src/uts/common/io/drm/drm_sunmod.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -98,7 +98,6 @@ struct cb_ops drm_cb_ops = {
 	D_NEW | D_MTSAFE |D_DEVMAP	/* cb_flag */
 };
 
-
 int
 _init(void)
 {
@@ -499,6 +498,9 @@ drm_sun_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
 	    ((ioctl->flags & DRM_MASTER) && !fpriv->master))
 		return (EACCES);
 
+	fpriv->dev = dev;
+	fpriv->credp = credp;
+
 	retval = func(dp, arg, fpriv, mode);
 
 	return (retval);
@@ -514,7 +516,7 @@ drm_sun_devmap(dev_t dev, devmap_cookie_t dhp, offset_t offset,
 	drm_inst_state_t	*mstate;
 	drm_device_t		*dp;
 	ddi_umem_cookie_t	cookie;
-	drm_local_map_t		*map;
+	drm_local_map_t		*map = NULL;
 	unsigned long	aperbase;
 	u_offset_t		handle;
 	offset_t		koff;
@@ -528,6 +530,11 @@ drm_sun_devmap(dev_t dev, devmap_cookie_t dhp, offset_t offset,
 		DDI_NEVERSWAP_ACC,
 		DDI_STRICTORDER_ACC,
 	};
+	static ddi_device_acc_attr_t gem_dev_attr = {
+		DDI_DEVICE_ATTR_V0,
+		DDI_NEVERSWAP_ACC,
+		DDI_MERGING_OK_ACC
+	};
 
 	mstate = drm_sup_devt_to_state(dev);
 	if (mstate == NULL)
@@ -560,29 +567,55 @@ drm_sun_devmap(dev_t dev, devmap_cookie_t dhp, offset_t offset,
 		return (EINVAL);
 	}
 
-	/*
-	 * We will solve 32-bit application on 64-bit kernel
-	 * issue later, now, we just use low 32-bit
-	 */
-	handle = (u_offset_t)offset;
-	handle &= 0xffffffff;
 	mutex_enter(&dp->dev_lock);
-	TAILQ_FOREACH(map, &dp->maplist, link) {
-		if (handle ==
-		    ((u_offset_t)((uintptr_t)map->handle) & 0xffffffff))
-			break;
+
+	if (dp->driver->use_gem == 1) {
+		struct idr_list *entry;
+		drm_cminor_t *mp;
+
+		mp = drm_find_file_by_minor(dp, minor);
+		if (!mp) {
+			mutex_exit(&dp->dev_lock);
+			DRM_ERROR("drm_sun_devmap: can't find authenticator");
+			return (EACCES);
+		}
+
+		spin_lock(&dp->struct_mutex);
+		idr_list_for_each(entry, &(mp->fpriv->object_idr)) {
+			if ((uintptr_t)entry->obj == (u_offset_t)offset) {
+				map = entry->obj->map;
+				goto goon;
+			}
+		}
+goon:
+		spin_unlock(&dp->struct_mutex);
 	}
 
-	/*
-	 * Temporarily, because offset is phys_addr for register
-	 * and framebuffer, is kernel virtual_addr for others
-	 * Maybe we will use hash table to solve this issue later.
-	 */
 	if (map == NULL) {
+		/*
+		 * We will solve 32-bit application on 64-bit kernel
+		 * issue later, now, we just use low 32-bit
+		 */
+		handle = (u_offset_t)offset;
+		handle &= 0xffffffff;
+
 		TAILQ_FOREACH(map, &dp->maplist, link) {
-			if (handle == (map->offset & 0xffffffff))
+			if (handle ==
+			    ((u_offset_t)((uintptr_t)map->handle) & 0xffffffff))
 				break;
 		}
+
+		/*
+		 * Temporarily, because offset is phys_addr for register
+		 * and framebuffer, is kernel virtual_addr for others
+		 * Maybe we will use hash table to solve this issue later.
+		 */
+		if (map == NULL) {
+			TAILQ_FOREACH(map, &dp->maplist, link) {
+				if (handle == (map->offset & 0xffffffff))
+					break;
+			}
+		}
 	}
 
 	if (map == NULL) {
@@ -704,6 +737,20 @@ drm_sun_devmap(dev_t dev, devmap_cookie_t dhp, offset_t offset,
 		*maplen = length;
 		break;
 
+	case _DRM_TTM:
+		if (map->drm_umem_cookie == NULL)
+			return (EINVAL);
+
+		if (gfxp_devmap_umem_setup(dhp, dp->dip,
+		    NULL, map->drm_umem_cookie, 0, map->size, PROT_ALL,
+		    IOMEM_DATA_UC_WR_COMBINE | DEVMAP_ALLOW_REMAP,
+		    &gem_dev_attr)) {
+			cmn_err(CE_WARN, "devmap:failed, retval=%d", ret);
+			return (DDI_FAILURE);
+		}
+		*maplen = map->size;
+		return (DDI_SUCCESS);
+
 	default:
 		return (DDI_FAILURE);
 	}
diff --git a/usr/src/uts/common/sys/agp/agpdefs.h b/usr/src/uts/common/sys/agp/agpdefs.h
index 507953f83c..5ed32cb122 100644
--- a/usr/src/uts/common/sys/agp/agpdefs.h
+++ b/usr/src/uts/common/sys/agp/agpdefs.h
@@ -20,6 +20,11 @@
  */
 
 /*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+/*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -103,6 +108,7 @@ extern "C" {
 #define	INTEL_BR_Q45			0x2e108086
 #define	INTEL_BR_G45			0x2e208086
 #define	INTEL_BR_G41			0x2e308086
+#define	INTEL_BR_B43			0x2e408086
 
 /* AGP common register offset in pci configuration space */
 #define	AGP_CONF_MISC			0x51 /* one byte */
@@ -172,6 +178,36 @@ extern "C" {
 #define	INTEL_IGD_Q45			0x2e128086
 #define	INTEL_IGD_G45			0x2e228086
 #define	INTEL_IGD_G41			0x2e328086
+#define	INTEL_IGD_B43			0x2e428086
+
+/* Intel 915 and 945 series */
+#define	IS_INTEL_915(device) ((device == INTEL_IGD_915) ||	\
+	(device == INTEL_IGD_915GM) ||	\
+	(device == INTEL_IGD_945) ||	\
+	(device == INTEL_IGD_945GM) ||	\
+	(device == INTEL_IGD_945GME))
+
+/* Intel 965 series */
+#define	IS_INTEL_965(device) ((device == INTEL_IGD_946GZ) ||	\
+	(device == INTEL_IGD_965G1) ||	\
+	(device == INTEL_IGD_965Q) ||	\
+	(device == INTEL_IGD_965G2) ||	\
+	(device == INTEL_IGD_965GM) ||	\
+	(device == INTEL_IGD_965GME) ||	\
+	(device == INTEL_IGD_GM45) ||	\
+	IS_INTEL_G4X(device))
+
+/* Intel G33 series */
+#define	IS_INTEL_X33(device) ((device == INTEL_IGD_Q35) ||	\
+	(device == INTEL_IGD_G33) ||	\
+	(device == INTEL_IGD_Q33))
+
+/* Intel G4X series */
+#define	IS_INTEL_G4X(device) ((device == INTEL_IGD_EL) ||	\
+	(device == INTEL_IGD_Q45) ||	\
+	(device == INTEL_IGD_G45) ||	\
+	(device == INTEL_IGD_G41) ||	\
+	(device == INTEL_IGD_B43))
 
 /* register offsets in PCI config space */
 #define	I8XX_CONF_GMADR			0x10 /* GMADR of i8xx series */
@@ -296,6 +332,7 @@ extern "C" {
 #define	GIGA_MASK		0xC0000000
 #define	UI32_MASK		0xffffffffU
 #define	MAXAPERMEGAS		0x1000 /* Aper size no more than 4G */
+#define	MINAPERMEGAS		192
 
 #endif /* _KERNEL */
 
diff --git a/usr/src/uts/common/sys/agp/agpgart_impl.h b/usr/src/uts/common/sys/agp/agpgart_impl.h
index 997e0da948..a8f5b6bda6 100644
--- a/usr/src/uts/common/sys/agp/agpgart_impl.h
+++ b/usr/src/uts/common/sys/agp/agpgart_impl.h
@@ -1,13 +1,16 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_SYS_AGPGART_IMPL_H
 #define	_SYS_AGPGART_IMPL_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -123,6 +126,12 @@ typedef struct _agp_info32 {
 } agp_info32_t;
 #endif /* _MULTI_DATAMODEL */
 
+struct list_head {
+	struct list_head *next, *prev;
+	struct igd_gtt_seg  *gttseg;
+};
+
+
 typedef struct	agpgart_softstate {
 	dev_info_t	*asoft_dip;
 	kmutex_t	asoft_instmutex;
@@ -147,6 +156,7 @@ typedef struct	agpgart_softstate {
 	/* all registered agp device in here */
 	agp_registered_dev_t	asoft_devreg;
 	kstat_t			*asoft_ksp;
+	struct		list_head	mapped_list;
 } agpgart_softstate_t;
 
 typedef struct agpgart_ctx {
diff --git a/usr/src/uts/common/sys/agp/agpmaster_io.h b/usr/src/uts/common/sys/agp/agpmaster_io.h
index c608c9a647..1202e76a6c 100644
--- a/usr/src/uts/common/sys/agp/agpmaster_io.h
+++ b/usr/src/uts/common/sys/agp/agpmaster_io.h
@@ -20,15 +20,18 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_SYS_AGPMASTER_IO_H
 #define	_SYS_AGPMASTER_IO_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/usr/src/uts/common/sys/agp/agptarget_io.h b/usr/src/uts/common/sys/agp/agptarget_io.h
index 75c21dbaf7..316e2aba84 100644
--- a/usr/src/uts/common/sys/agp/agptarget_io.h
+++ b/usr/src/uts/common/sys/agp/agptarget_io.h
@@ -1,13 +1,11 @@
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_SYS_AGPTARGET_IO_H
 #define	_SYS_AGPTARGET_IO_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -27,6 +25,9 @@ extern "C" {
 #define	AGP_TARGET_FLUSH_GTLB	_IO(AGPTARGETIOC_BASE, 35)
 #define	AGP_TARGET_CONFIGURE	_IO(AGPTARGETIOC_BASE, 36)
 #define	AGP_TARGET_UNCONFIG	_IO(AGPTARGETIOC_BASE, 37)
+#define	INTEL_CHIPSET_FLUSH_SETUP	_IO(AGPTARGETIOC_BASE, 38)
+#define	INTEL_CHIPSET_FLUSH	_IO(AGPTARGETIOC_BASE, 39)
+#define	INTEL_CHIPSET_FLUSH_FREE	_IO(AGPTARGETIOC_BASE, 40)
 
 /* Internal agp info struct */
 typedef struct _i_agp_info {
diff --git a/usr/src/uts/common/sys/agpgart.h b/usr/src/uts/common/sys/agpgart.h
index 2798f3d306..c8770aea88 100644
--- a/usr/src/uts/common/sys/agpgart.h
+++ b/usr/src/uts/common/sys/agpgart.h
@@ -1,10 +1,11 @@
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 /*
  * Copyright (c) 2000 Doug Rabson
+ * Copyright (c) 2009, Intel Corporation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,8 +34,6 @@
 #ifndef	_SYS_AGPGART_H
 #define	_SYS_AGPGART_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -69,6 +68,18 @@ typedef struct _agp_allocate {
 	uint32_t	agpa_physical;	/* for i810 only, private */
 } agp_allocate_t;
 
+typedef struct _agp_bind_pages {
+	uint32_t	agpb_pgstart;
+	pfn_t		*agpb_pages;
+	unsigned long 	agpb_pgcount;
+} agp_bind_pages_t;
+
+typedef struct _agp_unbind_pages {
+	uint32_t	agpb_pgstart;
+	unsigned long	agpb_pgcount;
+	uint32_t	agpb_type;
+} agp_unbind_pages_t;
+
 typedef struct _agp_bind {
 	int32_t		agpb_key;
 	uint32_t	agpb_pgstart;
@@ -92,6 +103,10 @@ typedef struct _agp_unbind {
 #define	AGPIOC_IOREMAP_FREE	_IO(AGPIOC_BASE, 9)
 #define	AGPIOC_READ		_IO(AGPIOC_BASE, 10)
 #define	AGPIOC_WRITE		_IO(AGPIOC_BASE, 11)
+#define	AGPIOC_FLUSHCHIPSET	_IO(AGPIOC_BASE, 12)
+#define	AGPIOC_PAGES_BIND	_IOW(AGPIOC_BASE, 13, agp_bind_pages_t)
+#define	AGPIOC_PAGES_UNBIND	_IOW(AGPIOC_BASE, 14, agp_unbind_pages_t)
+#define	AGPIOC_PAGES_REBIND	_IO(AGPIOC_BASE, 15)
 
 /* AGP status register bits definition */
 #define	AGPSTAT_RQ_MASK		0xff000000	/* target only */
diff --git a/usr/src/uts/intel/Makefile.files b/usr/src/uts/intel/Makefile.files
index a5c6c76cd5..502a99e4ab 100644
--- a/usr/src/uts/intel/Makefile.files
+++ b/usr/src/uts/intel/Makefile.files
@@ -146,7 +146,8 @@ FD_OBJS += fd.o
 GDA_OBJS += gda.o
 GHD_OBJS += ghd.o ghd_debug.o ghd_dma.o ghd_queue.o ghd_scsa.o \
 	ghd_scsi.o ghd_timer.o ghd_waitq.o ghd_gcmd.o
-I915_OBJS += i915_dma.o i915_drv.o i915_irq.o i915_mem.o
+I915_OBJS += i915_dma.o i915_drv.o i915_irq.o i915_mem.o \
+	i915_gem.o i915_gem_debug.o i915_gem_tiling.o
 NSKERN_OBJS += nsc_asm.o
 PCICFG_OBJS += pcicfg.o
 PCI_PCINEXUS_OBJS += pci_pci.o
diff --git a/usr/src/uts/intel/agptarget/Makefile b/usr/src/uts/intel/agptarget/Makefile
index 6d90d9f676..d8064ea72b 100644
--- a/usr/src/uts/intel/agptarget/Makefile
+++ b/usr/src/uts/intel/agptarget/Makefile
@@ -1,12 +1,10 @@
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # uts/intel/agptarget/Makefile
 #
 #
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-#
 #	This makefile drives the framework of agp protocol
 #	(agptarget) kernel module.
 #
@@ -24,6 +22,11 @@ LINTS		= $(AGPTARGET_OBJS:%.o=$(LINTS_DIR)/%.ln)
 ROOTMODULE	= $(ROOT_DRV_DIR)/$(MODULE)
 
 #
+#       dependency
+#
+LDFLAGS += -dy -Nmisc/busra
+
+#
 #	Include common rules.
 #
 include $(UTSBASE)/intel/Makefile.intel
diff --git a/usr/src/uts/intel/io/agpgart/agpgart.c b/usr/src/uts/intel/io/agpgart/agpgart.c
index 005dc57ac0..0f7ce74458 100644
--- a/usr/src/uts/intel/io/agpgart/agpgart.c
+++ b/usr/src/uts/intel/io/agpgart/agpgart.c
@@ -1,5 +1,10 @@
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -59,8 +64,55 @@ static void *agpgart_glob_soft_handle;
 #define	IS_TRUE_AGP(type)	(((type) == ARC_INTELAGP) || \
 	((type) == ARC_AMD64AGP))
 
+#define	AGP_HASH_NODE	1024
+
+static void
+list_head_init(struct list_head  *head) {
+	struct	list_head	*entry,	*tmp;
+	/* HASH for accelerate */
+	entry = kmem_zalloc(AGP_HASH_NODE *
+		sizeof (struct list_head), KM_NOSLEEP);
+	head->next = entry;
+	for (int i = 0; i < AGP_HASH_NODE; i++) {
+	tmp = &entry[i];
+	tmp->next = tmp;
+	tmp->prev = tmp;
+	tmp->gttseg = NULL;
+	}
+}
+
+static void
+list_head_add_new(struct list_head	*head,
+		igd_gtt_seg_t	*gttseg)
+{
+	struct list_head  *entry, *tmp;
+	int key;
+	entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
+	key = gttseg->igs_pgstart % AGP_HASH_NODE;
+	tmp = &head->next[key];
+	tmp->next->prev = entry;
+	entry->next = tmp->next;
+	entry->prev = tmp;
+	tmp->next = entry;
+	entry->gttseg = gttseg;
+}
+
+static void
+list_head_del(struct list_head	*entry) {
+	(entry)->next->prev = (entry)->prev;      \
+	(entry)->prev->next = (entry)->next;      \
+	(entry)->gttseg = NULL; \
+}
+
+#define	list_head_for_each_safe(entry,	temp,	head)	\
+	for (int key = 0; key < AGP_HASH_NODE; key++)	\
+	for (entry = (&(head)->next[key])->next, temp = (entry)->next;	\
+		entry != &(head)->next[key];	\
+		entry = temp, temp = temp->next)
+
+
 #define	agpinfo_default_to_32(v, v32)	\
-	{				\
+	{	\
 		(v32).agpi32_version = (v).agpi_version;	\
 		(v32).agpi32_devid = (v).agpi_devid;	\
 		(v32).agpi32_mode = (v).agpi_mode;	\
@@ -1134,7 +1186,7 @@ lyr_flush_gart_cache(agp_registered_dev_t *agp_regdev)
  *	based on the ammount of physical pages.
  * 	The algorithm is: compare the aperture size with 1/4 of total
  *	physical pages, and use the smaller one to for the max available
- * 	pages.
+ * 	pages. But the minimum video memory should be 192M.
  *
  * Arguments:
  * 	aper_size	system agp aperture size (in MB)
@@ -1145,14 +1197,18 @@ lyr_flush_gart_cache(agp_registered_dev_t *agp_regdev)
 static uint32_t
 get_max_pages(uint32_t aper_size)
 {
-	uint32_t i, j;
+	uint32_t i, j, size;
 
 	ASSERT(aper_size <= MAXAPERMEGAS);
 
 	i = AGP_MB2PAGES(aper_size);
 	j = (physmem >> 2);
 
-	return ((i < j) ? i : j);
+	size = ((i < j) ? i : j);
+
+	if (size < AGP_MB2PAGES(MINAPERMEGAS))
+		size = AGP_MB2PAGES(MINAPERMEGAS);
+	return (size);
 }
 
 /*
@@ -2438,6 +2494,8 @@ agpgart_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	    AGP_MAXKEYS * (sizeof (keytable_ent_t)),
 	    KM_SLEEP);
 
+	list_head_init(&softstate->mapped_list);
+
 	return (DDI_SUCCESS);
 err4:
 	agp_fini_kstats(softstate);
@@ -2484,6 +2542,21 @@ agpgart_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 		st->asoft_table = 0;
 	}
 
+	struct list_head	*entry,	*temp,	*head;
+	igd_gtt_seg_t	*gttseg;
+	list_head_for_each_safe(entry, temp, &st->mapped_list) {
+		gttseg = entry->gttseg;
+		list_head_del(entry);
+		kmem_free(entry, sizeof (*entry));
+		kmem_free(gttseg->igs_phyaddr,
+		    sizeof (uint32_t) * gttseg->igs_npage);
+		kmem_free(gttseg, sizeof (igd_gtt_seg_t));
+	}
+	head = &st->mapped_list;
+	kmem_free(head->next,
+	    AGP_HASH_NODE * sizeof (struct list_head));
+	head->next = NULL;
+
 	ddi_remove_minor_node(dip, AGPGART_DEVNODE);
 	agp_fini_kstats(st);
 	ldi_ident_release(st->asoft_li);
@@ -2557,6 +2630,7 @@ agpgart_open(dev_t *dev, int openflags, int otyp, cred_t *credp)
 	int instance = AGP_DEV2INST(*dev);
 	agpgart_softstate_t *softstate;
 	int rc = 0;
+	uint32_t devid;
 
 	if (secpolicy_gart_access(credp)) {
 		AGPDB_PRINT2((CE_WARN, "agpgart_open: permission denied"));
@@ -2612,6 +2686,21 @@ agpgart_open(dev_t *dev, int openflags, int otyp, cred_t *credp)
 
 			return (EIO);
 		}
+		devid = softstate->asoft_info.agpki_mdevid;
+		if (IS_INTEL_915(devid) ||
+		    IS_INTEL_965(devid) ||
+		    IS_INTEL_X33(devid) ||
+		    IS_INTEL_G4X(devid)) {
+			rc = ldi_ioctl(softstate->asoft_devreg.agprd_targethdl,
+			    INTEL_CHIPSET_FLUSH_SETUP, 0, FKIOCTL, kcred, 0);
+		}
+		if (rc) {
+			AGPDB_PRINT2((CE_WARN,
+			    "agpgart_open: Intel chipset flush setup error"));
+			lyr_end(&softstate->asoft_devreg);
+			mutex_exit(&softstate->asoft_instmutex);
+			return (EIO);
+		}
 		mutex_exit(&softstate->asoft_instmutex);
 		return (0);
 	}
@@ -2698,6 +2787,8 @@ agpgart_close(dev_t dev, int flag, int otyp, cred_t *credp)
 {
 	int instance = AGP_DEV2INST(dev);
 	agpgart_softstate_t *softstate;
+	int rc = 0;
+	uint32_t devid;
 
 	softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance);
 	if (softstate == NULL) {
@@ -2721,6 +2812,19 @@ agpgart_close(dev_t dev, int flag, int otyp, cred_t *credp)
 		release_control(softstate);
 	}
 
+	devid = softstate->asoft_info.agpki_mdevid;
+	if (IS_INTEL_915(devid) ||
+	    IS_INTEL_965(devid) ||
+	    IS_INTEL_X33(devid) ||
+	    IS_INTEL_G4X(devid)) {
+		rc = ldi_ioctl(softstate->asoft_devreg.agprd_targethdl,
+		    INTEL_CHIPSET_FLUSH_FREE, 0, FKIOCTL, kcred, 0);
+	}
+	if (rc) {
+		AGPDB_PRINT2((CE_WARN,
+		    "agpgart_open: Intel chipset flush free error"));
+	}
+
 	if (lyr_unconfig_devices(&softstate->asoft_devreg)) {
 		AGPDB_PRINT2((CE_WARN,
 		    "agpgart_close: lyr_unconfig_device error"));
@@ -3020,6 +3124,144 @@ ioctl_agpgart_unbind(agpgart_softstate_t  *st, void  *arg, int flags)
 	return (0);
 }
 
+static int
+ioctl_agpgart_flush_chipset(agpgart_softstate_t *st)
+{
+	ldi_handle_t	hdl;
+	uint32_t devid;
+	int rc = 0;
+	devid = st->asoft_info.agpki_mdevid;
+	hdl = st->asoft_devreg.agprd_targethdl;
+	if (IS_INTEL_915(devid) ||
+	    IS_INTEL_965(devid) ||
+	    IS_INTEL_X33(devid) ||
+	    IS_INTEL_G4X(devid)) {
+		rc = ldi_ioctl(hdl, INTEL_CHIPSET_FLUSH, 0, FKIOCTL, kcred, 0);
+	}
+	return	(rc);
+}
+
+static int
+ioctl_agpgart_pages_bind(agpgart_softstate_t  *st, void  *arg, int flags)
+{
+	agp_bind_pages_t 	bind_info;
+	uint32_t	pg_offset;
+	int err = 0;
+	ldi_handle_t hdl;
+	uint32_t npages;
+	igd_gtt_seg_t *gttseg;
+	uint32_t i;
+	int rval;
+	if (ddi_copyin(arg, &bind_info,
+	    sizeof (agp_bind_pages_t), flags) != 0) {
+		return (EFAULT);
+	}
+
+	gttseg = (igd_gtt_seg_t *)kmem_zalloc(sizeof (igd_gtt_seg_t),
+	    KM_SLEEP);
+
+	pg_offset = bind_info.agpb_pgstart;
+
+	gttseg->igs_pgstart =  pg_offset;
+	npages = (uint32_t)bind_info.agpb_pgcount;
+	gttseg->igs_npage = npages;
+
+	gttseg->igs_type = AGP_NORMAL;
+	gttseg->igs_phyaddr = (uint32_t *)kmem_zalloc
+	    (sizeof (uint32_t) * gttseg->igs_npage, KM_SLEEP);
+
+	for (i = 0; i < npages; i++) {
+		gttseg->igs_phyaddr[i] = bind_info.agpb_pages[i] <<
+		    GTT_PAGE_SHIFT;
+	}
+
+	hdl = st->asoft_devreg.agprd_masterhdl;
+	if (ldi_ioctl(hdl, I8XX_ADD2GTT, (intptr_t)gttseg, FKIOCTL,
+	    kcred, &rval)) {
+		AGPDB_PRINT2((CE_WARN, "ioctl_agpgart_pages_bind: start0x%x",
+		    gttseg->igs_pgstart));
+		AGPDB_PRINT2((CE_WARN, "ioctl_agpgart_pages_bind: pages=0x%x",
+		    gttseg->igs_npage));
+		AGPDB_PRINT2((CE_WARN, "ioctl_agpgart_pages_bind: type=0x%x",
+		    gttseg->igs_type));
+		err = -1;
+	}
+
+	list_head_add_new(&st->mapped_list, gttseg);
+	return (err);
+}
+
+static int
+ioctl_agpgart_pages_unbind(agpgart_softstate_t  *st, void  *arg, int flags)
+{
+	agp_unbind_pages_t unbind_info;
+	int	rval;
+	ldi_handle_t	hdl;
+	igd_gtt_seg_t	*gttseg;
+
+	if (ddi_copyin(arg, &unbind_info, sizeof (unbind_info), flags) != 0) {
+		return (EFAULT);
+	}
+
+	struct list_head  *entry, *temp;
+	list_head_for_each_safe(entry, temp, &st->mapped_list) {
+		if (entry->gttseg->igs_pgstart == unbind_info.agpb_pgstart) {
+			gttseg = entry->gttseg;
+			/* not unbind if VT switch */
+			if (unbind_info.agpb_type) {
+				list_head_del(entry);
+				kmem_free(entry, sizeof (*entry));
+			}
+			break;
+		}
+	}
+	ASSERT(gttseg != NULL);
+	gttseg->igs_pgstart =  unbind_info.agpb_pgstart;
+	ASSERT(gttseg->igs_npage == unbind_info.agpb_pgcount);
+
+	hdl = st->asoft_devreg.agprd_masterhdl;
+	if (ldi_ioctl(hdl, I8XX_REM_GTT, (intptr_t)gttseg, FKIOCTL,
+	    kcred, &rval))
+		return (-1);
+
+	if (unbind_info.agpb_type) {
+		kmem_free(gttseg->igs_phyaddr, sizeof (uint32_t) *
+		    gttseg->igs_npage);
+		kmem_free(gttseg, sizeof (igd_gtt_seg_t));
+	}
+
+	return (0);
+}
+
+static int
+ioctl_agpgart_pages_rebind(agpgart_softstate_t  *st)
+{
+	int	rval;
+	ldi_handle_t	hdl;
+	igd_gtt_seg_t	*gttseg;
+	int err = 0;
+
+	hdl = st->asoft_devreg.agprd_masterhdl;
+	struct list_head  *entry, *temp;
+	list_head_for_each_safe(entry, temp, &st->mapped_list) {
+		gttseg = entry->gttseg;
+		list_head_del(entry);
+		kmem_free(entry, sizeof (*entry));
+		if (ldi_ioctl(hdl, I8XX_ADD2GTT, (intptr_t)gttseg, FKIOCTL,
+		    kcred, &rval)) {
+			AGPDB_PRINT2((CE_WARN, "agpgart_pages_rebind errori"));
+			err = -1;
+			break;
+		}
+		kmem_free(gttseg->igs_phyaddr, sizeof (uint32_t) *
+		    gttseg->igs_npage);
+		kmem_free(gttseg, sizeof (igd_gtt_seg_t));
+
+	}
+	return (err);
+
+}
+
 /*ARGSUSED*/
 static int
 agpgart_ioctl(dev_t dev, int cmd, intptr_t intarg, int flags,
@@ -3065,6 +3307,18 @@ agpgart_ioctl(dev_t dev, int cmd, intptr_t intarg, int flags,
 	case AGPIOC_UNBIND:
 		retval = ioctl_agpgart_unbind(softstate, arg, flags);
 		break;
+	case AGPIOC_FLUSHCHIPSET:
+		retval = ioctl_agpgart_flush_chipset(softstate);
+		break;
+	case AGPIOC_PAGES_BIND:
+		retval = ioctl_agpgart_pages_bind(softstate, arg, flags);
+		break;
+	case AGPIOC_PAGES_UNBIND:
+		retval = ioctl_agpgart_pages_unbind(softstate, arg, flags);
+		break;
+	case AGPIOC_PAGES_REBIND:
+		retval = ioctl_agpgart_pages_rebind(softstate);
+		break;
 	default:
 		AGPDB_PRINT2((CE_WARN, "agpgart_ioctl: wrong argument"));
 		retval = ENXIO;
diff --git a/usr/src/uts/intel/io/agpgart/agptarget.c b/usr/src/uts/intel/io/agpgart/agptarget.c
index d9b710b842..f0c1808abb 100644
--- a/usr/src/uts/intel/io/agpgart/agptarget.c
+++ b/usr/src/uts/intel/io/agpgart/agptarget.c
@@ -20,6 +20,11 @@
  */
 
 /*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+/*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -32,6 +37,7 @@
 #include <sys/stat.h>
 #include <sys/ddi.h>
 #include <sys/sunddi.h>
+#include <sys/sunndi.h>
 #include <sys/modctl.h>
 #include <sys/sunldi.h>
 #include <sys/pci.h>
@@ -44,6 +50,37 @@ int agptarget_debug_var = 0;
 #define	INST2NODENUM(inst)	(inst)
 #define	DEV2INST(dev)		(getminor(dev))
 
+static ddi_device_acc_attr_t dev_attr = {
+	DDI_DEVICE_ATTR_V0,
+	DDI_NEVERSWAP_ACC,
+	DDI_STRICTORDER_ACC,
+};
+
+static struct _i9xx_private_compat {
+	uint64_t	physical;	/* physical address */
+	uint_t	size;	/* size of mapping */
+	uint_t	regnum;	/* register number */
+	caddr_t	flush_page;	/* kernel virtual address */
+	ddi_acc_handle_t	handle; /* data access handle */
+} i9xx_private;
+
+#define	I915_IFPADDR	0x60
+#define	I965_IFPADDR	0x70
+
+#define	HIADDR(n)	((uint32_t)(((uint64_t)(n) & \
+			0xFFFFFFFF00000000ULL) >> 32))
+#define	LOADDR(n)	((uint32_t)((uint64_t)(n) & 0x00000000FFFFFFFF))
+
+/*
+ * Using for GEM to flush the chipset global
+ * write buffers on certain intel chipset
+ */
+
+static void
+intel_chipset_flush_setup(dev_info_t *dip,
+		    ddi_acc_handle_t pci_acc_hdl,
+		    int gms_off);
+
 typedef struct agp_target_softstate {
 	dev_info_t		*tsoft_dip;
 	ddi_acc_handle_t	tsoft_pcihdl;
@@ -376,6 +413,8 @@ static gms_mode_t gms_modes[] = {
 	{INTEL_BR_G45, I8XX_CONF_GC, I8XX_GC_MODE_MASK,
 		GMS_SIZE(gms_G4X), gms_G4X},
 	{INTEL_BR_G41, I8XX_CONF_GC, I8XX_GC_MODE_MASK,
+		GMS_SIZE(gms_G4X), gms_G4X},
+	{INTEL_BR_B43, I8XX_CONF_GC, I8XX_GC_MODE_MASK,
 		GMS_SIZE(gms_G4X), gms_G4X}
 };
 static int
@@ -489,6 +528,145 @@ intel_br_suspend(agp_target_softstate_t *softstate)
 	return (DDI_SUCCESS);
 }
 
+static void
+intel_chipset_flush_setup(dev_info_t *dip,
+		    ddi_acc_handle_t pci_acc_hdl, int gms_off)
+{
+	uint32_t temp_hi, temp_lo;
+	ndi_ra_request_t	request;
+	uint64_t	answer;
+	uint64_t	alen;
+	pci_regspec_t	*regs, *regs2;
+	int	n_reg, length;
+	uint32_t	i, regnum, ret;
+	ddi_acc_handle_t	conf_hdl = pci_acc_hdl;
+	uint32_t phys_hi_mask = 0;
+
+	bzero((caddr_t)&request, sizeof (ndi_ra_request_t));
+	request.ra_flags  |= NDI_RA_ALIGN_SIZE | NDI_RA_ALLOC_BOUNDED;
+	request.ra_boundbase = 0;
+	request.ra_boundlen = 0xffffffff;
+	request.ra_len = AGP_PAGE_SIZE;
+
+	/* IS_I965 || IS_G33 || IS_G4X */
+	if (gms_off > 11) {
+		temp_hi = pci_config_get32(conf_hdl, I965_IFPADDR + 4);
+		temp_lo = pci_config_get32(conf_hdl, I965_IFPADDR);
+		phys_hi_mask |= PCI_ADDR_MEM64 | I965_IFPADDR;
+	} else {
+		temp_lo = pci_config_get32(conf_hdl, I915_IFPADDR);
+		phys_hi_mask |= PCI_ADDR_MEM32 | I915_IFPADDR;
+	}
+
+	if (!(temp_lo & 0x1)) {
+		/* allocate space from the allocator */
+		if (ndi_ra_alloc(ddi_get_parent(dip),
+		    &request, &answer, &alen,
+		    NDI_RA_TYPE_MEM, NDI_RA_PASS)
+		    != NDI_SUCCESS) {
+			return;
+		}
+		TARGETDB_PRINT2((CE_WARN, "addr = 0x%x.0x%x len [0x%x]\n",
+		    HIADDR(answer),
+		    LOADDR(answer),
+		    (uint32_t)alen));
+
+		if (gms_off > 11) {
+			pci_config_put32(conf_hdl, I965_IFPADDR + 4,
+			    HIADDR(answer));
+			pci_config_put32(conf_hdl, I965_IFPADDR,
+			    LOADDR(answer) | 0x1);
+		} else {
+			pci_config_put32(conf_hdl, I915_IFPADDR,
+			    LOADDR(answer) | 0x1);
+		}
+	}
+	else
+	{
+		temp_lo &= ~0x1;
+		answer = ((uint64_t)temp_hi << 32) | temp_lo;
+	}
+
+	temp_hi = pci_config_get32(conf_hdl, I965_IFPADDR + 4);
+	temp_lo = pci_config_get32(conf_hdl, I965_IFPADDR);
+
+	/* set pci props */
+	if (ddi_dev_nregs(dip, &n_reg) == DDI_FAILURE) {
+		TARGETDB_PRINT2((CE_WARN, "init_chipset_flush failed"));
+		n_reg = 0;
+		return;
+	}
+
+	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
+	    "reg", (caddr_t)&regs, &length) !=
+	    DDI_PROP_SUCCESS) {
+		TARGETDB_PRINT2((CE_WARN, "init_chipset_flush failed!"));
+		return;
+	}
+
+	regnum = length / sizeof (pci_regspec_t);
+
+	TARGETDB_PRINT2((CE_WARN, "reg regnum %d", regnum));
+
+	regs2 = kmem_alloc((regnum + 1) * sizeof (pci_regspec_t), KM_SLEEP);
+	if (regs2 == NULL) {
+
+		TARGETDB_PRINT2((CE_WARN, "init_chipset_flush failed"));
+		goto error;
+	}
+	if (memcpy(regs2, regs, (size_t)length) == NULL) {
+		TARGETDB_PRINT2((CE_WARN, "init_chipset_flush failed"));
+		kmem_free(regs2, (regnum + 1) * sizeof (pci_regspec_t));
+		goto error;
+	}
+
+	/* Bus=0, Dev=0, Func=0 0x82001000 */
+	regs2[regnum].pci_phys_hi = PCI_REG_REL_M | phys_hi_mask;
+	regs2[regnum].pci_phys_mid = HIADDR(answer);
+	regs2[regnum].pci_phys_low = LOADDR(answer);
+	regs2[regnum].pci_size_hi = 0x00000000;
+	regs2[regnum].pci_size_low = AGP_PAGE_SIZE;
+	kmem_free(regs, (size_t)length);
+	regs = regs2;
+
+	i = ndi_prop_update_int_array(DDI_DEV_T_NONE,
+	    dip,  "reg",  (int *)regs, (uint_t)5 * (regnum + 1));
+	if (i != DDI_PROP_SUCCESS) {
+		TARGETDB_PRINT2((CE_WARN, "Failed to update reg %d", i));
+		kmem_free(regs2, (regnum + 1) * sizeof (pci_regspec_t));
+		return;
+	}
+	kmem_free(regs2, (regnum + 1) * sizeof (pci_regspec_t));
+	regs = NULL;
+
+	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
+	    "reg", (caddr_t)&regs, &length) !=
+	    DDI_PROP_SUCCESS) {
+		TARGETDB_PRINT2((CE_WARN, "init_chipset_flush: failed1!"));
+		goto error;
+	}
+	regnum = length / sizeof (pci_regspec_t);
+	kmem_free(regs, (size_t)length);
+
+	i9xx_private.physical = answer;
+	i9xx_private.size = AGP_PAGE_SIZE;
+	i9xx_private.regnum = regnum - 1;
+	ret = ddi_regs_map_setup(dip, i9xx_private.regnum,
+	    (caddr_t *)&(i9xx_private.flush_page), 0,
+	    i9xx_private.size, &dev_attr,
+	    (ddi_acc_handle_t *)&i9xx_private.handle);
+
+	if (ret != DDI_SUCCESS) {
+		TARGETDB_PRINT2((CE_WARN, "chipset_flush do_ioremap failed "));
+		i9xx_private.handle = NULL;
+		return;
+	}
+	return;
+error:
+	if (regs)
+		kmem_free(regs, (size_t)length);
+}
+
 static int
 agp_target_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 {
@@ -795,6 +973,29 @@ agp_target_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
 		break;
 	}
 
+	case INTEL_CHIPSET_FLUSH_SETUP:
+	{
+		intel_chipset_flush_setup(st->tsoft_dip,
+		    st->tsoft_pcihdl, st->tsoft_gms_off);
+		break;
+	}
+	case INTEL_CHIPSET_FLUSH:
+	{
+		if (i9xx_private.handle != NULL)
+			ddi_put32(i9xx_private.handle,
+			    (uint32_t *)(uintptr_t)i9xx_private.flush_page, 1);
+
+		break;
+	}
+	case INTEL_CHIPSET_FLUSH_FREE:
+	{
+		if (i9xx_private.handle != NULL) {
+			ddi_regs_map_free(
+			    (ddi_acc_handle_t *)&i9xx_private.handle);
+			i9xx_private.handle = NULL;
+		}
+		break;
+	}
 	default:
 		mutex_exit(&st->tsoft_lock);
 		return (ENXIO);
diff --git a/usr/src/uts/intel/io/agpmaster/agpmaster.c b/usr/src/uts/intel/io/agpmaster/agpmaster.c
index 20e48b0e8c..24b77a4201 100644
--- a/usr/src/uts/intel/io/agpmaster/agpmaster.c
+++ b/usr/src/uts/intel/io/agpmaster/agpmaster.c
@@ -20,6 +20,11 @@
  */
 
 /*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+/*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
@@ -87,35 +92,6 @@ int agpm_debug = 0;
 #define	IS_IGD(agpmaster) ((agpmaster->agpm_dev_type == DEVICE_IS_I810) || \
 	(agpmaster->agpm_dev_type == DEVICE_IS_I830))
 
-
-/* Intel 915 and 945 series */
-#define	IS_INTEL_915(agpmaster) ((agpmaster->agpm_id == INTEL_IGD_915) || \
-	(agpmaster->agpm_id == INTEL_IGD_915GM) || \
-	(agpmaster->agpm_id == INTEL_IGD_945) || \
-	(agpmaster->agpm_id == INTEL_IGD_945GM) || \
-	(agpmaster->agpm_id == INTEL_IGD_945GME))
-
-/* Intel 965 series */
-#define	IS_INTEL_965(agpmaster) ((agpmaster->agpm_id == INTEL_IGD_946GZ) || \
-	(agpmaster->agpm_id == INTEL_IGD_965G1) || \
-	(agpmaster->agpm_id == INTEL_IGD_965Q) || \
-	(agpmaster->agpm_id == INTEL_IGD_965G2) || \
-	(agpmaster->agpm_id == INTEL_IGD_965GM) || \
-	(agpmaster->agpm_id == INTEL_IGD_965GME) || \
-	(agpmaster->agpm_id == INTEL_IGD_GM45) || \
-	IS_INTEL_G4X(agpmaster))
-
-/* Intel G33 series */
-#define	IS_INTEL_X33(agpmaster) ((agpmaster->agpm_id == INTEL_IGD_Q35) || \
-	(agpmaster->agpm_id == INTEL_IGD_G33) || \
-	(agpmaster->agpm_id == INTEL_IGD_Q33))
-
-/* Intel G4X series */
-#define	IS_INTEL_G4X(agpmaster) ((agpmaster->agpm_id == INTEL_IGD_EL) || \
-	(agpmaster->agpm_id == INTEL_IGD_Q45) || \
-	(agpmaster->agpm_id == INTEL_IGD_G45) || \
-	(agpmaster->agpm_id == INTEL_IGD_G41))
-
 static struct modlmisc modlmisc = {
 	&mod_miscops, "AGP master interfaces"
 };
@@ -288,7 +264,7 @@ set_gtt_mmio(dev_info_t *devi, agp_master_softc_t *agpmaster,
 	off_t gmadr_off;  /* GMADR offset in PCI config space */
 	int status;
 
-	if (IS_INTEL_X33(agpmaster)) {
+	if (IS_INTEL_X33(agpmaster->agpm_id)) {
 		/* Intel 3 series are similar with 915/945 series */
 		status = ddi_regs_map_setup(devi, I915_GTTADDR,
 		    &GTT_ADDR(agpmaster), 0, 0, &i8xx_dev_access,
@@ -303,13 +279,13 @@ set_gtt_mmio(dev_info_t *devi, agp_master_softc_t *agpmaster,
 		gmadr_off = I915_CONF_GMADR;
 		/* Different computing method used in getting aperture size. */
 		apersize = i3XX_apersize(pci_acc_hdl);
-	} else if (IS_INTEL_965(agpmaster)) {
+	} else if (IS_INTEL_965(agpmaster->agpm_id)) {
 		status = ddi_regs_map_setup(devi, I965_GTTMMADR,
 		    &MMIO_BASE(agpmaster), 0, 0, &i8xx_dev_access,
 		    &MMIO_HANDLE(agpmaster));
 		CHECK_STATUS(status);
 		if ((agpmaster->agpm_id == INTEL_IGD_GM45) ||
-		    IS_INTEL_G4X(agpmaster))
+		    IS_INTEL_G4X(agpmaster->agpm_id))
 			GTT_ADDR(agpmaster) =
 			    MMIO_BASE(agpmaster) + GM45_GTT_OFFSET;
 		else
@@ -319,7 +295,7 @@ set_gtt_mmio(dev_info_t *devi, agp_master_softc_t *agpmaster,
 
 		gmadr_off = I915_CONF_GMADR;
 		apersize = i965_apersize(agpmaster);
-	} else if (IS_INTEL_915(agpmaster)) {
+	} else if (IS_INTEL_915(agpmaster->agpm_id)) {
 		/* I915/945 series */
 		status = ddi_regs_map_setup(devi, I915_GTTADDR,
 		    &GTT_ADDR(agpmaster), 0, 0, &i8xx_dev_access,
@@ -650,6 +626,7 @@ detect_i8xx_device(agp_master_softc_t *master_softc)
 	case INTEL_IGD_Q45:
 	case INTEL_IGD_G45:
 	case INTEL_IGD_G41:
+	case INTEL_IGD_B43:
 		master_softc->agpm_dev_type = DEVICE_IS_I830;
 		break;
 	default:		/* unknown id */
diff --git a/usr/src/uts/intel/io/drm/drm_pciids.h b/usr/src/uts/intel/io/drm/drm_pciids.h
index 7d7f114392..16de72ca61 100644
--- a/usr/src/uts/intel/io/drm/drm_pciids.h
+++ b/usr/src/uts/intel/io/drm/drm_pciids.h
@@ -207,6 +207,7 @@ extern "C" {
 	{0x8086, 0x2E12, CHIP_I9XX|CHIP_I965, "Intel Q45"}, \
 	{0x8086, 0x2E22, CHIP_I9XX|CHIP_I965, "Intel G45"}, \
 	{0x8086, 0x2E32, CHIP_I9XX|CHIP_I965, "Intel G41"}, \
+	{0x8086, 0x2E42, CHIP_I9XX|CHIP_I965, "Intel B43"}, \
 	{0, 0, 0, NULL}
 
 #ifdef	__cplusplus
diff --git a/usr/src/uts/intel/io/drm/i915_dma.c b/usr/src/uts/intel/io/drm/i915_dma.c
index a29b298767..f737a1c38d 100644
--- a/usr/src/uts/intel/io/drm/i915_dma.c
+++ b/usr/src/uts/intel/io/drm/i915_dma.c
@@ -4,6 +4,7 @@
  */
 /*
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -99,7 +100,9 @@ int i915_init_hardware_status(drm_device_t *dev)
        (void) memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
 
        I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
-       DRM_DEBUG("Enabled hardware status page\n");
+       (void) I915_READ(HWS_PGA);
+
+       DRM_DEBUG("Enabled hardware status page add 0x%lx read GEM HWS 0x%x\n",dev_priv->hw_status_page, READ_HWSP(dev_priv, 0x20));
        return 0;
 }
 
@@ -108,6 +111,7 @@ void i915_free_hardware_status(drm_device_t *dev)
        drm_i915_private_t *dev_priv = dev->dev_private;
 	if (!I915_NEED_GFX_HWS(dev)) {
 		if (dev_priv->status_page_dmah) {
+			DRM_DEBUG("free status_page_dmal %x", dev_priv->status_page_dmah);
 			drm_pci_free(dev, dev_priv->status_page_dmah);
 			dev_priv->status_page_dmah = NULL;
 			/* Need to rewrite hardware status page */
@@ -115,35 +119,14 @@ void i915_free_hardware_status(drm_device_t *dev)
 		}
        	} else {
 		if (dev_priv->status_gfx_addr) {
+			DRM_DEBUG("free status_gfx_addr %x", dev_priv->status_gfx_addr);
 			dev_priv->status_gfx_addr = 0;
 			drm_core_ioremapfree(&dev_priv->hws_map, dev);
 			I915_WRITE(HWS_PGA, 0x1ffff000);
 		}
 	}
-}
 
-#if I915_RING_VALIDATE
-/**
- * Validate the cached ring tail value
- *
- * If the X server writes to the ring and DRM doesn't
- * reload the head and tail pointers, it will end up writing
- * data to the wrong place in the ring, causing havoc.
- */
-void i915_ring_validate(struct drm_device *dev, const char *func, int line)
-{
-       drm_i915_private_t *dev_priv = dev->dev_private;
-       drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
-       u32     tail = I915_READ(PRB0_TAIL) & HEAD_ADDR;
-       u32     head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
-
-       if (tail != ring->tail) {
-               DRM_ERROR("%s:%d head sw %x, hw %x. tail sw %x hw %x\n",
-                         func, line,
-                         ring->head, head, ring->tail, tail);
-       }
 }
-#endif
 
 void i915_kernel_lost_context(drm_device_t * dev)
 {
@@ -177,11 +160,13 @@ static int i915_dma_cleanup(drm_device_t * dev)
 		dev_priv->ring.map.size = 0;
 	}
 
+#ifdef I915_HAVE_GEM
+	if (I915_NEED_GFX_HWS(dev))    
+#endif
 	i915_free_hardware_status(dev);
 
 	dev_priv->sarea = NULL;
 	dev_priv->sarea_priv = NULL;
-	dev_priv->mmio_map = NULL;
 
 	return 0;
 }
@@ -200,30 +185,18 @@ static int i915_initialize(drm_device_t * dev,
 		return (EINVAL);
 	}
 
-	/*
-	 * mmio_map will be destoried after DMA clean up.  We should not
-	 * access mmio_map in suspend or resume process.
-	 */
-
- 	dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
+	dev_priv->sarea_priv = (drm_i915_sarea_t *)(uintptr_t)
+			((u8 *) dev_priv->sarea->handle +
+			 init->sarea_priv_offset);
 
-	 if (!dev_priv->mmio_map) {
-			dev->dev_private = (void *)dev_priv;
+	if (init->ring_size != 0) {
+		if (dev_priv->ring.ring_obj != NULL) {
 			(void) i915_dma_cleanup(dev);
-			DRM_ERROR("can not find mmio map!\n");
-			return (EINVAL);
-	 }
-
-       if (init->sarea_priv_offset)
-               dev_priv->sarea_priv = (drm_i915_sarea_t *)
-                       ((unsigned long) dev_priv->sarea->handle +
-                        init->sarea_priv_offset);
-       else {
-               /* No sarea_priv for you! */
-               dev_priv->sarea_priv = NULL;
-        }
+			DRM_ERROR("Client tried to initialize ringbuffer in "
+				  "GEM mode\n");
+			return -EINVAL;
+		}
 
-	if (init->ring_size != 0) {
 		dev_priv->ring.Size = init->ring_size;
 		dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
 
@@ -241,35 +214,18 @@ static int i915_initialize(drm_device_t * dev,
 			  " ring buffer\n");
 			return (ENOMEM);
 		}
-
-		dev_priv->ring.virtual_start = (u8 *)dev_priv->ring.map.dev_addr;
 	}
 
+	dev_priv->ring.virtual_start = (u8 *)dev_priv->ring.map.dev_addr;
 	dev_priv->cpp = init->cpp;
-
-	if (dev_priv->sarea_priv)
-		dev_priv->sarea_priv->pf_current_page = 0;
-
-	/* We are using separate values as placeholders for mechanisms for
-	 * private backbuffer/depthbuffer usage.
-	 */
+	dev_priv->back_offset = init->back_offset;
+	dev_priv->front_offset = init->front_offset;
+	dev_priv->current_page = 0;
+	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
 
 	/* Allow hardware batchbuffers unless told otherwise.
 	 */
 	dev_priv->allow_batchbuffer = 1;
-
-	/* Init HWS */
-	if (!I915_NEED_GFX_HWS(dev)) {
-		(void) i915_init_hardware_status(dev);
-	}
-
-	/* Enable vblank on pipe A for older X servers
-	*/
-	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A;
-
-#ifdef I915_HAVE_BUFFER
-	drm_bo_driver_init(dev);
-#endif
 	return 0;
 }
 
@@ -277,8 +233,6 @@ static int i915_dma_resume(drm_device_t * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
-	DRM_DEBUG("i915_dma_resume\n");
-
 	if (!dev_priv->sarea) {
 		DRM_ERROR("can not find sarea!\n");
 		return (EINVAL);
@@ -295,7 +249,7 @@ static int i915_dma_resume(drm_device_t * dev)
 		DRM_ERROR("Can not find hardware status page\n");
 		return (EINVAL);
 	}
-	DRM_DEBUG("hw status page @ %p\n", dev_priv->hw_status_page);
+	DRM_DEBUG("i915_dma_resume hw status page @ %p\n", dev_priv->hw_status_page);
 
 	if (!I915_NEED_GFX_HWS(dev))
 		I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
@@ -419,26 +373,30 @@ static int i915_emit_cmds(drm_device_t * dev, int __user * buffer, int dwords)
 	int i;
 	RING_LOCALS;
 
-	if ((dwords+1) * sizeof(int) >= dev_priv->ring.Size - 8)
+	if ((dwords+1) * sizeof(int) >= dev_priv->ring.Size - 8) {
+		DRM_ERROR(" emit cmds invalid arg");
 		return (EINVAL);
-
+	}
 	BEGIN_LP_RING((dwords+1)&~1);
 
 	for (i = 0; i < dwords;) {
 		int cmd, sz;
 
-		if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd)))
+		if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd))) {
+			DRM_ERROR("emit cmds failed to get cmd from user");
 			return (EINVAL);
+		}
 
-
-		if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
+		if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords) {
+			DRM_ERROR("emit cmds invalid");
 			return (EINVAL);
-
+		}
 		OUT_RING(cmd);
 
 		while (++i, --sz) {
 			if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i],
 							 sizeof(cmd))) {
+				DRM_ERROR("emit cmds failed get cmds");
 				return (EINVAL);
 			}
 			OUT_RING(cmd);
@@ -462,6 +420,7 @@ int i915_emit_box(drm_device_t * dev,
 	RING_LOCALS;
 
 	if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) {
+		DRM_ERROR("emit box failed to copy from user");
 		return (EFAULT);
 	}
 
@@ -501,49 +460,25 @@ void i915_emit_breadcrumb(drm_device_t *dev)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
 
-	if (++dev_priv->counter > BREADCRUMB_MASK) {
-		 dev_priv->counter = 1;
-		 DRM_DEBUG("Breadcrumb counter wrapped around\n");
-	}
-
+	dev_priv->counter++;
+	if (dev_priv->counter > 0x7FFFFFFFUL)
+		dev_priv->counter = 0;
 	if (dev_priv->sarea_priv)
 		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
 
 	BEGIN_LP_RING(4);
 	OUT_RING(MI_STORE_DWORD_INDEX);
-	OUT_RING(5 << MI_STORE_DWORD_INDEX_SHIFT);
+	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 	OUT_RING(dev_priv->counter);
 	OUT_RING(0);
 	ADVANCE_LP_RING();
 
 }
 
-
-void i915_emit_mi_flush(drm_device_t *dev, uint32_t flush)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	uint32_t flush_cmd = MI_FLUSH;
-	RING_LOCALS;
-
-	flush_cmd |= flush;
-
-	i915_kernel_lost_context(dev);
-
-	BEGIN_LP_RING(4);
-	OUT_RING(flush_cmd);
-	OUT_RING(0);
-	OUT_RING(0);
-	OUT_RING(0);
-	ADVANCE_LP_RING();
-}
-
 static int i915_dispatch_cmdbuffer(drm_device_t * dev,
 				   drm_i915_cmdbuffer_t * cmd)
 {
-#ifdef I915_HAVE_FENCE
-	drm_i915_private_t *dev_priv = dev->dev_private;
-#endif
 	int nbox = cmd->num_cliprects;
 	int i = 0, count, ret;
 
@@ -570,10 +505,6 @@ static int i915_dispatch_cmdbuffer(drm_device_t * dev,
 	}
 
 	i915_emit_breadcrumb( dev );
-#ifdef I915_HAVE_FENCE
-	if (unlikely((dev_priv->counter & 0xFF) == 0))
-		drm_fence_flush_old(dev, 0, dev_priv->counter);
-#endif
 	return 0;
 }
 
@@ -628,78 +559,53 @@ static int i915_dispatch_batchbuffer(drm_device_t * dev,
 	return 0;
 }
 
-static void i915_do_dispatch_flip(struct drm_device * dev, int plane, int sync)
+static int i915_dispatch_flip(struct drm_device * dev, int planes)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 num_pages, current_page, next_page, dspbase;
-	int shift = 2 * plane, x, y;
 	RING_LOCALS;
 
-	/* Calculate display base offset */
-	num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
-	current_page = (dev_priv->sarea_priv->pf_current_page >> shift) & 0x3;
-	next_page = (current_page + 1) % num_pages;
+	if (!dev_priv->sarea_priv)
+		return -EINVAL;
 
-	switch (next_page) {
-	default:
-	case 0:
-		dspbase = dev_priv->sarea_priv->front_offset;
-		break;
-	case 1:
-		dspbase = dev_priv->sarea_priv->back_offset;
-		break;
-	case 2:
-		dspbase = dev_priv->sarea_priv->third_offset;
-		break;
-	}
+	DRM_DEBUG("planes=0x%x pfCurrentPage=%d\n",
+		planes, dev_priv->sarea_priv->pf_current_page);
+
+	i915_kernel_lost_context(dev);
 
-	if (plane == 0) {
-		x = dev_priv->sarea_priv->planeA_x;
-		y = dev_priv->sarea_priv->planeA_y;
+	BEGIN_LP_RING(2);
+	OUT_RING(MI_FLUSH | MI_READ_FLUSH);
+	OUT_RING(0);
+	ADVANCE_LP_RING();
+
+	BEGIN_LP_RING(6);
+	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
+	OUT_RING(0);
+	if (dev_priv->current_page == 0) {
+		OUT_RING(dev_priv->back_offset);
+		dev_priv->current_page = 1;
 	} else {
-		x = dev_priv->sarea_priv->planeB_x;
-		y = dev_priv->sarea_priv->planeB_y;
+		OUT_RING(dev_priv->front_offset);
+		dev_priv->current_page = 0;
 	}
+	OUT_RING(0);
+	ADVANCE_LP_RING();
 
-	dspbase += (y * dev_priv->sarea_priv->pitch + x) * dev_priv->cpp;
-
+	BEGIN_LP_RING(2);
+	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
+	OUT_RING(0);
+	ADVANCE_LP_RING();
 
-	DRM_DEBUG("plane=%d current_page=%d dspbase=0x%x\n", plane, current_page, dspbase);
+	dev_priv->sarea_priv->last_enqueue = dev_priv->counter++;
 
 	BEGIN_LP_RING(4);
-	OUT_RING(sync ? 0 :
-		(MI_WAIT_FOR_EVENT | (plane ? MI_WAIT_FOR_PLANE_B_FLIP :
-					MI_WAIT_FOR_PLANE_A_FLIP)));
-	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | (sync ? 0 : ASYNC_FLIP) |
-		(plane ? DISPLAY_PLANE_B : DISPLAY_PLANE_A));
-	OUT_RING(dev_priv->sarea_priv->pitch * dev_priv->cpp);
-	OUT_RING(dspbase);
+	OUT_RING(MI_STORE_DWORD_INDEX);
+	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+	OUT_RING(dev_priv->counter);
+	OUT_RING(0);
 	ADVANCE_LP_RING();
 
-	dev_priv->sarea_priv->pf_current_page &= ~(0x3 << shift);
-	dev_priv->sarea_priv->pf_current_page |= next_page << shift;
-}
-
-void i915_dispatch_flip(struct drm_device * dev, int planes, int sync)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	int i;
-
-	DRM_DEBUG("planes=0x%x pfCurrentPage=%d\n",
-		planes, dev_priv->sarea_priv->pf_current_page);
-
-	i915_emit_mi_flush(dev, MI_READ_FLUSH | MI_EXE_FLUSH);
-
-	for (i = 0; i < 2; i++)
-		if (planes & (1 << i))
-			i915_do_dispatch_flip(dev, i, sync);
-
-	i915_emit_breadcrumb(dev);
-#ifdef I915_HAVE_FENCE
-	if (unlikely(!sync && ((dev_priv->counter & 0xFF) == 0)))
-		drm_fence_flush_old(dev, 0, dev_priv->counter);
-#endif
-
+	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
+	return 0;
 }
 
 static int i915_quiescent(drm_device_t * dev)
@@ -723,11 +629,16 @@ static int i915_quiescent(drm_device_t * dev)
 /*ARGSUSED*/
 static int i915_flush_ioctl(DRM_IOCTL_ARGS)
 {
+	int ret;
 	DRM_DEVICE;
 
 	LOCK_TEST_WITH_RETURN(dev, fpriv);
 
-	return i915_quiescent(dev);
+	spin_lock(&dev->struct_mutex);
+	ret = i915_quiescent(dev);
+	spin_unlock(&dev->struct_mutex);
+
+	return ret;
 }
 
 /*ARGSUSED*/
@@ -762,7 +673,6 @@ static int i915_batchbuffer(DRM_IOCTL_ARGS)
 		DRM_COPYFROM_WITH_RETURN(&batch, (void *) data,
 			sizeof(batch));
 
-
 	DRM_DEBUG("i915 batchbuffer, start %x used %d cliprects %d, counter %d\n",
 		  batch.start, batch.used, batch.num_cliprects, dev_priv->counter);
 
@@ -773,10 +683,12 @@ static int i915_batchbuffer(DRM_IOCTL_ARGS)
 						       batch.num_cliprects *
 						       sizeof(drm_clip_rect_t)))
 		return (EFAULT);
-*/		
-
+		
+*/
 
+	spin_lock(&dev->struct_mutex);
 	ret = i915_dispatch_batchbuffer(dev, &batch);
+	spin_unlock(&dev->struct_mutex);
 	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 
 	return ret;
@@ -825,7 +737,9 @@ static int i915_cmdbuffer(DRM_IOCTL_ARGS)
 	}
 */	
 
+	spin_lock(&dev->struct_mutex);
 	ret = i915_dispatch_cmdbuffer(dev, &cmdbuf);
+	spin_unlock(&dev->struct_mutex);
 	if (ret) {
 		DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
 		return ret;
@@ -835,46 +749,23 @@ static int i915_cmdbuffer(DRM_IOCTL_ARGS)
 	return 0;
 }
 
-static void i915_do_cleanup_pageflip(drm_device_t * dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	int i, planes, num_pages = dev_priv->sarea_priv->third_handle ? 3 : 2;
-
-	DRM_DEBUG("i915_do_cleanup_pageflip\n");
-
-	for (i = 0, planes = 0; i < 2; i++)
-		if (dev_priv->sarea_priv->pf_current_page & (0x3 << (2 * i))) {
-			dev_priv->sarea_priv->pf_current_page =
-				(dev_priv->sarea_priv->pf_current_page &
-				 ~(0x3 << (2 * i))) | ((num_pages - 1) << (2 * i));
-
-			planes |= 1 << i;
-		}
-
-	if (planes)
-		i915_dispatch_flip(dev, planes, 0);
-
-}
-
 /*ARGSUSED*/
 static int i915_flip_bufs(DRM_IOCTL_ARGS)
 {
 	DRM_DEVICE;
 	drm_i915_flip_t param;
+	int ret;
         DRM_COPYFROM_WITH_RETURN(&param, (drm_i915_flip_t *) data,
                                  sizeof(param));
 
 	DRM_DEBUG("i915_flip_bufs\n");
 
 	LOCK_TEST_WITH_RETURN(dev, fpriv);
-	/* This is really planes */
-	if (param.pipes & ~0x3) {
-		DRM_ERROR("Invalid planes 0x%x, only <= 0x3 is valid\n",
-			  param.pipes);
-		return -EINVAL;
-	}
-	i915_dispatch_flip(dev, param.pipes, 0);
-	return 0;
+
+	spin_lock(&dev->struct_mutex);
+	ret = i915_dispatch_flip(dev, param.pipes);
+	spin_unlock(&dev->struct_mutex);
+	return ret; 
 }
 
 /*ARGSUSED*/
@@ -916,8 +807,11 @@ static int i915_getparam(DRM_IOCTL_ARGS)
 	case I915_PARAM_CHIPSET_ID:
 		value = dev->pci_device;
 		break;
+	case I915_PARAM_HAS_GEM:
+		value = dev->driver->use_gem;
+		break;
 	default:
-		DRM_ERROR("Unknown parameter %d\n", param.param);
+		DRM_ERROR("Unknown get parameter %d\n", param.param);
 		return (EINVAL);
 	}
 
@@ -953,7 +847,7 @@ static int i915_setparam(DRM_IOCTL_ARGS)
 		dev_priv->allow_batchbuffer = param.value;
 		break;
 	default:
-		DRM_ERROR("unknown parameter %d\n", param.param);
+		DRM_ERROR("unknown set parameter %d\n", param.param);
 		return (EINVAL);
 	}
 
@@ -976,7 +870,7 @@ static int i915_set_status_page(DRM_IOCTL_ARGS)
 	}
 	DRM_COPYFROM_WITH_RETURN(&hws, (drm_i915_hws_addr_t __user *) data,
 			sizeof(hws));
-	DRM_DEBUG("set status page addr 0x%08x\n", (u32)hws.addr);
+DRM_ERROR("i915_set_status_page set status page addr 0x%08x\n", (u32)hws.addr);
 
 	dev_priv->status_gfx_addr = hws.addr & (0x1ffff<<12);
 	DRM_DEBUG("set gfx_addr 0x%08x\n", dev_priv->status_gfx_addr);
@@ -1013,7 +907,8 @@ static int i915_set_status_page(DRM_IOCTL_ARGS)
 int i915_driver_load(drm_device_t *dev, unsigned long flags)
 {
 	struct drm_i915_private *dev_priv;
-	int ret = 0;
+	unsigned long base, size;
+	int ret = 0, mmio_bar = IS_I9XX(dev) ? 0 : 1;
 
 	/* i915 has 4 more counters */
 	dev->counters += 4;
@@ -1030,8 +925,54 @@ int i915_driver_load(drm_device_t *dev, unsigned long flags)
 	dev->dev_private = (void *)dev_priv;
 	dev_priv->dev = dev;
 
-	mutex_init(&dev_priv->swaps_lock, "swap", MUTEX_DRIVER, NULL);
+	/* Add register map (needed for suspend/resume) */
+
+	base = drm_get_resource_start(dev, mmio_bar);
+	size = drm_get_resource_len(dev, mmio_bar);
+	dev_priv->mmio_map = drm_alloc(sizeof (drm_local_map_t), DRM_MEM_MAPS);
+	dev_priv->mmio_map->offset = base;
+	dev_priv->mmio_map->size = size;
+	dev_priv->mmio_map->type = _DRM_REGISTERS;
+	dev_priv->mmio_map->flags = _DRM_REMOVABLE;
+	drm_ioremap(dev, dev_priv->mmio_map);
+
+	DRM_DEBUG("i915_driverload mmio %p mmio_map->dev_addr %x", dev_priv->mmio_map, dev_priv->mmio_map->dev_addr);
+
+#if defined(__i386)
+	dev->driver->use_gem = 0;
+#else
+	if (IS_I965G(dev)) {
+		dev->driver->use_gem = 1;
+	} else {
+		dev->driver->use_gem = 0;
+	}
+#endif	/* __i386 */
+
+	dev->driver->get_vblank_counter = i915_get_vblank_counter;
+	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
+#if defined(__i386)
+	if (IS_G4X(dev) || IS_GM45(dev))
+#else
+	if (IS_G4X(dev))
+#endif
+	{
+		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
+		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
+	}
+
+
+#ifdef I915_HAVE_GEM
+	i915_gem_load(dev);
+#endif
+
+	if (!I915_NEED_GFX_HWS(dev)) {
+		ret = i915_init_hardware_status(dev);
+		if(ret)
+			return ret;
+	}
+
 	mutex_init(&dev_priv->user_irq_lock, "userirq", MUTEX_DRIVER, NULL);
+	mutex_init(&dev_priv->error_lock, "error_lock", MUTEX_DRIVER, NULL);
 
 	ret = drm_vblank_init(dev, I915_NUM_PIPE);
 	if (ret) {
@@ -1048,8 +989,8 @@ int i915_driver_unload(struct drm_device *dev)
 
        i915_free_hardware_status(dev);
 
-	DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
-        mutex_destroy(&dev_priv->swaps_lock);
+	drm_rmmap(dev, dev_priv->mmio_map);
+
         mutex_destroy(&dev_priv->user_irq_lock);
 
 	drm_free(dev->dev_private, sizeof(drm_i915_private_t),
@@ -1059,6 +1000,25 @@ int i915_driver_unload(struct drm_device *dev)
 	return 0;
 }
 
+/*ARGSUSED*/
+int i915_driver_open(drm_device_t * dev, struct drm_file *file_priv)
+{
+        struct drm_i915_file_private *i915_file_priv;
+
+        DRM_DEBUG("\n");
+        i915_file_priv = (struct drm_i915_file_private *)
+            drm_alloc(sizeof(*i915_file_priv), DRM_MEM_FILES);
+
+        if (!i915_file_priv)
+                return -ENOMEM;
+
+        file_priv->driver_priv = i915_file_priv;
+
+        i915_file_priv->mm.last_gem_seqno = 0;
+        i915_file_priv->mm.last_gem_throttle_seqno = 0;
+
+        return 0;
+}
 
 void i915_driver_lastclose(drm_device_t * dev)
 {
@@ -1068,43 +1028,28 @@ void i915_driver_lastclose(drm_device_t * dev)
 	if (!dev_priv)
 		return;
 
-#ifdef I915_HAVE_BUFFER
-	if (dev_priv->val_bufs) {
-		vfree(dev_priv->val_bufs);
-		dev_priv->val_bufs = NULL;
-	}
+#ifdef I915_HAVE_GEM
+	i915_gem_lastclose(dev);
 #endif
 
-
 	DRM_GETSAREA();
-	if (dev_priv->sarea_priv)
-		i915_do_cleanup_pageflip(dev);
 	if (dev_priv->agp_heap)
 		i915_mem_takedown(&(dev_priv->agp_heap));
-#if defined(I915_HAVE_BUFFER)
-	if (dev_priv->sarea_kmap.virtual) {
-		drm_bo_kunmap(&dev_priv->sarea_kmap);
-		dev_priv->sarea_kmap.virtual = NULL;
-		dev->lock.hw_lock = NULL;
-		dev->sigdata.lock = NULL;
-	}
-
-	if (dev_priv->sarea_bo) {
-		mutex_lock(&dev->struct_mutex);
-		drm_bo_usage_deref_locked(&dev_priv->sarea_bo);
-		mutex_unlock(&dev->struct_mutex);
-		dev_priv->sarea_bo = NULL;
-	}
-#endif
 	(void) i915_dma_cleanup(dev);
 }
 
 void i915_driver_preclose(drm_device_t * dev, drm_file_t *fpriv)
 {
-	if (dev->dev_private) {
-		drm_i915_private_t *dev_priv = dev->dev_private;
-		i915_mem_release(dev, fpriv, dev_priv->agp_heap);
-	}
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	i915_mem_release(dev, fpriv, dev_priv->agp_heap);
+}
+
+/*ARGSUSED*/
+void i915_driver_postclose(drm_device_t * dev, struct drm_file *file_priv)
+{
+	struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
+
+	drm_free(i915_file_priv, sizeof(*i915_file_priv), DRM_MEM_FILES);
 }
 
 drm_ioctl_desc_t i915_ioctls[] = {
@@ -1135,13 +1080,49 @@ drm_ioctl_desc_t i915_ioctls[] = {
 	[DRM_IOCTL_NR(DRM_I915_DESTROY_HEAP)] =
 	    {i915_mem_destroy_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
 	[DRM_IOCTL_NR(DRM_I915_SET_VBLANK_PIPE)] =
-	    {i915_vblank_pipe_set, DRM_AUTH},
+	    {i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
 	[DRM_IOCTL_NR(DRM_I915_GET_VBLANK_PIPE)] =
 	    {i915_vblank_pipe_get, DRM_AUTH},
 	[DRM_IOCTL_NR(DRM_I915_VBLANK_SWAP)] =
 	    {i915_vblank_swap, DRM_AUTH},
 	[DRM_IOCTL_NR(DRM_I915_HWS_ADDR)] =
 	    {i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY},
+#ifdef I915_HAVE_GEM
+        [DRM_IOCTL_NR(DRM_I915_GEM_INIT)] =
+            {i915_gem_init_ioctl, DRM_AUTH},
+        [DRM_IOCTL_NR(DRM_I915_GEM_EXECBUFFER)] =
+            {i915_gem_execbuffer, DRM_AUTH},
+        [DRM_IOCTL_NR(DRM_I915_GEM_PIN)] =
+            {i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY},
+        [DRM_IOCTL_NR(DRM_I915_GEM_UNPIN)] =
+            {i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY},
+        [DRM_IOCTL_NR(DRM_I915_GEM_BUSY)] =
+            {i915_gem_busy_ioctl, DRM_AUTH},
+        [DRM_IOCTL_NR(DRM_I915_GEM_THROTTLE)] =
+            {i915_gem_throttle_ioctl, DRM_AUTH},
+        [DRM_IOCTL_NR(DRM_I915_GEM_ENTERVT)] =
+            {i915_gem_entervt_ioctl, DRM_AUTH},
+        [DRM_IOCTL_NR(DRM_I915_GEM_LEAVEVT)] =
+            {i915_gem_leavevt_ioctl, DRM_AUTH},
+        [DRM_IOCTL_NR(DRM_I915_GEM_CREATE)] =
+            {i915_gem_create_ioctl, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_PREAD)] =
+            {i915_gem_pread_ioctl, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_PWRITE)] =
+            {i915_gem_pwrite_ioctl, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_MMAP)] =
+            {i915_gem_mmap_ioctl, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_SET_DOMAIN)] =
+            {i915_gem_set_domain_ioctl, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_SW_FINISH)] =
+            {i915_gem_sw_finish_ioctl, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_SET_TILING)] =
+            {i915_gem_set_tiling, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_GET_TILING)] =
+            {i915_gem_get_tiling, 0},
+        [DRM_IOCTL_NR(DRM_I915_GEM_GET_APERTURE)] =
+            {i915_gem_get_aperture_ioctl, 0},
+#endif
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
@@ -1163,12 +1144,3 @@ int i915_driver_device_is_agp(drm_device_t * dev)
 	return 1;
 }
 
-/*ARGSUSED*/
-int i915_driver_firstopen(struct drm_device *dev)
-{
-#ifdef I915_HAVE_BUFFER
-	drm_bo_driver_init(dev);
-#endif
-	return 0;
-}
-
diff --git a/usr/src/uts/intel/io/drm/i915_drm.h b/usr/src/uts/intel/io/drm/i915_drm.h
index 448e4534e6..e6b967aa39 100644
--- a/usr/src/uts/intel/io/drm/i915_drm.h
+++ b/usr/src/uts/intel/io/drm/i915_drm.h
@@ -2,6 +2,7 @@
 
 /*
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -113,21 +114,26 @@ typedef struct _drm_i915_sarea {
 	unsigned int rotated_tiled;
 	unsigned int rotated2_tiled;
 
-	int planeA_x;
-	int planeA_y;
-	int planeA_w;
-	int planeA_h;
-	int planeB_x;
-	int planeB_y;
-	int planeB_w;
-	int planeB_h;
+	int pipeA_x;
+	int pipeA_y;
+	int pipeA_w;
+	int pipeA_h;
+	int pipeB_x;
+	int pipeB_y;
+	int pipeB_w;
+	int pipeB_h;
 
+	int pad1;
 	/* Triple buffering */
 	drm_handle_t third_handle;
 	int third_offset;
 	int third_size;
 	unsigned int third_tiled;
 
+	unsigned int front_bo_handle;
+	unsigned int back_bo_handle;
+	unsigned int third_bo_handle;
+	unsigned int depth_bo_handle;
 } drm_i915_sarea_t;
 
 /* Driver specific fence types and classes.
@@ -168,6 +174,24 @@ typedef struct _drm_i915_sarea {
 #define DRM_I915_GET_VBLANK_PIPE	0x0e
 #define DRM_I915_VBLANK_SWAP	0x0f
 #define	DRM_I915_HWS_ADDR	0x11
+#define DRM_I915_GEM_INIT	0x13
+#define DRM_I915_GEM_EXECBUFFER	0x14
+#define DRM_I915_GEM_PIN	0x15
+#define DRM_I915_GEM_UNPIN	0x16
+#define DRM_I915_GEM_BUSY	0x17
+#define DRM_I915_GEM_THROTTLE	0x18
+#define DRM_I915_GEM_ENTERVT	0x19
+#define DRM_I915_GEM_LEAVEVT	0x1a
+#define DRM_I915_GEM_CREATE	0x1b
+#define DRM_I915_GEM_PREAD	0x1c
+#define DRM_I915_GEM_PWRITE	0x1d
+#define DRM_I915_GEM_MMAP	0x1e
+#define DRM_I915_GEM_SET_DOMAIN	0x1f
+#define DRM_I915_GEM_SW_FINISH	0x20
+#define DRM_I915_GEM_SET_TILING	0x21
+#define DRM_I915_GEM_GET_TILING	0x22
+#define DRM_I915_GEM_GET_APERTURE 0x23
+#define DRM_I915_GEM_MMAP_GTT	0x24
 
 #define DRM_IOCTL_I915_INIT		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
 #define DRM_IOCTL_I915_FLUSH		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
@@ -185,6 +209,24 @@ typedef struct _drm_i915_sarea {
 #define DRM_IOCTL_I915_SET_VBLANK_PIPE	DRM_IOW( DRM_COMMAND_BASE + DRM_I915_SET_VBLANK_PIPE, drm_i915_vblank_pipe_t)
 #define DRM_IOCTL_I915_GET_VBLANK_PIPE	DRM_IOR( DRM_COMMAND_BASE + DRM_I915_GET_VBLANK_PIPE, drm_i915_vblank_pipe_t)
 #define DRM_IOCTL_I915_VBLANK_SWAP	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_VBLANK_SWAP, drm_i915_vblank_swap_t)
+#define DRM_IOCTL_I915_GEM_INIT		DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_INIT, struct drm_i915_gem_init)
+#define DRM_IOCTL_I915_GEM_EXECBUFFER	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_EXECBUFFER, struct drm_i915_gem_execbuffer)
+#define DRM_IOCTL_I915_GEM_PIN		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_PIN, struct drm_i915_gem_pin)
+#define DRM_IOCTL_I915_GEM_UNPIN	DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_UNPIN, struct drm_i915_gem_unpin)
+#define DRM_IOCTL_I915_GEM_BUSY		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_BUSY, struct drm_i915_gem_busy)
+#define DRM_IOCTL_I915_GEM_THROTTLE	DRM_IO ( DRM_COMMAND_BASE + DRM_I915_GEM_THROTTLE)
+#define DRM_IOCTL_I915_GEM_ENTERVT	DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_ENTERVT)
+#define DRM_IOCTL_I915_GEM_LEAVEVT	DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_LEAVEVT)
+#define DRM_IOCTL_I915_GEM_CREATE	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_CREATE, struct drm_i915_gem_create)
+#define DRM_IOCTL_I915_GEM_PREAD	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PREAD, struct drm_i915_gem_pread)
+#define DRM_IOCTL_I915_GEM_PWRITE	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_PWRITE, struct drm_i915_gem_pwrite)
+#define DRM_IOCTL_I915_GEM_MMAP		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct drm_i915_gem_mmap)
+#define DRM_IOCTL_I915_GEM_MMAP_GTT	DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP_GTT, struct drm_i915_gem_mmap_gtt)
+#define DRM_IOCTL_I915_GEM_SET_DOMAIN	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SET_DOMAIN, struct drm_i915_gem_set_domain)
+#define DRM_IOCTL_I915_GEM_SW_FINISH	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_SW_FINISH, struct drm_i915_gem_sw_finish)
+#define DRM_IOCTL_I915_GEM_SET_TILING	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_SET_TILING, struct drm_i915_gem_set_tiling)
+#define DRM_IOCTL_I915_GEM_GET_TILING	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_GET_TILING, struct drm_i915_gem_get_tiling)
+#define DRM_IOCTL_I915_GEM_GET_APERTURE	DRM_IOR  (DRM_COMMAND_BASE + DRM_I915_GEM_GET_APERTURE, struct drm_i915_gem_get_aperture)
 
 /* Asynchronous page flipping:
  */
@@ -260,6 +302,7 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_ALLOW_BATCHBUFFER     2
 #define I915_PARAM_LAST_DISPATCH         3
 #define I915_PARAM_CHIPSET_ID            4
+#define I915_PARAM_HAS_GEM               5
 
 typedef struct drm_i915_getparam {
 	int param;
@@ -335,9 +378,365 @@ typedef struct drm_i915_vblank_swap {
 	unsigned int sequence;
 } drm_i915_vblank_swap_t;
 
+#define I915_MMIO_READ	0
+#define I915_MMIO_WRITE 1
+
+#define I915_MMIO_MAY_READ	0x1
+#define I915_MMIO_MAY_WRITE	0x2
+
+#define MMIO_REGS_IA_PRIMATIVES_COUNT		0
+#define MMIO_REGS_IA_VERTICES_COUNT		1
+#define MMIO_REGS_VS_INVOCATION_COUNT		2
+#define MMIO_REGS_GS_PRIMITIVES_COUNT		3
+#define MMIO_REGS_GS_INVOCATION_COUNT		4
+#define MMIO_REGS_CL_PRIMITIVES_COUNT		5
+#define MMIO_REGS_CL_INVOCATION_COUNT		6
+#define MMIO_REGS_PS_INVOCATION_COUNT		7
+#define MMIO_REGS_PS_DEPTH_COUNT		8
+
+typedef struct drm_i915_mmio_entry {
+	unsigned int flag;
+	unsigned int offset;
+	unsigned int size;
+} drm_i915_mmio_entry_t;
+
+typedef struct drm_i915_mmio {
+	unsigned int read_write:1;
+	unsigned int reg:31;
+	void __user *data;
+} drm_i915_mmio_t;
+
 typedef struct drm_i915_hws_addr {
 	uint64_t addr;
 } drm_i915_hws_addr_t;
 
 
+struct drm_i915_gem_init {
+	/**
+	 * Beginning offset in the GTT to be managed by the DRM memory
+	 * manager.
+	 */
+	uint64_t gtt_start;
+	/**
+	 * Ending offset in the GTT to be managed by the DRM memory
+	 * manager.
+	 */
+	uint64_t gtt_end;
+
+};
+
+struct drm_i915_gem_create {
+	/**
+	 * Requested size for the object.
+	 *
+	 * The (page-aligned) allocated size for the object will be returned.
+	 */
+	uint64_t size;
+	/**
+	 * Returned handle for the object.
+	 *
+	 * Object handles are nonzero.
+	 */
+	uint32_t handle;
+	uint32_t pad;
+};
+
+struct drm_i915_gem_pread {
+	/** Handle for the object being read. */
+	uint32_t handle;
+	uint32_t pad;
+	/** Offset into the object to read from */
+	uint64_t offset;
+	/** Length of data to read */
+	uint64_t size;
+	/**
+	 * Pointer to write the data into.
+	 *
+	 * This is a fixed-size type for 32/64 compatibility.
+	 */
+	uint64_t data_ptr;
+};
+
+struct drm_i915_gem_pwrite {
+	/** Handle for the object being written to. */
+	uint32_t handle;
+	uint32_t pad;
+	/** Offset into the object to write to */
+	uint64_t offset;
+	/** Length of data to write */
+	uint64_t size;
+	/**
+	 * Pointer to read the data from.
+	 *
+	 * This is a fixed-size type for 32/64 compatibility.
+	 */
+	uint64_t data_ptr;
+};
+
+struct drm_i915_gem_mmap {
+	/** Handle for the object being mapped. */
+	uint32_t handle;
+	uint32_t pad;
+	/** Offset in the object to map. */
+	uint64_t offset;
+	/**
+	 * Length of data to map.
+	 *
+	 * The value will be page-aligned.
+	 */
+	uint64_t size;
+	/**
+	 * Returned pointer the data was mapped at.
+	 *
+	 * This is a fixed-size type for 32/64 compatibility.
+	 */
+	uint64_t addr_ptr;
+};
+
+struct drm_i915_gem_mmap_gtt {
+	/** Handle for the object being mapped. */
+	uint32_t handle;
+	uint32_t pad;
+	/**
+	 * Fake offset to use for subsequent mmap call
+	 *
+	 * This is a fixed-size type for 32/64 compatibility.
+	 */
+	uint64_t offset;
+};
+
+struct drm_i915_gem_set_domain {
+	/** Handle for the object */
+	uint32_t handle;
+
+	/** New read domains */
+	uint32_t read_domains;
+
+	/** New write domain */
+	uint32_t write_domain;
+};
+
+struct drm_i915_gem_sw_finish {
+	/** Handle for the object */
+	uint32_t handle;
+};
+
+struct drm_i915_gem_relocation_entry {
+	/**
+	 * Handle of the buffer being pointed to by this relocation entry.
+	 *
+	 * It's appealing to make this be an index into the mm_validate_entry
+	 * list to refer to the buffer, but this allows the driver to create
+	 * a relocation list for state buffers and not re-write it per
+	 * exec using the buffer.
+	 */
+	uint32_t target_handle;
+
+	/**
+	 * Value to be added to the offset of the target buffer to make up
+	 * the relocation entry.
+	 */
+	uint32_t delta;
+
+	/** Offset in the buffer the relocation entry will be written into */
+	uint64_t offset;
+
+	/**
+	 * Offset value of the target buffer that the relocation entry was last
+	 * written as.
+	 *
+	 * If the buffer has the same offset as last time, we can skip syncing
+	 * and writing the relocation.  This value is written back out by
+	 * the execbuffer ioctl when the relocation is written.
+	 */
+	uint64_t presumed_offset;
+
+	/**
+	 * Target memory domains read by this operation.
+	 */
+	uint32_t read_domains;
+
+	/**
+	 * Target memory domains written by this operation.
+	 *
+	 * Note that only one domain may be written by the whole
+	 * execbuffer operation, so that where there are conflicts,
+	 * the application will get -EINVAL back.
+	 */
+	uint32_t write_domain;
+};
+
+/** @{
+ * Intel memory domains
+ *
+ * Most of these just align with the various caches in
+ * the system and are used to flush and invalidate as
+ * objects end up cached in different domains.
+ */
+/** CPU cache */
+#define I915_GEM_DOMAIN_CPU		0x00000001
+/** Render cache, used by 2D and 3D drawing */
+#define I915_GEM_DOMAIN_RENDER		0x00000002
+/** Sampler cache, used by texture engine */
+#define I915_GEM_DOMAIN_SAMPLER		0x00000004
+/** Command queue, used to load batch buffers */
+#define I915_GEM_DOMAIN_COMMAND		0x00000008
+/** Instruction cache, used by shader programs */
+#define I915_GEM_DOMAIN_INSTRUCTION	0x00000010
+/** Vertex address cache */
+#define I915_GEM_DOMAIN_VERTEX		0x00000020
+/** GTT domain - aperture and scanout */
+#define I915_GEM_DOMAIN_GTT		0x00000040
+/** @} */
+
+struct drm_i915_gem_exec_object {
+	/**
+	 * User's handle for a buffer to be bound into the GTT for this
+	 * operation.
+	 */
+	uint32_t handle;
+
+	/** Number of relocations to be performed on this buffer */
+	uint32_t relocation_count;
+
+	/**
+	 * Pointer to array of struct drm_i915_gem_relocation_entry containing
+	 * the relocations to be performed in this buffer.
+	 */
+	uint64_t relocs_ptr;
+
+	/** Required alignment in graphics aperture */
+	uint64_t alignment;
+
+	/**
+         * Returned value of the updated offset of the object, for future
+         * presumed_offset writes.
+         */
+        uint64_t offset;
+
+};
+
+struct drm_i915_gem_execbuffer {
+	/**
+	 * List of buffers to be validated with their relocations to be
+	 * performend on them.
+	 *
+	 * This is a pointer to an array of struct drm_i915_gem_validate_entry.
+	 *
+	 * These buffers must be listed in an order such that all relocations
+	 * a buffer is performing refer to buffers that have already appeared
+	 * in the validate list.
+	 */
+	uint64_t buffers_ptr;
+	uint32_t buffer_count;
+
+	/** Offset in the batchbuffer to start execution from. */
+	uint32_t batch_start_offset;
+	/** Bytes used in batchbuffer from batch_start_offset */
+	uint32_t batch_len;
+	uint32_t DR1;
+	uint32_t DR4;
+	uint32_t num_cliprects;
+	/** This is a struct drm_clip_rect *cliprects */
+	uint64_t cliprects_ptr;
+};
+
+struct drm_i915_gem_pin {
+	/** Handle of the buffer to be pinned. */
+	uint32_t handle;
+	uint32_t pad;
+
+	/** alignment required within the aperture */
+	uint64_t alignment;
+
+	/** Returned GTT offset of the buffer. */
+	uint64_t offset;
+};
+
+
+struct drm_i915_gem_unpin {
+	/** Handle of the buffer to be unpinned. */
+	uint32_t handle;
+	uint32_t pad;
+};
+
+struct drm_i915_gem_busy {
+	/** Handle of the buffer to check for busy */
+	uint32_t handle;
+
+	/** Return busy status (1 if busy, 0 if idle) */
+	uint32_t busy;
+};
+
+#define I915_TILING_NONE	0
+#define I915_TILING_X		1
+#define I915_TILING_Y		2
+
+#define I915_BIT_6_SWIZZLE_NONE		0
+#define I915_BIT_6_SWIZZLE_9		1
+#define I915_BIT_6_SWIZZLE_9_10		2
+#define I915_BIT_6_SWIZZLE_9_11		3
+#define I915_BIT_6_SWIZZLE_9_10_11	4
+/* Not seen by userland */
+#define I915_BIT_6_SWIZZLE_UNKNOWN	5
+
+struct drm_i915_gem_set_tiling {
+	/** Handle of the buffer to have its tiling state updated */
+	uint32_t handle;
+
+	/**
+	 * Tiling mode for the object (I915_TILING_NONE, I915_TILING_X,
+	 * I915_TILING_Y).
+	 *
+	 * This value is to be set on request, and will be updated by the
+	 * kernel on successful return with the actual chosen tiling layout.
+	 *
+	 * The tiling mode may be demoted to I915_TILING_NONE when the system
+	 * has bit 6 swizzling that can't be managed correctly by GEM.
+	 *
+	 * Buffer contents become undefined when changing tiling_mode.
+	 */
+	uint32_t tiling_mode;
+
+	/**
+	 * Stride in bytes for the object when in I915_TILING_X or
+	 * I915_TILING_Y.
+	 */
+	uint32_t stride;
+
+	/**
+	 * Returned address bit 6 swizzling required for CPU access through
+	 * mmap mapping.
+	 */
+	uint32_t swizzle_mode;
+};
+
+struct drm_i915_gem_get_tiling {
+	/** Handle of the buffer to get tiling state for. */
+	uint32_t handle;
+
+	/**
+	 * Current tiling mode for the object (I915_TILING_NONE, I915_TILING_X,
+	 * I915_TILING_Y).
+	 */
+	uint32_t tiling_mode;
+
+	/**
+	 * Returned address bit 6 swizzling required for CPU access through
+	 * mmap mapping.
+	 */
+	uint32_t swizzle_mode;
+};
+
+struct drm_i915_gem_get_aperture {
+	/** Total size of the aperture used by i915_gem_execbuffer, in bytes */
+	uint64_t aper_size;
+
+	/**
+	 * Available space in the aperture used by i915_gem_execbuffer, in
+	 * bytes
+	 */
+	uint64_t aper_available_size;
+};
+
 #endif /* _I915_DRM_H */
diff --git a/usr/src/uts/intel/io/drm/i915_drv.c b/usr/src/uts/intel/io/drm/i915_drv.c
index 20529e5c4a..d8eaf8c60a 100644
--- a/usr/src/uts/intel/io/drm/i915_drv.c
+++ b/usr/src/uts/intel/io/drm/i915_drv.c
@@ -7,6 +7,7 @@
 
 /*
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -431,23 +432,120 @@ i915_restore_vga(struct drm_device *dev)
 		vga_reg_put8(&regmap, VGA_DACDATA, s3_priv->saveDACDATA[i]);
 }
 
-static int
-i915_resume(struct drm_device *dev)
+/**
+ * i915_save_display - save display & mode info
+ * @dev: DRM device
+ *
+ * Save mode timings and display info.
+ */
+void i915_save_display(struct drm_device *dev)
 {
-	ddi_acc_handle_t conf_hdl;
 	struct s3_i915_private *s3_priv = dev->s3_private;
-	int i;
 
-	if (pci_config_setup(dev->dip, &conf_hdl) != DDI_SUCCESS) {
-		DRM_ERROR(("i915_resume: pci_config_setup fail"));
-		return (DDI_FAILURE);
+	/* Display arbitration control */
+	s3_priv->saveDSPARB = S3_READ(DSPARB);
+
+	/*
+	 * Pipe & plane A info.
+	 */
+	s3_priv->savePIPEACONF = S3_READ(PIPEACONF);
+	s3_priv->savePIPEASRC = S3_READ(PIPEASRC);
+	s3_priv->saveFPA0 = S3_READ(FPA0);
+	s3_priv->saveFPA1 = S3_READ(FPA1);
+	s3_priv->saveDPLL_A = S3_READ(DPLL_A);
+	if (IS_I965G(dev))
+		s3_priv->saveDPLL_A_MD = S3_READ(DPLL_A_MD);
+	s3_priv->saveHTOTAL_A = S3_READ(HTOTAL_A);
+	s3_priv->saveHBLANK_A = S3_READ(HBLANK_A);
+	s3_priv->saveHSYNC_A = S3_READ(HSYNC_A);
+	s3_priv->saveVTOTAL_A = S3_READ(VTOTAL_A);
+	s3_priv->saveVBLANK_A = S3_READ(VBLANK_A);
+	s3_priv->saveVSYNC_A = S3_READ(VSYNC_A);
+	s3_priv->saveBCLRPAT_A = S3_READ(BCLRPAT_A);
+
+	s3_priv->saveDSPACNTR = S3_READ(DSPACNTR);
+	s3_priv->saveDSPASTRIDE = S3_READ(DSPASTRIDE);
+	s3_priv->saveDSPASIZE = S3_READ(DSPASIZE);
+	s3_priv->saveDSPAPOS = S3_READ(DSPAPOS);
+	s3_priv->saveDSPABASE = S3_READ(DSPABASE);
+	if (IS_I965G(dev)) {
+		s3_priv->saveDSPASURF = S3_READ(DSPASURF);
+		s3_priv->saveDSPATILEOFF = S3_READ(DSPATILEOFF);
 	}
+	i915_save_palette(dev, PIPE_A);
+	s3_priv->savePIPEASTAT = S3_READ(PIPEASTAT);
+
 	/*
-	 * Nexus driver will resume pci config space and set the power state
-	 * for its children. So we needn't resume them explicitly here.
-	 * see pci_pre_resume for detail.
+	 * Pipe & plane B info
 	 */
-	pci_config_put8(conf_hdl, LBB, s3_priv->saveLBB);
+	s3_priv->savePIPEBCONF = S3_READ(PIPEBCONF);
+	s3_priv->savePIPEBSRC = S3_READ(PIPEBSRC);
+	s3_priv->saveFPB0 = S3_READ(FPB0);
+	s3_priv->saveFPB1 = S3_READ(FPB1);
+	s3_priv->saveDPLL_B = S3_READ(DPLL_B);
+	if (IS_I965G(dev))
+		s3_priv->saveDPLL_B_MD = S3_READ(DPLL_B_MD);
+	s3_priv->saveHTOTAL_B = S3_READ(HTOTAL_B);
+	s3_priv->saveHBLANK_B = S3_READ(HBLANK_B);
+	s3_priv->saveHSYNC_B = S3_READ(HSYNC_B);
+	s3_priv->saveVTOTAL_B = S3_READ(VTOTAL_B);
+	s3_priv->saveVBLANK_B = S3_READ(VBLANK_B);
+	s3_priv->saveVSYNC_B = S3_READ(VSYNC_B);
+	s3_priv->saveBCLRPAT_A = S3_READ(BCLRPAT_A);
+
+	s3_priv->saveDSPBCNTR = S3_READ(DSPBCNTR);
+	s3_priv->saveDSPBSTRIDE = S3_READ(DSPBSTRIDE);
+	s3_priv->saveDSPBSIZE = S3_READ(DSPBSIZE);
+	s3_priv->saveDSPBPOS = S3_READ(DSPBPOS);
+	s3_priv->saveDSPBBASE = S3_READ(DSPBBASE);
+	if (IS_I965GM(dev) || IS_GM45(dev)) {
+		s3_priv->saveDSPBSURF = S3_READ(DSPBSURF);
+		s3_priv->saveDSPBTILEOFF = S3_READ(DSPBTILEOFF);
+	}
+	i915_save_palette(dev, PIPE_B);
+	s3_priv->savePIPEBSTAT = S3_READ(PIPEBSTAT);
+
+	/*
+	 * CRT state
+	 */
+	s3_priv->saveADPA = S3_READ(ADPA);
+
+	/*
+	 * LVDS state
+	 */
+	s3_priv->savePP_CONTROL = S3_READ(PP_CONTROL);
+	s3_priv->savePFIT_PGM_RATIOS = S3_READ(PFIT_PGM_RATIOS);
+	s3_priv->saveBLC_PWM_CTL = S3_READ(BLC_PWM_CTL);
+	if (IS_I965G(dev))
+		s3_priv->saveBLC_PWM_CTL2 = S3_READ(BLC_PWM_CTL2);
+	if (IS_MOBILE(dev) && !IS_I830(dev))
+		s3_priv->saveLVDS = S3_READ(LVDS);
+	if (!IS_I830(dev) && !IS_845G(dev))
+		s3_priv->savePFIT_CONTROL = S3_READ(PFIT_CONTROL);
+	s3_priv->saveLVDSPP_ON = S3_READ(LVDSPP_ON);
+	s3_priv->saveLVDSPP_OFF = S3_READ(LVDSPP_OFF);
+	s3_priv->savePP_CYCLE = S3_READ(PP_CYCLE);
+
+	/* FIXME: save TV & SDVO state */
+
+	/* FBC state */
+	s3_priv->saveFBC_CFB_BASE = S3_READ(FBC_CFB_BASE);
+	s3_priv->saveFBC_LL_BASE = S3_READ(FBC_LL_BASE);
+	s3_priv->saveFBC_CONTROL2 = S3_READ(FBC_CONTROL2);
+	s3_priv->saveFBC_CONTROL = S3_READ(FBC_CONTROL);
+
+	/* VGA state */
+	s3_priv->saveVCLK_DIVISOR_VGA0 = S3_READ(VCLK_DIVISOR_VGA0);
+	s3_priv->saveVCLK_DIVISOR_VGA1 = S3_READ(VCLK_DIVISOR_VGA1);
+	s3_priv->saveVCLK_POST_DIV = S3_READ(VCLK_POST_DIV);
+	s3_priv->saveVGACNTRL = S3_READ(VGACNTRL);
+
+	i915_save_vga(dev);
+}
+
+void i915_restore_display(struct drm_device *dev)
+{
+        struct s3_i915_private *s3_priv = dev->s3_private;
 
 	S3_WRITE(DSPARB, s3_priv->saveDSPARB);
 
@@ -566,6 +664,37 @@ i915_resume(struct drm_device *dev)
 	S3_WRITE(VCLK_DIVISOR_VGA1, s3_priv->saveVCLK_DIVISOR_VGA1);
 	S3_WRITE(VCLK_POST_DIV, s3_priv->saveVCLK_POST_DIV);
 	drv_usecwait(150);
+	
+	i915_restore_vga(dev);
+}
+static int
+i915_resume(struct drm_device *dev)
+{
+	ddi_acc_handle_t conf_hdl;
+	struct s3_i915_private *s3_priv = dev->s3_private;
+	int i;
+
+	if (pci_config_setup(dev->dip, &conf_hdl) != DDI_SUCCESS) {
+		DRM_ERROR(("i915_resume: pci_config_setup fail"));
+		return (DDI_FAILURE);
+	}
+	/*
+	 * Nexus driver will resume pci config space and set the power state
+	 * for its children. So we needn't resume them explicitly here.
+	 * see pci_pre_resume for detail.
+	 */
+	pci_config_put8(conf_hdl, LBB, s3_priv->saveLBB);
+
+	if (IS_I965G(dev) && IS_MOBILE(dev))
+		S3_WRITE(MCHBAR_RENDER_STANDBY, s3_priv->saveRENDERSTANDBY);
+	if (IS_I965GM(dev))
+		(void) S3_READ(MCHBAR_RENDER_STANDBY);
+
+	S3_WRITE(HWS_PGA, s3_priv->saveHWS);
+	if (IS_I965GM(dev))
+		(void) S3_READ(HWS_PGA);
+
+	i915_restore_display(dev);
 
 	 /* Clock gating state */
 	S3_WRITE (D_STATE, s3_priv->saveD_STATE);
@@ -584,14 +713,18 @@ i915_resume(struct drm_device *dev)
 	for (i = 0; i < 3; i++)
 		S3_WRITE(SWF30 + (i << 2), s3_priv->saveSWF2[i]);
 
-	i915_restore_vga(dev);
+	if (IS_I965GM(dev)) {
+		S3_WRITE(I915REG_PGTBL_CTRL, s3_priv->pgtbl_ctl);
+		(void) S3_READ(I915REG_PGTBL_CTRL);
+	}
 
-	S3_WRITE(I915REG_PGTBL_CTRL, s3_priv->pgtbl_ctl);
-	
 	(void) pci_config_teardown(&conf_hdl);
 
+	drm_agp_rebind(dev);
+
 	return (DDI_SUCCESS);
 }
+
 static int
 i915_suspend(struct drm_device *dev)
 {
@@ -599,7 +732,6 @@ i915_suspend(struct drm_device *dev)
 	struct s3_i915_private *s3_priv = dev->s3_private;
 	int i;
 
-
 	if (pci_config_setup(dev->dip, &conf_hdl) != DDI_SUCCESS) {
 		DRM_ERROR(("i915_suspend: pci_config_setup fail"));
 		return (DDI_FAILURE);
@@ -611,109 +743,19 @@ i915_suspend(struct drm_device *dev)
 	 */
 	s3_priv->saveLBB = pci_config_get8(conf_hdl, LBB);
 
-	/* Display arbitration control */
-	s3_priv->saveDSPARB = S3_READ(DSPARB);
+	if (IS_I965G(dev) && IS_MOBILE(dev))
+		s3_priv->saveRENDERSTANDBY = S3_READ(MCHBAR_RENDER_STANDBY);
 
-	/*
-	 * Pipe & plane A info.
-	 */
-	s3_priv->savePIPEACONF = S3_READ(PIPEACONF);
-	s3_priv->savePIPEASRC = S3_READ(PIPEASRC);
-	s3_priv->saveFPA0 = S3_READ(FPA0);
-	s3_priv->saveFPA1 = S3_READ(FPA1);
-	s3_priv->saveDPLL_A = S3_READ(DPLL_A);
-	if (IS_I965G(dev))
-		s3_priv->saveDPLL_A_MD = S3_READ(DPLL_A_MD);
-	s3_priv->saveHTOTAL_A = S3_READ(HTOTAL_A);
-	s3_priv->saveHBLANK_A = S3_READ(HBLANK_A);
-	s3_priv->saveHSYNC_A = S3_READ(HSYNC_A);
-	s3_priv->saveVTOTAL_A = S3_READ(VTOTAL_A);
-	s3_priv->saveVBLANK_A = S3_READ(VBLANK_A);
-	s3_priv->saveVSYNC_A = S3_READ(VSYNC_A);
-	s3_priv->saveBCLRPAT_A = S3_READ(BCLRPAT_A);
+	/* Hardware status page */
+	s3_priv->saveHWS = S3_READ(HWS_PGA);
 
-	s3_priv->saveDSPACNTR = S3_READ(DSPACNTR);
-	s3_priv->saveDSPASTRIDE = S3_READ(DSPASTRIDE);
-	s3_priv->saveDSPASIZE = S3_READ(DSPASIZE);
-	s3_priv->saveDSPAPOS = S3_READ(DSPAPOS);
-	s3_priv->saveDSPABASE = S3_READ(DSPABASE);
-	if (IS_I965G(dev)) {
-		s3_priv->saveDSPASURF = S3_READ(DSPASURF);
-		s3_priv->saveDSPATILEOFF = S3_READ(DSPATILEOFF);
-	}
-	i915_save_palette(dev, PIPE_A);
-	s3_priv->savePIPEASTAT = S3_READ(PIPEASTAT);
-
-	/*
-	 * Pipe & plane B info
-	 */
-	s3_priv->savePIPEBCONF = S3_READ(PIPEBCONF);
-	s3_priv->savePIPEBSRC = S3_READ(PIPEBSRC);
-	s3_priv->saveFPB0 = S3_READ(FPB0);
-	s3_priv->saveFPB1 = S3_READ(FPB1);
-	s3_priv->saveDPLL_B = S3_READ(DPLL_B);
-	if (IS_I965G(dev))
-		s3_priv->saveDPLL_B_MD = S3_READ(DPLL_B_MD);
-	s3_priv->saveHTOTAL_B = S3_READ(HTOTAL_B);
-	s3_priv->saveHBLANK_B = S3_READ(HBLANK_B);
-	s3_priv->saveHSYNC_B = S3_READ(HSYNC_B);
-	s3_priv->saveVTOTAL_B = S3_READ(VTOTAL_B);
-	s3_priv->saveVBLANK_B = S3_READ(VBLANK_B);
-	s3_priv->saveVSYNC_B = S3_READ(VSYNC_B);
-	s3_priv->saveBCLRPAT_A = S3_READ(BCLRPAT_A);
-
-	s3_priv->saveDSPBCNTR = S3_READ(DSPBCNTR);
-	s3_priv->saveDSPBSTRIDE = S3_READ(DSPBSTRIDE);
-	s3_priv->saveDSPBSIZE = S3_READ(DSPBSIZE);
-	s3_priv->saveDSPBPOS = S3_READ(DSPBPOS);
-	s3_priv->saveDSPBBASE = S3_READ(DSPBBASE);
-	if (IS_I965GM(dev) || IS_GM45(dev)) {
-		s3_priv->saveDSPBSURF = S3_READ(DSPBSURF);
-		s3_priv->saveDSPBTILEOFF = S3_READ(DSPBTILEOFF);
-	}
-	i915_save_palette(dev, PIPE_B);
-	s3_priv->savePIPEBSTAT = S3_READ(PIPEBSTAT);
-
-	/*
-	 * CRT state
-	 */
-	s3_priv->saveADPA = S3_READ(ADPA);
-
-	/*
-	 * LVDS state
-	 */
-	s3_priv->savePP_CONTROL = S3_READ(PP_CONTROL);
-	s3_priv->savePFIT_PGM_RATIOS = S3_READ(PFIT_PGM_RATIOS);
-	s3_priv->saveBLC_PWM_CTL = S3_READ(BLC_PWM_CTL);
-	if (IS_I965G(dev))
-		s3_priv->saveBLC_PWM_CTL2 = S3_READ(BLC_PWM_CTL2);
-	if (IS_MOBILE(dev) && !IS_I830(dev))
-		s3_priv->saveLVDS = S3_READ(LVDS);
-	if (!IS_I830(dev) && !IS_845G(dev))
-		s3_priv->savePFIT_CONTROL = S3_READ(PFIT_CONTROL);
-	s3_priv->saveLVDSPP_ON = S3_READ(LVDSPP_ON);
-	s3_priv->saveLVDSPP_OFF = S3_READ(LVDSPP_OFF);
-	s3_priv->savePP_CYCLE = S3_READ(PP_CYCLE);
-
-	/* FIXME: save TV & SDVO state */
-
-	/* FBC state */
-	s3_priv->saveFBC_CFB_BASE = S3_READ(FBC_CFB_BASE);
-	s3_priv->saveFBC_LL_BASE = S3_READ(FBC_LL_BASE);
-	s3_priv->saveFBC_CONTROL2 = S3_READ(FBC_CONTROL2);
-	s3_priv->saveFBC_CONTROL = S3_READ(FBC_CONTROL);
+	i915_save_display(dev);
 
 	/* Interrupt state */
 	s3_priv->saveIIR = S3_READ(IIR);
 	s3_priv->saveIER = S3_READ(IER);
 	s3_priv->saveIMR = S3_READ(IMR);
 
-	/* VGA state */
-	s3_priv->saveVCLK_DIVISOR_VGA0 = S3_READ(VCLK_DIVISOR_VGA0);
-	s3_priv->saveVCLK_DIVISOR_VGA1 = S3_READ(VCLK_DIVISOR_VGA1);
-	s3_priv->saveVCLK_POST_DIV = S3_READ(VCLK_POST_DIV);
-	s3_priv->saveVGACNTRL = S3_READ(VGACNTRL);
-
 	/* Clock gating state */
 	s3_priv->saveD_STATE = S3_READ(D_STATE);
 	s3_priv->saveCG_2D_DIS = S3_READ(CG_2D_DIS);
@@ -732,12 +774,11 @@ i915_suspend(struct drm_device *dev)
 	for (i = 0; i < 3; i++)
 		s3_priv->saveSWF2[i] = S3_READ(SWF30 + (i << 2));
 
-	
-	i915_save_vga(dev);
 	/*
 	 * Save page table control register
 	 */
-	s3_priv->pgtbl_ctl = S3_READ(I915REG_PGTBL_CTRL);
+	if (IS_I965GM(dev))
+		s3_priv->pgtbl_ctl = S3_READ(I915REG_PGTBL_CTRL);
 
 	(void) pci_config_teardown(&conf_hdl);
 
@@ -960,16 +1001,20 @@ static void i915_configure(drm_driver_t *driver)
 	driver->buf_priv_size	=	1;	/* No dev_priv */
 	driver->load	=	i915_driver_load;
 	driver->unload	=	i915_driver_unload;
+	driver->open	=	i915_driver_open;
 	driver->preclose	=	i915_driver_preclose;
+	driver->postclose	=	i915_driver_postclose;
 	driver->lastclose	=	i915_driver_lastclose;
 	driver->device_is_agp	=	i915_driver_device_is_agp;
-	driver->get_vblank_counter	= i915_get_vblank_counter;
 	driver->enable_vblank	= 	i915_enable_vblank;
 	driver->disable_vblank	= 	i915_disable_vblank;
 	driver->irq_preinstall	=	i915_driver_irq_preinstall;
 	driver->irq_postinstall	=	i915_driver_irq_postinstall;
 	driver->irq_uninstall	=	i915_driver_irq_uninstall;
-	driver->irq_handler 		=	i915_driver_irq_handler;
+	driver->irq_handler 	=	i915_driver_irq_handler;
+
+	driver->gem_init_object = 	i915_gem_init_object;
+	driver->gem_free_object = 	i915_gem_free_object;
 
 	driver->driver_ioctls	=	i915_ioctls;
 	driver->max_driver_ioctl	=	i915_max_ioctl;
diff --git a/usr/src/uts/intel/io/drm/i915_drv.h b/usr/src/uts/intel/io/drm/i915_drv.h
index 20e43639ff..e96840064e 100644
--- a/usr/src/uts/intel/io/drm/i915_drv.h
+++ b/usr/src/uts/intel/io/drm/i915_drv.h
@@ -5,6 +5,7 @@
 /*
  *
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -44,12 +45,14 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20060929"
+#define DRIVER_DATE		"20080730"
 
 #if defined(__SVR4) && defined(__sun)
 #define spinlock_t kmutex_t 
 #endif
 
+#define I915_NUM_PIPE	2
+
 #define I915_NUM_PIPE  2
 
 /* Interface history:
@@ -58,15 +61,19 @@
  * 1.2: Add Power Management
  * 1.3: Add vblank support
  * 1.4: Fix cmdbuffer path, add heap destroy
+ * 1.5: Add vblank pipe configuration
+ * 1.6: - New ioctl for scheduling buffer swaps on vertical blank
+ * 	    - Support vertical blank on secondary display pipe
  */
 #define DRIVER_MAJOR		1
-#define DRIVER_MINOR		5	
+#define DRIVER_MINOR		6	
 #define DRIVER_PATCHLEVEL	0
 
 #if defined(__linux__)
 #define I915_HAVE_FENCE
 #define I915_HAVE_BUFFER
 #endif
+#define	I915_HAVE_GEM	1
 
 typedef struct _drm_i915_ring_buffer {
 	int tail_mask;
@@ -76,6 +83,7 @@ typedef struct _drm_i915_ring_buffer {
 	int tail;
 	int space;
 	drm_local_map_t map;
+	struct drm_gem_object *ring_obj;
 } drm_i915_ring_buffer_t;
 
 struct mem_block {
@@ -102,6 +110,8 @@ typedef struct s3_i915_private {
 	uint32_t saveDSPACNTR;
 	uint32_t saveDSPBCNTR;
 	uint32_t saveDSPARB;
+	uint32_t saveRENDERSTANDBY;
+	uint32_t saveHWS;
 	uint32_t savePIPEACONF;
 	uint32_t savePIPEBCONF;
 	uint32_t savePIPEASRC;
@@ -187,6 +197,22 @@ typedef struct s3_i915_private {
 	uint8_t saveCR[37];
 } s3_i915_private_t;
 
+struct drm_i915_error_state {
+	u32 eir;
+	u32 pgtbl_er;
+	u32 pipeastat;
+	u32 pipebstat;
+	u32 ipeir;
+	u32 ipehr;
+	u32 instdone;
+	u32 acthd;
+	u32 instpm;
+	u32 instps;
+	u32 instdone1;
+	u32 seqno;
+	struct timeval time;
+};
+
 typedef struct drm_i915_private {
 	struct drm_device *dev;
 
@@ -202,38 +228,214 @@ typedef struct drm_i915_private {
 	uint32_t counter;
 	unsigned int status_gfx_addr;
 	drm_local_map_t hws_map;
-
+	struct drm_gem_object *hws_obj;
+	
 	unsigned int cpp;
+	int back_offset;
+	int front_offset;
+	int current_page;
+	int page_flipping;
 
 	wait_queue_head_t irq_queue;
 	atomic_t irq_received;
+        /** Protects user_irq_refcount and irq_mask_reg */
+        spinlock_t user_irq_lock;
+        /** Refcount for i915_user_irq_get() versus i915_user_irq_put(). */
+        int user_irq_refcount;
+        /** Cached value of IMR to avoid reads in updating the bitfield */
+        int irq_mask_reg;
+	uint32_t pipestat[2];
 
 	int tex_lru_log_granularity;
 	int allow_batchbuffer;
 	struct mem_block *agp_heap;
 	unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
 	int vblank_pipe;
-	spinlock_t user_irq_lock;
-        int user_irq_refcount;
-        int fence_irq_on;
-        uint32_t irq_mask_reg;
-        int irq_enabled;
 
-#ifdef I915_HAVE_FENCE
-        uint32_t flush_sequence;
-	uint32_t flush_flags;
-	uint32_t flush_pending;
-	uint32_t saved_flush_status;
-#endif
-#ifdef I915_HAVE_BUFFER
-	void *agp_iomap;
-#endif
-	spinlock_t swaps_lock;
-	drm_i915_vbl_swap_t vbl_swaps;
-	unsigned int swaps_pending;
+	spinlock_t error_lock;
+	struct drm_i915_error_state *first_error;
+
+	struct {
+		struct drm_mm gtt_space;
+
+		drm_local_map_t gtt_mapping;
+		/**
+		 * List of objects currently involved in rendering from the
+		 * ringbuffer.
+		 *
+		 * A reference is held on the buffer while on this list.
+		 */
+		struct list_head active_list;
+
+		/**
+		 * List of objects which are not in the ringbuffer but which
+		 * still have a write_domain which needs to be flushed before
+		 * unbinding.
+		 *
+		 * A reference is held on the buffer while on this list.
+		 */
+		struct list_head flushing_list;
+
+		/**
+		 * LRU list of objects which are not in the ringbuffer and
+		 * are ready to unbind, but are still in the GTT.
+		 *
+		 * A reference is not held on the buffer while on this list,
+		 * as merely being GTT-bound shouldn't prevent its being
+		 * freed, and we'll pull it off the list in the free path.
+		 */
+		struct list_head inactive_list;
+
+		/**
+		 * List of breadcrumbs associated with GPU requests currently
+		 * outstanding.
+		 */
+		struct list_head request_list;
+
+		uint32_t next_gem_seqno;
+
+		/**
+		 * Waiting sequence number, if any
+		 */
+		uint32_t waiting_gem_seqno;
+
+		/**
+		 * Last seq seen at irq time
+		 */
+		uint32_t irq_gem_seqno;
+
+		/**
+		 * Flag if the X Server, and thus DRM, is not currently in
+		 * control of the device.
+		 *
+		 * This is set between LeaveVT and EnterVT.  It needs to be
+		 * replaced with a semaphore.  It also needs to be
+		 * transitioned away from for kernel modesetting.
+		 */
+		int suspended;
+
+		/**
+		 * Flag if the hardware appears to be wedged.
+		 *
+		 * This is set when attempts to idle the device timeout.
+		 * It prevents command submission from occuring and makes
+		 * every pending request fail
+		 */
+		int wedged;
+
+		/** Bit 6 swizzling required for X tiling */
+		uint32_t bit_6_swizzle_x;
+		/** Bit 6 swizzling required for Y tiling */
+		uint32_t bit_6_swizzle_y;
+	} mm;
 
 } drm_i915_private_t;
 
+struct drm_track {
+	struct drm_track *next, *prev;
+	caddr_t contain_ptr;
+	struct drm_gem_object *obj;
+	uint32_t name;
+	uint64_t offset;
+	
+};
+
+/** driver private structure attached to each drm_gem_object */
+struct drm_i915_gem_object {
+	/** This object's place on the active/flushing/inactive lists */
+	struct list_head list;
+
+	struct drm_gem_object *obj;
+
+	/** Current space allocated to this object in the GTT, if any. */
+	struct drm_mm_node *gtt_space;
+
+
+	/**
+	 * This is set if the object is on the active or flushing lists
+	 * (has pending rendering), and is not set if it's on inactive (ready
+	 * to be unbound).
+	 */
+	int active;
+
+	/**
+	 * This is set if the object has been written to since last bound
+	 * to the GTT
+	 */
+	int dirty;
+
+	/** AGP memory structure for our GTT binding. */
+	int	agp_mem;
+
+	caddr_t *page_list;
+
+	pfn_t	*pfnarray;
+	/**
+	 * Current offset of the object in GTT space.
+	 *
+	 * This is the same as gtt_space->start
+	 */
+	uint32_t gtt_offset;
+
+	/** Boolean whether this object has a valid gtt offset. */
+	int gtt_bound;
+
+	/** How many users have pinned this object in GTT space */
+	int pin_count;
+
+	/** Breadcrumb of last rendering to the buffer. */
+	uint32_t last_rendering_seqno;
+
+	/** Current tiling mode for the object. */
+	uint32_t tiling_mode;
+	uint32_t stride;
+	/**
+	 * Flagging of which individual pages are valid in GEM_DOMAIN_CPU when
+	 * GEM_DOMAIN_CPU is not in the object's read domain.
+	 */
+	uint8_t *page_cpu_valid;
+	/** User space pin count and filp owning the pin */
+	uint32_t user_pin_count;
+	struct drm_file *pin_filp;
+	/**
+	 * Used for checking the object doesn't appear more than once
+	 * in an execbuffer object list.
+	 */
+	int in_execbuffer;
+};
+
+/**
+ * Request queue structure.
+ *
+ * The request queue allows us to note sequence numbers that have been emitted
+ * and may be associated with active buffers to be retired.
+ *
+ * By keeping this list, we can avoid having to do questionable
+ * sequence-number comparisons on buffer last_rendering_seqnos, and associate
+ * an emission time with seqnos for tracking how far ahead of the GPU we are.
+ */
+struct drm_i915_gem_request {
+	struct list_head list;
+
+	/** GEM sequence number associated with this request. */
+	uint32_t seqno;
+
+	/** Time at which this request was emitted, in jiffies. */
+	unsigned long emitted_jiffies;
+
+	/** Cache domains that were flushed at the start of the request. */
+	uint32_t flush_domains;
+
+};
+
+struct drm_i915_file_private {
+	struct {
+		uint32_t last_gem_seqno;
+		uint32_t last_gem_throttle_seqno;
+	} mm;
+};
+
+
 enum intel_chip_family {
 	CHIP_I8XX = 0x01,
 	CHIP_I9XX = 0x02,
@@ -243,13 +445,18 @@ enum intel_chip_family {
 
 extern drm_ioctl_desc_t i915_ioctls[];
 extern int i915_max_ioctl;
+extern void i915_save_display(struct drm_device *dev);
+extern void i915_restore_display(struct drm_device *dev);
 
 				/* i915_dma.c */
 extern void i915_kernel_lost_context(drm_device_t * dev);
 extern int i915_driver_load(struct drm_device *, unsigned long flags);
 extern int i915_driver_unload(struct drm_device *dev);
+extern int i915_driver_open(drm_device_t * dev, drm_file_t *file_priv);
 extern void i915_driver_lastclose(drm_device_t * dev);
 extern void i915_driver_preclose(drm_device_t * dev, drm_file_t *filp);
+extern void i915_driver_postclose(drm_device_t * dev,
+		    struct drm_file *file_priv);
 extern int i915_driver_device_is_agp(drm_device_t * dev);
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
@@ -257,9 +464,8 @@ extern int i915_emit_box(struct drm_device *dev,
 			struct drm_clip_rect __user *boxes,
 			int i, int DR1, int DR4);
 extern void i915_emit_breadcrumb(struct drm_device *dev);
-extern void i915_dispatch_flip(struct drm_device * dev, int pipes, int sync);
 extern void i915_emit_mi_flush(drm_device_t *dev, uint32_t flush);
-
+extern void i915_handle_error(struct drm_device *dev);
 
 /* i915_irq.c */
 extern int i915_irq_emit(DRM_IOCTL_ARGS);
@@ -268,14 +474,15 @@ extern int i915_irq_wait(DRM_IOCTL_ARGS);
 extern int i915_enable_vblank(struct drm_device *dev, int crtc);
 extern void i915_disable_vblank(struct drm_device *dev, int crtc);
 extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc);
+extern u32 gm45_get_vblank_counter(struct drm_device *dev, int crtc);
 extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS);
 extern int i915_driver_irq_preinstall(drm_device_t * dev);
 extern void i915_driver_irq_postinstall(drm_device_t * dev);
 extern void i915_driver_irq_uninstall(drm_device_t * dev);
 extern int i915_emit_irq(drm_device_t * dev);
 extern int i915_vblank_swap(DRM_IOCTL_ARGS);
-extern void i915_user_irq_on(drm_i915_private_t *dev_priv);
-extern void i915_user_irq_off(drm_i915_private_t *dev_priv);
+extern void i915_user_irq_on(drm_device_t * dev);
+extern void i915_user_irq_off(drm_device_t * dev);
 extern int i915_vblank_pipe_set(DRM_IOCTL_ARGS);
 extern int i915_vblank_pipe_get(DRM_IOCTL_ARGS);
 
@@ -292,6 +499,45 @@ extern struct mem_block *find_block_by_proc(struct mem_block *, drm_file_t *);
 extern void mark_block(drm_device_t *, struct mem_block *, int);
 extern void free_block(struct mem_block *);
 
+/* i915_gem.c */
+int i915_gem_init_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_create_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_pread_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_pwrite_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_mmap_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_set_domain_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_sw_finish_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_execbuffer(DRM_IOCTL_ARGS);
+int i915_gem_pin_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_unpin_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_busy_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_throttle_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_entervt_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_leavevt_ioctl(DRM_IOCTL_ARGS);
+int i915_gem_set_tiling(DRM_IOCTL_ARGS);
+int i915_gem_get_tiling(DRM_IOCTL_ARGS);
+int i915_gem_get_aperture_ioctl(DRM_IOCTL_ARGS);
+void i915_gem_load(struct drm_device *dev);
+int i915_gem_init_object(struct drm_gem_object *obj);
+void i915_gem_free_object(struct drm_gem_object *obj);
+int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment);
+void i915_gem_object_unpin(struct drm_gem_object *obj);
+int i915_gem_object_unbind(struct drm_gem_object *obj, uint32_t type);
+void i915_gem_lastclose(struct drm_device *dev);
+uint32_t i915_get_gem_seqno(struct drm_device *dev);
+void i915_gem_retire_requests(struct drm_device *dev);
+void i915_gem_retire_work_handler(void *dev);
+void i915_gem_clflush_object(struct drm_gem_object *obj);
+int i915_gem_init_ringbuffer(struct drm_device *dev);
+
+/* i915_gem_tiling.c */
+void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
+
+/* i915_gem_debug.c */
+void i915_gem_command_decode(uint32_t *data, int count, 
+				uint32_t hw_offset, struct drm_device *dev);
+/* i915_gem_regdump.c */
+int i915_reg_dump_show(struct drm_device *dev, void *v);
 #ifdef I915_HAVE_FENCE
 /* i915_fence.c */
 
@@ -327,7 +573,7 @@ extern int i915_move(drm_buffer_object_t *bo, int evict,
 #define	S3_WRITE(reg, val) \
 	*(uint32_t volatile *)((uintptr_t)s3_priv->saveAddr + (reg)) = (val)
 
-#define I915_VERBOSE 0
+#define I915_VERBOSE 0 
 #define I915_RING_VALIDATE 0
 
 #if I915_RING_VALIDATE
@@ -340,9 +586,20 @@ void i915_ring_validate(struct drm_device *dev, const char *func, int line);
 #define RING_LOCALS	unsigned int outring, ringmask, outcount; \
                         volatile unsigned char *virt;
 
+
+#define I915_RING_VALIDATE 0
+
+#if I915_RING_VALIDATE
+void i915_ring_validate(struct drm_device *dev, const char *func, int line);
+#define I915_RING_DO_VALIDATE(dev) i915_ring_validate(dev, __FUNCTION__, __LINE__)
+#else
+#define I915_RING_DO_VALIDATE(dev)
+#endif
+
 #if I915_VERBOSE
 #define BEGIN_LP_RING(n) do {				\
 	DRM_DEBUG("BEGIN_LP_RING(%d)\n", (n));		\
+	DRM_DEBUG("dev_priv->ring.virtual_start (%lx)\n", (dev_priv->ring.virtual_start));		\
 	I915_RING_DO_VALIDATE(dev);			\
 	if (dev_priv->ring.space < (n)*4)			\
 		(void) i915_wait_ring(dev, (n)*4, __FUNCTION__);		\
@@ -401,6 +658,10 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 
 /* Extended config space */
 #define LBB 0xf4
+#define GDRST 0xc0
+#define GDRST_FULL	(0<<2)
+#define GDRST_RENDER	(1<<2)
+#define GDRST_MEDIA	(3<<2)
 
 /* VGA stuff */
 
@@ -469,6 +730,7 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define BB2_END_ADDR_MASK     (~0x7)
 
 #define	I915REG_PGTBL_CTRL	0x2020
+#define IPEIR			0x02088
 #define HWSTAM			0x02098
 #define IIR			0x020a4
 #define IMR		 	0x020a8
@@ -479,6 +741,22 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define PIPEBSTAT		0x71024
 #define ACTHD_I965		0x02074
 #define HWS_PGA			0x02080
+#define IPEIR_I965	0x02064
+#define IPEHR_I965	0x02068
+#define INSTDONE_I965	0x0206c
+#define INSTPS		0x02070 /* 965+ only */
+#define INSTDONE1	0x0207c /* 965+ only */
+#define IPEHR		0x0208c
+#define INSTDONE	0x02090
+#define EIR		0x020b0
+#define EMR		0x020b4
+#define ESR		0x020b8
+#define   GM45_ERROR_PAGE_TABLE				(1<<5)
+#define   GM45_ERROR_MEM_PRIV				(1<<4)
+#define   I915_ERROR_PAGE_TABLE				(1<<4)
+#define   GM45_ERROR_CP_PRIV				(1<<3)
+#define   I915_ERROR_MEMORY_REFRESH			(1<<1)
+#define   I915_ERROR_INSTRUCTION			(1<<0)
 
 #define PIPE_VBLANK_INTERRUPT_ENABLE	(1UL<<17)
 #define I915_VBLANK_CLEAR		(1UL<<1)
@@ -526,8 +804,10 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define RING_VALID_MASK     	0x00000001
 #define RING_VALID          	0x00000001
 #define RING_INVALID        	0x00000000
+#define PGTBL_ER		0x02024
 #define PRB0_TAIL              0x02030
 #define PRB0_HEAD              0x02034
+#define PRB0_START		0x02038
 #define PRB0_CTL               0x0203c
 #define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
 #define SC_UPDATE_SCISSOR       (0x1<<1)
@@ -578,11 +858,25 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 
 #define CMD_OP_DESTBUFFER_INFO	 ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
 
-#define BREADCRUMB_BITS 31
-#define BREADCRUMB_MASK ((1U << BREADCRUMB_BITS) - 1)
-
-#define READ_BREADCRUMB(dev_priv)  (((volatile u32*)(dev_priv->hw_status_page))[5])
+/**
+ * Reads a dword out of the status page, which is written to from the command
+ * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
+ * MI_STORE_DATA_IMM.
+ *
+ * The following dwords have a reserved meaning:
+ * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
+ * 0x04: ring 0 head pointer
+ * 0x05: ring 1 head pointer (915-class)
+ * 0x06: ring 2 head pointer (915-class)
+ * 0x10-0x1b: Context status DWords (GM45)
+ * 0x1f: Last written status offset. (GM45)
+ *
+ * The area from dword 0x20 to 0x3ff is available for driver usage.
+ */
 #define READ_HWSP(dev_priv, reg)  (((volatile u32*)(dev_priv->hw_status_page))[reg])
+#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
+#define I915_GEM_HWS_INDEX		0x20
+#define I915_BREADCRUMB_INDEX		0x21
 
 /*
  * add here for S3 support
@@ -702,6 +996,34 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define PALETTE_A               0x0a000
 #define PALETTE_B               0x0a800
 
+/* MCH MMIO space */
+
+/*
+ * MCHBAR mirror.
+ *
+ * This mirrors the MCHBAR MMIO space whose location is determined by
+ * device 0 function 0's pci config register 0x44 or 0x48 and matches it in
+ * every way.  It is not accessible from the CP register read instructions.
+ *
+ */
+#define MCHBAR_MIRROR_BASE	0x10000
+
+/** 915-945 and GM965 MCH register controlling DRAM channel access */
+#define DCC			0x10200
+#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL		(0 << 0)
+#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC	(1 << 0)
+#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED	(2 << 0)
+#define DCC_ADDRESSING_MODE_MASK			(3 << 0)
+#define DCC_CHANNEL_XOR_DISABLE				(1 << 10)
+#define DCC_CHANNEL_XOR_BIT_17				(1 << 9)
+
+/** 965 MCH register controlling DRAM channel configuration */
+#define C0DRB3			0x10206
+#define C1DRB3			0x10606
+
+/** GM965 GM45 render standby register */
+#define MCHBAR_RENDER_STANDBY	0x111B8
+
 #define FPA0            0x06040
 #define FPA1            0x06044
 #define FPB0            0x06048
@@ -834,11 +1156,13 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
  * - pipe enabled
  * - LVDS/DVOB/DVOC on
  */
-# define PP_READY                               (1 << 30) # define PP_SEQUENCE_NONE                       (0 << 28)
-# define PP_SEQUENCE_ON                         (1 << 28) # define PP_SEQUENCE_OFF                        (2 << 28)
-# define PP_SEQUENCE_MASK                       0x30000000
+#define PP_READY                               (1 << 30)
+#define PP_SEQUENCE_NONE                       (0 << 28)
+#define PP_SEQUENCE_ON                         (1 << 28)
+#define PP_SEQUENCE_OFF                        (2 << 28)
+#define PP_SEQUENCE_MASK                       0x30000000
 #define PP_CONTROL      0x61204
-# define POWER_TARGET_ON                        (1 << 0)
+#define POWER_TARGET_ON                        (1 << 0)
 
 #define LVDSPP_ON       0x61208
 #define LVDSPP_OFF      0x6120c
@@ -864,8 +1188,10 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define   FBC_CTL_INTERVAL_SHIFT (16)
 #define   FBC_CTL_UNCOMPRESSIBLE (1<<14)
 #define   FBC_CTL_STRIDE_SHIFT  (5)
-#define   FBC_CTL_FENCENO       (1<<0) #define FBC_COMMAND             0x0320c
-#define   FBC_CMD_COMPRESS      (1<<0) #define FBC_STATUS              0x03210
+#define   FBC_CTL_FENCENO       (1<<0)
+#define	FBC_COMMAND             0x0320c
+#define   FBC_CMD_COMPRESS      (1<<0)
+#define FBC_STATUS              0x03210
 #define   FBC_STAT_COMPRESSING  (1<<31)
 #define   FBC_STAT_COMPRESSED   (1<<30)
 #define   FBC_STAT_MODIFIED     (1<<29)
@@ -914,6 +1240,7 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT		(1<<4)
 #define I915_DEBUG_INTERRUPT				(1<<2)
 #define I915_USER_INTERRUPT				(1<<1)
+#define	I915_ASLE_INTERRUPT				(1<<0)
 
 #define I915_FIFO_UNDERRUN_STATUS		(1UL<<31)
 #define I915_CRC_ERROR_ENABLE			(1UL<<29)
@@ -941,6 +1268,12 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define PIPE_VBLANK_INTERRUPT_STATUS		(1UL<<1)
 #define I915_OVERLAY_UPDATED_STATUS		(1UL<<0)
 
+/* GM45+ just has to be different */
+#define PIPEA_FRMCOUNT_GM45    0x70040
+#define PIPEA_FLIPCOUNT_GM45   0x70044
+#define PIPEB_FRMCOUNT_GM45    0x71040
+#define PIPEB_FLIPCOUNT_GM45   0x71044
+
 /*
  * Some BIOS scratch area registers.  The 845 (and 830?) store the amount
  * of video memory available to the BIOS in SWF1.
@@ -978,6 +1311,7 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define	PCI_DEVICE_ID_INTEL_82Q45_IG	0x2e12
 #define	PCI_DEVICE_ID_INTEL_82G45_IG	0x2e22
 #define	PCI_DEVICE_ID_INTEL_82G41_IG	0x2e32
+#define	PCI_DEVICE_ID_INTEL_82B43_IG	0x2e42
 
 
 #define IS_I830(dev) ((dev)->pci_device == PCI_DEVICE_ID_INTEL_82830_CGC)
@@ -1002,6 +1336,7 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
                        (dev)->pci_device == PCI_DEVICE_ID_INTEL_EL_IG || \
                        (dev)->pci_device == PCI_DEVICE_ID_INTEL_82Q45_IG || \
                        (dev)->pci_device == PCI_DEVICE_ID_INTEL_82G45_IG || \
+			(dev)->pci_device == PCI_DEVICE_ID_INTEL_82B43_IG || \
 			(dev)->pci_device == PCI_DEVICE_ID_INTEL_82G41_IG)
 
 #define IS_I965GM(dev) ((dev)->pci_device == PCI_DEVICE_ID_INTEL_GM965_IG)
@@ -1011,6 +1346,7 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define IS_G4X(dev) ((dev)->pci_device == PCI_DEVICE_ID_INTEL_EL_IG || \
                      (dev)->pci_device == PCI_DEVICE_ID_INTEL_82Q45_IG || \
                      (dev)->pci_device == PCI_DEVICE_ID_INTEL_82G45_IG || \
+                     (dev)->pci_device == PCI_DEVICE_ID_INTEL_82B43_IG || \
                      (dev)->pci_device == PCI_DEVICE_ID_INTEL_82G41_IG)
 
 #define IS_G33(dev)    ((dev)->pci_device == PCI_DEVICE_ID_INTEL_82G33_IG ||  \
@@ -1023,6 +1359,15 @@ extern int i915_wait_ring(drm_device_t * dev, int n, const char *caller);
 #define IS_MOBILE(dev) (IS_I830(dev) || IS_I85X(dev) || IS_I915GM(dev) || \
                         IS_I945GM(dev) || IS_I965GM(dev) || IS_GM45(dev))
 
+#define IS_IGDG(dev) ((dev)->pci_device == 0xa001)
+#define IS_IGDGM(dev) ((dev)->pci_device == 0xa011)
+#define IS_IGD(dev) (IS_IGDG(dev) || IS_IGDGM(dev))
+
 #define I915_NEED_GFX_HWS(dev) (IS_G33(dev) || IS_GM45(dev) || IS_G4X(dev))
+/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
+ * rows, which changed the alignment requirements and fence programming.
+ */
+#define HAS_128_BYTE_Y_TILING(dev) (IS_I9XX(dev) && !(IS_I915G(dev) || \
+						      IS_I915GM(dev)))
 
 #endif /* _I915_DRV_H */
diff --git a/usr/src/uts/intel/io/drm/i915_gem.c b/usr/src/uts/intel/io/drm/i915_gem.c
new file mode 100644
index 0000000000..28a534f406
--- /dev/null
+++ b/usr/src/uts/intel/io/drm/i915_gem.c
@@ -0,0 +1,2906 @@
+/* BEGIN CSTYLED */
+
+/*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/x86_archext.h>
+#include <sys/vfs_opreg.h>
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+
+#ifndef roundup
+#define	roundup(x, y)   ((((x)+((y)-1))/(y))*(y))
+#endif /* !roundup */
+
+#define I915_GEM_GPU_DOMAINS	(~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+
+static timeout_id_t worktimer_id = NULL;
+
+extern int drm_mm_init(struct drm_mm *mm,
+		    unsigned long start, unsigned long size);
+extern void drm_mm_put_block(struct drm_mm_node *cur);
+extern int choose_addr(struct as *as, caddr_t *addrp, size_t len, offset_t off,
+    int vacalign, uint_t flags);
+
+static void
+i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
+				  uint32_t read_domains,
+				  uint32_t write_domain);
+static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
+static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
+static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
+static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
+					     int write);
+static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
+					     int write);
+static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
+						     uint64_t offset,
+						     uint64_t size);
+static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
+static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
+static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
+static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
+
+static void
+i915_gem_cleanup_ringbuffer(struct drm_device *dev);
+
+/*ARGSUSED*/
+int
+i915_gem_init_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_init args;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_init *) data, sizeof(args));
+
+	spin_lock(&dev->struct_mutex);
+
+	if ((args.gtt_start >= args.gtt_end) ||
+	    ((args.gtt_start & (PAGE_SIZE - 1)) != 0) ||
+	    ((args.gtt_end & (PAGE_SIZE - 1)) != 0)) {
+		spin_unlock(&dev->struct_mutex);
+		DRM_ERROR("i915_gem_init_ioctel invalid arg 0x%lx args.start 0x%lx end 0x%lx", &args, args.gtt_start, args.gtt_end);
+		return EINVAL;
+	}
+
+	dev->gtt_total = (uint32_t) (args.gtt_end - args.gtt_start);
+
+	drm_mm_init(&dev_priv->mm.gtt_space, (unsigned long) args.gtt_start,
+	    dev->gtt_total);
+	DRM_DEBUG("i915_gem_init_ioctl dev->gtt_total %x, dev_priv->mm.gtt_space 0x%x gtt_start 0x%lx", dev->gtt_total, dev_priv->mm.gtt_space, args.gtt_start);
+	ASSERT(dev->gtt_total != 0);
+
+	spin_unlock(&dev->struct_mutex);
+
+
+	return 0;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_get_aperture_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;	
+	struct drm_i915_gem_get_aperture args;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	args.aper_size = (uint64_t)dev->gtt_total;
+	args.aper_available_size = (args.aper_size -
+				     atomic_read(&dev->pin_memory));
+
+        ret = DRM_COPY_TO_USER((struct drm_i915_gem_get_aperture __user *) data, &args, sizeof(args));
+
+        if ( ret != 0)
+                DRM_ERROR(" i915_gem_get_aperture_ioctl error! %d", ret);
+
+	DRM_DEBUG("i915_gem_get_aaperture_ioctl called sizeof %d, aper_size 0x%x, aper_available_size 0x%x\n", sizeof(args), dev->gtt_total, args.aper_available_size);
+
+	return 0;
+}
+
+/**
+ * Creates a new mm object and returns a handle to it.
+ */
+/*ARGSUSED*/
+int
+i915_gem_create_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_create args;
+	struct drm_gem_object *obj;
+	int handlep;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+	    (struct drm_i915_gem_create *) data, sizeof(args));
+
+
+	args.size = (uint64_t) roundup(args.size, PAGE_SIZE);
+
+	if (args.size == 0) {
+		DRM_ERROR("Invalid obj size %d", args.size);
+		return EINVAL;
+	}
+	/* Allocate the new object */
+	obj = drm_gem_object_alloc(dev, args.size);
+	if (obj == NULL) {
+		DRM_ERROR("Failed to alloc obj");
+		return ENOMEM;
+	}
+
+	ret = drm_gem_handle_create(fpriv, obj, &handlep);
+	spin_lock(&dev->struct_mutex);
+	drm_gem_object_handle_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	args.handle = handlep;
+
+	ret = DRM_COPY_TO_USER((struct drm_i915_gem_create *) data, &args, sizeof(args));
+
+	if ( ret != 0)
+		DRM_ERROR(" gem create error! %d", ret);
+
+	DRM_DEBUG("i915_gem_create_ioctl object name %d, size 0x%lx, list 0x%lx, obj 0x%lx",handlep, args.size, &fpriv->object_idr, obj);
+
+	return 0;
+}
+
+/**
+ * Reads data from the object referenced by handle.
+ *
+ * On error, the contents of *data are undefined.
+ */
+/*ARGSUSED*/
+int
+i915_gem_pread_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_pread args;
+	struct drm_gem_object *obj;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+	    (struct drm_i915_gem_pread __user *) data, sizeof(args));
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return EBADF;
+
+	/* Bounds check source.
+	 *
+	 * XXX: This could use review for overflow issues...
+	 */
+	if (args.offset > obj->size || args.size > obj->size ||
+	    args.offset + args.size > obj->size) {
+		drm_gem_object_unreference(obj);
+		DRM_ERROR("i915_gem_pread_ioctl invalid args");
+		return EINVAL;
+	}
+
+	spin_lock(&dev->struct_mutex);
+
+	ret = i915_gem_object_set_cpu_read_domain_range(obj, args.offset, args.size);
+	if (ret != 0) {
+		drm_gem_object_unreference(obj);
+		spin_unlock(&dev->struct_mutex);
+		DRM_ERROR("pread failed to read domain range ret %d!!!", ret);
+		return EFAULT;
+	}
+
+	unsigned long unwritten = 0;
+	uint32_t *user_data;
+	user_data = (uint32_t *) (uintptr_t) args.data_ptr;
+
+	unwritten = DRM_COPY_TO_USER(user_data, obj->kaddr + args.offset, args.size);
+        if (unwritten) {
+                ret = EFAULT;
+                DRM_ERROR("i915_gem_pread error!!! unwritten %d", unwritten);
+        }
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	return ret;
+}
+
+/*ARGSUSED*/
+static int
+i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
+		    struct drm_i915_gem_pwrite *args,
+		    struct drm_file *file_priv)
+{
+	uint32_t *user_data;	
+	int ret = 0;
+	unsigned long unwritten = 0;
+
+	user_data = (uint32_t *) (uintptr_t) args->data_ptr;
+	spin_lock(&dev->struct_mutex);
+	ret = i915_gem_object_pin(obj, 0);
+	if (ret) {
+		spin_unlock(&dev->struct_mutex);
+		DRM_ERROR("i915_gem_gtt_pwrite failed to pin ret %d", ret);
+		return ret;
+	}
+
+	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+	if (ret)
+		goto err;
+	
+	DRM_DEBUG("obj %d write domain 0x%x read domain 0x%x", obj->name, obj->write_domain, obj->read_domains);
+	
+	unwritten = DRM_COPY_FROM_USER(obj->kaddr + args->offset, user_data, args->size);
+        if (unwritten) {
+                ret = EFAULT;
+                DRM_ERROR("i915_gem_gtt_pwrite error!!! unwritten %d", unwritten);
+                goto err;
+        }
+
+err:
+	i915_gem_object_unpin(obj);
+	spin_unlock(&dev->struct_mutex);
+	if (ret)
+		DRM_ERROR("i915_gem_gtt_pwrite error %d", ret);
+	return ret;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
+		      struct drm_i915_gem_pwrite *args,
+		      struct drm_file *file_priv)
+{
+	DRM_ERROR(" i915_gem_shmem_pwrite Not support");
+	return -1;
+}
+
+/**
+ * Writes data to the object referenced by handle.
+ *
+ * On error, the contents of the buffer that were to be modified are undefined.
+ */
+/*ARGSUSED*/
+int
+i915_gem_pwrite_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_pwrite args;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret = 0;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	ret = DRM_COPY_FROM_USER(&args,
+            (struct drm_i915_gem_pwrite __user *) data, sizeof(args));
+	if (ret)
+		DRM_ERROR("i915_gem_pwrite_ioctl failed to copy from user");
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return EBADF;
+	obj_priv = obj->driver_private;
+	DRM_DEBUG("i915_gem_pwrite_ioctl, obj->name %d",obj->name);
+
+	/* Bounds check destination.
+	 *
+	 * XXX: This could use review for overflow issues...
+	 */
+	if (args.offset > obj->size || args.size > obj->size ||
+	    args.offset + args.size > obj->size) {
+		drm_gem_object_unreference(obj);
+		DRM_ERROR("i915_gem_pwrite_ioctl invalid arg");
+		return EINVAL;
+	}
+
+	/* We can only do the GTT pwrite on untiled buffers, as otherwise
+	 * it would end up going through the fenced access, and we'll get
+	 * different detiling behavior between reading and writing.
+	 * pread/pwrite currently are reading and writing from the CPU
+	 * perspective, requiring manual detiling by the client.
+	 */
+	if (obj_priv->tiling_mode == I915_TILING_NONE &&
+	    dev->gtt_total != 0)
+		ret = i915_gem_gtt_pwrite(dev, obj, &args, fpriv);
+	else
+		ret = i915_gem_shmem_pwrite(dev, obj, &args, fpriv);
+
+	if (ret)
+		DRM_ERROR("pwrite failed %d\n", ret);
+
+	drm_gem_object_unreference(obj);
+
+	return ret;
+}
+
+/**
+ * Called when user space prepares to use an object with the CPU, either
+ * through the mmap ioctl's mapping or a GTT mapping.
+ */
+/*ARGSUSED*/
+int
+i915_gem_set_domain_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_set_domain args;
+	struct drm_gem_object *obj;
+	int ret = 0;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+        DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_set_domain __user *) data, sizeof(args));
+
+	uint32_t read_domains = args.read_domains;
+	uint32_t write_domain = args.write_domain;
+
+	/* Only handle setting domains to types used by the CPU. */
+	if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+		ret = EINVAL;
+
+	if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+		ret = EINVAL;
+
+	/* Having something in the write domain implies it's in the read
+	 * domain, and only that read domain.  Enforce that in the request.
+	 */
+	if (write_domain != 0 && read_domains != write_domain)
+		ret = EINVAL;
+	if (ret) {
+		DRM_ERROR("set_domain invalid read or write");
+		return EINVAL;
+	}
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return EBADF;
+
+	spin_lock(&dev->struct_mutex);
+	DRM_DEBUG("set_domain_ioctl %p(name %d size 0x%x), %08x %08x\n",
+		 obj, obj->name, obj->size, args.read_domains, args.write_domain);
+
+	if (read_domains & I915_GEM_DOMAIN_GTT) {
+		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
+
+		/* Silently promote "you're not bound, there was nothing to do"
+		 * to success, since the client was just asking us to
+		 * make sure everything was done.
+		 */
+		if (ret == EINVAL)
+			ret = 0;
+	} else {
+		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
+	}
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+	if (ret)
+		DRM_ERROR("i915_set_domain_ioctl ret %d", ret);	
+	return ret;
+}
+
+/**
+ * Called when user space has done writes to this buffer
+ */
+/*ARGSUSED*/
+int
+i915_gem_sw_finish_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_sw_finish args;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret = 0;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+        DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_sw_finish __user *) data, sizeof(args));
+
+	spin_lock(&dev->struct_mutex);
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL) {
+		spin_unlock(&dev->struct_mutex);
+		return EBADF;
+	}
+
+	DRM_DEBUG("%s: sw_finish %d (%p name %d size 0x%x)\n",
+		 __func__, args.handle, obj, obj->name, obj->size);
+
+	obj_priv = obj->driver_private;
+	/* Pinned buffers may be scanout, so flush the cache */
+	if (obj_priv->pin_count)
+	{
+		i915_gem_object_flush_cpu_write_domain(obj);
+	}
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+	return ret;
+}
+
+/**
+ * Maps the contents of an object, returning the address it is mapped
+ * into.
+ *
+ * While the mapping holds a reference on the contents of the object, it doesn't
+ * imply a ref on the object itself.
+ */
+/*ARGSUSED*/
+int
+i915_gem_mmap_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_mmap args;
+	struct drm_gem_object *obj;
+	caddr_t vvaddr = NULL;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	DRM_COPYFROM_WITH_RETURN(
+	    &args, (struct drm_i915_gem_mmap __user *)data,
+	    sizeof (struct drm_i915_gem_mmap));
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return EBADF;
+
+	ret = ddi_devmap_segmap(fpriv->dev, (off_t)obj->map->handle,
+	    ttoproc(curthread)->p_as, &vvaddr, obj->map->size,
+	    PROT_ALL, PROT_ALL, MAP_SHARED, fpriv->credp);
+	if (ret)
+		return ret;
+
+	spin_lock(&dev->struct_mutex);
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	args.addr_ptr = (uint64_t)(uintptr_t)vvaddr;
+
+	DRM_COPYTO_WITH_RETURN(
+	    (struct drm_i915_gem_mmap __user *)data,
+	    &args, sizeof (struct drm_i915_gem_mmap));
+
+	return 0;
+}
+
+static void
+i915_gem_object_free_page_list(struct drm_gem_object *obj)
+{
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	if (obj_priv->page_list == NULL)
+		return;
+
+        kmem_free(obj_priv->page_list,
+                 btop(obj->size) * sizeof(caddr_t));
+
+        obj_priv->page_list = NULL;
+}
+
+static void
+i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
+{
+	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+	/* Add a reference if we're newly entering the active list. */
+	if (!obj_priv->active) {
+		drm_gem_object_reference(obj);
+		obj_priv->active = 1;
+	}
+	/* Move from whatever list we were on to the tail of execution. */
+	list_move_tail(&obj_priv->list,
+		       &dev_priv->mm.active_list, (caddr_t)obj_priv);
+	obj_priv->last_rendering_seqno = seqno;
+}
+
+static void
+i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+	list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list, (caddr_t)obj_priv);
+	obj_priv->last_rendering_seqno = 0;
+}
+
+static void
+i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+	if (obj_priv->pin_count != 0)
+	{	
+		list_del_init(&obj_priv->list);
+	} else {
+		list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list, (caddr_t)obj_priv);
+	}
+	obj_priv->last_rendering_seqno = 0;	
+	if (obj_priv->active) {
+		obj_priv->active = 0;
+		drm_gem_object_unreference(obj);
+	}
+}
+
+/**
+ * Creates a new sequence number, emitting a write of it to the status page
+ * plus an interrupt, which will trigger i915_user_interrupt_handler.
+ *
+ * Must be called with struct_lock held.
+ *
+ * Returned sequence numbers are nonzero on success.
+ */
+static uint32_t
+i915_add_request(struct drm_device *dev, uint32_t flush_domains)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_request *request;
+	uint32_t seqno;
+	int was_empty;
+	RING_LOCALS;
+
+	request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
+	if (request == NULL) {
+		DRM_ERROR("Failed to alloc request");
+		return 0;
+	}
+	/* Grab the seqno we're going to make this request be, and bump the
+	 * next (skipping 0 so it can be the reserved no-seqno value).
+	 */
+	seqno = dev_priv->mm.next_gem_seqno;
+	dev_priv->mm.next_gem_seqno++;
+	if (dev_priv->mm.next_gem_seqno == 0)
+		dev_priv->mm.next_gem_seqno++;
+
+	DRM_DEBUG("add_request seqno = %d dev 0x%lx", seqno, dev);
+
+	BEGIN_LP_RING(4);
+	OUT_RING(MI_STORE_DWORD_INDEX);
+	OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+	OUT_RING(seqno);
+        OUT_RING(0);
+        ADVANCE_LP_RING();
+
+	BEGIN_LP_RING(2);
+	OUT_RING(0);
+	OUT_RING(MI_USER_INTERRUPT);
+	ADVANCE_LP_RING();
+	
+	request->seqno = seqno;
+	request->emitted_jiffies = jiffies;
+	was_empty = list_empty(&dev_priv->mm.request_list);
+	list_add_tail(&request->list, &dev_priv->mm.request_list, (caddr_t)request);
+
+	/* Associate any objects on the flushing list matching the write
+	 * domain we're flushing with our flush.
+	 */
+	if (flush_domains != 0) {
+		struct drm_i915_gem_object *obj_priv, *next;
+
+		obj_priv = list_entry(dev_priv->mm.flushing_list.next, struct drm_i915_gem_object, list),
+		next = list_entry(obj_priv->list.next, struct drm_i915_gem_object, list);
+		for(; &obj_priv->list != &dev_priv->mm.flushing_list;
+			obj_priv = next,
+			next = list_entry(next->list.next, struct drm_i915_gem_object, list)) {
+			struct drm_gem_object *obj = obj_priv->obj;
+
+			if ((obj->write_domain & flush_domains) ==
+			    obj->write_domain) {
+				obj->write_domain = 0;
+				i915_gem_object_move_to_active(obj, seqno);
+			}
+		}
+
+	}
+
+	if (was_empty && !dev_priv->mm.suspended)
+	{
+		/* change to delay HZ and then run work (not insert to workqueue of Linux) */ 
+		worktimer_id = timeout(i915_gem_retire_work_handler, (void *) dev, DRM_HZ);
+		DRM_DEBUG("i915_gem: schedule_delayed_work");
+	}
+	return seqno;
+}
+
+/**
+ * Command execution barrier
+ *
+ * Ensures that all commands in the ring are finished
+ * before signalling the CPU
+ */
+uint32_t
+i915_retire_commands(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
+	uint32_t flush_domains = 0;
+	RING_LOCALS;
+
+	/* The sampler always gets flushed on i965 (sigh) */
+	if (IS_I965G(dev))
+		flush_domains |= I915_GEM_DOMAIN_SAMPLER;
+	BEGIN_LP_RING(3);
+	OUT_RING(cmd);
+	OUT_RING(0); /* noop */
+	ADVANCE_LP_RING();
+
+	return flush_domains;
+}
+
+/**
+ * Moves buffers associated only with the given active seqno from the active
+ * to inactive list, potentially freeing them.
+ */
+static void
+i915_gem_retire_request(struct drm_device *dev,
+			struct drm_i915_gem_request *request)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	/* Move any buffers on the active list that are no longer referenced
+	 * by the ringbuffer to the flushing/inactive lists as appropriate.
+	 */
+	while (!list_empty(&dev_priv->mm.active_list)) {
+		struct drm_gem_object *obj;
+		struct drm_i915_gem_object *obj_priv;
+
+		obj_priv = list_entry(dev_priv->mm.active_list.next,
+					    struct drm_i915_gem_object,
+					    list);
+		obj = obj_priv->obj;
+
+		/* If the seqno being retired doesn't match the oldest in the
+		 * list, then the oldest in the list must still be newer than
+		 * this seqno.
+		 */
+		if (obj_priv->last_rendering_seqno != request->seqno)
+			return;
+
+		DRM_DEBUG("%s: retire %d moves to inactive list %p\n",
+			 __func__, request->seqno, obj);
+
+		if (obj->write_domain != 0) {
+			i915_gem_object_move_to_flushing(obj);
+		} else {
+			i915_gem_object_move_to_inactive(obj);
+		}
+	}
+}
+
+/**
+ * Returns true if seq1 is later than seq2.
+ */
+static int
+i915_seqno_passed(uint32_t seq1, uint32_t seq2)
+{
+	return (int32_t)(seq1 - seq2) >= 0;
+}
+
+uint32_t
+i915_get_gem_seqno(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
+}
+
+/**
+ * This function clears the request list as sequence numbers are passed.
+ */
+void
+i915_gem_retire_requests(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t seqno;
+
+	seqno = i915_get_gem_seqno(dev);
+
+	while (!list_empty(&dev_priv->mm.request_list)) {
+		struct drm_i915_gem_request *request;
+		uint32_t retiring_seqno;
+		request = (struct drm_i915_gem_request *)(uintptr_t)(dev_priv->mm.request_list.next->contain_ptr);
+		retiring_seqno = request->seqno;
+
+		if (i915_seqno_passed(seqno, retiring_seqno) ||
+		    dev_priv->mm.wedged) {
+			i915_gem_retire_request(dev, request);
+
+			list_del(&request->list);
+			drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
+		} else
+			break;
+	}
+}
+
+void
+i915_gem_retire_work_handler(void *device)
+{
+	struct drm_device *dev = (struct drm_device *)device;	
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	spin_lock(&dev->struct_mutex);
+
+	/* Return if gem idle */
+	if (worktimer_id == NULL) {
+		spin_unlock(&dev->struct_mutex);
+		return;
+	}
+
+	i915_gem_retire_requests(dev);
+	if (!dev_priv->mm.suspended && !list_empty(&dev_priv->mm.request_list))
+	{	
+		DRM_DEBUG("i915_gem: schedule_delayed_work");
+		worktimer_id = timeout(i915_gem_retire_work_handler, (void *) dev, DRM_HZ);
+	}
+	spin_unlock(&dev->struct_mutex);
+}
+
+/**
+ * i965_reset - reset chip after a hang
+ * @dev: drm device to reset
+ * @flags: reset domains
+ *
+ * Reset the chip.  Useful if a hang is detected.
+ *
+ * Procedure is fairly simple:
+ *   - reset the chip using the reset reg
+ *   - re-init context state
+ *   - re-init hardware status page
+ *   - re-init ring buffer
+ *   - re-init interrupt state
+ *   - re-init display
+ */
+void i965_reset(struct drm_device *dev, u8 flags)
+{
+	ddi_acc_handle_t conf_hdl;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int timeout = 0;
+	uint8_t gdrst;
+
+	if (flags & GDRST_FULL)
+		i915_save_display(dev);
+
+	if (pci_config_setup(dev->dip, &conf_hdl) != DDI_SUCCESS) {
+		DRM_ERROR(("i915_reset: pci_config_setup fail"));
+		return;
+	}
+
+	/*
+	 * Set the reset bit, wait for reset, then clear it.  Hardware
+	 * will clear the status bit (bit 1) when it's actually ready
+	 * for action again.
+	 */
+	gdrst = pci_config_get8(conf_hdl, GDRST);
+	pci_config_put8(conf_hdl, GDRST, gdrst | flags);
+	drv_usecwait(50);	
+	pci_config_put8(conf_hdl, GDRST, gdrst | 0xfe);
+
+	/* ...we don't want to loop forever though, 500ms should be plenty */
+	do {
+		drv_usecwait(100);
+		gdrst = pci_config_get8(conf_hdl, GDRST);
+	} while ((gdrst & 2) && (timeout++ < 5));
+
+	/* Ok now get things going again... */
+
+	/*
+	 * Everything depends on having the GTT running, so we need to start
+	 * there.  Fortunately we don't need to do this unless we reset the
+	 * chip at a PCI level.
+	 *
+	 * Next we need to restore the context, but we don't use those
+	 * yet either...
+	 *
+	 * Ring buffer needs to be re-initialized in the KMS case, or if X
+	 * was running at the time of the reset (i.e. we weren't VT
+	 * switched away).
+	 */
+	 if (!dev_priv->mm.suspended) {
+		drm_i915_ring_buffer_t *ring = &dev_priv->ring;
+		struct drm_gem_object *obj = ring->ring_obj;
+		struct drm_i915_gem_object *obj_priv = obj->driver_private;
+		dev_priv->mm.suspended = 0;
+
+		/* Stop the ring if it's running. */
+		I915_WRITE(PRB0_CTL, 0);
+		I915_WRITE(PRB0_TAIL, 0);
+		I915_WRITE(PRB0_HEAD, 0);
+
+		/* Initialize the ring. */
+		I915_WRITE(PRB0_START, obj_priv->gtt_offset);
+		I915_WRITE(PRB0_CTL,
+			   ((obj->size - 4096) & RING_NR_PAGES) |
+			   RING_NO_REPORT |
+			   RING_VALID);
+		i915_kernel_lost_context(dev);
+
+		drm_irq_install(dev);
+	}
+
+	/*
+	 * Display needs restore too...
+	 */
+	if (flags & GDRST_FULL)
+		i915_restore_display(dev);
+}
+
+/**
+ * Waits for a sequence number to be signaled, and cleans up the
+ * request and object lists appropriately for that event.
+ */
+int
+i915_wait_request(struct drm_device *dev, uint32_t seqno)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret = 0;
+
+	ASSERT(seqno != 0);
+
+	if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
+		dev_priv->mm.waiting_gem_seqno = seqno;
+		i915_user_irq_on(dev);
+		DRM_WAIT(ret, &dev_priv->irq_queue,
+		    (i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
+				dev_priv->mm.wedged));
+		i915_user_irq_off(dev);
+		dev_priv->mm.waiting_gem_seqno = 0;
+	}
+	if (dev_priv->mm.wedged) {
+		ret = EIO;
+	}
+
+	/* GPU maybe hang, reset needed*/
+	if (ret == -2 && (seqno > i915_get_gem_seqno(dev))) {
+		if (IS_I965G(dev)) {
+			DRM_ERROR("GPU hang detected try to reset ... wait for irq_queue seqno %d, now seqno %d", seqno, i915_get_gem_seqno(dev));
+			dev_priv->mm.wedged = 1;
+			i965_reset(dev, GDRST_RENDER);
+			i915_gem_retire_requests(dev);
+			dev_priv->mm.wedged = 0;	
+		}
+		else
+			DRM_ERROR("GPU hang detected.... reboot required");
+		return 0;
+	}
+	/* Directly dispatch request retiring.  While we have the work queue
+	 * to handle this, the waiter on a request often wants an associated
+	 * buffer to have made it to the inactive list, and we would need
+	 * a separate wait queue to handle that.
+	 */
+	if (ret == 0)
+		i915_gem_retire_requests(dev);
+
+	return ret;
+}
+
+static void
+i915_gem_flush(struct drm_device *dev,
+	       uint32_t invalidate_domains,
+	       uint32_t flush_domains)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t cmd;
+	RING_LOCALS;
+
+	DRM_DEBUG("%s: invalidate %08x flush %08x\n", __func__,
+		  invalidate_domains, flush_domains);
+
+	if (flush_domains & I915_GEM_DOMAIN_CPU)
+		drm_agp_chipset_flush(dev);
+
+	if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
+						     I915_GEM_DOMAIN_GTT)) {
+		/*
+		 * read/write caches:
+		 *
+		 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
+		 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
+		 * also flushed at 2d versus 3d pipeline switches.
+		 *
+		 * read-only caches:
+		 *
+		 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
+		 * MI_READ_FLUSH is set, and is always flushed on 965.
+		 *
+		 * I915_GEM_DOMAIN_COMMAND may not exist?
+		 *
+		 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
+		 * invalidated when MI_EXE_FLUSH is set.
+		 *
+		 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
+		 * invalidated with every MI_FLUSH.
+		 *
+		 * TLBs:
+		 *
+		 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
+		 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
+		 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
+		 * are flushed at any MI_FLUSH.
+		 */
+
+		cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
+		if ((invalidate_domains|flush_domains) &
+		    I915_GEM_DOMAIN_RENDER)
+			cmd &= ~MI_NO_WRITE_FLUSH;
+		if (!IS_I965G(dev)) {
+			/*
+			 * On the 965, the sampler cache always gets flushed
+			 * and this bit is reserved.
+			 */
+			if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+				cmd |= MI_READ_FLUSH;
+		}
+		if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
+			cmd |= MI_EXE_FLUSH;
+
+		DRM_DEBUG("%s: queue flush %08x to ring\n", __func__, cmd);
+
+		BEGIN_LP_RING(2);
+		OUT_RING(cmd);
+		OUT_RING(0); /* noop */
+		ADVANCE_LP_RING();
+	}
+}
+
+/**
+ * Ensures that all rendering to the object has completed and the object is
+ * safe to unbind from the GTT or access from the CPU.
+ */
+static int
+i915_gem_object_wait_rendering(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int ret, seqno;
+
+	/* This function only exists to support waiting for existing rendering,
+	 * not for emitting required flushes.
+	 */
+
+	if((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0) {
+		DRM_ERROR("write domain should not be GPU DOMAIN %d", obj_priv->active);
+		return 0;
+	}
+
+	/* If there is rendering queued on the buffer being evicted, wait for
+	 * it.
+	 */
+	if (obj_priv->active) {
+		DRM_DEBUG("%s: object %d %p wait for seqno %08x\n",
+			  __func__, obj->name, obj, obj_priv->last_rendering_seqno);
+
+		seqno = obj_priv->last_rendering_seqno;
+		if (seqno == 0) {
+			DRM_DEBUG("last rendering maybe finished");
+			return 0;
+		}
+		ret = i915_wait_request(dev, seqno);
+		if (ret != 0) {
+			DRM_ERROR("%s: i915_wait_request request->seqno %d now %d\n", __func__, seqno, i915_get_gem_seqno(dev));
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Unbinds an object from the GTT aperture.
+ */
+int
+i915_gem_object_unbind(struct drm_gem_object *obj, uint32_t type)
+{
+	struct drm_device *dev = obj->dev;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int ret = 0;
+
+	if (obj_priv->gtt_space == NULL)
+		return 0;
+
+	if (obj_priv->pin_count != 0) {
+		DRM_ERROR("Attempting to unbind pinned buffer\n");
+		return EINVAL;
+	}
+
+	/* Wait for any rendering to complete
+	 */
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret) {
+		DRM_ERROR("wait_rendering failed: %d\n", ret);
+		return ret;
+	}
+
+	/* Move the object to the CPU domain to ensure that
+	 * any possible CPU writes while it's not in the GTT
+	 * are flushed when we go to remap it. This will
+	 * also ensure that all pending GPU writes are finished
+	 * before we unbind.
+	 */
+	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+	if (ret) {
+		DRM_ERROR("set_domain failed: %d\n", ret);
+		return ret;
+	}
+
+	if (!obj_priv->agp_mem) {
+		drm_agp_unbind_pages(dev, obj->size / PAGE_SIZE, obj_priv->gtt_offset, type);
+		obj_priv->agp_mem = -1;
+	}
+
+	ASSERT(!obj_priv->active);
+
+	i915_gem_object_free_page_list(obj);
+
+	if (obj_priv->gtt_space) {
+		atomic_dec(&dev->gtt_count);
+		atomic_sub(obj->size, &dev->gtt_memory);
+		drm_mm_put_block(obj_priv->gtt_space);
+		obj_priv->gtt_space = NULL;
+	}
+
+	/* Remove ourselves from the LRU list if present. */
+	if (!list_empty(&obj_priv->list))
+		list_del_init(&obj_priv->list);
+
+	return 0;
+}
+
+static int
+i915_gem_evict_something(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret = 0;
+
+	for (;;) {
+		/* If there's an inactive buffer available now, grab it
+		 * and be done.
+		 */
+		if (!list_empty(&dev_priv->mm.inactive_list)) {
+			obj_priv = list_entry(dev_priv->mm.inactive_list.next,
+						    struct drm_i915_gem_object,
+						    list);
+			obj = obj_priv->obj;
+			ASSERT(!(obj_priv->pin_count != 0));
+			DRM_DEBUG("%s: evicting %d\n", __func__, obj->name);
+			ASSERT(!(obj_priv->active));
+			/* Wait on the rendering and unbind the buffer. */
+			ret = i915_gem_object_unbind(obj, 1);
+			break;
+		}
+		/* If we didn't get anything, but the ring is still processing
+		 * things, wait for one of those things to finish and hopefully
+		 * leave us a buffer to evict.
+		 */
+		if (!list_empty(&dev_priv->mm.request_list)) {
+			struct drm_i915_gem_request *request;
+
+			request = list_entry(dev_priv->mm.request_list.next,
+						   struct drm_i915_gem_request,
+						   list);
+
+			ret = i915_wait_request(dev, request->seqno);
+			if (ret) {
+				break;
+			}
+			/* if waiting caused an object to become inactive,
+			 * then loop around and wait for it. Otherwise, we
+			 * assume that waiting freed and unbound something,
+			 * so there should now be some space in the GTT
+			 */
+			if (!list_empty(&dev_priv->mm.inactive_list))
+				continue;
+			break;
+		}
+
+		/* If we didn't have anything on the request list but there
+		 * are buffers awaiting a flush, emit one and try again.
+		 * When we wait on it, those buffers waiting for that flush
+		 * will get moved to inactive.
+		 */
+		if (!list_empty(&dev_priv->mm.flushing_list)) {
+			obj_priv = list_entry(dev_priv->mm.flushing_list.next,
+						    struct drm_i915_gem_object,
+						    list);
+			obj = obj_priv->obj;
+
+			i915_gem_flush(dev,
+				       obj->write_domain,
+				       obj->write_domain);
+			(void) i915_add_request(dev, obj->write_domain);
+
+			obj = NULL;
+			continue;
+		}
+
+		DRM_ERROR("inactive empty %d request empty %d "
+			  "flushing empty %d\n",
+			  list_empty(&dev_priv->mm.inactive_list),
+			  list_empty(&dev_priv->mm.request_list),
+			  list_empty(&dev_priv->mm.flushing_list));
+		/* If we didn't do any of the above, there's nothing to be done
+		 * and we just can't fit it in.
+		 */
+		return ENOMEM;
+	}
+	return ret;
+}
+
+static int
+i915_gem_evict_everything(struct drm_device *dev)
+{
+	int ret;
+
+	for (;;) {
+		ret = i915_gem_evict_something(dev);
+		if (ret != 0)
+			break;
+	}
+	if (ret == ENOMEM)
+		return 0;
+	else
+		DRM_ERROR("evict_everything ret %d", ret);
+	return ret;
+}
+
+/**
+ * Finds free space in the GTT aperture and binds the object there.
+ */
+static int
+i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, uint32_t alignment)
+{
+	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	struct drm_mm_node *free_space;
+	int page_count, ret;
+
+	if (dev_priv->mm.suspended)
+		return EBUSY;
+	if (alignment == 0)
+		alignment = PAGE_SIZE;
+	if (alignment & (PAGE_SIZE - 1)) {
+		DRM_ERROR("Invalid object alignment requested %u\n", alignment);
+		return EINVAL;
+	}
+
+	if (obj_priv->gtt_space) {
+		DRM_ERROR("Already bind!!");
+		return 0;
+	}
+search_free:
+	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
+					(unsigned long) obj->size, alignment, 0);
+	if (free_space != NULL) {
+		obj_priv->gtt_space = drm_mm_get_block(free_space, (unsigned long) obj->size,
+						       alignment);
+		if (obj_priv->gtt_space != NULL) {
+			obj_priv->gtt_space->private = obj;
+			obj_priv->gtt_offset = obj_priv->gtt_space->start;
+		}
+	}
+	if (obj_priv->gtt_space == NULL) {
+		/* If the gtt is empty and we're still having trouble
+		 * fitting our object in, we're out of memory.
+		 */
+		if (list_empty(&dev_priv->mm.inactive_list) &&
+		    list_empty(&dev_priv->mm.flushing_list) &&
+		    list_empty(&dev_priv->mm.active_list)) {
+			DRM_ERROR("GTT full, but LRU list empty\n");
+			return ENOMEM;
+		}
+
+		ret = i915_gem_evict_something(dev);
+		if (ret != 0) {
+			DRM_ERROR("Failed to evict a buffer %d\n", ret);
+			return ret;
+		}
+		goto search_free;
+	}
+
+	ret = i915_gem_object_get_page_list(obj);
+	if (ret) {
+		drm_mm_put_block(obj_priv->gtt_space);
+		obj_priv->gtt_space = NULL;
+		DRM_ERROR("bind to gtt failed to get page list");
+		return ret;
+	}
+
+	page_count = obj->size / PAGE_SIZE;
+	/* Create an AGP memory structure pointing at our pages, and bind it
+	 * into the GTT.
+	 */
+	DRM_DEBUG("Binding object %d of page_count %d at gtt_offset 0x%x obj->pfnarray = 0x%lx", 
+		 obj->name, page_count, obj_priv->gtt_offset, obj->pfnarray);
+
+	obj_priv->agp_mem = drm_agp_bind_pages(dev,
+					       obj->pfnarray,
+					       page_count,
+					       obj_priv->gtt_offset);
+	if (obj_priv->agp_mem) {
+		i915_gem_object_free_page_list(obj);
+		drm_mm_put_block(obj_priv->gtt_space);
+		obj_priv->gtt_space = NULL;
+		DRM_ERROR("Failed to bind pages obj %d, obj 0x%lx", obj->name, obj);
+		return ENOMEM;
+	}
+	atomic_inc(&dev->gtt_count);
+	atomic_add(obj->size, &dev->gtt_memory);
+
+	/* Assert that the object is not currently in any GPU domain. As it
+	 * wasn't in the GTT, there shouldn't be any way it could have been in
+	 * a GPU cache
+	 */
+	ASSERT(!(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)));
+	ASSERT(!(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)));
+
+	return 0;
+}
+
+void
+i915_gem_clflush_object(struct drm_gem_object *obj)
+{
+	struct drm_i915_gem_object	*obj_priv = obj->driver_private;
+
+	/* If we don't have a page list set up, then we're not pinned
+	 * to GPU, and we can ignore the cache flush because it'll happen
+	 * again at bind time.
+	 */
+
+	if (obj_priv->page_list == NULL)
+		return;
+	drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
+}
+
+/** Flushes any GPU write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	uint32_t seqno;
+
+	if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
+		return;
+
+	/* Queue the GPU write cache flushing we need. */
+	i915_gem_flush(dev, 0, obj->write_domain);
+	seqno = i915_add_request(dev, obj->write_domain);
+	DRM_DEBUG("flush_gpu_write_domain seqno = %d", seqno);
+	obj->write_domain = 0;
+	i915_gem_object_move_to_active(obj, seqno);
+}
+
+/** Flushes the GTT write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
+{
+	if (obj->write_domain != I915_GEM_DOMAIN_GTT)
+		return;
+
+	/* No actual flushing is required for the GTT write domain.   Writes
+	 * to it immediately go to main memory as far as we know, so there's
+	 * no chipset flush.  It also doesn't land in render cache.
+	 */
+	obj->write_domain = 0;
+}
+
+/** Flushes the CPU write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+
+	if (obj->write_domain != I915_GEM_DOMAIN_CPU)
+		return;
+
+	i915_gem_clflush_object(obj);
+	drm_agp_chipset_flush(dev);
+	obj->write_domain = 0;
+}
+
+/**
+ * Moves a single object to the GTT read, and possibly write domain.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
+{
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int ret;
+
+	/* Not valid to be called on unbound objects. */
+	if (obj_priv->gtt_space == NULL)
+		return EINVAL;
+
+	i915_gem_object_flush_gpu_write_domain(obj);
+	/* Wait on any GPU rendering and flushing to occur. */
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret != 0) {
+		DRM_ERROR("set_to_gtt_domain wait_rendering ret %d", ret);
+		return ret;
+	}
+	/* If we're writing through the GTT domain, then CPU and GPU caches
+	 * will need to be invalidated at next use.
+	 */
+	if (write)
+		obj->read_domains &= I915_GEM_DOMAIN_GTT;
+	i915_gem_object_flush_cpu_write_domain(obj);
+
+	DRM_DEBUG("i915_gem_object_set_to_gtt_domain obj->read_domains %x ", obj->read_domains);
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0));
+	obj->read_domains |= I915_GEM_DOMAIN_GTT;
+	if (write) {
+		obj->write_domain = I915_GEM_DOMAIN_GTT;
+		obj_priv->dirty = 1;
+	}
+
+	return 0;
+}
+
+/**
+ * Moves a single object to the CPU read, and possibly write domain.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
+{
+	struct drm_device *dev = obj->dev;
+	int ret;
+
+
+	i915_gem_object_flush_gpu_write_domain(obj);
+	/* Wait on any GPU rendering and flushing to occur. */
+
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret != 0)
+		return ret;
+
+	i915_gem_object_flush_gtt_write_domain(obj);
+
+	/* If we have a partially-valid cache of the object in the CPU,
+	 * finish invalidating it and free the per-page flags.
+	 */
+	i915_gem_object_set_to_full_cpu_read_domain(obj);
+
+	/* Flush the CPU cache if it's still invalid. */
+	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
+		i915_gem_clflush_object(obj);
+		drm_agp_chipset_flush(dev);
+		obj->read_domains |= I915_GEM_DOMAIN_CPU;
+	}
+
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0));
+
+	/* If we're writing through the CPU, then the GPU read domains will
+	 * need to be invalidated at next use.
+	 */
+	if (write) {
+		obj->read_domains &= I915_GEM_DOMAIN_CPU;
+		obj->write_domain = I915_GEM_DOMAIN_CPU;
+	}
+
+	return 0;
+}
+
+/*
+ * Set the next domain for the specified object. This
+ * may not actually perform the necessary flushing/invaliding though,
+ * as that may want to be batched with other set_domain operations
+ *
+ * This is (we hope) the only really tricky part of gem. The goal
+ * is fairly simple -- track which caches hold bits of the object
+ * and make sure they remain coherent. A few concrete examples may
+ * help to explain how it works. For shorthand, we use the notation
+ * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
+ * a pair of read and write domain masks.
+ *
+ * Case 1: the batch buffer
+ *
+ *	1. Allocated
+ *	2. Written by CPU
+ *	3. Mapped to GTT
+ *	4. Read by GPU
+ *	5. Unmapped from GTT
+ *	6. Freed
+ *
+ *	Let's take these a step at a time
+ *
+ *	1. Allocated
+ *		Pages allocated from the kernel may still have
+ *		cache contents, so we set them to (CPU, CPU) always.
+ *	2. Written by CPU (using pwrite)
+ *		The pwrite function calls set_domain (CPU, CPU) and
+ *		this function does nothing (as nothing changes)
+ *	3. Mapped by GTT
+ *		This function asserts that the object is not
+ *		currently in any GPU-based read or write domains
+ *	4. Read by GPU
+ *		i915_gem_execbuffer calls set_domain (COMMAND, 0).
+ *		As write_domain is zero, this function adds in the
+ *		current read domains (CPU+COMMAND, 0).
+ *		flush_domains is set to CPU.
+ *		invalidate_domains is set to COMMAND
+ *		clflush is run to get data out of the CPU caches
+ *		then i915_dev_set_domain calls i915_gem_flush to
+ *		emit an MI_FLUSH and drm_agp_chipset_flush
+ *	5. Unmapped from GTT
+ *		i915_gem_object_unbind calls set_domain (CPU, CPU)
+ *		flush_domains and invalidate_domains end up both zero
+ *		so no flushing/invalidating happens
+ *	6. Freed
+ *		yay, done
+ *
+ * Case 2: The shared render buffer
+ *
+ *	1. Allocated
+ *	2. Mapped to GTT
+ *	3. Read/written by GPU
+ *	4. set_domain to (CPU,CPU)
+ *	5. Read/written by CPU
+ *	6. Read/written by GPU
+ *
+ *	1. Allocated
+ *		Same as last example, (CPU, CPU)
+ *	2. Mapped to GTT
+ *		Nothing changes (assertions find that it is not in the GPU)
+ *	3. Read/written by GPU
+ *		execbuffer calls set_domain (RENDER, RENDER)
+ *		flush_domains gets CPU
+ *		invalidate_domains gets GPU
+ *		clflush (obj)
+ *		MI_FLUSH and drm_agp_chipset_flush
+ *	4. set_domain (CPU, CPU)
+ *		flush_domains gets GPU
+ *		invalidate_domains gets CPU
+ *		wait_rendering (obj) to make sure all drawing is complete.
+ *		This will include an MI_FLUSH to get the data from GPU
+ *		to memory
+ *		clflush (obj) to invalidate the CPU cache
+ *		Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
+ *	5. Read/written by CPU
+ *		cache lines are loaded and dirtied
+ *	6. Read written by GPU
+ *		Same as last GPU access
+ *
+ * Case 3: The constant buffer
+ *
+ *	1. Allocated
+ *	2. Written by CPU
+ *	3. Read by GPU
+ *	4. Updated (written) by CPU again
+ *	5. Read by GPU
+ *
+ *	1. Allocated
+ *		(CPU, CPU)
+ *	2. Written by CPU
+ *		(CPU, CPU)
+ *	3. Read by GPU
+ *		(CPU+RENDER, 0)
+ *		flush_domains = CPU
+ *		invalidate_domains = RENDER
+ *		clflush (obj)
+ *		MI_FLUSH
+ *		drm_agp_chipset_flush
+ *	4. Updated (written) by CPU again
+ *		(CPU, CPU)
+ *		flush_domains = 0 (no previous write domain)
+ *		invalidate_domains = 0 (no new read domains)
+ *	5. Read by GPU
+ *		(CPU+RENDER, 0)
+ *		flush_domains = CPU
+ *		invalidate_domains = RENDER
+ *		clflush (obj)
+ *		MI_FLUSH
+ *		drm_agp_chipset_flush
+ */
+static void 
+i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
+			    uint32_t read_domains,
+			    uint32_t write_domain)
+{
+	struct drm_device		*dev = obj->dev;
+	struct drm_i915_gem_object	*obj_priv = obj->driver_private;
+	uint32_t			invalidate_domains = 0;
+	uint32_t			flush_domains = 0;
+
+	DRM_DEBUG("%s: object %p read %08x -> %08x write %08x -> %08x\n",
+		 __func__, obj,
+		 obj->read_domains, read_domains,
+		 obj->write_domain, write_domain);
+	/*
+	 * If the object isn't moving to a new write domain,
+	 * let the object stay in multiple read domains
+	 */
+	if (write_domain == 0)
+		read_domains |= obj->read_domains;
+	else
+		obj_priv->dirty = 1;
+
+	/*
+	 * Flush the current write domain if
+	 * the new read domains don't match. Invalidate
+	 * any read domains which differ from the old
+	 * write domain
+	 */
+	if (obj->write_domain && obj->write_domain != read_domains) {
+		flush_domains |= obj->write_domain;
+		invalidate_domains |= read_domains & ~obj->write_domain;
+	}
+	/*
+	 * Invalidate any read caches which may have
+	 * stale data. That is, any new read domains.
+	 */
+	invalidate_domains |= read_domains & ~obj->read_domains;
+	if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
+		DRM_DEBUG("%s: CPU domain flush %08x invalidate %08x\n",
+			 __func__, flush_domains, invalidate_domains);
+	i915_gem_clflush_object(obj);
+	}
+
+	if ((write_domain | flush_domains) != 0)
+		obj->write_domain = write_domain;
+	obj->read_domains = read_domains;
+
+	dev->invalidate_domains |= invalidate_domains;
+	dev->flush_domains |= flush_domains;
+
+	DRM_DEBUG("%s: read %08x write %08x invalidate %08x flush %08x\n",
+		 __func__,
+		 obj->read_domains, obj->write_domain,
+		 dev->invalidate_domains, dev->flush_domains);
+
+}
+
+/**
+ * Moves the object from a partially CPU read to a full one.
+ *
+ * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
+ * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
+ */
+static void
+i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+	if (!obj_priv->page_cpu_valid)
+		return;
+
+	/* If we're partially in the CPU read domain, finish moving it in.
+	 */
+	if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
+		int i;
+
+		for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
+			if (obj_priv->page_cpu_valid[i])
+				continue;
+			drm_clflush_pages(obj_priv->page_list + i, 1);
+		}
+		drm_agp_chipset_flush(dev);
+	}
+
+	/* Free the page_cpu_valid mappings which are now stale, whether
+	 * or not we've got I915_GEM_DOMAIN_CPU.
+	 */
+	drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
+		 DRM_MEM_DRIVER);
+	obj_priv->page_cpu_valid = NULL;
+}
+
+/**
+ * Set the CPU read domain on a range of the object.
+ *
+ * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
+ * not entirely valid.  The page_cpu_valid member of the object flags which
+ * pages have been flushed, and will be respected by
+ * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
+ * of the whole object.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
+					  uint64_t offset, uint64_t size)
+{
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int i, ret;
+
+	if (offset == 0 && size == obj->size)
+		return i915_gem_object_set_to_cpu_domain(obj, 0);
+
+	i915_gem_object_flush_gpu_write_domain(obj);
+	/* Wait on any GPU rendering and flushing to occur. */
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret != 0)
+		return ret;
+	i915_gem_object_flush_gtt_write_domain(obj);
+
+	/* If we're already fully in the CPU read domain, we're done. */
+	if (obj_priv->page_cpu_valid == NULL &&
+	    (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
+		return 0;
+
+	/* Otherwise, create/clear the per-page CPU read domain flag if we're
+	 * newly adding I915_GEM_DOMAIN_CPU
+	 */
+	if (obj_priv->page_cpu_valid == NULL) {
+		obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
+						      DRM_MEM_DRIVER);
+		if (obj_priv->page_cpu_valid == NULL)
+			return ENOMEM;
+	} else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
+		(void) memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
+
+	/* Flush the cache on any pages that are still invalid from the CPU's
+	 * perspective.
+	 */
+	for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
+	     i++) {
+		if (obj_priv->page_cpu_valid[i])
+			continue;
+
+		drm_clflush_pages(obj_priv->page_list + i, 1);
+		obj_priv->page_cpu_valid[i] = 1;
+	}
+
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0));
+
+	obj->read_domains |= I915_GEM_DOMAIN_CPU;
+
+	return 0;
+}
+
+/**
+ * Pin an object to the GTT and evaluate the relocations landing in it.
+ */
+static int
+i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
+				 struct drm_file *file_priv,
+				 struct drm_i915_gem_exec_object *entry)
+{
+	struct drm_i915_gem_relocation_entry reloc;
+	struct drm_i915_gem_relocation_entry __user *relocs;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int i, ret;
+
+	/* Choose the GTT offset for our buffer and put it there. */
+	ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
+	if (ret) {
+		DRM_ERROR("failed to pin");
+		return ret;
+	}
+	entry->offset = obj_priv->gtt_offset;
+
+	relocs = (struct drm_i915_gem_relocation_entry __user *)
+		 (uintptr_t) entry->relocs_ptr;
+	/* Apply the relocations, using the GTT aperture to avoid cache
+	 * flushing requirements.
+	 */
+	for (i = 0; i < entry->relocation_count; i++) {
+		struct drm_gem_object *target_obj;
+		struct drm_i915_gem_object *target_obj_priv;
+		uint32_t reloc_val, reloc_offset, *reloc_entry;
+
+		ret = DRM_COPY_FROM_USER(&reloc, relocs + i, sizeof(reloc));
+		if (ret != 0) {
+			i915_gem_object_unpin(obj);
+			DRM_ERROR("failed to copy from user");	
+			return ret;
+		}
+
+		target_obj = drm_gem_object_lookup(file_priv,
+						   reloc.target_handle);
+		if (target_obj == NULL) {
+			i915_gem_object_unpin(obj);
+			return EBADF;
+		}
+		target_obj_priv = target_obj->driver_private;
+
+		/* The target buffer should have appeared before us in the
+		 * exec_object list, so it should have a GTT space bound by now.
+		 */
+		if (target_obj_priv->gtt_space == NULL) {
+			DRM_ERROR("No GTT space found for object %d\n",
+				  reloc.target_handle);
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return EINVAL;
+		}
+
+		if (reloc.offset > obj->size - 4) {
+			DRM_ERROR("Relocation beyond object bounds: "
+				  "obj %p target %d offset %d size %d.\n",
+				  obj, reloc.target_handle,
+				  (int) reloc.offset, (int) obj->size);
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return EINVAL;
+		}
+		if (reloc.offset & 3) {
+			DRM_ERROR("Relocation not 4-byte aligned: "
+				  "obj %p target %d offset %d.\n",
+				  obj, reloc.target_handle,
+				  (int) reloc.offset);
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return EINVAL;
+		}
+
+		if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
+		    reloc.read_domains & I915_GEM_DOMAIN_CPU) {
+			DRM_ERROR("reloc with read/write CPU domains: "
+				  "obj %p target %d offset %d "
+				  "read %08x write %08x",
+				  obj, reloc.target_handle,
+				  (int) reloc.offset,
+				  reloc.read_domains,
+				  reloc.write_domain);
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return EINVAL;
+		}
+
+		if (reloc.write_domain && target_obj->pending_write_domain &&
+		    reloc.write_domain != target_obj->pending_write_domain) {
+			DRM_ERROR("Write domain conflict: "
+				  "obj %p target %d offset %d "
+				  "new %08x old %08x\n",
+				  obj, reloc.target_handle,
+				  (int) reloc.offset,
+				  reloc.write_domain,
+				  target_obj->pending_write_domain);
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return EINVAL;
+		}
+		DRM_DEBUG("%s: obj %p offset %08x target %d "
+			 "read %08x write %08x gtt %08x "
+			 "presumed %08x delta %08x\n",
+			 __func__,
+			 obj,
+			 (int) reloc.offset,
+			 (int) reloc.target_handle,
+			 (int) reloc.read_domains,
+			 (int) reloc.write_domain,
+			 (int) target_obj_priv->gtt_offset,
+			 (int) reloc.presumed_offset,
+			 reloc.delta);
+
+		target_obj->pending_read_domains |= reloc.read_domains;
+		target_obj->pending_write_domain |= reloc.write_domain;
+
+		/* If the relocation already has the right value in it, no
+		 * more work needs to be done.
+		 */
+		if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
+			drm_gem_object_unreference(target_obj);
+			continue;
+		}
+
+		ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+		if (ret != 0) {
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			return EINVAL;
+		}
+
+               /* Map the page containing the relocation we're going to
+                * perform.
+                */
+
+		int reloc_base = (reloc.offset & ~(PAGE_SIZE-1));
+		reloc_offset = reloc.offset & (PAGE_SIZE-1);
+		reloc_entry = (uint32_t *)(uintptr_t)(obj_priv->page_list[reloc_base/PAGE_SIZE] + reloc_offset);
+		reloc_val = target_obj_priv->gtt_offset + reloc.delta;
+		*reloc_entry = reloc_val;
+
+		/* Write the updated presumed offset for this entry back out
+		 * to the user.
+		 */
+		reloc.presumed_offset = target_obj_priv->gtt_offset;
+		ret = DRM_COPY_TO_USER(relocs + i, &reloc, sizeof(reloc));
+		if (ret != 0) {
+			drm_gem_object_unreference(target_obj);
+			i915_gem_object_unpin(obj);
+			DRM_ERROR("%s: Failed to copy to user ret %d", __func__, ret);
+			return ret;
+		}
+
+		drm_gem_object_unreference(target_obj);
+	}
+
+	return 0;
+}
+
+/** Dispatch a batchbuffer to the ring
+ */
+static int
+i915_dispatch_gem_execbuffer(struct drm_device *dev,
+			      struct drm_i915_gem_execbuffer *exec,
+			      uint64_t exec_offset)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
+					     (uintptr_t) exec->cliprects_ptr;
+	int nbox = exec->num_cliprects;
+	int i = 0, count;
+	uint64_t	exec_start, exec_len;
+	RING_LOCALS;
+
+	exec_start = exec_offset + exec->batch_start_offset;
+	exec_len = exec->batch_len;
+
+	if ((exec_start | exec_len) & 0x7) {
+		DRM_ERROR("alignment\n");
+		return EINVAL;
+	}
+
+	if (!exec_start) {
+		DRM_ERROR("wrong arg");
+		return EINVAL;
+	}
+
+	count = nbox ? nbox : 1;
+
+	for (i = 0; i < count; i++) {
+		if (i < nbox) {
+			int ret = i915_emit_box(dev, boxes, i,
+						exec->DR1, exec->DR4);
+			if (ret) {
+				DRM_ERROR("i915_emit_box %d DR1 0x%lx DRI2 0x%lx", ret, exec->DR1, exec->DR4);
+				return ret;
+			}
+		}
+		if (IS_I830(dev) || IS_845G(dev)) {
+			BEGIN_LP_RING(4);
+			OUT_RING(MI_BATCH_BUFFER);
+			OUT_RING(exec_start | MI_BATCH_NON_SECURE);
+			OUT_RING(exec_start + exec_len - 4);
+			OUT_RING(0);
+			ADVANCE_LP_RING();
+		} else {
+			BEGIN_LP_RING(2);
+			if (IS_I965G(dev)) {
+				OUT_RING(MI_BATCH_BUFFER_START |
+					 (2 << 6) |
+					 (3 << 9) |
+					 MI_BATCH_NON_SECURE_I965);
+				OUT_RING(exec_start);
+
+			} else {
+				OUT_RING(MI_BATCH_BUFFER_START |
+					 (2 << 6));
+				OUT_RING(exec_start | MI_BATCH_NON_SECURE);
+			}
+			ADVANCE_LP_RING();
+		}
+	}
+	/* XXX breadcrumb */
+	return 0;
+}
+
+/* Throttle our rendering by waiting until the ring has completed our requests
+ * emitted over 20 msec ago.
+ *
+ * This should get us reasonable parallelism between CPU and GPU but also
+ * relatively low latency when blocking on a particular request to finish.
+ */
+static int
+i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
+{
+	struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
+	int ret = 0;
+	uint32_t seqno;
+
+	spin_lock(&dev->struct_mutex);
+	seqno = i915_file_priv->mm.last_gem_throttle_seqno;
+	i915_file_priv->mm.last_gem_throttle_seqno =
+		i915_file_priv->mm.last_gem_seqno;
+	if (seqno) {
+		ret = i915_wait_request(dev, seqno);
+		if (ret != 0)
+			DRM_ERROR("%s: i915_wait_request request->seqno %d now %d\n", __func__, seqno, i915_get_gem_seqno(dev));
+	}
+	spin_unlock(&dev->struct_mutex);
+	return ret;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_execbuffer(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_file_private *i915_file_priv = fpriv->driver_priv;
+	struct drm_i915_gem_execbuffer args;
+	struct drm_i915_gem_exec_object *exec_list = NULL;
+	struct drm_gem_object **object_list = NULL;
+	struct drm_gem_object *batch_obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret = 0, i, pinned = 0;
+	uint64_t exec_offset;
+	uint32_t seqno, flush_domains;
+	int pin_tries;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+        DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_execbuffer __user *) data, sizeof(args));
+
+	DRM_DEBUG("buffer_count %d len %x\n", args.buffer_count, args.batch_len);
+
+	if (args.buffer_count < 1) {
+		DRM_ERROR("execbuf with %d buffers\n", args.buffer_count);
+		return EINVAL;
+	}
+	/* Copy in the exec list from userland */
+	exec_list = drm_calloc(sizeof(*exec_list), args.buffer_count,
+			       DRM_MEM_DRIVER);
+	object_list = drm_calloc(sizeof(*object_list), args.buffer_count,
+				 DRM_MEM_DRIVER);
+	if (exec_list == NULL || object_list == NULL) {
+		DRM_ERROR("Failed to allocate exec or object list "
+			  "for %d buffers\n",
+			  args.buffer_count);
+		ret = ENOMEM;
+		goto pre_mutex_err;
+	}
+
+	ret = DRM_COPY_FROM_USER(exec_list,
+			     (struct drm_i915_gem_exec_object __user *)
+			     (uintptr_t) args.buffers_ptr,
+			     sizeof(*exec_list) * args.buffer_count);
+	if (ret != 0) {
+		DRM_ERROR("copy %d exec entries failed %d\n",
+			  args.buffer_count, ret);
+		goto pre_mutex_err;
+	}
+	spin_lock(&dev->struct_mutex);
+
+	if (dev_priv->mm.wedged) {
+		DRM_ERROR("Execbuf while wedged\n");
+		spin_unlock(&dev->struct_mutex);
+		return EIO;
+	}
+
+	if (dev_priv->mm.suspended) {
+		DRM_ERROR("Execbuf while VT-switched.\n");
+		spin_unlock(&dev->struct_mutex);
+		return EBUSY;
+	}
+
+	/* Look up object handles */
+	for (i = 0; i < args.buffer_count; i++) {
+		object_list[i] = drm_gem_object_lookup(fpriv,
+						       exec_list[i].handle);
+		if (object_list[i] == NULL) {
+			DRM_ERROR("Invalid object handle %d at index %d\n",
+				   exec_list[i].handle, i);
+			ret = EBADF;
+			goto err;
+		}
+		obj_priv = object_list[i]->driver_private;
+		if (obj_priv->in_execbuffer) {
+			DRM_ERROR("Object[%d] (%d) %p appears more than once in object list in args.buffer_count %d \n",
+				   i, object_list[i]->name, object_list[i], args.buffer_count);
+
+			ret = EBADF;
+			goto err;
+		}
+
+		obj_priv->in_execbuffer = 1;
+	}
+
+	/* Pin and relocate */
+	for (pin_tries = 0; ; pin_tries++) {
+		ret = 0;
+		for (i = 0; i < args.buffer_count; i++) {
+			object_list[i]->pending_read_domains = 0;
+			object_list[i]->pending_write_domain = 0;
+			ret = i915_gem_object_pin_and_relocate(object_list[i],
+							       fpriv,
+							       &exec_list[i]);
+			if (ret) {
+				DRM_ERROR("Not all object pinned");
+				break;
+			}
+			pinned = i + 1;
+		}
+		/* success */
+		if (ret == 0)
+		{
+			DRM_DEBUG("gem_execbuffer pin_relocate success");	
+			break;
+		}
+		/* error other than GTT full, or we've already tried again */
+		if (ret != ENOMEM || pin_tries >= 1) {
+			if (ret != ERESTART)
+				DRM_ERROR("Failed to pin buffers %d\n", ret);
+			goto err;
+		}
+
+		/* unpin all of our buffers */
+		for (i = 0; i < pinned; i++)
+			i915_gem_object_unpin(object_list[i]);
+		pinned = 0;
+
+		/* evict everyone we can from the aperture */
+		ret = i915_gem_evict_everything(dev);
+		if (ret)
+			goto err;
+	}
+
+	/* Set the pending read domains for the batch buffer to COMMAND */
+	batch_obj = object_list[args.buffer_count-1];
+	batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
+	batch_obj->pending_write_domain = 0;
+
+	/* Zero the gloabl flush/invalidate flags. These
+	 * will be modified as each object is bound to the
+	 * gtt
+	 */
+	dev->invalidate_domains = 0;
+	dev->flush_domains = 0;
+
+	for (i = 0; i < args.buffer_count; i++) {
+		struct drm_gem_object *obj = object_list[i];
+
+		/* Compute new gpu domains and update invalidate/flush */
+		i915_gem_object_set_to_gpu_domain(obj,
+						  obj->pending_read_domains,
+						  obj->pending_write_domain);
+	}
+
+	if (dev->invalidate_domains | dev->flush_domains) {
+
+		DRM_DEBUG("%s: invalidate_domains %08x flush_domains %08x Then flush\n",
+			  __func__,
+			 dev->invalidate_domains,
+			 dev->flush_domains);
+                i915_gem_flush(dev,
+                               dev->invalidate_domains,
+                               dev->flush_domains);
+                if (dev->flush_domains) {
+                        (void) i915_add_request(dev, dev->flush_domains);
+
+		}
+	}
+
+	for (i = 0; i < args.buffer_count; i++) {
+		struct drm_gem_object *obj = object_list[i];
+
+		obj->write_domain = obj->pending_write_domain;
+	}
+
+	exec_offset = exec_list[args.buffer_count - 1].offset;
+
+	/* Exec the batchbuffer */
+	ret = i915_dispatch_gem_execbuffer(dev, &args, exec_offset);
+	if (ret) {
+		DRM_ERROR("dispatch failed %d\n", ret);
+		goto err;
+	}
+
+	/*
+	 * Ensure that the commands in the batch buffer are
+	 * finished before the interrupt fires
+	 */
+	flush_domains = i915_retire_commands(dev);
+
+	/*
+	 * Get a seqno representing the execution of the current buffer,
+	 * which we can wait on.  We would like to mitigate these interrupts,
+	 * likely by only creating seqnos occasionally (so that we have
+	 * *some* interrupts representing completion of buffers that we can
+	 * wait on when trying to clear up gtt space).
+	 */
+	seqno = i915_add_request(dev, flush_domains);
+	ASSERT(!(seqno == 0));
+	i915_file_priv->mm.last_gem_seqno = seqno;
+	for (i = 0; i < args.buffer_count; i++) {
+		struct drm_gem_object *obj = object_list[i];
+		i915_gem_object_move_to_active(obj, seqno);
+		DRM_DEBUG("%s: move to exec list %p\n", __func__, obj);
+	}
+
+err:
+	if (object_list != NULL) {
+		for (i = 0; i < pinned; i++)
+			i915_gem_object_unpin(object_list[i]);
+
+		for (i = 0; i < args.buffer_count; i++) {
+			if (object_list[i]) {
+				obj_priv = object_list[i]->driver_private;
+				obj_priv->in_execbuffer = 0;
+			}
+			drm_gem_object_unreference(object_list[i]);
+		}
+	}
+	spin_unlock(&dev->struct_mutex);
+
+	if (!ret) {
+	        /* Copy the new buffer offsets back to the user's exec list. */
+	        ret = DRM_COPY_TO_USER((struct drm_i915_relocation_entry __user *)
+	                           (uintptr_t) args.buffers_ptr,
+	                           exec_list,
+	                           sizeof(*exec_list) * args.buffer_count);
+	        if (ret)
+	                DRM_ERROR("failed to copy %d exec entries "
+	                          "back to user (%d)\n",
+	                           args.buffer_count, ret);
+	}
+
+pre_mutex_err:
+	drm_free(object_list, sizeof(*object_list) * args.buffer_count,
+		 DRM_MEM_DRIVER);
+	drm_free(exec_list, sizeof(*exec_list) * args.buffer_count,
+		 DRM_MEM_DRIVER);
+
+	return ret;
+}
+
+int
+i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
+{
+	struct drm_device *dev = obj->dev;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	int ret;
+
+	if (obj_priv->gtt_space == NULL) {
+		ret = i915_gem_object_bind_to_gtt(obj, alignment);
+		if (ret != 0) {
+			DRM_ERROR("Failure to bind: %d", ret);
+			return ret;
+		}
+	}
+	obj_priv->pin_count++;
+
+	/* If the object is not active and not pending a flush,
+	 * remove it from the inactive list
+	 */
+	if (obj_priv->pin_count == 1) {
+		atomic_inc(&dev->pin_count);
+		atomic_add(obj->size, &dev->pin_memory);
+		if (!obj_priv->active &&
+		    (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
+					   I915_GEM_DOMAIN_GTT)) == 0 &&
+		    !list_empty(&obj_priv->list))
+			list_del_init(&obj_priv->list);
+	}
+	return 0;
+}
+
+void
+i915_gem_object_unpin(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	obj_priv->pin_count--;
+	ASSERT(!(obj_priv->pin_count < 0));
+	ASSERT(!(obj_priv->gtt_space == NULL));
+
+	/* If the object is no longer pinned, and is
+	 * neither active nor being flushed, then stick it on
+	 * the inactive list
+	 */
+	if (obj_priv->pin_count == 0) {
+		if (!obj_priv->active &&
+		    (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
+					   I915_GEM_DOMAIN_GTT)) == 0)
+			list_move_tail(&obj_priv->list,
+				       &dev_priv->mm.inactive_list, (caddr_t)obj_priv);
+		atomic_dec(&dev->pin_count);
+		atomic_sub(obj->size, &dev->pin_memory);
+	}
+}
+
+/*ARGSUSED*/
+int
+i915_gem_pin_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_pin args;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+        DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_pin __user *) data, sizeof(args));
+
+	spin_lock(&dev->struct_mutex);
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL) {
+		DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
+			  args.handle);
+		spin_unlock(&dev->struct_mutex);
+		return EBADF;
+	}
+	DRM_DEBUG("i915_gem_pin_ioctl obj->name %d", obj->name);
+	obj_priv = obj->driver_private;
+
+	if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != fpriv) {
+		DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
+			  args.handle);
+		drm_gem_object_unreference(obj);
+		spin_unlock(&dev->struct_mutex);
+		return EINVAL;
+	}
+
+	obj_priv->user_pin_count++;
+	obj_priv->pin_filp = fpriv;
+	if (obj_priv->user_pin_count == 1) {
+		ret = i915_gem_object_pin(obj, args.alignment);
+		if (ret != 0) {
+			drm_gem_object_unreference(obj);
+			spin_unlock(&dev->struct_mutex);
+			return ret;
+		}
+	}
+
+	/* XXX - flush the CPU caches for pinned objects
+	 * as the X server doesn't manage domains yet
+	 */
+	i915_gem_object_flush_cpu_write_domain(obj);
+	args.offset = obj_priv->gtt_offset;
+
+	ret = DRM_COPY_TO_USER((struct drm_i915_gem_pin __user *) data, &args, sizeof(args));
+	if ( ret != 0)
+		DRM_ERROR(" gem pin ioctl error! %d", ret);
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	return 0;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_unpin_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_pin args;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+        DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_pin __user *) data, sizeof(args));
+
+	spin_lock(&dev->struct_mutex);
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL) {
+		DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
+			  args.handle);
+		spin_unlock(&dev->struct_mutex);
+		return EBADF;
+	}
+	obj_priv = obj->driver_private;	
+	DRM_DEBUG("i915_gem_unpin_ioctl, obj->name %d", obj->name);
+	if (obj_priv->pin_filp != fpriv) {
+		DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
+			  args.handle);
+		drm_gem_object_unreference(obj);
+		spin_unlock(&dev->struct_mutex);
+		return EINVAL;
+	}
+	obj_priv->user_pin_count--;
+	if (obj_priv->user_pin_count == 0) {
+		obj_priv->pin_filp = NULL;
+		i915_gem_object_unpin(obj);
+	}
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+	return 0;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_busy_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_busy args;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+        DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_busy __user *) data, sizeof(args));
+
+	spin_lock(&dev->struct_mutex);
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL) {
+		DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
+			  args.handle);
+		spin_unlock(&dev->struct_mutex);
+		return EBADF;
+	}
+
+	obj_priv = obj->driver_private;
+	/* Don't count being on the flushing list against the object being
+	 * done.  Otherwise, a buffer left on the flushing list but not getting
+	 * flushed (because nobody's flushing that domain) won't ever return
+	 * unbusy and get reused by libdrm's bo cache.  The other expected
+	 * consumer of this interface, OpenGL's occlusion queries, also specs
+	 * that the objects get unbusy "eventually" without any interference.
+	 */
+	args.busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
+	DRM_DEBUG("i915_gem_busy_ioctl call obj->name %d busy %d", obj->name, args.busy);
+
+        ret = DRM_COPY_TO_USER((struct drm_i915_gem_busy __user *) data, &args, sizeof(args));
+        if ( ret != 0)
+                DRM_ERROR(" gem busy error! %d", ret);
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+	return 0;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_throttle_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	return i915_gem_ring_throttle(dev, fpriv);
+}
+
+static int
+i915_gem_object_get_page_list(struct drm_gem_object *obj)
+{
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+        caddr_t va;
+        long i;
+
+	if (obj_priv->page_list)
+		return 0;
+        pgcnt_t np = btop(obj->size);
+
+        obj_priv->page_list = kmem_zalloc(np * sizeof(caddr_t), KM_SLEEP);
+        if (obj_priv->page_list == NULL) {
+                DRM_ERROR("Faled to allocate page list\n");
+                return ENOMEM;
+        }
+
+	for (i = 0, va = obj->kaddr; i < np; i++, va += PAGESIZE) {
+		obj_priv->page_list[i] = va;
+	}
+	return 0;
+}
+
+
+int i915_gem_init_object(struct drm_gem_object *obj)
+{
+	struct drm_i915_gem_object *obj_priv;
+
+	obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
+	if (obj_priv == NULL)
+		return ENOMEM;
+
+	/*
+	 * We've just allocated pages from the kernel,
+	 * so they've just been written by the CPU with
+	 * zeros. They'll need to be clflushed before we
+	 * use them with the GPU.
+	 */
+	obj->write_domain = I915_GEM_DOMAIN_CPU;
+	obj->read_domains = I915_GEM_DOMAIN_CPU;
+
+	obj->driver_private = obj_priv;
+	obj_priv->obj = obj;
+	INIT_LIST_HEAD(&obj_priv->list);
+	return 0;
+}
+
+void i915_gem_free_object(struct drm_gem_object *obj)
+{
+	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+	while (obj_priv->pin_count > 0)
+		i915_gem_object_unpin(obj);
+
+	DRM_DEBUG("%s: obj %d",__func__, obj->name);
+
+	(void) i915_gem_object_unbind(obj, 1);
+	if (obj_priv->page_cpu_valid != NULL)
+		drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE, DRM_MEM_DRIVER);
+	drm_free(obj->driver_private, sizeof(*obj_priv), DRM_MEM_DRIVER);
+}
+
+/** Unbinds all objects that are on the given buffer list. */
+static int
+i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head, uint32_t type)
+{
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	while (!list_empty(head)) {
+		obj_priv = list_entry(head->next,
+				struct drm_i915_gem_object,
+			    	list);
+		obj = obj_priv->obj;
+
+		if (obj_priv->pin_count != 0) {
+			DRM_ERROR("Pinned object in unbind list\n");
+			spin_unlock(&dev->struct_mutex);
+			return EINVAL;
+		}
+		DRM_DEBUG("%s: obj %d type %d",__func__, obj->name, type);
+		ret = i915_gem_object_unbind(obj, type);
+		if (ret != 0) {
+			DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
+				  ret);
+			spin_unlock(&dev->struct_mutex);
+			return ret;
+		}
+	}
+
+
+	return 0;
+}
+
+static int
+i915_gem_idle(struct drm_device *dev, uint32_t type)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t seqno, cur_seqno, last_seqno;
+	int stuck, ret;
+
+	spin_lock(&dev->struct_mutex);
+
+	if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
+		spin_unlock(&dev->struct_mutex);
+		return 0;
+	}
+
+	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
+	 * We need to replace this with a semaphore, or something.
+	 */
+	dev_priv->mm.suspended = 1;
+
+	/* Cancel the retire work handler, wait for it to finish if running
+	 */
+	if (worktimer_id != NULL) {
+		(void) untimeout(worktimer_id);
+		worktimer_id = NULL;
+	}
+
+	i915_kernel_lost_context(dev);
+
+	/* Flush the GPU along with all non-CPU write domains
+	 */
+	i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
+		       ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
+	seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
+					I915_GEM_DOMAIN_GTT));
+	if (seqno == 0) {
+		spin_unlock(&dev->struct_mutex);
+		return ENOMEM;
+	}
+
+	dev_priv->mm.waiting_gem_seqno = seqno;
+	last_seqno = 0;
+	stuck = 0;
+	for (;;) {
+		cur_seqno = i915_get_gem_seqno(dev);
+		if (i915_seqno_passed(cur_seqno, seqno))
+			break;
+		if (last_seqno == cur_seqno) {
+			if (stuck++ > 100) {
+				DRM_ERROR("hardware wedged\n");
+				dev_priv->mm.wedged = 1;
+				DRM_WAKEUP(&dev_priv->irq_queue);
+				break;
+			}
+		}
+		DRM_UDELAY(10);
+		last_seqno = cur_seqno;
+	}
+	dev_priv->mm.waiting_gem_seqno = 0;
+
+	i915_gem_retire_requests(dev);
+
+	/* Empty the active and flushing lists to inactive.  If there's
+	 * anything left at this point, it means that we're wedged and
+	 * nothing good's going to happen by leaving them there.  So strip
+	 * the GPU domains and just stuff them onto inactive.
+	 */
+	while (!list_empty(&dev_priv->mm.active_list)) {
+		struct drm_i915_gem_object *obj_priv;
+
+		obj_priv = list_entry(dev_priv->mm.active_list.next,
+					    struct drm_i915_gem_object,
+					    list);
+		obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
+		i915_gem_object_move_to_inactive(obj_priv->obj);
+	}
+
+	while (!list_empty(&dev_priv->mm.flushing_list)) {
+		struct drm_i915_gem_object *obj_priv;
+
+		obj_priv = list_entry(dev_priv->mm.flushing_list.next,
+					    struct drm_i915_gem_object,
+					    list);
+		obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
+		i915_gem_object_move_to_inactive(obj_priv->obj);
+	}
+	
+	/* Move all inactive buffers out of the GTT. */
+	ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list, type);
+	ASSERT(list_empty(&dev_priv->mm.inactive_list));
+	if (ret) {
+		spin_unlock(&dev->struct_mutex);
+		return ret;
+	}
+
+	i915_gem_cleanup_ringbuffer(dev);
+	spin_unlock(&dev->struct_mutex);
+
+	return 0;
+}
+
+static int
+i915_gem_init_hws(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	/* If we need a physical address for the status page, it's already
+	 * initialized at driver load time.
+	 */
+	if (!I915_NEED_GFX_HWS(dev))
+		return 0;
+
+
+	obj = drm_gem_object_alloc(dev, 4096);
+	if (obj == NULL) {
+		DRM_ERROR("Failed to allocate status page\n");
+		return ENOMEM;
+	}
+
+	obj_priv = obj->driver_private;
+
+	ret = i915_gem_object_pin(obj, 4096);
+	if (ret != 0) {
+		drm_gem_object_unreference(obj);
+		return ret;
+	}
+
+	dev_priv->status_gfx_addr = obj_priv->gtt_offset;
+	dev_priv->hws_map.offset = dev->agp->agp_info.agpi_aperbase + obj_priv->gtt_offset;
+	dev_priv->hws_map.size = 4096;
+	dev_priv->hws_map.type = 0;
+	dev_priv->hws_map.flags = 0;
+	dev_priv->hws_map.mtrr = 0;
+
+	drm_core_ioremap(&dev_priv->hws_map, dev);
+	if (dev_priv->hws_map.handle == NULL) {
+		DRM_ERROR("Failed to map status page.\n");
+		(void) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
+		drm_gem_object_unreference(obj);
+		return EINVAL;
+	}
+
+	dev_priv->hws_obj = obj;
+
+	dev_priv->hw_status_page = dev_priv->hws_map.handle;
+
+	(void) memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
+	I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
+	(void) I915_READ(HWS_PGA); /* posting read */
+	DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
+
+	return 0;
+}
+
+static void
+i915_gem_cleanup_hws(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+
+	if (dev_priv->hws_obj == NULL)
+		return;
+
+	obj = dev_priv->hws_obj;
+
+	drm_core_ioremapfree(&dev_priv->hws_map, dev);
+	i915_gem_object_unpin(obj);
+	drm_gem_object_unreference(obj);
+	dev_priv->hws_obj = NULL;
+
+	(void) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
+	dev_priv->hw_status_page = NULL;
+
+	/* Write high address into HWS_PGA when disabling. */
+	I915_WRITE(HWS_PGA, 0x1ffff000);
+}
+
+int
+i915_gem_init_ringbuffer(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+	u32 head;
+
+	ret = i915_gem_init_hws(dev);
+	if (ret != 0)
+		return ret;
+	obj = drm_gem_object_alloc(dev, 128 * 1024);
+	if (obj == NULL) {
+		DRM_ERROR("Failed to allocate ringbuffer\n");
+		i915_gem_cleanup_hws(dev);
+		return ENOMEM;
+	}
+
+	obj_priv = obj->driver_private;
+	ret = i915_gem_object_pin(obj, 4096);
+	if (ret != 0) {
+		drm_gem_object_unreference(obj);
+		i915_gem_cleanup_hws(dev);
+		return ret;
+	}
+
+	/* Set up the kernel mapping for the ring. */
+	dev_priv->ring.Size = obj->size;
+	dev_priv->ring.tail_mask = obj->size - 1;
+
+	dev_priv->ring.map.offset = dev->agp->agp_info.agpi_aperbase + obj_priv->gtt_offset;
+	dev_priv->ring.map.size = obj->size;
+	dev_priv->ring.map.type = 0;
+	dev_priv->ring.map.flags = 0;
+	dev_priv->ring.map.mtrr = 0;
+
+	drm_core_ioremap(&dev_priv->ring.map, dev);
+	if (dev_priv->ring.map.handle == NULL) {
+		DRM_ERROR("Failed to map ringbuffer.\n");
+		(void) memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
+		drm_gem_object_unreference(obj);
+		i915_gem_cleanup_hws(dev);
+		return EINVAL;
+	}
+
+	dev_priv->ring.ring_obj = obj;
+
+	dev_priv->ring.virtual_start = (u8 *) dev_priv->ring.map.handle;
+
+	/* Stop the ring if it's running. */
+	I915_WRITE(PRB0_CTL, 0);
+	I915_WRITE(PRB0_HEAD, 0);
+	I915_WRITE(PRB0_TAIL, 0);
+
+
+	/* Initialize the ring. */
+	I915_WRITE(PRB0_START, obj_priv->gtt_offset);
+	head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+
+	/* G45 ring initialization fails to reset head to zero */
+	if (head != 0) {
+		DRM_ERROR("Ring head not reset to zero "
+			  "ctl %08x head %08x tail %08x start %08x\n",
+			  I915_READ(PRB0_CTL),
+			  I915_READ(PRB0_HEAD),
+			  I915_READ(PRB0_TAIL),
+			  I915_READ(PRB0_START));
+		I915_WRITE(PRB0_HEAD, 0);
+
+		DRM_ERROR("Ring head forced to zero "
+			  "ctl %08x head %08x tail %08x start %08x\n",
+			  I915_READ(PRB0_CTL),
+			  I915_READ(PRB0_HEAD),
+			  I915_READ(PRB0_TAIL),
+			  I915_READ(PRB0_START));
+	}
+
+	I915_WRITE(PRB0_CTL,
+		   ((obj->size - 4096) & RING_NR_PAGES) |
+		   RING_NO_REPORT |
+		   RING_VALID);
+
+	head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+
+	/* If the head is still not zero, the ring is dead */
+	if (head != 0) {
+		DRM_ERROR("Ring initialization failed "
+			  "ctl %08x head %08x tail %08x start %08x\n",
+			  I915_READ(PRB0_CTL),
+			  I915_READ(PRB0_HEAD),
+			  I915_READ(PRB0_TAIL),
+			  I915_READ(PRB0_START));
+		return EIO;
+	}
+
+	/* Update our cache of the ring state */
+	i915_kernel_lost_context(dev);
+
+	return 0;
+}
+
+static void
+i915_gem_cleanup_ringbuffer(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	if (dev_priv->ring.ring_obj == NULL)
+		return;
+
+	drm_core_ioremapfree(&dev_priv->ring.map, dev);
+
+	i915_gem_object_unpin(dev_priv->ring.ring_obj);
+	drm_gem_object_unreference(dev_priv->ring.ring_obj);
+	dev_priv->ring.ring_obj = NULL;
+	(void) memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
+	i915_gem_cleanup_hws(dev);
+}
+
+/*ARGSUSED*/
+int
+i915_gem_entervt_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	if (dev_priv->mm.wedged) {
+		DRM_ERROR("Reenabling wedged hardware, good luck\n");
+		dev_priv->mm.wedged = 0;
+	}
+        /* Set up the kernel mapping for the ring. */
+        dev_priv->mm.gtt_mapping.offset = dev->agp->agp_info.agpi_aperbase;
+        dev_priv->mm.gtt_mapping.size = dev->agp->agp_info.agpi_apersize;
+        dev_priv->mm.gtt_mapping.type = 0;
+        dev_priv->mm.gtt_mapping.flags = 0;
+        dev_priv->mm.gtt_mapping.mtrr = 0;
+
+        drm_core_ioremap(&dev_priv->mm.gtt_mapping, dev);
+
+	spin_lock(&dev->struct_mutex);
+	dev_priv->mm.suspended = 0;
+	ret = i915_gem_init_ringbuffer(dev);
+	if (ret != 0)
+		return ret;
+
+	spin_unlock(&dev->struct_mutex);
+
+	drm_irq_install(dev);
+
+	return 0;
+}
+
+/*ARGSUSED*/
+int
+i915_gem_leavevt_ioctl(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	ret = i915_gem_idle(dev, 0);
+	drm_irq_uninstall(dev);
+
+	drm_core_ioremapfree(&dev_priv->mm.gtt_mapping, dev);
+	return ret;
+}
+
+void
+i915_gem_lastclose(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret;
+
+	ret = i915_gem_idle(dev, 1);
+	if (ret)
+		DRM_ERROR("failed to idle hardware: %d\n", ret);
+
+	drm_mm_clean_ml(&dev_priv->mm.gtt_space);
+}
+
+void
+i915_gem_load(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	INIT_LIST_HEAD(&dev_priv->mm.active_list);
+	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
+	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
+	INIT_LIST_HEAD(&dev_priv->mm.request_list);
+	dev_priv->mm.next_gem_seqno = 1;
+
+	i915_gem_detect_bit_6_swizzle(dev);
+
+}
+
diff --git a/usr/src/uts/intel/io/drm/i915_gem_debug.c b/usr/src/uts/intel/io/drm/i915_gem_debug.c
new file mode 100644
index 0000000000..d16c3df0c0
--- /dev/null
+++ b/usr/src/uts/intel/io/drm/i915_gem_debug.c
@@ -0,0 +1,1120 @@
+/* BEGIN CSTYLED */
+
+/*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Keith Packard <keithp@keithp.com>
+ *
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+
+#define BUFFER_FAIL(_count, _len, _name) { 			\
+	DRM_ERROR("Buffer size too small in %s (%d < %d)\n",	\
+	    (_name), (_count), (_len));				\
+	(*failures)++;						\
+	return count;						\
+}
+
+
+static uint32_t saved_s2 = 0, saved_s4 = 0;
+static char saved_s2_set = 0, saved_s4_set = 0;
+
+static float
+int_as_float(uint32_t intval)
+{
+    union intfloat {
+	uint32_t i;
+	float f;
+    } uval;
+
+    uval.i = intval;
+    return uval.f;
+}
+
+static void
+instr_out(uint32_t *data, uint32_t hw_offset, unsigned int index,
+	  const char *fmt, ...)
+{
+
+    DRM_ERROR("0x%08x: 0x%08x:%s ", hw_offset + index * 4, data[index],
+	    index == 0 ? "" : "  ");
+        va_list ap;
+
+        va_start(ap, fmt);
+        vcmn_err(CE_WARN, fmt, ap);
+        va_end(ap);
+
+}
+
+static int
+decode_mi(uint32_t *data, int count, uint32_t hw_offset, int *failures)
+{
+    unsigned int opcode;
+
+    struct {
+	uint32_t opcode;
+	int min_len;
+	int max_len;
+	char *name;
+    } opcodes_mi[] = {
+	{ 0x08, 1, 1, "MI_ARB_ON_OFF" },
+	{ 0x0a, 1, 1, "MI_BATCH_BUFFER_END" },
+	{ 0x31, 2, 2, "MI_BATCH_BUFFER_START" },
+	{ 0x14, 3, 3, "MI_DISPLAY_BUFFER_INFO" },
+	{ 0x04, 1, 1, "MI_FLUSH" },
+	{ 0x22, 3, 3, "MI_LOAD_REGISTER_IMM" },
+	{ 0x13, 2, 2, "MI_LOAD_SCAN_LINES_EXCL" },
+	{ 0x12, 2, 2, "MI_LOAD_SCAN_LINES_INCL" },
+	{ 0x00, 1, 1, "MI_NOOP" },
+	{ 0x11, 2, 2, "MI_OVERLAY_FLIP" },
+	{ 0x07, 1, 1, "MI_REPORT_HEAD" },
+	{ 0x18, 2, 2, "MI_SET_CONTEXT" },
+	{ 0x20, 3, 4, "MI_STORE_DATA_IMM" },
+	{ 0x21, 3, 4, "MI_STORE_DATA_INDEX" },
+	{ 0x24, 3, 3, "MI_STORE_REGISTER_MEM" },
+	{ 0x02, 1, 1, "MI_USER_INTERRUPT" },
+	{ 0x03, 1, 1, "MI_WAIT_FOR_EVENT" },
+    };
+
+
+    for (opcode = 0; opcode < sizeof(opcodes_mi) / sizeof(opcodes_mi[0]);
+	 opcode++) {
+	if ((data[0] & 0x1f800000) >> 23 == opcodes_mi[opcode].opcode) {
+	    unsigned int len = 1, i;
+
+	    instr_out(data, hw_offset, 0, "%s\n", opcodes_mi[opcode].name);
+	    if (opcodes_mi[opcode].max_len > 1) {
+		len = (data[0] & 0x000000ff) + 2;
+		if (len < opcodes_mi[opcode].min_len ||
+		    len > opcodes_mi[opcode].max_len)
+		{
+		    DRM_ERROR("Bad length in %s\n",
+			    opcodes_mi[opcode].name);
+		}
+	    }
+
+	    for (i = 1; i < len; i++) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len, opcodes_mi[opcode].name);
+		instr_out(data, hw_offset, i, "dword %d\n", i);
+	    }
+
+	    return len;
+	}
+    }
+
+    instr_out(data, hw_offset, 0, "MI UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+static int
+decode_2d(uint32_t *data, int count, uint32_t hw_offset, int *failures)
+{
+    unsigned int opcode, len;
+    char *format = NULL;
+
+    struct {
+	uint32_t opcode;
+	int min_len;
+	int max_len;
+	char *name;
+    } opcodes_2d[] = {
+	{ 0x40, 5, 5, "COLOR_BLT" },
+	{ 0x43, 6, 6, "SRC_COPY_BLT" },
+	{ 0x01, 8, 8, "XY_SETUP_BLT" },
+	{ 0x11, 9, 9, "XY_SETUP_MONO_PATTERN_SL_BLT" },
+	{ 0x03, 3, 3, "XY_SETUP_CLIP_BLT" },
+	{ 0x24, 2, 2, "XY_PIXEL_BLT" },
+	{ 0x25, 3, 3, "XY_SCANLINES_BLT" },
+	{ 0x26, 4, 4, "Y_TEXT_BLT" },
+	{ 0x31, 5, 134, "XY_TEXT_IMMEDIATE_BLT" },
+	{ 0x50, 6, 6, "XY_COLOR_BLT" },
+	{ 0x51, 6, 6, "XY_PAT_BLT" },
+	{ 0x76, 8, 8, "XY_PAT_CHROMA_BLT" },
+	{ 0x72, 7, 135, "XY_PAT_BLT_IMMEDIATE" },
+	{ 0x77, 9, 137, "XY_PAT_CHROMA_BLT_IMMEDIATE" },
+	{ 0x52, 9, 9, "XY_MONO_PAT_BLT" },
+	{ 0x59, 7, 7, "XY_MONO_PAT_FIXED_BLT" },
+	{ 0x53, 8, 8, "XY_SRC_COPY_BLT" },
+	{ 0x54, 8, 8, "XY_MONO_SRC_COPY_BLT" },
+	{ 0x71, 9, 137, "XY_MONO_SRC_COPY_IMMEDIATE_BLT" },
+	{ 0x55, 9, 9, "XY_FULL_BLT" },
+	{ 0x55, 9, 137, "XY_FULL_IMMEDIATE_PATTERN_BLT" },
+	{ 0x56, 9, 9, "XY_FULL_MONO_SRC_BLT" },
+	{ 0x75, 10, 138, "XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT" },
+	{ 0x57, 12, 12, "XY_FULL_MONO_PATTERN_BLT" },
+	{ 0x58, 12, 12, "XY_FULL_MONO_PATTERN_MONO_SRC_BLT" },
+    };
+
+    switch ((data[0] & 0x1fc00000) >> 22) {
+    case 0x50:
+	instr_out(data, hw_offset, 0,
+		  "XY_COLOR_BLT (rgb %sabled, alpha %sabled, dst tile %d)\n",
+		  (data[0] & (1 << 20)) ? "en" : "dis",
+		  (data[0] & (1 << 21)) ? "en" : "dis",
+		  (data[0] >> 11) & 1);
+
+	len = (data[0] & 0x000000ff) + 2;
+	if (len != 6)
+	    DRM_ERROR("Bad count in XY_COLOR_BLT\n");
+	if (count < 6)
+	    BUFFER_FAIL(count, len, "XY_COLOR_BLT");
+
+	switch ((data[1] >> 24) & 0x3) {
+	case 0:
+	    format="8";
+	    break;
+	case 1:
+	    format="565";
+	    break;
+	case 2:
+	    format="1555";
+	    break;
+	case 3:
+	    format="8888";
+	    break;
+	}
+
+	instr_out(data, hw_offset, 1, "format %s, pitch %d, "
+		  "clipping %sabled\n", format,
+		  (short)(data[1] & 0xffff),
+		  data[1] & (1 << 30) ? "en" : "dis");
+	instr_out(data, hw_offset, 2, "(%d,%d)\n",
+		  data[2] & 0xffff, data[2] >> 16);
+	instr_out(data, hw_offset, 3, "(%d,%d)\n",
+		  data[3] & 0xffff, data[3] >> 16);
+	instr_out(data, hw_offset, 4, "offset 0x%08x\n", data[4]);
+	instr_out(data, hw_offset, 5, "color\n");
+	return len;
+    case 0x53:
+	instr_out(data, hw_offset, 0,
+		  "XY_SRC_COPY_BLT (rgb %sabled, alpha %sabled, "
+		  "src tile %d, dst tile %d)\n",
+		  (data[0] & (1 << 20)) ? "en" : "dis",
+		  (data[0] & (1 << 21)) ? "en" : "dis",
+		  (data[0] >> 15) & 1,
+		  (data[0] >> 11) & 1);
+
+	len = (data[0] & 0x000000ff) + 2;
+	if (len != 8)
+	    DRM_ERROR("Bad count in XY_SRC_COPY_BLT\n");
+	if (count < 8)
+	    BUFFER_FAIL(count, len, "XY_SRC_COPY_BLT");
+
+	switch ((data[1] >> 24) & 0x3) {
+	case 0:
+	    format="8";
+	    break;
+	case 1:
+	    format="565";
+	    break;
+	case 2:
+	    format="1555";
+	    break;
+	case 3:
+	    format="8888";
+	    break;
+	}
+
+	instr_out(data, hw_offset, 1, "format %s, dst pitch %d, "
+		  "clipping %sabled\n", format,
+		  (short)(data[1] & 0xffff),
+		  data[1] & (1 << 30) ? "en" : "dis");
+	instr_out(data, hw_offset, 2, "dst (%d,%d)\n",
+		  data[2] & 0xffff, data[2] >> 16);
+	instr_out(data, hw_offset, 3, "dst (%d,%d)\n",
+		  data[3] & 0xffff, data[3] >> 16);
+	instr_out(data, hw_offset, 4, "dst offset 0x%08x\n", data[4]);
+	instr_out(data, hw_offset, 5, "src (%d,%d)\n",
+		  data[5] & 0xffff, data[5] >> 16);
+	instr_out(data, hw_offset, 6, "src pitch %d\n",
+		  (short)(data[6] & 0xffff));
+	instr_out(data, hw_offset, 7, "src offset 0x%08x\n", data[7]);
+	return len;
+    }
+
+    for (opcode = 0; opcode < sizeof(opcodes_2d) / sizeof(opcodes_2d[0]);
+	 opcode++) {
+	if ((data[0] & 0x1fc00000) >> 22 == opcodes_2d[opcode].opcode) {
+	    unsigned int i;
+
+	    len = 1;
+	    instr_out(data, hw_offset, 0, "%s\n", opcodes_2d[opcode].name);
+	    if (opcodes_2d[opcode].max_len > 1) {
+		len = (data[0] & 0x000000ff) + 2;
+		if (len < opcodes_2d[opcode].min_len ||
+		    len > opcodes_2d[opcode].max_len)
+		{
+		    DRM_ERROR("Bad count in %s\n", opcodes_2d[opcode].name);
+		}
+	    }
+
+	    for (i = 1; i < len; i++) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len, opcodes_2d[opcode].name);
+		instr_out(data, hw_offset, i, "dword %d\n", i);
+	    }
+
+	    return len;
+	}
+    }
+
+    instr_out(data, hw_offset, 0, "2D UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+/*ARGSUSED*/
+static int
+decode_3d_1c(uint32_t *data, int count, uint32_t hw_offset, int *failures)
+{
+    switch ((data[0] & 0x00f80000) >> 19) {
+    case 0x11:
+	instr_out(data, hw_offset, 0, "3DSTATE_DEPTH_SUBRECTANGLE_DISALBE\n");
+	return 1;
+    case 0x10:
+	instr_out(data, hw_offset, 0, "3DSTATE_SCISSOR_ENABLE\n");
+	return 1;
+    case 0x01:
+	instr_out(data, hw_offset, 0, "3DSTATE_MAP_COORD_SET_I830\n");
+	return 1;
+    case 0x0a:
+	instr_out(data, hw_offset, 0, "3DSTATE_MAP_CUBE_I830\n");
+	return 1;
+    case 0x05:
+	instr_out(data, hw_offset, 0, "3DSTATE_MAP_TEX_STREAM_I830\n");
+	return 1;
+    }
+
+    instr_out(data, hw_offset, 0, "3D UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+static int
+decode_3d_1d(uint32_t *data, int count, uint32_t hw_offset, int *failures, int i830)
+{
+    unsigned int len, i, c, opcode, word, map, sampler, instr;
+
+    struct {
+	uint32_t opcode;
+	int i830_only;
+	int min_len;
+	int max_len;
+	char *name;
+    } opcodes_3d_1d[] = {
+	{ 0x8e, 0, 3, 3, "3DSTATE_BUFFER_INFO" },
+	{ 0x86, 0, 4, 4, "3DSTATE_CHROMA_KEY" },
+	{ 0x9c, 0, 1, 1, "3DSTATE_CLEAR_PARAMETERS" },
+	{ 0x88, 0, 2, 2, "3DSTATE_CONSTANT_BLEND_COLOR" },
+	{ 0x99, 0, 2, 2, "3DSTATE_DEFAULT_DIFFUSE" },
+	{ 0x9a, 0, 2, 2, "3DSTATE_DEFAULT_SPECULAR" },
+	{ 0x98, 0, 2, 2, "3DSTATE_DEFAULT_Z" },
+	{ 0x97, 0, 2, 2, "3DSTATE_DEPTH_OFFSET_SCALE" },
+	{ 0x85, 0, 2, 2, "3DSTATE_DEST_BUFFER_VARIABLES" },
+	{ 0x80, 0, 5, 5, "3DSTATE_DRAWING_RECTANGLE" },
+	{ 0x8e, 0, 3, 3, "3DSTATE_BUFFER_INFO" },
+	{ 0x9d, 0, 65, 65, "3DSTATE_FILTER_COEFFICIENTS_4X4" },
+	{ 0x9e, 0, 4, 4, "3DSTATE_MONO_FILTER" },
+	{ 0x89, 0, 4, 4, "3DSTATE_FOG_MODE" },
+	{ 0x8f, 0, 2, 16, "3DSTATE_MAP_PALLETE_LOAD_32" },
+	{ 0x81, 0, 3, 3, "3DSTATE_SCISSOR_RECTANGLE" },
+	{ 0x83, 0, 2, 2, "3DSTATE_SPAN_STIPPLE" },
+	{ 0x8c, 1, 2, 2, "3DSTATE_MAP_COORD_TRANSFORM_I830" },
+	{ 0x8b, 1, 2, 2, "3DSTATE_MAP_VERTEX_TRANSFORM_I830" },
+	{ 0x8d, 1, 3, 3, "3DSTATE_W_STATE_I830" },
+	{ 0x01, 1, 2, 2, "3DSTATE_COLOR_FACTOR_I830" },
+	{ 0x02, 1, 2, 2, "3DSTATE_MAP_COORD_SETBIND_I830" },
+    };
+
+    switch ((data[0] & 0x00ff0000) >> 16) {
+    case 0x07:
+	/* This instruction is unusual.  A 0 length means just 1 DWORD instead of
+	 * 2.  The 0 length is specified in one place to be unsupported, but
+	 * stated to be required in another, and 0 length LOAD_INDIRECTs appear
+	 * to cause no harm at least.
+	 */
+	instr_out(data, hw_offset, 0, "3DSTATE_LOAD_INDIRECT\n");
+	len = (data[0] & 0x000000ff) + 1;
+	i = 1;
+	if (data[0] & (0x01 << 8)) {
+	    if (i + 2 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_LOAD_INDIRECT");
+	    instr_out(data, hw_offset, i++, "SIS.0\n");
+	    instr_out(data, hw_offset, i++, "SIS.1\n");
+	}
+	if (data[0] & (0x02 << 8)) {
+	    if (i + 1 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_LOAD_INDIRECT");
+	    instr_out(data, hw_offset, i++, "DIS.0\n");
+	}
+	if (data[0] & (0x04 << 8)) {
+	    if (i + 2 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_LOAD_INDIRECT");
+	    instr_out(data, hw_offset, i++, "SSB.0\n");
+	    instr_out(data, hw_offset, i++, "SSB.1\n");
+	}
+	if (data[0] & (0x08 << 8)) {
+	    if (i + 2 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_LOAD_INDIRECT");
+	    instr_out(data, hw_offset, i++, "MSB.0\n");
+	    instr_out(data, hw_offset, i++, "MSB.1\n");
+	}
+	if (data[0] & (0x10 << 8)) {
+	    if (i + 2 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_LOAD_INDIRECT");
+	    instr_out(data, hw_offset, i++, "PSP.0\n");
+	    instr_out(data, hw_offset, i++, "PSP.1\n");
+	}
+	if (data[0] & (0x20 << 8)) {
+	    if (i + 2 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_LOAD_INDIRECT");
+	    instr_out(data, hw_offset, i++, "PSC.0\n");
+	    instr_out(data, hw_offset, i++, "PSC.1\n");
+	}
+	if (len != i) {
+	    DRM_ERROR("Bad count in 3DSTATE_LOAD_INDIRECT\n");
+	    (*failures)++;
+	    return len;
+	}
+	return len;
+    case 0x04:
+	instr_out(data, hw_offset, 0, "3DSTATE_LOAD_STATE_IMMEDIATE_1\n");
+	len = (data[0] & 0x0000000f) + 2;
+	i = 1;
+	for (word = 0; word <= 7; word++) {
+	    if (data[0] & (1 << (4 + word))) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len, "3DSTATE_LOAD_STATE_IMMEDIATE_1");
+
+		/* save vertex state for decode */
+		if (word == 2) {
+		    saved_s2_set = 1;
+		    saved_s2 = data[i];
+		}
+		if (word == 4) {
+		    saved_s4_set = 1;
+		    saved_s4 = data[i];
+		}
+
+		instr_out(data, hw_offset, i++, "S%d\n", word);
+	    }
+	}
+	if (len != i) {
+	    DRM_ERROR("Bad count in 3DSTATE_LOAD_INDIRECT\n");
+	    (*failures)++;
+	}
+	return len;
+    case 0x00:
+	instr_out(data, hw_offset, 0, "3DSTATE_MAP_STATE\n");
+	len = (data[0] & 0x0000003f) + 2;
+
+	i = 1;
+	for (map = 0; map <= 15; map++) {
+	    if (data[1] & (1 << map)) {
+		if (i + 3 >= count)
+		    BUFFER_FAIL(count, len, "3DSTATE_MAP_STATE");
+		instr_out(data, hw_offset, i++, "map %d MS2\n", map);
+		instr_out(data, hw_offset, i++, "map %d MS3\n", map);
+		instr_out(data, hw_offset, i++, "map %d MS4\n", map);
+	    }
+	}
+	if (len != i) {
+	    DRM_ERROR("Bad count in 3DSTATE_MAP_STATE\n");
+	    (*failures)++;
+	    return len;
+	}
+	return len;
+    case 0x06:
+	instr_out(data, hw_offset, 0, "3DSTATE_PIXEL_SHADER_CONSTANTS\n");
+	len = (data[0] & 0x000000ff) + 2;
+
+	i = 1;
+	for (c = 0; c <= 31; c++) {
+	    if (data[1] & (1 << c)) {
+		if (i + 4 >= count)
+		    BUFFER_FAIL(count, len, "3DSTATE_PIXEL_SHADER_CONSTANTS");
+		instr_out(data, hw_offset, i, "C%d.X = %f\n",
+			  c, int_as_float(data[i]));
+		i++;
+		instr_out(data, hw_offset, i, "C%d.Y = %f\n",
+			  c, int_as_float(data[i]));
+		i++;
+		instr_out(data, hw_offset, i, "C%d.Z = %f\n",
+			  c, int_as_float(data[i]));
+		i++;
+		instr_out(data, hw_offset, i, "C%d.W = %f\n",
+			  c, int_as_float(data[i]));
+		i++;
+	    }
+	}
+	if (len != i) {
+	    DRM_ERROR("Bad count in 3DSTATE_MAP_STATE\n");
+	    (*failures)++;
+	}
+	return len;
+    case 0x05:
+	instr_out(data, hw_offset, 0, "3DSTATE_PIXEL_SHADER_PROGRAM\n");
+	len = (data[0] & 0x000000ff) + 2;
+	if ((len - 1) % 3 != 0 || len > 370) {
+	    DRM_ERROR("Bad count in 3DSTATE_PIXEL_SHADER_PROGRAM\n");
+	    (*failures)++;
+	}
+	i = 1;
+	for (instr = 0; instr < (len - 1) / 3; instr++) {
+	    if (i + 3 >= count)
+		BUFFER_FAIL(count, len, "3DSTATE_MAP_STATE");
+	    instr_out(data, hw_offset, i++, "PS%03x\n", instr);
+	    instr_out(data, hw_offset, i++, "PS%03x\n", instr);
+	    instr_out(data, hw_offset, i++, "PS%03x\n", instr);
+	}
+	return len;
+    case 0x01:
+	if (i830)
+	    break;
+	instr_out(data, hw_offset, 0, "3DSTATE_SAMPLER_STATE\n");
+	len = (data[0] & 0x0000003f) + 2;
+	i = 1;
+	for (sampler = 0; sampler <= 15; sampler++) {
+	    if (data[1] & (1 << sampler)) {
+		if (i + 3 >= count)
+		    BUFFER_FAIL(count, len, "3DSTATE_SAMPLER_STATE");
+		instr_out(data, hw_offset, i++, "sampler %d SS2\n",
+			  sampler);
+		instr_out(data, hw_offset, i++, "sampler %d SS3\n",
+			  sampler);
+		instr_out(data, hw_offset, i++, "sampler %d SS4\n",
+			  sampler);
+	    }
+	}
+	if (len != i) {
+	    DRM_ERROR("Bad count in 3DSTATE_SAMPLER_STATE\n");
+	    (*failures)++;
+	}
+	return len;
+    }
+
+    for (opcode = 0; opcode < sizeof(opcodes_3d_1d) / sizeof(opcodes_3d_1d[0]);
+	 opcode++)
+    {
+	if (opcodes_3d_1d[opcode].i830_only && !i830)
+	    continue;
+
+	if (((data[0] & 0x00ff0000) >> 16) == opcodes_3d_1d[opcode].opcode) {
+	    len = 1;
+
+	    instr_out(data, hw_offset, 0, "%s\n", opcodes_3d_1d[opcode].name);
+	    if (opcodes_3d_1d[opcode].max_len > 1) {
+		len = (data[0] & 0x0000ffff) + 2;
+		if (len < opcodes_3d_1d[opcode].min_len ||
+		    len > opcodes_3d_1d[opcode].max_len)
+		{
+		    DRM_ERROR("Bad count in %s\n",
+			    opcodes_3d_1d[opcode].name);
+		    (*failures)++;
+		}
+	    }
+
+	    for (i = 1; i < len; i++) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len,  opcodes_3d_1d[opcode].name);
+		instr_out(data, hw_offset, i, "dword %d\n", i);
+	    }
+
+	    return len;
+	}
+    }
+
+    instr_out(data, hw_offset, 0, "3D UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+static int
+decode_3d_primitive(uint32_t *data, int count, uint32_t hw_offset,
+		    int *failures)
+{
+    char immediate = (data[0] & (1 << 23)) == 0;
+    unsigned int len, i;
+    char *primtype;
+
+    switch ((data[0] >> 18) & 0xf) {
+    case 0x0: primtype = "TRILIST"; break;
+    case 0x1: primtype = "TRISTRIP"; break;
+    case 0x2: primtype = "TRISTRIP_REVERSE"; break;
+    case 0x3: primtype = "TRIFAN"; break;
+    case 0x4: primtype = "POLYGON"; break;
+    case 0x5: primtype = "LINELIST"; break;
+    case 0x6: primtype = "LINESTRIP"; break;
+    case 0x7: primtype = "RECTLIST"; break;
+    case 0x8: primtype = "POINTLIST"; break;
+    case 0x9: primtype = "DIB"; break;
+    case 0xa: primtype = "CLEAR_RECT"; break;
+    default: primtype = "unknown"; break;
+    }
+
+    /* XXX: 3DPRIM_DIB not supported */
+    if (immediate) {
+	len = (data[0] & 0x0003ffff) + 2;
+	instr_out(data, hw_offset, 0, "3DPRIMITIVE inline %s\n", primtype);
+	if (count < len)
+	    BUFFER_FAIL(count, len,  "3DPRIMITIVE inline");
+	if (!saved_s2_set || !saved_s4_set) {
+	    DRM_ERROR("unknown vertex format\n");
+	    for (i = 1; i < len; i++) {
+		instr_out(data, hw_offset, i,
+			  "           vertex data (%f float)\n",
+			  int_as_float(data[i]));
+	    }
+	} else {
+	    unsigned int vertex = 0;
+	    for (i = 1; i < len;) {
+		unsigned int tc;
+
+#define VERTEX_OUT(fmt, ...) {					\
+    if (i < len)							\
+	instr_out(data, hw_offset, i, " V%d."fmt"\n", vertex, __VA_ARGS__); \
+    else								\
+	DRM_ERROR(" missing data in V%d\n", vertex);			\
+    i++;								\
+}
+
+		VERTEX_OUT("X = %f", int_as_float(data[i]));
+		VERTEX_OUT("Y = %f", int_as_float(data[i]));
+	        switch (saved_s4 >> 6 & 0x7) {
+		case 0x1:
+		    VERTEX_OUT("Z = %f", int_as_float(data[i]));
+		    break;
+		case 0x2:
+		    VERTEX_OUT("Z = %f", int_as_float(data[i]));
+		    VERTEX_OUT("W = %f", int_as_float(data[i]));
+		    break;
+		case 0x3:
+		    break;
+		case 0x4:
+		    VERTEX_OUT("W = %f", int_as_float(data[i]));
+		    break;
+		default:
+		    DRM_ERROR("bad S4 position mask\n");
+		}
+
+		if (saved_s4 & (1 << 10)) {
+		    VERTEX_OUT("color = (A=0x%02x, R=0x%02x, G=0x%02x, "
+			       "B=0x%02x)",
+			       data[i] >> 24,
+			       (data[i] >> 16) & 0xff,
+			       (data[i] >> 8) & 0xff,
+			       data[i] & 0xff);
+		}
+		if (saved_s4 & (1 << 11)) {
+		    VERTEX_OUT("spec = (A=0x%02x, R=0x%02x, G=0x%02x, "
+			       "B=0x%02x)",
+			       data[i] >> 24,
+			       (data[i] >> 16) & 0xff,
+			       (data[i] >> 8) & 0xff,
+			       data[i] & 0xff);
+		}
+		if (saved_s4 & (1 << 12))
+		    VERTEX_OUT("width = 0x%08x)", data[i]);
+
+		for (tc = 0; tc <= 7; tc++) {
+		    switch ((saved_s2 >> (tc * 4)) & 0xf) {
+		    case 0x0:
+			VERTEX_OUT("T%d.X = %f", tc, int_as_float(data[i]));
+			VERTEX_OUT("T%d.Y = %f", tc, int_as_float(data[i]));
+			break;
+		    case 0x1:
+			VERTEX_OUT("T%d.X = %f", tc, int_as_float(data[i]));
+			VERTEX_OUT("T%d.Y = %f", tc, int_as_float(data[i]));
+			VERTEX_OUT("T%d.Z = %f", tc, int_as_float(data[i]));
+			break;
+		    case 0x2:
+			VERTEX_OUT("T%d.X = %f", tc, int_as_float(data[i]));
+			VERTEX_OUT("T%d.Y = %f", tc, int_as_float(data[i]));
+			VERTEX_OUT("T%d.Z = %f", tc, int_as_float(data[i]));
+			VERTEX_OUT("T%d.W = %f", tc, int_as_float(data[i]));
+			break;
+		    case 0x3:
+			VERTEX_OUT("T%d.X = %f", tc, int_as_float(data[i]));
+			break;
+		    case 0x4:
+			VERTEX_OUT("T%d.XY = 0x%08x half-float", tc, data[i]);
+			break;
+		    case 0x5:
+			VERTEX_OUT("T%d.XY = 0x%08x half-float", tc, data[i]);
+			VERTEX_OUT("T%d.ZW = 0x%08x half-float", tc, data[i]);
+			break;
+		    case 0xf:
+			break;
+		    default:
+			DRM_ERROR("bad S2.T%d format\n", tc);
+		    }
+		}
+		vertex++;
+	    }
+	}
+    } else {
+	/* indirect vertices */
+	len = data[0] & 0x0000ffff; /* index count */
+	if (data[0] & (1 << 17)) {
+	    /* random vertex access */
+	    if (count < (len + 1) / 2 + 1)
+		BUFFER_FAIL(count, (len + 1) / 2 + 1, "3DPRIMITIVE random indirect");
+	    instr_out(data, hw_offset, 0,
+		      "3DPRIMITIVE random indirect %s (%d)\n", primtype, len);
+	    if (len == 0) {
+		/* vertex indices continue until 0xffff is found */
+		for (i = 1; i < count; i++) {
+		    if ((data[i] & 0xffff) == 0xffff) {
+			instr_out(data, hw_offset, i,
+				  "            indices: (terminator)\n");
+			return i;
+		    } else if ((data[i] >> 16) == 0xffff) {
+			instr_out(data, hw_offset, i,
+				  "            indices: 0x%04x, "
+				  "(terminator)\n",
+				  data[i] & 0xffff);
+			return i;
+		    } else {
+			instr_out(data, hw_offset, i,
+				  "            indices: 0x%04x, 0x%04x\n",
+				  data[i] & 0xffff, data[i] >> 16);
+		    }
+		}
+		DRM_ERROR("3DPRIMITIVE: no terminator found in index buffer\n");
+		(*failures)++;
+		return count;
+	    } else {
+		/* fixed size vertex index buffer */
+		for (i = 0; i < len; i += 2) {
+		    if (i * 2 == len - 1) {
+			instr_out(data, hw_offset, i,
+				  "            indices: 0x%04x\n",
+				  data[i] & 0xffff);
+		    } else {
+			instr_out(data, hw_offset, i,
+				  "            indices: 0x%04x, 0x%04x\n",
+				  data[i] & 0xffff, data[i] >> 16);
+		    }
+		}
+	    }
+	    return (len + 1) / 2 + 1;
+	} else {
+	    /* sequential vertex access */
+	    if (count < 2)
+		BUFFER_FAIL(count, 2, "3DPRIMITIVE seq indirect");
+	    instr_out(data, hw_offset, 0,
+		      "3DPRIMITIVE sequential indirect %s, %d starting from "
+		      "%d\n", primtype, len, data[1] & 0xffff);
+	    instr_out(data, hw_offset, 1, "           start\n");
+	    return 2;
+	}
+    }
+
+    return len;
+}
+
+static int
+decode_3d(uint32_t *data, int count, uint32_t hw_offset, int *failures)
+{
+    unsigned int opcode;
+
+    struct {
+	uint32_t opcode;
+	int min_len;
+	int max_len;
+	char *name;
+    } opcodes_3d[] = {
+	{ 0x06, 1, 1, "3DSTATE_ANTI_ALIASING" },
+	{ 0x08, 1, 1, "3DSTATE_BACKFACE_STENCIL_OPS" },
+	{ 0x09, 1, 1, "3DSTATE_BACKFACE_STENCIL_MASKS" },
+	{ 0x16, 1, 1, "3DSTATE_COORD_SET_BINDINGS" },
+	{ 0x15, 1, 1, "3DSTATE_FOG_COLOR" },
+	{ 0x0b, 1, 1, "3DSTATE_INDEPENDENT_ALPHA_BLEND" },
+	{ 0x0d, 1, 1, "3DSTATE_MODES_4" },
+	{ 0x0c, 1, 1, "3DSTATE_MODES_5" },
+	{ 0x07, 1, 1, "3DSTATE_RASTERIZATION_RULES" },
+    };
+
+    switch ((data[0] & 0x1f000000) >> 24) {
+    case 0x1f:
+	return decode_3d_primitive(data, count, hw_offset, failures);
+    case 0x1d:
+	return decode_3d_1d(data, count, hw_offset, failures, 0);
+    case 0x1c:
+	return decode_3d_1c(data, count, hw_offset, failures);
+    }
+
+    for (opcode = 0; opcode < sizeof(opcodes_3d) / sizeof(opcodes_3d[0]);
+	 opcode++) {
+	if ((data[0] & 0x1f000000) >> 24 == opcodes_3d[opcode].opcode) {
+	    unsigned int len = 1, i;
+
+	    instr_out(data, hw_offset, 0, "%s\n", opcodes_3d[opcode].name);
+	    if (opcodes_3d[opcode].max_len > 1) {
+		len = (data[0] & 0xff) + 2;
+		if (len < opcodes_3d[opcode].min_len ||
+		    len > opcodes_3d[opcode].max_len)
+		{
+		    DRM_ERROR("Bad count in %s\n", opcodes_3d[opcode].name);
+		}
+	    }
+
+	    for (i = 1; i < len; i++) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len, opcodes_3d[opcode].name);
+		instr_out(data, hw_offset, i, "dword %d\n", i);
+	    }
+	    return len;
+	}
+    }
+
+    instr_out(data, hw_offset, 0, "3D UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+static const char *
+get_965_surfacetype(unsigned int surfacetype)
+{
+    switch (surfacetype) {
+    case 0: return "1D";
+    case 1: return "2D";
+    case 2: return "3D";
+    case 3: return "CUBE";
+    case 4: return "BUFFER";
+    case 7: return "NULL";
+    default: return "unknown";
+    }
+}
+
+static const char *
+get_965_depthformat(unsigned int depthformat)
+{
+    switch (depthformat) {
+    case 0: return "s8_z24float";
+    case 1: return "z32float";
+    case 2: return "z24s8";
+    case 5: return "z16";
+    default: return "unknown";
+    }
+}
+
+static int
+decode_3d_965(uint32_t *data, int count, uint32_t hw_offset, int *failures)
+{
+    unsigned int opcode, len;
+
+    struct {
+	uint32_t opcode;
+	int min_len;
+	int max_len;
+	char *name;
+    } opcodes_3d[] = {
+	{ 0x6000, 3, 3, "URB_FENCE" },
+	{ 0x6001, 2, 2, "CS_URB_STATE" },
+	{ 0x6002, 2, 2, "CONSTANT_BUFFER" },
+	{ 0x6101, 6, 6, "STATE_BASE_ADDRESS" },
+	{ 0x6102, 2, 2 , "STATE_SIP" },
+	{ 0x6104, 1, 1, "3DSTATE_PIPELINE_SELECT" },
+	{ 0x680b, 1, 1, "3DSTATE_VF_STATISTICS" },
+	{ 0x6904, 1, 1, "3DSTATE_PIPELINE_SELECT" },
+	{ 0x7800, 7, 7, "3DSTATE_PIPELINED_POINTERS" },
+	{ 0x7801, 6, 6, "3DSTATE_BINDING_TABLE_POINTERS" },
+	{ 0x780b, 1, 1, "3DSTATE_VF_STATISTICS" },
+	{ 0x7808, 5, 257, "3DSTATE_VERTEX_BUFFERS" },
+	{ 0x7809, 3, 256, "3DSTATE_VERTEX_ELEMENTS" },
+	/* 0x7808: 3DSTATE_VERTEX_BUFFERS */
+	/* 0x7809: 3DSTATE_VERTEX_ELEMENTS */
+	{ 0x7900, 4, 4, "3DSTATE_DRAWING_RECTANGLE" },
+	{ 0x7901, 5, 5, "3DSTATE_CONSTANT_COLOR" },
+	{ 0x7905, 5, 7, "3DSTATE_DEPTH_BUFFER" },
+	{ 0x7906, 2, 2, "3DSTATE_POLY_STIPPLE_OFFSET" },
+	{ 0x7907, 33, 33, "3DSTATE_POLY_STIPPLE_PATTERN" },
+	{ 0x7908, 3, 3, "3DSTATE_LINE_STIPPLE" },
+	{ 0x7909, 2, 2, "3DSTATE_GLOBAL_DEPTH_OFFSET_CLAMP" },
+	{ 0x790a, 3, 3, "3DSTATE_AA_LINE_PARAMETERS" },
+	{ 0x7b00, 6, 6, "3DPRIMITIVE" },
+    };
+
+    len = (data[0] & 0x0000ffff) + 2;
+
+    switch ((data[0] & 0xffff0000) >> 16) {
+    case 0x6101:
+	if (len != 6)
+	    DRM_ERROR("Bad count in STATE_BASE_ADDRESS\n");
+	if (count < 6)
+	    BUFFER_FAIL(count, len, "STATE_BASE_ADDRESS");
+
+	instr_out(data, hw_offset, 0,
+		  "STATE_BASE_ADDRESS\n");
+
+	if (data[1] & 1) {
+	    instr_out(data, hw_offset, 1, "General state at 0x%08x\n",
+		      data[1] & ~1);
+	} else
+	    instr_out(data, hw_offset, 1, "General state not updated\n");
+
+	if (data[2] & 1) {
+	    instr_out(data, hw_offset, 2, "Surface state at 0x%08x\n",
+		      data[2] & ~1);
+	} else
+	    instr_out(data, hw_offset, 2, "Surface state not updated\n");
+
+	if (data[3] & 1) {
+	    instr_out(data, hw_offset, 3, "Indirect state at 0x%08x\n",
+		      data[3] & ~1);
+	} else
+	    instr_out(data, hw_offset, 3, "Indirect state not updated\n");
+
+	if (data[4] & 1) {
+	    instr_out(data, hw_offset, 4, "General state upper bound 0x%08x\n",
+		      data[4] & ~1);
+	} else
+	    instr_out(data, hw_offset, 4, "General state not updated\n");
+
+	if (data[5] & 1) {
+	    instr_out(data, hw_offset, 5, "Indirect state upper bound 0x%08x\n",
+		      data[5] & ~1);
+	} else
+	    instr_out(data, hw_offset, 5, "Indirect state not updated\n");
+
+	return len;
+    case 0x7800:
+	if (len != 7)
+	    DRM_ERROR("Bad count in 3DSTATE_PIPELINED_POINTERS\n");
+	if (count < 7)
+	    BUFFER_FAIL(count, len, "3DSTATE_PIPELINED_POINTERS");
+
+	instr_out(data, hw_offset, 0,
+		  "3DSTATE_PIPELINED_POINTERS\n");
+	instr_out(data, hw_offset, 1, "VS state\n");
+	instr_out(data, hw_offset, 2, "GS state\n");
+	instr_out(data, hw_offset, 3, "Clip state\n");
+	instr_out(data, hw_offset, 4, "SF state\n");
+	instr_out(data, hw_offset, 5, "WM state\n");
+	instr_out(data, hw_offset, 6, "CC state\n");
+	return len;
+    case 0x7801:
+	if (len != 6)
+	    DRM_ERROR("Bad count in 3DSTATE_BINDING_TABLE_POINTERS\n");
+	if (count < 6)
+	    BUFFER_FAIL(count, len, "3DSTATE_BINDING_TABLE_POINTERS");
+
+	instr_out(data, hw_offset, 0,
+		  "3DSTATE_BINDING_TABLE_POINTERS\n");
+	instr_out(data, hw_offset, 1, "VS binding table\n");
+	instr_out(data, hw_offset, 2, "GS binding table\n");
+	instr_out(data, hw_offset, 3, "Clip binding table\n");
+	instr_out(data, hw_offset, 4, "SF binding table\n");
+	instr_out(data, hw_offset, 5, "WM binding table\n");
+
+	return len;
+
+    case 0x7900:
+	if (len != 4)
+	    DRM_ERROR("Bad count in 3DSTATE_DRAWING_RECTANGLE\n");
+	if (count < 4)
+	    BUFFER_FAIL(count, len, "3DSTATE_DRAWING_RECTANGLE");
+
+	instr_out(data, hw_offset, 0,
+		  "3DSTATE_DRAWING_RECTANGLE\n");
+	instr_out(data, hw_offset, 1, "top left: %d,%d\n",
+		  data[1] & 0xffff,
+		  (data[1] >> 16) & 0xffff);
+	instr_out(data, hw_offset, 2, "bottom right: %d,%d\n",
+		  data[2] & 0xffff,
+		  (data[2] >> 16) & 0xffff);
+	instr_out(data, hw_offset, 3, "origin: %d,%d\n",
+		  (int)data[3] & 0xffff,
+		  ((int)data[3] >> 16) & 0xffff);
+
+	return len;
+
+    case 0x7905:
+	if (len != 5)
+	    DRM_ERROR("Bad count in 3DSTATE_DEPTH_BUFFER\n");
+	if (count < 5)
+	    BUFFER_FAIL(count, len, "3DSTATE_DEPTH_BUFFER");
+
+	instr_out(data, hw_offset, 0,
+		  "3DSTATE_DEPTH_BUFFER\n");
+	instr_out(data, hw_offset, 1, "%s, %s, pitch = %d bytes, %stiled\n",
+		  get_965_surfacetype(data[1] >> 29),
+		  get_965_depthformat((data[1] >> 18) & 0x7),
+		  (data[1] & 0x0001ffff) + 1,
+		  data[1] & (1 << 27) ? "" : "not ");
+	instr_out(data, hw_offset, 2, "depth offset\n");
+	instr_out(data, hw_offset, 3, "%dx%d\n",
+		  ((data[3] & 0x0007ffc0) >> 6) + 1,
+		  ((data[3] & 0xfff80000) >> 19) + 1);
+	instr_out(data, hw_offset, 4, "volume depth\n");
+
+	return len;
+    }
+
+    for (opcode = 0; opcode < sizeof(opcodes_3d) / sizeof(opcodes_3d[0]);
+	 opcode++) {
+	if ((data[0] & 0xffff0000) >> 16 == opcodes_3d[opcode].opcode) {
+	    unsigned int i;
+	    len = 1;
+
+	    instr_out(data, hw_offset, 0, "%s\n", opcodes_3d[opcode].name);
+	    if (opcodes_3d[opcode].max_len > 1) {
+		len = (data[0] & 0xff) + 2;
+		if (len < opcodes_3d[opcode].min_len ||
+		    len > opcodes_3d[opcode].max_len)
+		{
+		    DRM_ERROR("Bad count in %s\n", opcodes_3d[opcode].name);
+		}
+	    }
+
+	    for (i = 1; i < len; i++) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len, opcodes_3d[opcode].name);
+		instr_out(data, hw_offset, i, "dword %d\n", i);
+	    }
+	    return len;
+	}
+    }
+
+    instr_out(data, hw_offset, 0, "3D UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+
+static int
+decode_3d_i830(uint32_t *data, int count, uint32_t hw_offset, int *failures)
+{
+    unsigned int opcode;
+
+    struct {
+	uint32_t opcode;
+	int min_len;
+	int max_len;
+	char *name;
+    } opcodes_3d[] = {
+	{ 0x02, 1, 1, "3DSTATE_MODES_3" },
+	{ 0x03, 1, 1, "3DSTATE_ENABLES_1"},
+	{ 0x04, 1, 1, "3DSTATE_ENABLES_2"},
+	{ 0x05, 1, 1, "3DSTATE_VFT0"},
+	{ 0x06, 1, 1, "3DSTATE_AA"},
+	{ 0x07, 1, 1, "3DSTATE_RASTERIZATION_RULES" },
+	{ 0x08, 1, 1, "3DSTATE_MODES_1" },
+	{ 0x09, 1, 1, "3DSTATE_STENCIL_TEST" },
+	{ 0x0a, 1, 1, "3DSTATE_VFT1"},
+	{ 0x0b, 1, 1, "3DSTATE_INDPT_ALPHA_BLEND" },
+	{ 0x0c, 1, 1, "3DSTATE_MODES_5" },
+	{ 0x0d, 1, 1, "3DSTATE_MAP_BLEND_OP" },
+	{ 0x0e, 1, 1, "3DSTATE_MAP_BLEND_ARG" },
+	{ 0x0f, 1, 1, "3DSTATE_MODES_2" },
+	{ 0x15, 1, 1, "3DSTATE_FOG_COLOR" },
+	{ 0x16, 1, 1, "3DSTATE_MODES_4" },
+    };
+
+    switch ((data[0] & 0x1f000000) >> 24) {
+    case 0x1f:
+	return decode_3d_primitive(data, count, hw_offset, failures);
+    case 0x1d:
+	return decode_3d_1d(data, count, hw_offset, failures, 1);
+    case 0x1c:
+	return decode_3d_1c(data, count, hw_offset, failures);
+    }
+
+    for (opcode = 0; opcode < sizeof(opcodes_3d) / sizeof(opcodes_3d[0]);
+	 opcode++) {
+	if ((data[0] & 0x1f000000) >> 24 == opcodes_3d[opcode].opcode) {
+	    unsigned int len = 1, i;
+
+	    instr_out(data, hw_offset, 0, "%s\n", opcodes_3d[opcode].name);
+	    if (opcodes_3d[opcode].max_len > 1) {
+		len = (data[0] & 0xff) + 2;
+		if (len < opcodes_3d[opcode].min_len ||
+		    len > opcodes_3d[opcode].max_len)
+		{
+		    DRM_ERROR("Bad count in %s\n", opcodes_3d[opcode].name);
+		}
+	    }
+
+	    for (i = 1; i < len; i++) {
+		if (i >= count)
+		    BUFFER_FAIL(count, len, opcodes_3d[opcode].name);
+		instr_out(data, hw_offset, i, "dword %d\n", i);
+	    }
+	    return len;
+	}
+    }
+
+    instr_out(data, hw_offset, 0, "3D UNKNOWN\n");
+    (*failures)++;
+    return 1;
+}
+
+void i915_gem_command_decode(uint32_t *data, int count, uint32_t hw_offset, struct drm_device *dev)
+{
+    int index = 0;
+    int failures = 0;
+
+   while (index < count) {
+	switch ((data[index] & 0xe0000000) >> 29) {
+	case 0x0:
+	    index += decode_mi(data + index, count - index,
+			       hw_offset + index * 4, &failures);
+	    break;
+	case 0x2:
+	    index += decode_2d(data + index, count - index,
+			       hw_offset + index * 4, &failures);
+	    break;
+	case 0x3:
+	    if (IS_I965G(dev)) {
+		index += decode_3d_965(data + index, count - index,
+				       hw_offset + index * 4, &failures);
+	    } else if (IS_I9XX(dev)) {
+		index += decode_3d(data + index, count - index,
+				   hw_offset + index * 4, &failures);
+	    } else {
+		index += decode_3d_i830(data + index, count - index,
+					hw_offset + index * 4, &failures);
+	    }
+	    break;
+	default:
+	    instr_out(data, hw_offset, index, "UNKNOWN\n");
+	    failures++;
+	    index++;
+	    break;
+	}
+    }
+}
+
diff --git a/usr/src/uts/intel/io/drm/i915_gem_tiling.c b/usr/src/uts/intel/io/drm/i915_gem_tiling.c
new file mode 100644
index 0000000000..9978463c24
--- /dev/null
+++ b/usr/src/uts/intel/io/drm/i915_gem_tiling.c
@@ -0,0 +1,383 @@
+/* BEGIN CSTYLED */
+
+/*
+ * Copyright (c) 2009, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+
+/** @file i915_gem_tiling.c
+ *
+ * Support for managing tiling state of buffer objects.
+ *
+ * The idea behind tiling is to increase cache hit rates by rearranging
+ * pixel data so that a group of pixel accesses are in the same cacheline.
+ * Performance improvement from doing this on the back/depth buffer are on
+ * the order of 30%.
+ *
+ * Intel architectures make this somewhat more complicated, though, by
+ * adjustments made to addressing of data when the memory is in interleaved
+ * mode (matched pairs of DIMMS) to improve memory bandwidth.
+ * For interleaved memory, the CPU sends every sequential 64 bytes
+ * to an alternate memory channel so it can get the bandwidth from both.
+ *
+ * The GPU also rearranges its accesses for increased bandwidth to interleaved
+ * memory, and it matches what the CPU does for non-tiled.  However, when tiled
+ * it does it a little differently, since one walks addresses not just in the
+ * X direction but also Y.  So, along with alternating channels when bit
+ * 6 of the address flips, it also alternates when other bits flip --  Bits 9
+ * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines)
+ * are common to both the 915 and 965-class hardware.
+ *
+ * The CPU also sometimes XORs in higher bits as well, to improve
+ * bandwidth doing strided access like we do so frequently in graphics.  This
+ * is called "Channel XOR Randomization" in the MCH documentation.  The result
+ * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address
+ * decode.
+ *
+ * All of this bit 6 XORing has an effect on our memory management,
+ * as we need to make sure that the 3d driver can correctly address object
+ * contents.
+ *
+ * If we don't have interleaved memory, all tiling is safe and no swizzling is
+ * required.
+ *
+ * When bit 17 is XORed in, we simply refuse to tile at all.  Bit
+ * 17 is not just a page offset, so as we page an objet out and back in,
+ * individual pages in it will have different bit 17 addresses, resulting in
+ * each 64 bytes being swapped with its neighbor!
+ *
+ * Otherwise, if interleaved, we have to tell the 3d driver what the address
+ * swizzling it needs to do is, since it's writing with the CPU to the pages
+ * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the
+ * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling
+ * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order
+ * to match what the GPU expects.
+ */
+
+/**
+ * Detects bit 6 swizzling of address lookup between IGD access and CPU
+ * access through main memory.
+ */
+void
+i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
+	uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
+
+	if (!IS_I9XX(dev)) {
+		/* As far as we know, the 865 doesn't have these bit 6
+		 * swizzling issues.
+		 */
+		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
+		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
+		} else if (IS_MOBILE(dev)) {
+		uint32_t dcc;
+
+		/* On mobile 9xx chipsets, channel interleave by the CPU is
+		 * determined by DCC.  For single-channel, neither the CPU
+		 * nor the GPU do swizzling.  For dual channel interleaved,
+		 * the GPU's interleave is bit 9 and 10 for X tiled, and bit
+		 * 9 for Y tiled.  The CPU's interleave is independent, and
+		 * can be based on either bit 11 (haven't seen this yet) or
+		 * bit 17 (common).
+		 */
+
+		dcc = I915_READ(DCC);
+		switch (dcc & DCC_ADDRESSING_MODE_MASK) {
+		case DCC_ADDRESSING_MODE_SINGLE_CHANNEL:
+		case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC:
+			swizzle_x = I915_BIT_6_SWIZZLE_NONE;
+			swizzle_y = I915_BIT_6_SWIZZLE_NONE;
+			break;
+		case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED:
+			if (dcc & DCC_CHANNEL_XOR_DISABLE) {
+				/* This is the base swizzling by the GPU for
+				 * tiled buffers.
+				 */
+				swizzle_x = I915_BIT_6_SWIZZLE_9_10;
+				swizzle_y = I915_BIT_6_SWIZZLE_9;
+			} else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
+				/* Bit 11 swizzling by the CPU in addition. */
+				swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
+				swizzle_y = I915_BIT_6_SWIZZLE_9_11;
+			} else {
+				/* Bit 17 swizzling by the CPU in addition. */	
+				swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
+				swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
+			}
+			break;
+		}
+		if (dcc == 0xffffffff) {
+			DRM_ERROR("Couldn't read from MCHBAR.  "
+				  "Disabling tiling.\n");
+			swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
+			swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
+		}
+	} else {
+		/* The 965, G33, and newer, have a very flexible memory
+		 * configuration.  It will enable dual-channel mode
+		 * (interleaving) on as much memory as it can, and the GPU
+		 * will additionally sometimes enable different bit 6
+		 * swizzling for tiled objects from the CPU.
+		 *
+		 * Here's what I found on the G965:
+		 *    slot fill         memory size  swizzling
+		 * 0A   0B   1A   1B    1-ch   2-ch
+		 * 512  0    0    0     512    0     O
+		 * 512  0    512  0     16     1008  X
+		 * 512  0    0    512   16     1008  X
+		 * 0    512  0    512   16     1008  X
+		 * 1024 1024 1024 0     2048   1024  O
+		 *
+		 * We could probably detect this based on either the DRB
+		 * matching, which was the case for the swizzling required in
+		 * the table above, or from the 1-ch value being less than
+		 * the minimum size of a rank.
+		 */
+		if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) {
+			swizzle_x = I915_BIT_6_SWIZZLE_NONE;
+			swizzle_y = I915_BIT_6_SWIZZLE_NONE;
+		} else {
+			swizzle_x = I915_BIT_6_SWIZZLE_9_10;
+			swizzle_y = I915_BIT_6_SWIZZLE_9;
+		}
+	}
+
+	dev_priv->mm.bit_6_swizzle_x = swizzle_x;
+	dev_priv->mm.bit_6_swizzle_y = swizzle_y;
+}
+
+
+/**
+ * Returns the size of the fence for a tiled object of the given size.
+ */
+static int
+i915_get_fence_size(struct drm_device *dev, int size)
+{
+	int i;
+	int start;
+
+	if (IS_I965G(dev)) {
+		/* The 965 can have fences at any page boundary. */
+
+		return (size + PAGE_SIZE-1) & ~(PAGE_SIZE-1);
+	} else {
+		/* Align the size to a power of two greater than the smallest
+		 * fence size.
+		 */
+		if (IS_I9XX(dev))
+			start = 1024 * 1024;
+		else
+			start = 512 * 1024;
+
+		for (i = start; i < size; i <<= 1)
+			;
+
+		return i;
+	}
+}
+
+/* Check pitch constriants for all chips & tiling formats */
+static int 
+i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
+{
+	int tile_width;
+
+	/* Linear is always fine */
+	if (tiling_mode == I915_TILING_NONE)
+		return 1;
+
+	if (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
+		tile_width = 128;
+	else
+		tile_width = 512;
+
+	if (stride == 0)
+		return 0;
+
+	/* 965+ just needs multiples of tile width */
+	if (IS_I965G(dev)) {
+		if (stride & (tile_width - 1))
+			return 0;
+		return 1;
+	}
+
+	/* Pre-965 needs power of two tile widths */
+	if (stride < tile_width)
+		return 0;
+
+	if (stride & (stride - 1))
+		return 0;
+
+	/* We don't handle the aperture area covered by the fence being bigger
+	 * than the object size.
+	 */
+	if (i915_get_fence_size(dev, size) != size)
+		return 0;
+
+	return 1;
+}
+
+/**
+ * Sets the tiling mode of an object, returning the required swizzling of
+ * bit 6 of addresses in the object.
+ */
+/*ARGSUSED*/
+int
+i915_gem_set_tiling(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_set_tiling args;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+            (struct drm_i915_gem_set_tiling __user *) data, sizeof(args));
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return EINVAL;
+	obj_priv = obj->driver_private;
+
+	if (!i915_tiling_ok(dev, args.stride, obj->size, args.tiling_mode)) {
+		drm_gem_object_unreference(obj);
+		DRM_DEBUG("i915 tiling is not OK");
+		return EINVAL;
+	}
+
+	spin_lock(&dev->struct_mutex);
+
+	if (args.tiling_mode == I915_TILING_NONE) {
+		args.swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
+	} else {
+		if (args.tiling_mode == I915_TILING_X)
+			args.swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
+		else
+			args.swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
+		/* If we can't handle the swizzling, make it untiled. */
+		if (args.swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
+			args.tiling_mode = I915_TILING_NONE;
+			args.swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
+		}
+	}
+
+	if (args.tiling_mode != obj_priv->tiling_mode) {
+		int ret;
+
+		/* Unbind the object, as switching tiling means we're
+		 * switching the cache organization due to fencing, probably.
+		 */
+		ret = i915_gem_object_unbind(obj, 1);
+		if (ret != 0) {
+			args.tiling_mode = obj_priv->tiling_mode;
+			spin_unlock(&dev->struct_mutex);
+			drm_gem_object_unreference(obj);
+			DRM_ERROR("tiling switch!! unbind error %d", ret);
+			return ret;
+		}
+		obj_priv->tiling_mode = args.tiling_mode;
+	}
+	obj_priv->stride = args.stride;
+
+	ret = DRM_COPY_TO_USER((struct drm_i915_gem_set_tiling __user *) data, &args, sizeof(args));
+        if ( ret != 0)
+                DRM_ERROR(" gem set tiling error! %d", ret);
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	return 0;
+}
+
+/**
+ * Returns the current tiling mode and required bit 6 swizzling for the object.
+ */
+/*ARGSUSED*/
+int
+i915_gem_get_tiling(DRM_IOCTL_ARGS)
+{
+	DRM_DEVICE;
+	struct drm_i915_gem_get_tiling args;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj_priv;
+	int ret;
+
+	if (dev->driver->use_gem != 1)
+		return ENODEV;
+
+	DRM_COPYFROM_WITH_RETURN(&args,
+	    (struct drm_i915_gem_get_tiling __user *) data, sizeof(args));
+
+	obj = drm_gem_object_lookup(fpriv, args.handle);
+	if (obj == NULL)
+		return EINVAL;
+	obj_priv = obj->driver_private;
+
+	spin_lock(&dev->struct_mutex);
+
+	args.tiling_mode = obj_priv->tiling_mode;
+	switch (obj_priv->tiling_mode) {
+	case I915_TILING_X:
+		args.swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
+		break;
+	case I915_TILING_Y:
+		args.swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
+		break;
+	case I915_TILING_NONE:
+		args.swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
+		break;
+	default:
+		DRM_ERROR("unknown tiling mode\n");
+	}
+
+
+
+	ret = DRM_COPY_TO_USER((struct drm_i915_gem_get_tiling __user *) data, &args, sizeof(args));
+        if ( ret != 0)
+                DRM_ERROR(" gem get tiling error! %d", ret);
+
+	drm_gem_object_unreference(obj);
+	spin_unlock(&dev->struct_mutex);
+
+	return 0;
+}
diff --git a/usr/src/uts/intel/io/drm/i915_irq.c b/usr/src/uts/intel/io/drm/i915_irq.c
index 664a82d76d..6596fdd094 100644
--- a/usr/src/uts/intel/io/drm/i915_irq.c
+++ b/usr/src/uts/intel/io/drm/i915_irq.c
@@ -4,6 +4,7 @@
  */
 /*
  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * Copyright (c) 2009, Intel Corporation.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -41,67 +42,79 @@
 
 #define MAX_NOPID ((u32)~0)
 
-/*
- * These are the interrupts used by the driver
+/**
+ * Interrupts that are always left unmasked.
+ *
+ * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
+ * we leave them always unmasked in IMR and then control enabling them through
+ * PIPESTAT alone.
  */
-#define I915_INTERRUPT_ENABLE_MASK (I915_USER_INTERRUPT | \
-				    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
-				    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
+
+#define I915_INTERRUPT_ENABLE_FIX (I915_ASLE_INTERRUPT |		 \
+				   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
+				   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | \
+				   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+
+/** Interrupts that we mask and unmask at runtime. */
+#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)
+
+/** These are all of the interrupts used by the driver */
+#define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \
+				    I915_INTERRUPT_ENABLE_VAR)
 
 static inline void
 i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
 {
-	if ((dev_priv->irq_mask_reg & mask) != 0) {
-		dev_priv->irq_mask_reg &= ~mask;
-		I915_WRITE(IMR, dev_priv->irq_mask_reg);
-		(void) I915_READ(IMR);
-	}
+        if ((dev_priv->irq_mask_reg & mask) != 0) {
+                dev_priv->irq_mask_reg &= ~mask;
+                I915_WRITE(IMR, dev_priv->irq_mask_reg);
+                (void) I915_READ(IMR);
+        }
 }
 
 static inline void
 i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
 {
 	if ((dev_priv->irq_mask_reg & mask) != mask) {
-		dev_priv->irq_mask_reg |= mask;
-		I915_WRITE(IMR, dev_priv->irq_mask_reg);
-		(void) I915_READ(IMR);
-	}
+                dev_priv->irq_mask_reg |= mask;
+                I915_WRITE(IMR, dev_priv->irq_mask_reg);
+                (void) I915_READ(IMR);
+        }
 }
- /**
- * i915_get_pipe - return the the pipe associated with a given plane
- * @dev: DRM device
- * @plane: plane to look for
- *
- * The Intel Mesa & 2D drivers call the vblank routines with a plane number
- * rather than a pipe number, since they may not always be equal.  This routine
- * maps the given @plane back to a pipe number.
- */
-static int
-i915_get_pipe(struct drm_device *dev, int plane)
+
+static inline uint32_t
+i915_pipestat(int pipe)
 {
-	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 dspcntr;
+	if (pipe == 0)
+		return PIPEASTAT;
+	if (pipe == 1)
+		return PIPEBSTAT;
+	return 0;
+}
 
-	dspcntr = plane ? I915_READ(DSPBCNTR) : I915_READ(DSPACNTR);
+void
+i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, uint32_t mask)
+{
+	if ((dev_priv->pipestat[pipe] & mask) != mask) {
+		u32 reg = i915_pipestat(pipe);
 
-	return dspcntr & DISPPLANE_SEL_PIPE_MASK ? 1 : 0;
+		dev_priv->pipestat[pipe] |= mask;
+		/* Enable the interrupt, clear any pending status */
+		I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
+		(void) I915_READ(reg);
+	}
 }
 
-/**
- * i915_get_plane - return the the plane associated with a given pipe
- * @dev: DRM device
- * @pipe: pipe to look for
- *
- * The Intel Mesa & 2D drivers call the vblank routines with a plane number
- * rather than a plane number, since they may not always be equal.  This routine
- * maps the given @pipe back to a plane number.
- */
-static int
-i915_get_plane(struct drm_device *dev, int pipe)
+void
+i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 {
-	if (i915_get_pipe(dev, 0) == pipe)
-		return 0;
-	return 1;
+	if ((dev_priv->pipestat[pipe] & mask) != 0) {
+		u32 reg = i915_pipestat(pipe);
+
+		dev_priv->pipestat[pipe] &= ~mask;
+		I915_WRITE(reg, dev_priv->pipestat[pipe]);
+		(void) I915_READ(reg);
+	}
 }
 
 /**
@@ -125,309 +138,240 @@ i915_pipe_enabled(struct drm_device *dev, int pipe)
 	return 0;
 }
 
-/**
- * Emit a synchronous flip.
- *
- * This function must be called with the drawable spinlock held.
- */
-static void
-i915_dispatch_vsync_flip(struct drm_device *dev, struct drm_drawable_info *drw,
-			 int plane)
+u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	u16 x1, y1, x2, y2;
-	int pf_planes = 1 << plane;
+	unsigned long high_frame;
+	unsigned long low_frame;
+	u32 high1, high2, low, count;
 
-	DRM_SPINLOCK_ASSERT(&dev->drw_lock);
+	high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
+	low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
 
-	/* If the window is visible on the other plane, we have to flip on that
-	 * plane as well.
-	 */
-	if (plane == 1) {
-		x1 = sarea_priv->planeA_x;
-		y1 = sarea_priv->planeA_y;
-		x2 = x1 + sarea_priv->planeA_w;
-		y2 = y1 + sarea_priv->planeA_h;
-	} else {
-		x1 = sarea_priv->planeB_x;
-		y1 = sarea_priv->planeB_y;
-		x2 = x1 + sarea_priv->planeB_w;
-		y2 = y1 + sarea_priv->planeB_h;
+	if (!i915_pipe_enabled(dev, pipe)) {
+	    DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
+	    return 0;
 	}
 
-	if (x2 > 0 && y2 > 0) {
-		int i, num_rects = drw->num_rects;
-		struct drm_clip_rect *rect = drw->rects;
-
-		for (i = 0; i < num_rects; i++)
-			if (!(rect[i].x1 >= x2 || rect[i].y1 >= y2 ||
-			      rect[i].x2 <= x1 || rect[i].y2 <= y1)) {
-				pf_planes = 0x3;
+	/*
+	 * High & low register fields aren't synchronized, so make sure
+	 * we get a low value that's stable across two reads of the high
+	 * register.
+	 */
+	do {
+		high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
+			 PIPE_FRAME_HIGH_SHIFT);
+		low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
+			PIPE_FRAME_LOW_SHIFT);
+		high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
+			 PIPE_FRAME_HIGH_SHIFT);
+	} while (high1 != high2);
 
-				break;
-			}
-	}
+	count = (high1 << 8) | low;
 
-	i915_dispatch_flip(dev, pf_planes, 1);
+	return count;
 }
 
 /**
- * Emit blits for scheduled buffer swaps.
+ * i915_capture_error_state - capture an error record for later analysis
+ * @dev: drm device
  *
- * This function will be called with the HW lock held.
+ * Should be called when an error is detected (either a hang or an error
+ * interrupt) to capture error state from the time of the error.  Fills
+ * out a structure which becomes available in debugfs for user level tools
+ * to pick up.
  */
-static void i915_vblank_tasklet(drm_device_t *dev)
+static void i915_capture_error_state(struct drm_device *dev)
 {
-	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	struct list_head *list, *tmp, hits, *hit;
-	int nhits, slice[2], upper[2], lower[2], i, num_pages;
-	unsigned counter[2];
-	struct drm_drawable_info *drw;
-	drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
-	u32 cpp = dev_priv->cpp,  offsets[3];
-	u32 cmd = (cpp == 4) ? (XY_SRC_COPY_BLT_CMD |
-				XY_SRC_COPY_BLT_WRITE_ALPHA |
-				XY_SRC_COPY_BLT_WRITE_RGB)
-			     : XY_SRC_COPY_BLT_CMD;
-	u32 src_pitch = sarea_priv->pitch * cpp;
-	u32 dst_pitch = sarea_priv->pitch * cpp;
-	/* COPY rop (0xcc), map cpp to magic color depth constants */
-	u32 ropcpp = (0xcc << 16) | ((cpp - 1) << 24);
-	RING_LOCALS;
-	
-	if (IS_I965G(dev) && sarea_priv->front_tiled) {
-		cmd |= XY_SRC_COPY_BLT_DST_TILED;
-		dst_pitch >>= 2;
-	}
-	if (IS_I965G(dev) && sarea_priv->back_tiled) {
-		cmd |= XY_SRC_COPY_BLT_SRC_TILED;
-		src_pitch >>= 2;
-	}
-	
-	counter[0] = drm_vblank_count(dev, 0);
-	counter[1] = drm_vblank_count(dev, 1);
-
-	INIT_LIST_HEAD(&hits);
-
-	nhits = 0;
-
-	/* No irqsave/restore necessary.  This tasklet may be run in an
-	 * interrupt context or normal context, but we don't have to worry
-	 * about getting interrupted by something acquiring the lock, because
-	 * we are the interrupt context thing that acquires the lock.
-	 */
-	DRM_SPINLOCK(&dev_priv->swaps_lock);
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_error_state *error;
 
-	/* Find buffer swaps scheduled for this vertical blank */
-	list_for_each_safe(list, tmp, &dev_priv->vbl_swaps.head) {
-		drm_i915_vbl_swap_t *vbl_swap =
-			list_entry(list, drm_i915_vbl_swap_t, head);
-		int pipe = i915_get_pipe(dev, vbl_swap->plane);
-
-		if ((counter[pipe] - vbl_swap->sequence) > (1<<23))
-			continue;
-
-		list_del(list);
-		dev_priv->swaps_pending--;
-		drm_vblank_put(dev, pipe);
+	spin_lock_irqsave(&dev_priv->error_lock, flags);
+#if 0
+	if (dev_priv->first_error)
+		goto out;
+#endif
+	error = drm_alloc(sizeof(*error), DRM_MEM_DRIVER);
+	if (!error) {
+		DRM_DEBUG("out ot memory, not capturing error state\n");
+		goto out;
+	}
+
+	error->eir = I915_READ(EIR);
+	error->pgtbl_er = I915_READ(PGTBL_ER);
+	error->pipeastat = I915_READ(PIPEASTAT);
+	error->pipebstat = I915_READ(PIPEBSTAT);
+	error->instpm = I915_READ(INSTPM);
+	if (!IS_I965G(dev)) {
+		error->ipeir = I915_READ(IPEIR);
+		error->ipehr = I915_READ(IPEHR);
+		error->instdone = I915_READ(INSTDONE);
+		error->acthd = I915_READ(ACTHD);
+	} else {
+		error->ipeir = I915_READ(IPEIR_I965);
+		error->ipehr = I915_READ(IPEHR_I965);
+		error->instdone = I915_READ(INSTDONE_I965);
+		error->instps = I915_READ(INSTPS);
+		error->instdone1 = I915_READ(INSTDONE1);
+		error->acthd = I915_READ(ACTHD_I965);
+	}
 
-		DRM_SPINUNLOCK(&dev_priv->swaps_lock);
-		DRM_SPINLOCK(&dev->drw_lock);
+	(void) uniqtime(&error->time);
 
-		drw = drm_get_drawable_info(dev, vbl_swap->drw_id);
+	dev_priv->first_error = error;
+	
+	DRM_DEBUG("Time: %ld s %ld us\n", error->time.tv_sec,
+		   error->time.tv_usec);
+	DRM_DEBUG("EIR: 0x%08x\n", error->eir);
+	DRM_DEBUG("  PGTBL_ER: 0x%08x\n", error->pgtbl_er);
+	DRM_DEBUG("  INSTPM: 0x%08x\n", error->instpm);
+	DRM_DEBUG("  IPEIR: 0x%08x\n", error->ipeir);
+	DRM_DEBUG("  IPEHR: 0x%08x\n", error->ipehr);
+	DRM_DEBUG("  INSTDONE: 0x%08x\n", error->instdone);
+	DRM_DEBUG("  ACTHD: 0x%08x\n", error->acthd);
+	DRM_DEBUG("  DMA_FADD_P: 0x%08x\n", I915_READ(0x2078));
+	if (IS_I965G(dev)) {
+		DRM_DEBUG("  INSTPS: 0x%08x\n", error->instps);
+		DRM_DEBUG("  INSTDONE1: 0x%08x\n", error->instdone1);
+	}
+	drm_free(error, sizeof(*error), DRM_MEM_DRIVER);
+out:
+	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
+}
 
-		if (!drw) {
-			DRM_SPINUNLOCK(&dev->drw_lock);
-			drm_free(vbl_swap, sizeof(*vbl_swap), DRM_MEM_DRIVER);
-			DRM_SPINLOCK(&dev_priv->swaps_lock);
-			continue;
+/**
+ * i915_handle_error - handle an error interrupt
+ * @dev: drm device
+ *
+ * Do some basic checking of regsiter state at error interrupt time and
+ * dump it to the syslog.  Also call i915_capture_error_state() to make
+ * sure we get a record and make it available in debugfs.  Fire a uevent
+ * so userspace knows something bad happened (should trigger collection
+ * of a ring dump etc.).
+ */
+void i915_handle_error(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 eir = I915_READ(EIR);
+	u32 pipea_stats = I915_READ(PIPEASTAT);
+	u32 pipeb_stats = I915_READ(PIPEBSTAT);
+
+	i915_capture_error_state(dev);
+
+	DRM_DEBUG("render error detected, EIR: 0x%08x\n",
+	       eir);
+
+	if (IS_G4X(dev)) {
+		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
+			u32 ipeir = I915_READ(IPEIR_I965);
+
+			DRM_DEBUG("  IPEIR: 0x%08x\n",
+			       I915_READ(IPEIR_I965));
+			DRM_DEBUG("  IPEHR: 0x%08x\n",
+			       I915_READ(IPEHR_I965));
+			DRM_DEBUG("  INSTDONE: 0x%08x\n",
+			       I915_READ(INSTDONE_I965));
+			DRM_DEBUG("  INSTPS: 0x%08x\n",
+			       I915_READ(INSTPS));
+			DRM_DEBUG("  INSTDONE1: 0x%08x\n",
+			       I915_READ(INSTDONE1));
+			DRM_DEBUG("  ACTHD: 0x%08x\n",
+			       I915_READ(ACTHD_I965));
+			I915_WRITE(IPEIR_I965, ipeir);
+			(void)I915_READ(IPEIR_I965);
 		}
-
-		list_for_each(hit, &hits) {
-			drm_i915_vbl_swap_t *swap_cmp =
-				list_entry(hit, drm_i915_vbl_swap_t, head);
-			struct drm_drawable_info *drw_cmp =
-				drm_get_drawable_info(dev, swap_cmp->drw_id);
-
-			if (drw_cmp &&
-			    drw_cmp->rects[0].y1 > drw->rects[0].y1) {
-				list_add_tail(list, hit);
-				break;
-			}
+		if (eir & GM45_ERROR_PAGE_TABLE) {
+			u32 pgtbl_err = I915_READ(PGTBL_ER);
+			DRM_DEBUG("page table error\n");
+			DRM_DEBUG("  PGTBL_ER: 0x%08x\n",
+			       pgtbl_err);
+			I915_WRITE(PGTBL_ER, pgtbl_err);
+			(void)I915_READ(PGTBL_ER);
 		}
-
-		DRM_SPINUNLOCK(&dev->drw_lock);
-
-		/* List of hits was empty, or we reached the end of it */
-		if (hit == &hits)
-			list_add_tail(list, hits.prev);
-
-		nhits++;
-
-		DRM_SPINLOCK(&dev_priv->swaps_lock);
 	}
 
-	DRM_SPINUNLOCK(&dev_priv->swaps_lock);
-
-	if (nhits == 0) {
-		return;
+	if (IS_I9XX(dev)) {
+		if (eir & I915_ERROR_PAGE_TABLE) {
+			u32 pgtbl_err = I915_READ(PGTBL_ER);
+			DRM_DEBUG("page table error\n");
+			DRM_DEBUG("PGTBL_ER: 0x%08x\n",
+			       pgtbl_err);
+			I915_WRITE(PGTBL_ER, pgtbl_err);
+			(void)I915_READ(PGTBL_ER);
+		}
 	}
 
-	i915_kernel_lost_context(dev);
-
-	upper[0] = upper[1] = 0;
-	slice[0] = max(sarea_priv->planeA_h / nhits, 1);
-	slice[1] = max(sarea_priv->planeB_h / nhits, 1);
-	lower[0] = sarea_priv->planeA_y + slice[0];
-	lower[1] = sarea_priv->planeB_y + slice[0];
-
-	offsets[0] = sarea_priv->front_offset;
-	offsets[1] = sarea_priv->back_offset;
-	offsets[2] = sarea_priv->third_offset;
-	num_pages = sarea_priv->third_handle ? 3 : 2;
-
-	DRM_SPINLOCK(&dev->drw_lock);
-
-	/* Emit blits for buffer swaps, partitioning both outputs into as many
-	 * slices as there are buffer swaps scheduled in order to avoid tearing
-	 * (based on the assumption that a single buffer swap would always
-	 * complete before scanout starts).
-	 */
-	for (i = 0; i++ < nhits;
-	     upper[0] = lower[0], lower[0] += slice[0],
-	     upper[1] = lower[1], lower[1] += slice[1]) {
-		int init_drawrect = 1;
-
-		if (i == nhits)
-			lower[0] = lower[1] = sarea_priv->height;
-
-		list_for_each(hit, &hits) {
-			drm_i915_vbl_swap_t *swap_hit =
-				list_entry(hit, drm_i915_vbl_swap_t, head);
-			struct drm_clip_rect *rect;
-			int num_rects, plane, front, back;
-			unsigned short top, bottom;
-
-			drw = drm_get_drawable_info(dev, swap_hit->drw_id);
-
-			if (!drw)
-				continue;
-
-			plane = swap_hit->plane;
-
-			if (swap_hit->flip) {
-				i915_dispatch_vsync_flip(dev, drw, plane);
-				continue;
-			}
-
-			if (init_drawrect) {
-				int width  = sarea_priv->width;
-				int height = sarea_priv->height;
-				if (IS_I965G(dev)) {
-					BEGIN_LP_RING(4);
-					OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
-					OUT_RING(0);
-					OUT_RING(((width - 1) & 0xffff) | ((height - 1) << 16));
-					OUT_RING(0);
-					ADVANCE_LP_RING();
-				} else {
-					BEGIN_LP_RING(6);
-					OUT_RING(GFX_OP_DRAWRECT_INFO);
-					OUT_RING(0);
-					OUT_RING(0);
-					OUT_RING(((width - 1) & 0xffff) | ((height - 1) << 16));
-					OUT_RING(0);
-					OUT_RING(0);
-					ADVANCE_LP_RING();
-				}
-
-				sarea_priv->ctxOwner = DRM_KERNEL_CONTEXT;
-
-				init_drawrect = 0;
-			}
-
-			rect = drw->rects;
-			top = upper[plane];
-			bottom = lower[plane];
-
-			front = (dev_priv->sarea_priv->pf_current_page >>
-				 (2 * plane)) & 0x3;
-			back = (front + 1) % num_pages;
-
-			for (num_rects = drw->num_rects; num_rects--; rect++) {
-				int y1 = max(rect->y1, top);
-				int y2 = min(rect->y2, bottom);
-
-				if (y1 >= y2)
-					continue;
-
-				BEGIN_LP_RING(8);
-				OUT_RING(cmd);
-				OUT_RING(ropcpp | dst_pitch);
-				OUT_RING((y1 << 16) | rect->x1);
-				OUT_RING((y2 << 16) | rect->x2);
-				OUT_RING(offsets[front]);
-				OUT_RING((y1 << 16) | rect->x1);
-				OUT_RING(src_pitch);
-				OUT_RING(offsets[back]);
-				ADVANCE_LP_RING();
-			}
+	if (eir & I915_ERROR_MEMORY_REFRESH) {
+		DRM_DEBUG("memory refresh error\n");
+		DRM_DEBUG("PIPEASTAT: 0x%08x\n",
+		       pipea_stats);
+		DRM_DEBUG("PIPEBSTAT: 0x%08x\n",
+		       pipeb_stats);
+		/* pipestat has already been acked */
+	}
+	if (eir & I915_ERROR_INSTRUCTION) {
+		DRM_DEBUG("instruction error\n");
+		DRM_DEBUG("  INSTPM: 0x%08x\n",
+		       I915_READ(INSTPM));
+		if (!IS_I965G(dev)) {
+			u32 ipeir = I915_READ(IPEIR);
+
+			DRM_DEBUG("  IPEIR: 0x%08x\n",
+			       I915_READ(IPEIR));
+			DRM_DEBUG("  IPEHR: 0x%08x\n",
+			       I915_READ(IPEHR));
+			DRM_DEBUG("  INSTDONE: 0x%08x\n",
+			       I915_READ(INSTDONE));
+			DRM_DEBUG("  ACTHD: 0x%08x\n",
+			       I915_READ(ACTHD));
+			I915_WRITE(IPEIR, ipeir);
+			(void)I915_READ(IPEIR);
+		} else {
+			u32 ipeir = I915_READ(IPEIR_I965);
+
+			DRM_DEBUG("  IPEIR: 0x%08x\n",
+			       I915_READ(IPEIR_I965));
+			DRM_DEBUG("  IPEHR: 0x%08x\n",
+			       I915_READ(IPEHR_I965));
+			DRM_DEBUG("  INSTDONE: 0x%08x\n",
+			       I915_READ(INSTDONE_I965));
+			DRM_DEBUG("  INSTPS: 0x%08x\n",
+			       I915_READ(INSTPS));
+			DRM_DEBUG("  INSTDONE1: 0x%08x\n",
+			       I915_READ(INSTDONE1));
+			DRM_DEBUG("  ACTHD: 0x%08x\n",
+			       I915_READ(ACTHD_I965));
+			I915_WRITE(IPEIR_I965, ipeir);
+			(void)I915_READ(IPEIR_I965);
 		}
 	}
 
-	DRM_SPINUNLOCK(&dev->drw_lock);
-
-	list_for_each_safe(hit, tmp, &hits) {
-		drm_i915_vbl_swap_t *swap_hit =
-			list_entry(hit, drm_i915_vbl_swap_t, head);
-
-		list_del(hit);
-
-		drm_free(swap_hit, sizeof(*swap_hit), DRM_MEM_DRIVER);
+	I915_WRITE(EIR, eir);
+	(void)I915_READ(EIR);
+	eir = I915_READ(EIR);
+	if (eir) {
+		/*
+		 * some errors might have become stuck,
+		 * mask them.
+		 */
+		DRM_DEBUG("EIR stuck: 0x%08x, masking\n", eir);
+		I915_WRITE(EMR, I915_READ(EMR) | eir);
+		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
 	}
+
 }
 
-u32 i915_get_vblank_counter(struct drm_device *dev, int plane)
+u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 {
-	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	unsigned long high_frame;
-	unsigned long low_frame;
-	u32 high1, high2, low, count;
-	int pipe;
+       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+       int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;
 
-	pipe = i915_get_pipe(dev, plane);
-	high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
-	low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
-
-	if (!i915_pipe_enabled(dev, pipe)) {
-	    DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe);
-	    return 0;
-	}
-
-	/*
-	 * High & low register fields aren't synchronized, so make sure
-	 * we get a low value that's stable across two reads of the high
-	 * register.
-	 */
-	do {
-		high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
-			 PIPE_FRAME_HIGH_SHIFT);
-		low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
-			PIPE_FRAME_LOW_SHIFT);
-		high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
-			 PIPE_FRAME_HIGH_SHIFT);
-	} while (high1 != high2);
-
-	count = (high1 << 8) | low;
+       if (!i915_pipe_enabled(dev, pipe)) {
+		DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
+               return 0;
+       }
 
-	/* count may be reset by other driver(e.g. 2D driver), 
-	   we have no way to know if it is wrapped or resetted 
-	   when count is zero. do a rough guess.
-	*/
-	if (count < dev->last_vblank[pipe] && dev->last_vblank[pipe] < dev->max_vblank_count/2)
-		dev->last_vblank[pipe]=0;	
-	return count;
+       return I915_READ(reg);
 }
 
 irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
@@ -437,69 +381,67 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
         u32 iir;
         u32 pipea_stats = 0, pipeb_stats = 0;
 	int vblank = 0;
-	iir = I915_READ(IIR);
 
-	atomic_inc(&dev_priv->irq_received);
+	iir = I915_READ(IIR);
 
 	if (iir == 0) {
 		return IRQ_NONE;
 	}
+start:
 
-	if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
-		pipea_stats = I915_READ(PIPEASTAT);
-
-		/* The vblank interrupt gets enabled even if we didn't ask for
-		   it, so make sure it's shut down again */
-		if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A))
-			pipea_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
-					 PIPE_VBLANK_INTERRUPT_ENABLE);
-		else if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
-					PIPE_VBLANK_INTERRUPT_STATUS))
-		{
-			vblank++;
-			drm_handle_vblank(dev, i915_get_plane(dev, 0));
-		}
-
-		I915_WRITE(PIPEASTAT, pipea_stats);
+	if (dev_priv->sarea_priv) {
+		if (dev_priv->hw_status_page)
+	    		dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 	}
-	if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
-		pipeb_stats = I915_READ(PIPEBSTAT);
-
-		/* The vblank interrupt gets enabled even if we didn't ask for
-		   it, so make sure it's shut down again */
-		if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
-			pipeb_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
-					 PIPE_VBLANK_INTERRUPT_ENABLE);
-		else if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
-					PIPE_VBLANK_INTERRUPT_STATUS))
-		{
-			vblank++;
-			drm_handle_vblank(dev, i915_get_plane(dev, 1));
-		}
-
-		I915_WRITE(PIPEBSTAT, pipeb_stats);
-	}
-
-	if (dev_priv->sarea_priv)
-	    dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 
 	I915_WRITE(IIR, iir);
 
 	(void) I915_READ(IIR); /* Flush posted writes */
 
+
+	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+		i915_handle_error(dev);
+
         if (iir & I915_USER_INTERRUPT) {
+		dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
                 DRM_WAKEUP(&dev_priv->irq_queue);
-#ifdef I915_HAVE_FENCE
-                i915_fence_handler(dev);
-#endif
         }
 
-	if (vblank) {
-		if (dev_priv->swaps_pending > 0)
-			drm_locked_tasklet(dev, i915_vblank_tasklet);
-	}
-
-        return IRQ_HANDLED;
+        if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
+                pipea_stats = I915_READ(PIPEASTAT);
+
+                /* The vblank interrupt gets enabled even if we didn't ask for
+                   it, so make sure it's shut down again */
+                if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A))
+                        pipea_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
+                                         PIPE_VBLANK_INTERRUPT_ENABLE);
+                else if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
+                                        PIPE_VBLANK_INTERRUPT_STATUS))
+                {
+                        vblank++;
+                        drm_handle_vblank(dev, 0);
+                }
+
+                I915_WRITE(PIPEASTAT, pipea_stats);
+        }
+        if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
+                pipeb_stats = I915_READ(PIPEBSTAT);
+
+                /* The vblank interrupt gets enabled even if we didn't ask for
+                   it, so make sure it's shut down again */
+                if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
+                        pipeb_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
+                                         PIPE_VBLANK_INTERRUPT_ENABLE);
+                else if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
+                                        PIPE_VBLANK_INTERRUPT_STATUS))
+                {
+                        vblank++;
+                        drm_handle_vblank(dev, 1);
+                }
+
+                I915_WRITE(PIPEBSTAT, pipeb_stats);
+        }
+       return IRQ_HANDLED;
 
 }
 
@@ -510,31 +452,71 @@ int i915_emit_irq(drm_device_t * dev)
 	RING_LOCALS;
 
 	i915_kernel_lost_context(dev);
-
-	i915_emit_breadcrumb(dev);
-
-	BEGIN_LP_RING(2);
-	OUT_RING(0);
+	
+	dev_priv->counter++;
+	if (dev_priv->counter > 0x7FFFFFFFUL)
+		dev_priv->counter = 1;
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
+
+#if defined(__i386)
+	if (IS_GM45(dev)) {
+		BEGIN_LP_RING(3);
+		OUT_RING(MI_STORE_DWORD_INDEX);
+		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+		OUT_RING(dev_priv->counter);
+		ADVANCE_LP_RING();
+
+		(void) READ_BREADCRUMB(dev_priv);
+		BEGIN_LP_RING(2);
+		OUT_RING(0);
+		OUT_RING(MI_USER_INTERRUPT);
+		ADVANCE_LP_RING();
+	} else {
+#endif  /* __i386 */
+	BEGIN_LP_RING(4);
+	OUT_RING(MI_STORE_DWORD_INDEX);
+	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+	OUT_RING(dev_priv->counter);
 	OUT_RING(MI_USER_INTERRUPT);
 	ADVANCE_LP_RING();
+#if defined(__i386)
+	}
+#endif  /* __i386 */
+
+#if defined(__i386)
+	if (IS_I965GM(dev) || IS_GM45(dev))
+#else
+	if (IS_I965GM(dev))
+#endif  /* __i386 */
+	{
+		(void) READ_BREADCRUMB(dev_priv);
+		BEGIN_LP_RING(2);
+		OUT_RING(0);
+		OUT_RING(0);
+		ADVANCE_LP_RING();
+		(void) READ_BREADCRUMB(dev_priv);
+	}
 
 	return dev_priv->counter;
 }
 
-void i915_user_irq_on(drm_i915_private_t *dev_priv)
+void i915_user_irq_on(struct drm_device *dev)
 {
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	spin_lock(&dev_priv->user_irq_lock);
-	if (dev_priv->irq_enabled && (++dev_priv->user_irq_refcount == 1)){
+	if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)){
 		i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 	}
 	spin_unlock(&dev_priv->user_irq_lock);
 
 }
 		
-void i915_user_irq_off(drm_i915_private_t *dev_priv)
+void i915_user_irq_off(struct drm_device *dev)
 {
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	spin_lock(&dev_priv->user_irq_lock);
-	if (dev_priv->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
+	if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
 		i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 	}
 	spin_unlock(&dev_priv->user_irq_lock);
@@ -546,30 +528,25 @@ static int i915_wait_irq(drm_device_t * dev, int irq_nr)
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	int ret = 0;
 
-
-
-       if (!dev_priv) {
-               DRM_ERROR("called with no initialization\n");
-               return -EINVAL;
-       }
-
-
-
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
 
 	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
-		if (dev_priv->sarea_priv)
+		if (dev_priv->sarea_priv) {
 			dev_priv->sarea_priv->last_dispatch =
 				READ_BREADCRUMB(dev_priv);
+		}
 		return 0;
 	}
-
 	DRM_DEBUG("i915_wait_irq: irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv));
-	i915_user_irq_on(dev_priv);
+	i915_user_irq_on(dev);
 	DRM_WAIT_ON(ret, &dev_priv->irq_queue, 3 * DRM_HZ,
 		    READ_BREADCRUMB(dev_priv) >= irq_nr);
-	i915_user_irq_off(dev_priv);
+	i915_user_irq_off(dev);
 
-	if (ret == EBUSY || ret == EINTR) {
+	if (ret == EBUSY) {
 		DRM_DEBUG("%d: EBUSY -- rec: %d emitted: %d\n",
 			  ret,
 			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
@@ -599,6 +576,7 @@ int i915_irq_emit(DRM_IOCTL_ARGS)
 		return (EINVAL);
 	}
 
+
 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
 		drm_i915_irq_emit32_t irq_emit32;
 
@@ -610,7 +588,9 @@ int i915_irq_emit(DRM_IOCTL_ARGS)
 		DRM_COPYFROM_WITH_RETURN(&emit,
 		    (drm_i915_irq_emit_t __user *) data, sizeof(emit));
 
+	spin_lock(&dev->struct_mutex);
 	result = i915_emit_irq(dev);
+	spin_unlock(&dev->struct_mutex);
 
 	if (DRM_COPY_TO_USER(emit.irq_seq, &result, sizeof(int))) {
 		DRM_ERROR("copy_to_user\n");
@@ -640,107 +620,37 @@ int i915_irq_wait(DRM_IOCTL_ARGS)
 	return i915_wait_irq(dev, irqwait.irq_seq);
 }
 
-int i915_enable_vblank(struct drm_device *dev, int plane)
+int i915_enable_vblank(struct drm_device *dev, int pipe)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	int pipe = i915_get_pipe(dev, plane);
-	u32	pipestat_reg = 0;
-	u32	mask_reg = 0;
-	u32	pipestat;
-
-	switch (pipe) {
-	case 0:
-		pipestat_reg = PIPEASTAT;
-		mask_reg |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
-		break;
-	case 1:
-		pipestat_reg = PIPEBSTAT;
-		mask_reg |= I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
-		break;
-	default:
-		DRM_ERROR("tried to enable vblank on non-existent pipe %d\n",
-			  pipe);
-		break;
-	}
-
-	if (pipestat_reg)
-	{
-		pipestat = I915_READ (pipestat_reg);
-		/*
-		 * Older chips didn't have the start vblank interrupt,
-		 * but 
-		 */
-		if (IS_I965G (dev))
-			pipestat |= PIPE_START_VBLANK_INTERRUPT_ENABLE;
-		else
-			pipestat |= PIPE_VBLANK_INTERRUPT_ENABLE;
-		/*
-		 * Clear any pending status
-		 */
-		pipestat |= (PIPE_START_VBLANK_INTERRUPT_STATUS |
-			     PIPE_VBLANK_INTERRUPT_STATUS);
-		I915_WRITE(pipestat_reg, pipestat);
-	}
-	DRM_SPINLOCK(&dev_priv->user_irq_lock);
-	i915_enable_irq(dev_priv, mask_reg);
-	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
+	int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
+	u32 pipeconf;
 
-	return 0;
-}
-
-void i915_disable_vblank(struct drm_device *dev, int plane)
-{
-	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	int pipe = i915_get_pipe(dev, plane);
-	u32	pipestat_reg = 0;
-	u32	mask_reg = 0;
-	u32	pipestat;
-
-	switch (pipe) {
-	case 0:
-		pipestat_reg = PIPEASTAT;
-		mask_reg |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
-		break;
-	case 1:
-		pipestat_reg = PIPEBSTAT;
-		mask_reg |= I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
-		break;
-	default:
-		DRM_ERROR("tried to disable vblank on non-existent pipe %d\n",
-			  pipe);
-		break;
-	}
+	pipeconf = I915_READ(pipeconf_reg);
+	if (!(pipeconf & PIPEACONF_ENABLE))
+		return -EINVAL;
 
-	DRM_SPINLOCK(&dev_priv->user_irq_lock);
-	i915_disable_irq(dev_priv, mask_reg);
-	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
+	spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+	if (IS_I965G(dev))
+		i915_enable_pipestat(dev_priv, pipe,
+				     PIPE_START_VBLANK_INTERRUPT_ENABLE);
+	else
+		i915_enable_pipestat(dev_priv, pipe,
+				     PIPE_VBLANK_INTERRUPT_ENABLE);
+	spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 
-	if (pipestat_reg)
-	{
-		pipestat = I915_READ (pipestat_reg);
-		pipestat &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
-			      PIPE_VBLANK_INTERRUPT_ENABLE);
-		/*
-		 * Clear any pending status
-		 */
-		pipestat |= (PIPE_START_VBLANK_INTERRUPT_STATUS |
-			     PIPE_VBLANK_INTERRUPT_STATUS);
-		I915_WRITE(pipestat_reg, pipestat);
-		(void) I915_READ(pipestat_reg);
-	}
+	return 0;
 }
 
-
-static void i915_enable_interrupt (drm_device_t *dev)
+void i915_disable_vblank(struct drm_device *dev, int pipe)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
-	dev_priv->irq_mask_reg = 0xffffffff;
-	I915_WRITE(IMR, dev_priv->irq_mask_reg);
-	I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
-	(void) I915_READ (IER);
-	
-	dev_priv->irq_enabled = 1;
+	spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+	i915_disable_pipestat(dev_priv, pipe,
+			      PIPE_VBLANK_INTERRUPT_ENABLE |
+			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
+	spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 }
 
 /* Set the vblank monitor pipe
@@ -771,7 +681,6 @@ int i915_vblank_pipe_get(DRM_IOCTL_ARGS)
 		return -EINVAL;
 	}
 
-
 	DRM_COPYFROM_WITH_RETURN(&pipe, (drm_i915_vblank_pipe_t __user *)data, sizeof (pipe));
 
 	pipe.pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
@@ -785,159 +694,22 @@ int i915_vblank_pipe_get(DRM_IOCTL_ARGS)
 /*ARGSUSED*/
 int i915_vblank_swap(DRM_IOCTL_ARGS)
 {
-	DRM_DEVICE;
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_vblank_swap_t *swap;
-	drm_i915_vbl_swap_t *vbl_swap;
-	unsigned int pipe, seqtype, curseq, plane;
-	struct list_head *list;
-	int ret;
-
-	if (!dev_priv) {
-		DRM_ERROR("%s called with no initialization\n", __func__);
-		return -EINVAL;
-	}
-
-	if (!dev_priv->sarea_priv || dev_priv->sarea_priv->rotation) {
-		DRM_DEBUG("Rotation not supported\n");
-		return -EINVAL;
-	}
-
-	DRM_COPYFROM_WITH_RETURN(&swap, (drm_i915_vblank_swap_t __user *)data, sizeof (swap));
-
-	if (swap->seqtype & ~(_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE |
-			     _DRM_VBLANK_SECONDARY | _DRM_VBLANK_NEXTONMISS |
-			     _DRM_VBLANK_FLIP)) {
-		DRM_ERROR("Invalid sequence type 0x%x\n", swap->seqtype);
-		return -EINVAL;
-	}
-
-	plane = (swap->seqtype & _DRM_VBLANK_SECONDARY) ? 1 : 0;
-	pipe = i915_get_pipe(dev, plane);
-
-	seqtype = swap->seqtype & (_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE);
-
-	if (!(dev_priv->vblank_pipe & (1 << pipe))) {
-		DRM_ERROR("Invalid pipe %d\n", pipe);
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&dev->drw_lock, irqflags);
-
-	/* It makes no sense to schedule a swap for a drawable that doesn't have
-	 * valid information at this point. E.g. this could mean that the X
-	 * server is too old to push drawable information to the DRM, in which
-	 * case all such swaps would become ineffective.
-	 */
-	if (!drm_get_drawable_info(dev, swap->drawable)) {
-		spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-		DRM_DEBUG("Invalid drawable ID %d\n", swap->drawable);
-		return -EINVAL;
-	}
-
-	spin_unlock_irqrestore(&dev->drw_lock, irqflags);
-
-	/*
-	 * We take the ref here and put it when the swap actually completes
-	 * in the tasklet.
-	 */
-	ret = drm_vblank_get(dev, pipe);
-	if (ret)
-		return ret;
-	curseq = drm_vblank_count(dev, pipe);
-
-	if (seqtype == _DRM_VBLANK_RELATIVE)
-		swap->sequence += curseq;
-
-	if ((curseq - swap->sequence) <= (1<<23)) {
-		if (swap->seqtype & _DRM_VBLANK_NEXTONMISS) {
-			swap->sequence = curseq + 1;
-		} else {
-			DRM_DEBUG("Missed target sequence\n");
-			drm_vblank_put(dev, pipe);
-			return -EINVAL;
-		}
-	}
-
-	if (swap->seqtype & _DRM_VBLANK_FLIP) {
-		swap->sequence--;
-
-		if ((curseq - swap->sequence) <= (1<<23)) {
-			struct drm_drawable_info *drw;
-
-			LOCK_TEST_WITH_RETURN(dev, fpriv);
-
-			spin_lock_irqsave(&dev->drw_lock, irqflags);
-
-			drw = drm_get_drawable_info(dev, swap->drawable);
-
-			if (!drw) {
-				spin_unlock_irqrestore(&dev->drw_lock,
-				    irqflags);
-				DRM_DEBUG("Invalid drawable ID %d\n",
-					  swap->drawable);
-				drm_vblank_put(dev, pipe);
-				return -EINVAL;
-			}
-
-			i915_dispatch_vsync_flip(dev, drw, plane);
-
-			spin_unlock_irqrestore(&dev->drw_lock, irqflags);
+        /* The delayed swap mechanism was fundamentally racy, and has been
+        * removed.  The model was that the client requested a delayed flip/swap
+        * from the kernel, then waited for vblank before continuing to perform
+        * rendering.  The problem was that the kernel might wake the client
+        * up before it dispatched the vblank swap (since the lock has to be
+        * held while touching the ringbuffer), in which case the client would
+        * clear and start the next frame before the swap occurred, and
+        * flicker would occur in addition to likely missing the vblank.
+        *
+        * In the absence of this ioctl, userland falls back to a correct path
+        * of waiting for a vblank, then dispatching the swap on its own.
+        * Context switching to userland and back is plenty fast enough for
+        * meeting the requirements of vblank swapping.
+        */
+	return -EINVAL;
 
-			drm_vblank_put(dev, pipe);
-			return 0;
-		}
-	}
-
-	spin_lock_irqsave(&dev_priv->swaps_lock, irqflags);
-
-	list_for_each(list, &dev_priv->vbl_swaps.head) {
-		vbl_swap = list_entry(list, drm_i915_vbl_swap_t, head);
-
-		if (vbl_swap->drw_id == swap->drawable &&
-		    vbl_swap->plane == plane &&
-		    vbl_swap->sequence == swap->sequence) {
-			vbl_swap->flip = (swap->seqtype & _DRM_VBLANK_FLIP);
-			spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
-			DRM_DEBUG("Already scheduled\n");
-			return 0;
-		}
-	}
-
-	spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
-
-	if (dev_priv->swaps_pending >= 100) {
-		DRM_DEBUG("Too many swaps queued\n");
-		drm_vblank_put(dev, pipe);
-		return -EBUSY;
-	}
-
-	vbl_swap = drm_calloc(1, sizeof(*vbl_swap), DRM_MEM_DRIVER);
-
-	if (!vbl_swap) {
-		DRM_ERROR("Failed to allocate memory to queue swap\n");
-		drm_vblank_put(dev, pipe);
-		return -ENOMEM;
-	}
-
-	DRM_DEBUG("vbl_swap\n");
-
-	vbl_swap->drw_id = swap->drawable;
-	vbl_swap->plane = plane;
-	vbl_swap->sequence = swap->sequence;
-	vbl_swap->flip = (swap->seqtype & _DRM_VBLANK_FLIP);
-
-	if (vbl_swap->flip)
-		swap->sequence++;
-
-	spin_lock_irqsave(&dev_priv->swaps_lock, irqflags);
-
-	list_add_tail(&vbl_swap->head, &dev_priv->vbl_swaps.head);
-	dev_priv->swaps_pending++;
-
-	spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
-
-	return 0;
 }
 
 /* drm_dma.h hooks
@@ -949,57 +721,76 @@ int i915_driver_irq_preinstall(drm_device_t * dev)
 	if (!dev_priv->mmio_map)
 		return -EINVAL;
 
-	I915_WRITE(HWSTAM, 0xeffe);
+	I915_WRITE16(HWSTAM, 0xeffe);
+	I915_WRITE(PIPEASTAT, 0);
+	I915_WRITE(PIPEBSTAT, 0);
 	I915_WRITE(IMR, 0xffffffff);
-	I915_WRITE(IER, 0x0);
+	I915_WRITE16(IER, 0x0);
+	(void) I915_READ(IER);
 
 	return 0;
 }
 
 void i915_driver_irq_postinstall(drm_device_t * dev)
 {
+	int error_mask;
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
-	INIT_LIST_HEAD(&dev_priv->vbl_swaps.head);
-	dev_priv->swaps_pending = 0;
-
-	dev_priv->user_irq_refcount = 0;
-	dev_priv->irq_mask_reg = 0xffffffff;
-
 	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
-	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
 
-	i915_enable_interrupt(dev);
+	/* Unmask the interrupts that we always want on. */
+	dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
 
-	DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
+	dev_priv->pipestat[0] = 0;
+	dev_priv->pipestat[1] = 0;
 
 	/*
-	 * Initialize the hardware status page IRQ location.
+	 * Enable some error detection, note the instruction error mask
+	 * bit is reserved, so we leave it masked.
 	 */
+	if (IS_G4X(dev)) {
+		error_mask = ~(GM45_ERROR_PAGE_TABLE |
+			       GM45_ERROR_MEM_PRIV |
+			       GM45_ERROR_CP_PRIV |
+			       I915_ERROR_MEMORY_REFRESH);
+	} else {
+		error_mask = ~(I915_ERROR_PAGE_TABLE |
+			       I915_ERROR_MEMORY_REFRESH);
+	}
+	I915_WRITE(EMR, error_mask);
+
+	/* Disable pipe interrupt enables, clear pending pipe status */
+	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
+	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
+	/* Clear pending interrupt status */
+	I915_WRITE(IIR, I915_READ(IIR));
+
+	I915_WRITE(IMR, dev_priv->irq_mask_reg);
+	I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
+	(void) I915_READ(IER);
+
+	DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
 
-	I915_WRITE(INSTPM, (1 << 5) | (1 << 21));
 	return;
 }
 
 void i915_driver_irq_uninstall(drm_device_t * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 temp;
-	if ((!dev_priv) || (dev_priv->irq_enabled == 0))
+	if ((!dev_priv) || (dev->irq_enabled == 0))
 		return;
 
 	dev_priv->vblank_pipe = 0;
-	dev_priv->irq_enabled = 0;
 
 	I915_WRITE(HWSTAM, 0xffffffff);
+	I915_WRITE(PIPEASTAT, 0);
+	I915_WRITE(PIPEBSTAT, 0);
 	I915_WRITE(IMR, 0xffffffff);
 	I915_WRITE(IER, 0x0);
 
-	temp = I915_READ(PIPEASTAT);
-	I915_WRITE(PIPEASTAT, temp);
-	temp = I915_READ(PIPEBSTAT);
-	I915_WRITE(PIPEBSTAT, temp);
-	temp = I915_READ(IIR);
-	I915_WRITE(IIR, temp);
+	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
+	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
+	I915_WRITE(IIR, I915_READ(IIR));
 
+	DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
 }