Merge branch 'master' of master.kernel.org:/pub/scm/linux/kernel/git/davem/sparc-2.6

5bc705e9 · Linus Torvalds <[email protected]> · 2007-08-18 09:34:28 -07:00

* 'master' of master.kernel.org:/pub/scm/linux/kernel/git/davem/sparc-2.6:
  [MATH-EMU]: Fix underflow exception reporting.
  [SPARC64]: Create a HWCAP_SPARC_N2 and report it to userspace on Niagara-2.
  [SPARC64]: SMP trampoline needs to avoid %tick_cmpr on sun4v too.
  [SPARC64]: Do not touch %tick_cmpr on sun4v cpus.
  [SPARC64]: Niagara-2 optimized copies.
  [SPARC64]: Allow userspace to get at the machine description.
  [SPARC32]: Remove superfluous 'kernel_end' alignment on sun4c.
  [SPARC32]: Fix bogus ramdisk image location check.
  [SPARC32]: Remove iommu from struct sbus_bus and use archdata like sparc64.

Commit: 5bc705e9fe774be9bb87584644c1f6752ee5ea7e
Parents: 605a494e · 40584961

Merge commit. Changes below are compared with the first parent: 605a494e.

Changes

21 files changed · +803 −29

File	Status	Add	Del	Content
arch/sparc/kernel/ebus.c	modified	+1	−0	View file
arch/sparc/mm/init.c	modified	+1	−2	View file
arch/sparc/mm/io-unit.c	modified	+9	−9	View file
arch/sparc/mm/iommu.c	modified	+6	−6	View file
arch/sparc/mm/sun4c.c	modified	+0	−2	View file
arch/sparc64/kernel/head.S	modified	+12	−2	View file
arch/sparc64/kernel/mdesc.c	modified	+38	−0	View file
arch/sparc64/kernel/trampoline.S	modified	+3	−4	View file
arch/sparc64/lib/Makefile	modified	+2	−0	View file
arch/sparc64/lib/NG2copy_from_user.S	added	+40	−0	View file
arch/sparc64/lib/NG2copy_to_user.S	added	+49	−0	View file
arch/sparc64/lib/NG2memcpy.S	added	+520	−0	View file
arch/sparc64/lib/NG2page.S	added	+61	−0	View file
arch/sparc64/lib/NG2patch.S	added	+33	−0	View file
arch/sparc64/lib/NGpage.S	modified	+1	−0	View file
include/asm-sparc/sbus.h	modified	+0	−1	View file
include/asm-sparc/sfp-machine.h	modified	+6	−0	View file
include/asm-sparc64/elf.h	modified	+8	−2	View file
include/asm-sparc64/sfp-machine.h	modified	+2	−0	View file
include/math-emu/op-common.h	modified	+4	−1	View file
include/math-emu/soft-fp.h	modified	+7	−0	View file

Only the first 600 diff lines are shown. Download the complete patch.

arch/sparc/kernel/ebus.c

View file

diff --git a/arch/sparc/kernel/ebus.c b/arch/sparc/kernel/ebus.c
index ac352eb..e2d02fd 100644
--- a/arch/sparc/kernel/ebus.c
+++ b/arch/sparc/kernel/ebus.c
@@ -238,6 +238,7 @@ void __init fill_ebus_device(struct device_node *dp, struct linux_ebus_device *d
 	sd = &dev->ofdev.dev.archdata;
 	sd->prom_node = dp;
 	sd->op = &dev->ofdev;
+	sd->iommu = dev->bus->ofdev.dev.parent->archdata.iommu;
 
 	dev->ofdev.node = dp;
 	dev->ofdev.dev.parent = &dev->bus->ofdev.dev;

arch/sparc/mm/init.c

View file

diff --git a/arch/sparc/mm/init.c b/arch/sparc/mm/init.c
index a1bef07..c13e6cd 100644
--- a/arch/sparc/mm/init.c
+++ b/arch/sparc/mm/init.c
@@ -206,8 +206,7 @@ unsigned long __init bootmem_init(unsigned long *pages_avail)
 #ifdef CONFIG_BLK_DEV_INITRD
 	/* Now have to check initial ramdisk, so that bootmap does not overwrite it */
 	if (sparc_ramdisk_image) {
-		if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
-			sparc_ramdisk_image -= KERNBASE;
+		sparc_ramdisk_image -= KERNBASE;
 		initrd_start = sparc_ramdisk_image + phys_base;
 		initrd_end = initrd_start + sparc_ramdisk_size;
 		if (initrd_end > end_of_phys_memory) {

arch/sparc/mm/io-unit.c

View file

diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c
index 4ccda77..7c89893 100644
--- a/arch/sparc/mm/io-unit.c
+++ b/arch/sparc/mm/io-unit.c
@@ -66,7 +66,7 @@ iounit_init(int sbi_node, int io_node, struct sbus_bus *sbus)
 	}
 	if(!xpt) panic("Cannot map External Page Table.");
 	
-	sbus->iommu = (struct iommu_struct *)iounit;
+	sbus->ofdev.dev.archdata.iommu = iounit;
 	iounit->page_table = xpt;
 	spin_lock_init(&iounit->lock);
 	
@@ -127,7 +127,7 @@ nexti:	scan = find_next_zero_bit(iounit->bmap, limit, scan);
 static __u32 iounit_get_scsi_one(char *vaddr, unsigned long len, struct sbus_bus *sbus)
 {
 	unsigned long ret, flags;
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 	
 	spin_lock_irqsave(&iounit->lock, flags);
 	ret = iounit_get_area(iounit, (unsigned long)vaddr, len);
@@ -138,7 +138,7 @@ static __u32 iounit_get_scsi_one(char *vaddr, unsigned long len, struct sbus_bus
 static void iounit_get_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
 {
 	unsigned long flags;
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 
 	/* FIXME: Cache some resolved pages - often several sg entries are to the same page */
 	spin_lock_irqsave(&iounit->lock, flags);
@@ -153,7 +153,7 @@ static void iounit_get_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus 
 static void iounit_release_scsi_one(__u32 vaddr, unsigned long len, struct sbus_bus *sbus)
 {
 	unsigned long flags;
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 	
 	spin_lock_irqsave(&iounit->lock, flags);
 	len = ((vaddr & ~PAGE_MASK) + len + (PAGE_SIZE-1)) >> PAGE_SHIFT;
@@ -168,7 +168,7 @@ static void iounit_release_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_
 {
 	unsigned long flags;
 	unsigned long vaddr, len;
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 
 	spin_lock_irqsave(&iounit->lock, flags);
 	while (sz != 0) {
@@ -211,7 +211,7 @@ static int iounit_map_dma_area(dma_addr_t *pba, unsigned long va, __u32 addr, in
 			i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT);
 
 			for_each_sbus(sbus) {
-				struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+				struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 
 				iopte = (iopte_t *)(iounit->page_table + i);
 				*iopte = MKIOPTE(__pa(page));
@@ -235,7 +235,7 @@ static void iounit_unmap_dma_area(unsigned long addr, int len)
 static struct page *iounit_translate_dvma(unsigned long addr)
 {
 	struct sbus_bus *sbus = sbus_root;	/* They are all the same */
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 	int i;
 	iopte_t *iopte;
 
@@ -279,7 +279,7 @@ __u32 iounit_map_dma_init(struct sbus_bus *sbus, int size)
 	unsigned long rotor, scan, limit;
 	unsigned long flags;
 	__u32 ret;
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 
         npages = (size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
 	i = 0x0213;
@@ -315,7 +315,7 @@ nexti:	scan = find_next_zero_bit(iounit->bmap, limit, scan);
 __u32 iounit_map_dma_page(__u32 vaddr, void *addr, struct sbus_bus *sbus)
 {
 	int scan = (vaddr - IOUNIT_DMA_BASE) >> PAGE_SHIFT;
-	struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu;
+	struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
 	
 	iounit->page_table[scan] = MKIOPTE(__pa(((unsigned long)addr) & PAGE_MASK));
 	return vaddr + (((unsigned long)addr) & ~PAGE_MASK);

arch/sparc/mm/iommu.c

View file

diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index be042ef..52e907a 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -132,7 +132,7 @@ iommu_init(int iommund, struct sbus_bus *sbus)
 	    impl, vers, iommu->page_table,
 	    (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
 
-	sbus->iommu = iommu;
+	sbus->ofdev.dev.archdata.iommu = iommu;
 }
 
 /* This begs to be btfixup-ed by srmmu. */
@@ -166,7 +166,7 @@ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
 
 static u32 iommu_get_one(struct page *page, int npages, struct sbus_bus *sbus)
 {
-	struct iommu_struct *iommu = sbus->iommu;
+	struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
 	int ioptex;
 	iopte_t *iopte, *iopte0;
 	unsigned int busa, busa0;
@@ -291,7 +291,7 @@ static void iommu_get_scsi_sgl_pflush(struct scatterlist *sg, int sz, struct sbu
 
 static void iommu_release_one(u32 busa, int npages, struct sbus_bus *sbus)
 {
-	struct iommu_struct *iommu = sbus->iommu;
+	struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
 	int ioptex;
 	int i;
 
@@ -334,7 +334,7 @@ static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va,
     unsigned long addr, int len)
 {
 	unsigned long page, end;
-	struct iommu_struct *iommu = sbus_root->iommu;
+	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
 	iopte_t *iopte = iommu->page_table;
 	iopte_t *first;
 	int ioptex;
@@ -399,7 +399,7 @@ static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va,
 
 static void iommu_unmap_dma_area(unsigned long busa, int len)
 {
-	struct iommu_struct *iommu = sbus_root->iommu;
+	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
 	iopte_t *iopte = iommu->page_table;
 	unsigned long end;
 	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
@@ -420,7 +420,7 @@ static void iommu_unmap_dma_area(unsigned long busa, int len)
 
 static struct page *iommu_translate_dvma(unsigned long busa)
 {
-	struct iommu_struct *iommu = sbus_root->iommu;
+	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
 	iopte_t *iopte = iommu->page_table;
 
 	iopte += ((busa - iommu->start) >> PAGE_SHIFT);

arch/sparc/mm/sun4c.c

View file

diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c
index 79d60d8..005a3e7 100644
--- a/arch/sparc/mm/sun4c.c
+++ b/arch/sparc/mm/sun4c.c
@@ -268,7 +268,6 @@ static inline void sun4c_init_clean_mmu(unsigned long kernel_end)
 	unsigned char savectx, ctx;
 
 	savectx = sun4c_get_context();
-	kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
 	for (ctx = 0; ctx < num_contexts; ctx++) {
 		sun4c_set_context(ctx);
 		for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE)
@@ -2064,7 +2063,6 @@ void __init sun4c_paging_init(void)
 	unsigned long end_pfn, pages_avail;
 
 	kernel_end = (unsigned long) &end;
-	kernel_end += (SUN4C_REAL_PGDIR_SIZE * 4);
 	kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
 
 	pages_avail = 0;

arch/sparc64/kernel/head.S

View file

diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index ac18bd8..63144ad 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -501,7 +501,7 @@ niagara_tlb_fixup:
 	cmp	%g1, SUN4V_CHIP_NIAGARA1
 	be,pt	%xcc, niagara_patch
 	 cmp	%g1, SUN4V_CHIP_NIAGARA2
-	be,pt	%xcc, niagara_patch
+	be,pt	%xcc, niagara2_patch
 	 nop
 
 	call	generic_patch_copyops
@@ -512,6 +512,15 @@ niagara_tlb_fixup:
 	 nop
 
 	ba,a,pt	%xcc, 80f
+niagara2_patch:
+	call	niagara2_patch_copyops
+	 nop
+	call	niagara_patch_bzero
+	 nop
+	call	niagara2_patch_pageops
+	 nop
+
+	ba,a,pt	%xcc, 80f
 
 niagara_patch:
 	call	niagara_patch_copyops
@@ -706,12 +715,13 @@ setup_trap_table:
 
 	membar	#Sync
 
+	BRANCH_IF_SUN4V(o2, 1f)
+
 	/* Kill PROM timer */
 	sethi	%hi(0x80000000), %o2
 	sllx	%o2, 32, %o2
 	wr	%o2, 0, %tick_cmpr
 
-	BRANCH_IF_SUN4V(o2, 1f)
 	BRANCH_IF_ANY_CHEETAH(o2, o3, 1f)
 
 	ba,pt	%xcc, 2f

arch/sparc64/kernel/mdesc.c

View file

diff --git a/arch/sparc64/kernel/mdesc.c b/arch/sparc64/kernel/mdesc.c
index 95059c2..9f22e4f 100644
--- a/arch/sparc64/kernel/mdesc.c
+++ b/arch/sparc64/kernel/mdesc.c
@@ -9,6 +9,7 @@
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
+#include <linux/miscdevice.h>
 
 #include <asm/hypervisor.h>
 #include <asm/mdesc.h>
@@ -836,6 +837,43 @@ void __devinit mdesc_fill_in_cpu_data(cpumask_t mask)
 	mdesc_release(hp);
 }
 
+static ssize_t mdesc_read(struct file *file, char __user *buf,
+			  size_t len, loff_t *offp)
+{
+	struct mdesc_handle *hp = mdesc_grab();
+	int err;
+
+	if (!hp)
+		return -ENODEV;
+
+	err = hp->handle_size;
+	if (len < hp->handle_size)
+		err = -EMSGSIZE;
+	else if (copy_to_user(buf, &hp->mdesc, hp->handle_size))
+		err = -EFAULT;
+	mdesc_release(hp);
+
+	return err;
+}
+
+static const struct file_operations mdesc_fops = {
+	.read	= mdesc_read,
+	.owner	= THIS_MODULE,
+};
+
+static struct miscdevice mdesc_misc = {
+	.minor	= MISC_DYNAMIC_MINOR,
+	.name	= "mdesc",
+	.fops	= &mdesc_fops,
+};
+
+static int __init mdesc_misc_init(void)
+{
+	return misc_register(&mdesc_misc);
+}
+
+__initcall(mdesc_misc_init);
+
 void __init sun4v_mdesc_init(void)
 {
 	struct mdesc_handle *hp;

arch/sparc64/kernel/trampoline.S

View file

diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 9448595..9533a25 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -95,14 +95,13 @@ spitfire_startup:
 	membar		#Sync
 
 startup_continue:
+	mov		%o0, %l0
+	BRANCH_IF_SUN4V(g1, niagara_lock_tlb)
+
 	sethi		%hi(0x80000000), %g2
 	sllx		%g2, 32, %g2
 	wr		%g2, 0, %tick_cmpr
 
-	mov		%o0, %l0
-
-	BRANCH_IF_SUN4V(g1, niagara_lock_tlb)
-
 	/* Call OBP by hand to lock KERNBASE into i/d tlbs.
 	 * We lock 2 consequetive entries if we are 'bigkernel'.
 	 */

arch/sparc64/lib/Makefile

View file

diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index f95fbfa..f095e13 100644
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -13,6 +13,8 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
 	 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
 	 NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \
 	 NGpage.o NGbzero.o \
+	 NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o NG2patch.o \
+	 NG2page.o \
 	 GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o GENpatch.o \
 	 GENpage.o GENbzero.o \
 	 copy_in_user.o user_fixup.o memmove.o \

arch/sparc64/lib/NG2copy_from_user.S

View file

diff --git a/arch/sparc64/lib/NG2copy_from_user.S b/arch/sparc64/lib/NG2copy_from_user.S
new file mode 100644
index 0000000..c77ef5f
--- /dev/null
+++ b/arch/sparc64/lib/NG2copy_from_user.S
@@ -0,0 +1,40 @@
+/* NG2copy_from_user.S: Niagara-2 optimized copy from userspace.
+ *
+ * Copyright (C) 2007 David S. Miller ([email protected])
+ */
+
+#define EX_LD(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	wr	%g0, ASI_AIUS, %asi;\
+	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table,"a";\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#ifndef ASI_AIUS
+#define ASI_AIUS	0x11
+#endif
+
+#ifndef ASI_BLK_AIUS_4V
+#define ASI_BLK_AIUS_4V	0x17
+#endif
+
+#define FUNC_NAME		NG2copy_from_user
+#define LOAD(type,addr,dest)	type##a [addr] %asi, dest
+#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_AIUS_4V, dest
+#define EX_RETVAL(x)		0
+
+#ifdef __KERNEL__
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop
+#endif
+
+#include "NG2memcpy.S"

arch/sparc64/lib/NG2copy_to_user.S

View file

diff --git a/arch/sparc64/lib/NG2copy_to_user.S b/arch/sparc64/lib/NG2copy_to_user.S
new file mode 100644
index 0000000..4bd4093
--- /dev/null
+++ b/arch/sparc64/lib/NG2copy_to_user.S
@@ -0,0 +1,49 @@
+/* NG2copy_to_user.S: Niagara-2 optimized copy to userspace.
+ *
+ * Copyright (C) 2007 David S. Miller ([email protected])
+ */
+
+#define EX_ST(x)		\
+98:	x;			\
+	.section .fixup;	\
+	.align 4;		\
+99:	wr	%g0, ASI_AIUS, %asi;\
+	retl;			\
+	 mov	1, %o0;		\
+	.section __ex_table,"a";\
+	.align 4;		\
+	.word 98b, 99b;		\
+	.text;			\
+	.align 4;
+
+#ifndef ASI_AIUS
+#define ASI_AIUS	0x11
+#endif
+
+#ifndef ASI_BLK_AIUS_4V
+#define ASI_BLK_AIUS_4V	0x17
+#endif
+
+#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS
+#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23
+#endif
+
+#define FUNC_NAME		NG2copy_to_user
+#define STORE(type,src,addr)	type##a src, [addr] ASI_AIUS
+#define STORE_ASI		ASI_BLK_INIT_QUAD_LDD_AIUS
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_AIUS_4V
+#define EX_RETVAL(x)		0
+
+#ifdef __KERNEL__
+	/* Writing to %asi is _expensive_ so we hardcode it.
+	 * Reading %asi to check for KERNEL_DS is comparatively
+	 * cheap.
+	 */
+#define PREAMBLE					\
+	rd		%asi, %g1;			\
+	cmp		%g1, ASI_AIUS;			\
+	bne,pn		%icc, memcpy_user_stub;		\
+	 nop
+#endif
+
+#include "NG2memcpy.S"

arch/sparc64/lib/NG2memcpy.S

View file

diff --git a/arch/sparc64/lib/NG2memcpy.S b/arch/sparc64/lib/NG2memcpy.S
new file mode 100644
index 0000000..0aed756
--- /dev/null
+++ b/arch/sparc64/lib/NG2memcpy.S
@@ -0,0 +1,520 @@
+/* NG2memcpy.S: Niagara-2 optimized memcpy.
+ *
+ * Copyright (C) 2007 David S. Miller ([email protected])
+ */
+
+#ifdef __KERNEL__
+#include <asm/visasm.h>
+#include <asm/asi.h>
+#define GLOBAL_SPARE	%g7
+#else
+#define ASI_PNF 0x82
+#define ASI_BLK_P 0xf0
+#define ASI_BLK_INIT_QUAD_LDD_P 0xe2
+#define FPRS_FEF  0x04
+#ifdef MEMCPY_DEBUG
+#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
+		     clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
+#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#else
+#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
+#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
+#endif
+#define GLOBAL_SPARE	%g5
+#endif
+
+#ifndef STORE_ASI
+#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
+#define STORE_ASI	ASI_BLK_INIT_QUAD_LDD_P
+#else
+#define STORE_ASI	0x80		/* ASI_P */
+#endif
+#endif
+
+#ifndef EX_LD
+#define EX_LD(x)	x
+#endif
+
+#ifndef EX_ST
+#define EX_ST(x)	x
+#endif
+
+#ifndef EX_RETVAL
+#define EX_RETVAL(x)	x
+#endif
+
+#ifndef LOAD
+#define LOAD(type,addr,dest)	type [addr], dest
+#endif
+
+#ifndef LOAD_BLK
+#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_P, dest
+#endif
+
+#ifndef STORE
+#ifndef MEMCPY_DEBUG
+#define STORE(type,src,addr)	type src, [addr]
+#else
+#define STORE(type,src,addr)	type##a src, [addr] 0x80
+#endif
+#endif
+
+#ifndef STORE_BLK
+#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
+#endif
+
+#ifndef STORE_INIT
+#define STORE_INIT(src,addr)	stxa src, [addr] STORE_ASI
+#endif
+
+#ifndef FUNC_NAME
+#define FUNC_NAME	NG2memcpy
+#endif
+
+#ifndef PREAMBLE
+#define PREAMBLE
+#endif
+
+#ifndef XCC
+#define XCC xcc
+#endif
+
+#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \
+	faligndata	%x0, %x1, %f0; \
+	faligndata	%x1, %x2, %f2; \
+	faligndata	%x2, %x3, %f4; \
+	faligndata	%x3, %x4, %f6; \
+	faligndata	%x4, %x5, %f8; \
+	faligndata	%x5, %x6, %f10; \
+	faligndata	%x6, %x7, %f12; \
+	faligndata	%x7, %x8, %f14;
+
+#define FREG_MOVE_1(x0) \
+	fmovd		%x0, %f0;
+#define FREG_MOVE_2(x0, x1) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2;
+#define FREG_MOVE_3(x0, x1, x2) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2; \
+	fmovd		%x2, %f4;
+#define FREG_MOVE_4(x0, x1, x2, x3) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2; \
+	fmovd		%x2, %f4; \
+	fmovd		%x3, %f6;
+#define FREG_MOVE_5(x0, x1, x2, x3, x4) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2; \
+	fmovd		%x2, %f4; \
+	fmovd		%x3, %f6; \
+	fmovd		%x4, %f8;
+#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2; \
+	fmovd		%x2, %f4; \
+	fmovd		%x3, %f6; \
+	fmovd		%x4, %f8; \
+	fmovd		%x5, %f10;
+#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2; \
+	fmovd		%x2, %f4; \
+	fmovd		%x3, %f6; \
+	fmovd		%x4, %f8; \
+	fmovd		%x5, %f10; \
+	fmovd		%x6, %f12;
+#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
+	fmovd		%x0, %f0; \
+	fmovd		%x1, %f2; \
+	fmovd		%x2, %f4; \
+	fmovd		%x3, %f6; \
+	fmovd		%x4, %f8; \
+	fmovd		%x5, %f10; \
+	fmovd		%x6, %f12; \
+	fmovd		%x7, %f14;
+#define FREG_LOAD_1(base, x0) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0))
+#define FREG_LOAD_2(base, x0, x1) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0)); \
+	EX_LD(LOAD(ldd, base + 0x08, %x1));
+#define FREG_LOAD_3(base, x0, x1, x2) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0)); \
+	EX_LD(LOAD(ldd, base + 0x08, %x1)); \
+	EX_LD(LOAD(ldd, base + 0x10, %x2));
+#define FREG_LOAD_4(base, x0, x1, x2, x3) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0)); \
+	EX_LD(LOAD(ldd, base + 0x08, %x1)); \
+	EX_LD(LOAD(ldd, base + 0x10, %x2)); \
+	EX_LD(LOAD(ldd, base + 0x18, %x3));
+#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0)); \
+	EX_LD(LOAD(ldd, base + 0x08, %x1)); \
+	EX_LD(LOAD(ldd, base + 0x10, %x2)); \
+	EX_LD(LOAD(ldd, base + 0x18, %x3)); \
+	EX_LD(LOAD(ldd, base + 0x20, %x4));
+#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0)); \
+	EX_LD(LOAD(ldd, base + 0x08, %x1)); \
+	EX_LD(LOAD(ldd, base + 0x10, %x2)); \
+	EX_LD(LOAD(ldd, base + 0x18, %x3)); \
+	EX_LD(LOAD(ldd, base + 0x20, %x4)); \
+	EX_LD(LOAD(ldd, base + 0x28, %x5));
+#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
+	EX_LD(LOAD(ldd, base + 0x00, %x0)); \
+	EX_LD(LOAD(ldd, base + 0x08, %x1)); \
+	EX_LD(LOAD(ldd, base + 0x10, %x2)); \
+	EX_LD(LOAD(ldd, base + 0x18, %x3)); \
+	EX_LD(LOAD(ldd, base + 0x20, %x4)); \
+	EX_LD(LOAD(ldd, base + 0x28, %x5)); \