path: root/usr/src/uts/sun4/ml/ip_ocsum.s

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/param.h>
#include <sys/errno.h>
#include <sys/asm_linkage.h>
#include <sys/vtrace.h>
#include <sys/machthread.h>
#include <sys/machparam.h>

#include "assym.h"

/*
 * Prefetch considerations
 * 
 * We prefetch one cacheline ahead.  This may not be enough on Serengeti
 * systems - see default_copyout() etc which prefetch 5 lines ahead.
 * On the other hand, we expect most of the source buffers to be
 * recently used enough to be cached.
 *
 * On US-I the prefetches are inoperative.  On US-II they preload the E$;
 * the mainloop unrolling and load-buffer should cover loads from E$.
 * The stores appear to be the slow point on US-II.
 * 
 * On US-IIICu the prefetch preloads the L2$ too, but there is no load
 * buffer so the loads will stall for D$ miss, L2$ hit.  The hardware
 * auto-prefetch is not activated by integer loads.  No solution
 * in sight for this, barring odd games with FP read, write, integer read.
 * 
 * US-IV (Panther) appears similar to US-IIICu, except that a strong
 * variant of prefetch is available which can take TLB traps.  We don't
 * use this.  The h/w prefetch stride can be set to 64, 128 or 192,
 * and they only reach to the L2$ (we don't use these either).
 * L2$ load-to-use latency is 15 cycles (best).
 */


/*
 * ip_ocsum(address, halfword_count, sum)
 * Do a 16 bit one's complement sum of a given number of (16-bit)
 * halfwords. The halfword pointer must not be odd.
 *	%o0 address; %o1 count; %o2 sum accumulator; %o4 temp
 * 	%g2 and %g3 used in main loop
 *
 * (from @(#)ocsum.s 1.3 89/02/24 SMI)
 *
 */

	ENTRY(ip_ocsum)

/*
 * On ttcp transmits, called once per ocsum_copyin but with a small
 * block ( >99.9% ).  Could be the tx hdrs?  How many acks/seg are we rxing?
 * On ttcp receives, called more than once per ocsum_copyout. Rx hdrs
 * and tx acks?
 *
 * To do: telnet and nfs traffic
 *
 * On an NCA'd webserver about 10% of the calls are >64 bytes
 *	about 10% of those start on a 64byte boundary
 *	about 30% are >5*64 bytes.
 * The NCA numbers & proportions don't change with h/w cksum on.
 *
 * Tx hdrs are likely to be already in cache.
 * Rx hdrs depends if already inspected.
 */

	!
	! Entry point for checksum-only.
	! %o0 contains buffer address
	! %o1 contains count of 16bit words
	! %o2 contains sum
	!
	! %o3 temporary
	! %o4 temporary
	! %g1 32bit mask
	! %g4 16bit mask
	! %g5 64bit mask (all 1s)
	!
	not	%g0, %g5	! all 1's
	prefetch [%o0], #n_reads	! first hword, dword, cacheline

	clruw	%g5, %g1	! 32 1's at low end
	srl	%g5, 16, %g4	! 16 1's at low end

	cmp	%o1, 32		! at least a cacheline (64 bytes)?
	bge,pn %icc, ip_ocsum_long	! yes, do the whole works
	andn	%o0, 7, %o5	! delay: base src addr


	cmp	%o1, 4		! < 4 halfwords?
	bl,pn	%icc, .tiny	! < 4 halfwords, just do them
	inc	8, %o5		! delay: next addr (no matter for .tiny)

	/* leading dword with 1-4 hwords: 9 clocks */
	/* Assumes ok to read the entire dword with the leading hwords */

	ldx	[%o5-8], %o3	! NB base addr
	sub	%o5, %o0, %g2	! byte count: 2/4/6/8
	mov	%o5, %o0

	sll	%g2, 2, %g2	! 8/16/24/32 for mask

	sllx	%g5, %g2, %o5

	sllx	%o5, %g2, %o5	! mask: 16/32/48/64 0's at low end

	srl	%g2, 3, %g2	! hw count
	andn	%o3, %o5, %o3	! select hw's from src

	srlx	%o3, 32, %o4	! hi32
	b	9f
	sub	%o1, %g2, %o1	! delay: decr count, 1-4 halfwords

.short_dw:			! max 7 iters of 4 clocks; 1 mispred of 4
	ldx	[%o0], %o3	! tmp64 = *src++ (groups with the branch)

	inc	8, %o0		! (D-cache load-use delay)
	dec	4, %o1		! decrement count, 4 halfwords

	srlx	%o3, 32, %o4	! hi32
9:	and	%o3, %g1, %o3	! lo32

	add	%o4, %o2, %o2	! accumulator
	andncc	%o1, 3, %g0	! more than 3 hwords left?

	bnz,pt %icc, .short_dw
	add	%o3, %o2, %o2	! accumulator

.short_hw:			! trailing dw: 0-3 hwords
	tst	%o1		! 0 seems fairly common...
	bz,a	.short_fold
	srlx	%o2, 32, %o4	! delay: hi32
				! mispredict 4 + 7 clocks for 1-3
	ldx	[%o0], %o3
	sll	%o1, 4, %o1	! bitcount: 16/32/48

	srlx	%g5, %o1, %o5	! mask: 16/32/48  0's at high end

	andn	%o3, %o5, %o3	! select hw's from src

	srlx	%o3, 32, %o4	! hi32
	and	%o3, %g1, %o3	! lo32

	add	%o4, %o2, %o2	! accumulator

	add	%o3, %o2, %o2	! accumulator

	! at this point the 64-bit accumulator
	! has the result that needs to be returned in 16-bits
	srlx	%o2, 32, %o4	! hi32
.short_fold:
	and	%o2, %g1, %o2	! lo32

	add	%o4, %o2, %o2	! 33b

	srlx	%o2, 16, %o3	! hi17
	and	%o2, %g4, %o2	! lo16

	add	%o3, %o2, %o2	! 18b

	srlx	%o2, 16, %o3	! hi2
	and	%o2, %g4, %o2	! lo16

	retl			! return
	add	%o3, %o2, %o0	! 16b result in %o0

.tiny:				! almost never: less than 4 halfwords total.
	tst	%o1
	bz,a	.short_fold

	srlx	%o2, 32, %o4	! delay: hi32

	lduh	[%o0], %o3	! tmp16 = *src++
1:	
	inc	2, %o0
				! stall for D-cache

	add	%o3, %o2, %o2	! accumulator

	deccc	%o1		! decrement count
	bnz,a,pt %icc, 1b
	lduh	[%o0], %o3	! tmp16 = *src++

	! at this point the 64-bit accumulator
	! has the result that needs to be returned in 16-bits
	b	.short_fold
	srlx	%o2, 32, %o4	! hi32

	SET_SIZE(ip_ocsum)	! 64-bit version


	ENTRY(ip_ocsum_long)	! 64-bit, large blocks
	save	%sp, -SA(MINFRAME), %sp	! get another window
	!
	! %i0 contains buffer address
	! %i1 contains count of 16bit words
	! %i2 contains sum
	! %i4 contains the mainloop count
	! %i5 comes in with the buffer address rounded down to the first dword
	!
	! %g1 32bit mask
	! %g4 16bit mask
	! %g5 64bit mask (all 1s)
	! %g6 fetch-ahead offset for Ecache
	!
	! %l0-7,%o0-5,%g2-3 mainloop temporaries
	!
	!
				! 1 clock overhead
	btst	63, %i0		! src 64-byte aligned?
	bz,a,pt	%icc, .mainsection	! aligned blocks are fairly common
	andncc	%i1, 31, %i4	! at least 64 bytes for main loop?


	! Leading dword, with 1-4 hwords: 9 clocks
	! Assumes ok to read the entire dword with the leading bytes
	ldx	[%i5], %l0	! NB base addr
	inc	8, %i5		! next addr

	sub	%i5, %i0, %l2	! byte count: 2/4/6/8
	mov	%i5, %i0

	sll	%l2, 2, %l2	! 8/16/24/32 for mask

	sllx	%g5, %l2, %l4

	sllx	%l4, %l2, %l4	! mask: 16, 32, 48, 64 0's at lsb

	srl	%l2, 3, %l2	! 1/2/3/4 for count
	andn	%l0, %l4, %l0	! select hw's from src

	srlx	%l0, 32, %o0	! hi32
	b	9f
	sub	%i1, %l2, %i1	! decr count, 1-4 halfwords

	! Do dwords until source is 64-byte aligned, 0-6 iterations
	! 4 clocks per + 4 for 1 mispred = 16 clocks avg
.dw:	ldx	[%i0], %l0	! tmp64 = *src++ (groups with the branch below)

	inc	8, %i0		! (Dcache load-use delay)
	dec	4, %i1		! decrement count, 4 halfwords

	srlx	%l0, 32, %o0	! hi32
9:	and	%l0, %g1, %l0	! lo32

	add	%o0, %i2, %i2	! accumulator
	btst	63, %i0		! src 64-byte aligned?

	bnz,pt	%icc, .dw
	add	%l0, %i2, %i2	! accumulator


	! At this point source address is 64 byte aligned
	! and we've dealt with 1-32 halfwords.
	andncc	%i1, 31, %i4	! at least 64 bytes for main loop?
.mainsection:				! total 18n + 21 clocks
	bz,pn	%icc, .postamble
	and	%i1, 31, %i1	! count for postamble

	! preload for main loop - 9 clocks assuming D$ hits at 1 per
	ldx	[%i0+0], %l0
	ldx	[%i0+8], %l1
	ldx	[%i0+16], %l2	! %l0 could be used here if Dcache hit
	ldx	[%i0+24], %l3	!  but US-II prefetch only loads Ecache
	ldx	[%i0+32], %l4	!  check on US-III: could mix preloads & splits?
	ldx	[%i0+40], %l5
	ldx	[%i0+48], %l6
	ldx	[%i0+56], %l7
	inc	64, %i0
	prefetch [%i0], #n_reads

	! main loop. Read 64 bytes at a time - 18 clocks per iteration
5:	!					plus 4 for the exit mispredict
	srlx	%l0, 32, %o0		! hi32 to %o0
	and	%l0, %g1, %l0		! lo32 to %l0

	srlx	%l1, 32, %o1		! hi32 to %o1
	and	%l1, %g1, %l1		! lo32 to %l1

	srlx	%l2, 32, %o2		! hi32 to %o2
	and	%l2, %g1, %l2		! lo32 to %l2

	srlx	%l3, 32, %o3		! hi32 to %o3
	and	%l3, %g1, %l3		! lo32 to %l3

	srlx	%l4, 32, %o4		! hi32 to %o4
	and	%l4, %g1, %l4		! lo32 to %l4

	srlx	%l5, 32, %o5		! hi32 to %o5
	and	%l5, %g1, %l5		! lo32 to %l5

	srlx	%l6, 32, %g2		! hi32 to %g2
	and	%l6, %g1, %l6		! lo32 to %l6

	srlx	%l7, 32, %g3		! hi32 to %g3
	and	%l7, %g1, %l7		! lo32 to %l7
				! splits gave 16 off 32b vals
	deccc	32, %i4		! mv early,avoid mispredicts? nohelp US-II.
	bz,pn	%icc, .looptidy	! count now zero?
	add	%l0, %o0, %o0	! delay

	ldx	[%i0+0], %l0
	add	%l1, %o1, %o1	! adds and loads
	add	%l2, %o2, %o2

	ldx	[%i0+8], %l1
	add	%l3, %o3, %o3
	add	%l4, %o4, %o4

	ldx	[%i0+16], %l2
	add	%l5, %o5, %o5
	add	%l6, %g2, %g2

	ldx	[%i0+24], %l3
	add	%l7, %g3, %g3		! now 8 off 33b vals
	add	%o0, %o1, %o0

	ldx	[%i0+32], %l4
	add	%o2, %o3, %o1
	add	%o4, %o5, %o2

	ldx	[%i0+40], %l5
	add	%g2, %g3, %o3		! now 4 off 34b vals
	add	%o0, %o1, %o0

	ldx	[%i0+48], %l6
	add	%o2, %o3, %o1		! 2 off 35b

	ldx	[%i0+56], %l7
	add	%o0, %o1, %o0		! 36b
	inc	64, %i0		! increment source address

	add	%o0, %i2, %i2	! accumulator
	ba	5b
	prefetch [%i0], #n_reads	! next cacheline
				! end of main loop
.looptidy:	! compute remaining partial sum - 8 clocks
	add	%l1, %o1, %o1
	add	%l2, %o2, %o2

	add	%l3, %o3, %o3
	add	%l4, %o4, %o4

	add	%l5, %o5, %o5
	add	%l6, %g2, %g2

	add	%l7, %g3, %g3		! 8 x 33b
	add	%o0, %o1, %o0

	add	%o2, %o3, %o1
	add	%o4, %o5, %o2

	add	%g2, %g3, %o3		! 4 x 34b
	add	%o0, %o1, %o0

	add	%o2, %o3, %o1		! 2 x 35b
	add	%o0, %i2, %i2	! accumulator

	add	%o1, %i2, %i2	! accumulator


.postamble:
	! postamble hword count is in %i1 (can be zero)
	! while at least 1 dword, do dwords.   Max 7 iterations.
	andncc	%i1, 3, %g0	! more than 3 hwords?
.dotail_dw:
	bz,a,pn	%icc, .dotail_hw
	tst	%i1		! delay: any at all left?
8:	
	ldx	[%i0], %l0	! tmp64 = *src++
	inc	8, %i0
	dec	4, %i1		! decrement count, 4 halfwords

				! stall for D-cache

	srlx	%l0, 32, %o0	! hi32
	and	%l0, %g1, %l0	! lo32

	add	%o0, %i2, %i2	! accumulator

	andncc	%i1, 3, %g0	! more than 3 hwords?
	bnz,pt	%icc, 8b
	add	%l0, %i2, %i2	! accumulator

	! while at least 1 hword, do hwords.   Max 3 iterations.
	tst	%i1
.dotail_hw:
	bz,a	.fold
	srlx	%i2, 32, %o0	! delay: hi32
	lduh	[%i0], %l0	! tmp16 = *src++
1:	
	inc	2, %i0
				! stall for D-cache

	add	%l0, %i2, %i2	! accumulator

	deccc	%i1		! decrement count
	bnz,a,pt %icc, 1b
	lduh	[%i0], %l0	! tmp16 = *src++

	! at this point the 64-bit accumulator
	! has the result that needs to be returned in 16-bits
	srlx	%i2, 32, %o0	! hi32
.fold:
	and	%i2, %g1, %o1	! lo32

	add	%o0, %o1, %o0	! 33b

	srlx	%o0, 16, %o1	! hi17
	and	%o0, %g4, %o0	! lo16

	add	%o1, %o0, %o0	! 18b

	srlx	%o0, 16, %o1	! hi2
	and	%o0, %g4, %o0	! lo16

	add	%o1, %o0, %i0	! 16b result in %i0

	ret			! return
	restore


	SET_SIZE(ip_ocsum_long)	! 64-bit version