WebSVN - shark - Blame - Rev 1618 - /shark/branches/xen/drivers/linuxc26/include/asm/xor.h

Rev	Author	Line No.	Line
422	giacomo	1	/*
		2	* include/asm-i386/xor.h
		3	*
		4	* Optimized RAID-5 checksumming functions for MMX and SSE.
		5	*
		6	* This program is free software; you can redistribute it and/or modify
		7	* it under the terms of the GNU General Public License as published by
		8	* the Free Software Foundation; either version 2, or (at your option)
		9	* any later version.
		10	*
		11	* You should have received a copy of the GNU General Public License
		12	* (for example /usr/src/linux/COPYING); if not, write to the Free
		13	* Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
		14	*/
		15
		16	/*
		17	* High-speed RAID5 checksumming functions utilizing MMX instructions.
		18	* Copyright (C) 1998 Ingo Molnar.
		19	*/
		20
		21	#define LD(x,y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
		22	#define ST(x,y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
		23	#define XO1(x,y) " pxor 8*("#x")(%2), %%mm"#y" ;\n"
		24	#define XO2(x,y) " pxor 8*("#x")(%3), %%mm"#y" ;\n"
		25	#define XO3(x,y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
		26	#define XO4(x,y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
		27
		28	#include <asm/i387.h>
		29
		30	static void
		31	xor_pII_mmx_2(unsigned long bytes, unsigned long p1, unsigned long p2)
		32	{
		33	unsigned long lines = bytes >> 7;
		34
		35	kernel_fpu_begin();
		36
		37	__asm__ __volatile__ (
		38	#undef BLOCK
		39	#define BLOCK(i) \
		40	LD(i,0) \
		41	LD(i+1,1) \
		42	LD(i+2,2) \
		43	LD(i+3,3) \
		44	XO1(i,0) \
		45	ST(i,0) \
		46	XO1(i+1,1) \
		47	ST(i+1,1) \
		48	XO1(i+2,2) \
		49	ST(i+2,2) \
		50	XO1(i+3,3) \
		51	ST(i+3,3)
		52
		53	" .align 32 ;\n"
		54	" 1: ;\n"
		55
		56	BLOCK(0)
		57	BLOCK(4)
		58	BLOCK(8)
		59	BLOCK(12)
		60
		61	" addl $128, %1 ;\n"
		62	" addl $128, %2 ;\n"
		63	" decl %0 ;\n"
		64	" jnz 1b ;\n"
		65	: "+r" (lines),
		66	"+r" (p1), "+r" (p2)
		67	:
		68	: "memory");
		69
		70	kernel_fpu_end();
		71	}
		72
		73	static void
		74	xor_pII_mmx_3(unsigned long bytes, unsigned long p1, unsigned long p2,
		75	unsigned long *p3)
		76	{
		77	unsigned long lines = bytes >> 7;
		78
		79	kernel_fpu_begin();
		80
		81	__asm__ __volatile__ (
		82	#undef BLOCK
		83	#define BLOCK(i) \
		84	LD(i,0) \
		85	LD(i+1,1) \
		86	LD(i+2,2) \
		87	LD(i+3,3) \
		88	XO1(i,0) \
		89	XO1(i+1,1) \
		90	XO1(i+2,2) \
		91	XO1(i+3,3) \
		92	XO2(i,0) \
		93	ST(i,0) \
		94	XO2(i+1,1) \
		95	ST(i+1,1) \
		96	XO2(i+2,2) \
		97	ST(i+2,2) \
		98	XO2(i+3,3) \
		99	ST(i+3,3)
		100
		101	" .align 32 ;\n"
		102	" 1: ;\n"
		103
		104	BLOCK(0)
		105	BLOCK(4)
		106	BLOCK(8)
		107	BLOCK(12)
		108
		109	" addl $128, %1 ;\n"
		110	" addl $128, %2 ;\n"
		111	" addl $128, %3 ;\n"
		112	" decl %0 ;\n"
		113	" jnz 1b ;\n"
		114	: "+r" (lines),
		115	"+r" (p1), "+r" (p2), "+r" (p3)
		116	:
		117	: "memory");
		118
		119	kernel_fpu_end();
		120	}
		121
		122	static void
		123	xor_pII_mmx_4(unsigned long bytes, unsigned long p1, unsigned long p2,
		124	unsigned long p3, unsigned long p4)
		125	{
		126	unsigned long lines = bytes >> 7;
		127
		128	kernel_fpu_begin();
		129
		130	__asm__ __volatile__ (
		131	#undef BLOCK
		132	#define BLOCK(i) \
		133	LD(i,0) \
		134	LD(i+1,1) \
		135	LD(i+2,2) \
		136	LD(i+3,3) \
		137	XO1(i,0) \
		138	XO1(i+1,1) \
		139	XO1(i+2,2) \
		140	XO1(i+3,3) \
		141	XO2(i,0) \
		142	XO2(i+1,1) \
		143	XO2(i+2,2) \
		144	XO2(i+3,3) \
		145	XO3(i,0) \
		146	ST(i,0) \
		147	XO3(i+1,1) \
		148	ST(i+1,1) \
		149	XO3(i+2,2) \
		150	ST(i+2,2) \
		151	XO3(i+3,3) \
		152	ST(i+3,3)
		153
		154	" .align 32 ;\n"
		155	" 1: ;\n"
		156
		157	BLOCK(0)
		158	BLOCK(4)
		159	BLOCK(8)
		160	BLOCK(12)
		161
		162	" addl $128, %1 ;\n"
		163	" addl $128, %2 ;\n"
		164	" addl $128, %3 ;\n"
		165	" addl $128, %4 ;\n"
		166	" decl %0 ;\n"
		167	" jnz 1b ;\n"
		168	: "+r" (lines),
		169	"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
		170	:
		171	: "memory");
		172
		173	kernel_fpu_end();
		174	}
		175
		176
		177	static void
		178	xor_pII_mmx_5(unsigned long bytes, unsigned long p1, unsigned long p2,
		179	unsigned long p3, unsigned long p4, unsigned long *p5)
		180	{
		181	unsigned long lines = bytes >> 7;
		182
		183	kernel_fpu_begin();
		184
		185	/* need to save/restore p4/p5 manually otherwise gcc's 10 argument
		186	limit gets exceeded (+ counts as two arguments) */
		187	__asm__ __volatile__ (
		188	" pushl %4\n"
		189	" pushl %5\n"
		190	#undef BLOCK
		191	#define BLOCK(i) \
		192	LD(i,0) \
		193	LD(i+1,1) \
		194	LD(i+2,2) \
		195	LD(i+3,3) \
		196	XO1(i,0) \
		197	XO1(i+1,1) \
		198	XO1(i+2,2) \
		199	XO1(i+3,3) \
		200	XO2(i,0) \
		201	XO2(i+1,1) \
		202	XO2(i+2,2) \
		203	XO2(i+3,3) \
		204	XO3(i,0) \
		205	XO3(i+1,1) \
		206	XO3(i+2,2) \
		207	XO3(i+3,3) \
		208	XO4(i,0) \
		209	ST(i,0) \
		210	XO4(i+1,1) \
		211	ST(i+1,1) \
		212	XO4(i+2,2) \
		213	ST(i+2,2) \
		214	XO4(i+3,3) \
		215	ST(i+3,3)
		216
		217	" .align 32 ;\n"
		218	" 1: ;\n"
		219
		220	BLOCK(0)
		221	BLOCK(4)
		222	BLOCK(8)
		223	BLOCK(12)
		224
		225	" addl $128, %1 ;\n"
		226	" addl $128, %2 ;\n"
		227	" addl $128, %3 ;\n"
		228	" addl $128, %4 ;\n"
		229	" addl $128, %5 ;\n"
		230	" decl %0 ;\n"
		231	" jnz 1b ;\n"
		232	" popl %5\n"
		233	" popl %4\n"
		234	: "+r" (lines),
		235	"+r" (p1), "+r" (p2), "+r" (p3)
		236	: "r" (p4), "r" (p5)
		237	: "memory");
		238
		239	kernel_fpu_end();
		240	}
		241
		242	#undef LD
		243	#undef XO1
		244	#undef XO2
		245	#undef XO3
		246	#undef XO4
		247	#undef ST
		248	#undef BLOCK
		249
		250	static void
		251	xor_p5_mmx_2(unsigned long bytes, unsigned long p1, unsigned long p2)
		252	{
		253	unsigned long lines = bytes >> 6;
		254
		255	kernel_fpu_begin();
		256
		257	__asm__ __volatile__ (
		258	" .align 32 ;\n"
		259	" 1: ;\n"
		260	" movq (%1), %%mm0 ;\n"
		261	" movq 8(%1), %%mm1 ;\n"
		262	" pxor (%2), %%mm0 ;\n"
		263	" movq 16(%1), %%mm2 ;\n"
		264	" movq %%mm0, (%1) ;\n"
		265	" pxor 8(%2), %%mm1 ;\n"
		266	" movq 24(%1), %%mm3 ;\n"
		267	" movq %%mm1, 8(%1) ;\n"
		268	" pxor 16(%2), %%mm2 ;\n"
		269	" movq 32(%1), %%mm4 ;\n"
		270	" movq %%mm2, 16(%1) ;\n"
		271	" pxor 24(%2), %%mm3 ;\n"
		272	" movq 40(%1), %%mm5 ;\n"
		273	" movq %%mm3, 24(%1) ;\n"
		274	" pxor 32(%2), %%mm4 ;\n"
		275	" movq 48(%1), %%mm6 ;\n"
		276	" movq %%mm4, 32(%1) ;\n"
		277	" pxor 40(%2), %%mm5 ;\n"
		278	" movq 56(%1), %%mm7 ;\n"
		279	" movq %%mm5, 40(%1) ;\n"
		280	" pxor 48(%2), %%mm6 ;\n"
		281	" pxor 56(%2), %%mm7 ;\n"
		282	" movq %%mm6, 48(%1) ;\n"
		283	" movq %%mm7, 56(%1) ;\n"
		284
		285	" addl $64, %1 ;\n"
		286	" addl $64, %2 ;\n"
		287	" decl %0 ;\n"
		288	" jnz 1b ;\n"
		289	: "+r" (lines),
		290	"+r" (p1), "+r" (p2)
		291	:
		292	: "memory");
		293
		294	kernel_fpu_end();
		295	}
		296
		297	static void
		298	xor_p5_mmx_3(unsigned long bytes, unsigned long p1, unsigned long p2,
		299	unsigned long *p3)
		300	{
		301	unsigned long lines = bytes >> 6;
		302
		303	kernel_fpu_begin();
		304
		305	__asm__ __volatile__ (
		306	" .align 32,0x90 ;\n"
		307	" 1: ;\n"
		308	" movq (%1), %%mm0 ;\n"
		309	" movq 8(%1), %%mm1 ;\n"
		310	" pxor (%2), %%mm0 ;\n"
		311	" movq 16(%1), %%mm2 ;\n"
		312	" pxor 8(%2), %%mm1 ;\n"
		313	" pxor (%3), %%mm0 ;\n"
		314	" pxor 16(%2), %%mm2 ;\n"
		315	" movq %%mm0, (%1) ;\n"
		316	" pxor 8(%3), %%mm1 ;\n"
		317	" pxor 16(%3), %%mm2 ;\n"
		318	" movq 24(%1), %%mm3 ;\n"
		319	" movq %%mm1, 8(%1) ;\n"
		320	" movq 32(%1), %%mm4 ;\n"
		321	" movq 40(%1), %%mm5 ;\n"
		322	" pxor 24(%2), %%mm3 ;\n"
		323	" movq %%mm2, 16(%1) ;\n"
		324	" pxor 32(%2), %%mm4 ;\n"
		325	" pxor 24(%3), %%mm3 ;\n"
		326	" pxor 40(%2), %%mm5 ;\n"
		327	" movq %%mm3, 24(%1) ;\n"
		328	" pxor 32(%3), %%mm4 ;\n"
		329	" pxor 40(%3), %%mm5 ;\n"
		330	" movq 48(%1), %%mm6 ;\n"
		331	" movq %%mm4, 32(%1) ;\n"
		332	" movq 56(%1), %%mm7 ;\n"
		333	" pxor 48(%2), %%mm6 ;\n"
		334	" movq %%mm5, 40(%1) ;\n"
		335	" pxor 56(%2), %%mm7 ;\n"
		336	" pxor 48(%3), %%mm6 ;\n"
		337	" pxor 56(%3), %%mm7 ;\n"
		338	" movq %%mm6, 48(%1) ;\n"
		339	" movq %%mm7, 56(%1) ;\n"
		340
		341	" addl $64, %1 ;\n"
		342	" addl $64, %2 ;\n"
		343	" addl $64, %3 ;\n"
		344	" decl %0 ;\n"
		345	" jnz 1b ;\n"
		346	: "+r" (lines),
		347	"+r" (p1), "+r" (p2), "+r" (p3)
		348	:
		349	: "memory" );
		350
		351	kernel_fpu_end();
		352	}
		353
		354	static void
		355	xor_p5_mmx_4(unsigned long bytes, unsigned long p1, unsigned long p2,
		356	unsigned long p3, unsigned long p4)
		357	{
		358	unsigned long lines = bytes >> 6;
		359
		360	kernel_fpu_begin();
		361
		362	__asm__ __volatile__ (
		363	" .align 32,0x90 ;\n"
		364	" 1: ;\n"
		365	" movq (%1), %%mm0 ;\n"
		366	" movq 8(%1), %%mm1 ;\n"
		367	" pxor (%2), %%mm0 ;\n"
		368	" movq 16(%1), %%mm2 ;\n"
		369	" pxor 8(%2), %%mm1 ;\n"
		370	" pxor (%3), %%mm0 ;\n"
		371	" pxor 16(%2), %%mm2 ;\n"
		372	" pxor 8(%3), %%mm1 ;\n"
		373	" pxor (%4), %%mm0 ;\n"
		374	" movq 24(%1), %%mm3 ;\n"
		375	" pxor 16(%3), %%mm2 ;\n"
		376	" pxor 8(%4), %%mm1 ;\n"
		377	" movq %%mm0, (%1) ;\n"
		378	" movq 32(%1), %%mm4 ;\n"
		379	" pxor 24(%2), %%mm3 ;\n"
		380	" pxor 16(%4), %%mm2 ;\n"
		381	" movq %%mm1, 8(%1) ;\n"
		382	" movq 40(%1), %%mm5 ;\n"
		383	" pxor 32(%2), %%mm4 ;\n"
		384	" pxor 24(%3), %%mm3 ;\n"
		385	" movq %%mm2, 16(%1) ;\n"
		386	" pxor 40(%2), %%mm5 ;\n"
		387	" pxor 32(%3), %%mm4 ;\n"
		388	" pxor 24(%4), %%mm3 ;\n"
		389	" movq %%mm3, 24(%1) ;\n"
		390	" movq 56(%1), %%mm7 ;\n"
		391	" movq 48(%1), %%mm6 ;\n"
		392	" pxor 40(%3), %%mm5 ;\n"
		393	" pxor 32(%4), %%mm4 ;\n"
		394	" pxor 48(%2), %%mm6 ;\n"
		395	" movq %%mm4, 32(%1) ;\n"
		396	" pxor 56(%2), %%mm7 ;\n"
		397	" pxor 40(%4), %%mm5 ;\n"
		398	" pxor 48(%3), %%mm6 ;\n"
		399	" pxor 56(%3), %%mm7 ;\n"
		400	" movq %%mm5, 40(%1) ;\n"
		401	" pxor 48(%4), %%mm6 ;\n"
		402	" pxor 56(%4), %%mm7 ;\n"
		403	" movq %%mm6, 48(%1) ;\n"
		404	" movq %%mm7, 56(%1) ;\n"
		405
		406	" addl $64, %1 ;\n"
		407	" addl $64, %2 ;\n"
		408	" addl $64, %3 ;\n"
		409	" addl $64, %4 ;\n"
		410	" decl %0 ;\n"
		411	" jnz 1b ;\n"
		412	: "+r" (lines),
		413	"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
		414	:
		415	: "memory");
		416
		417	kernel_fpu_end();
		418	}
		419
		420	static void
		421	xor_p5_mmx_5(unsigned long bytes, unsigned long p1, unsigned long p2,
		422	unsigned long p3, unsigned long p4, unsigned long *p5)
		423	{
		424	unsigned long lines = bytes >> 6;
		425
		426	kernel_fpu_begin();
		427
		428	/* need to save p4/p5 manually to not exceed gcc's 10 argument limit */
		429	__asm__ __volatile__ (
		430	" pushl %4\n"
		431	" pushl %5\n"
		432	" .align 32,0x90 ;\n"
		433	" 1: ;\n"
		434	" movq (%1), %%mm0 ;\n"
		435	" movq 8(%1), %%mm1 ;\n"
		436	" pxor (%2), %%mm0 ;\n"
		437	" pxor 8(%2), %%mm1 ;\n"
		438	" movq 16(%1), %%mm2 ;\n"
		439	" pxor (%3), %%mm0 ;\n"
		440	" pxor 8(%3), %%mm1 ;\n"
		441	" pxor 16(%2), %%mm2 ;\n"
		442	" pxor (%4), %%mm0 ;\n"
		443	" pxor 8(%4), %%mm1 ;\n"
		444	" pxor 16(%3), %%mm2 ;\n"
		445	" movq 24(%1), %%mm3 ;\n"
		446	" pxor (%5), %%mm0 ;\n"
		447	" pxor 8(%5), %%mm1 ;\n"
		448	" movq %%mm0, (%1) ;\n"
		449	" pxor 16(%4), %%mm2 ;\n"
		450	" pxor 24(%2), %%mm3 ;\n"
		451	" movq %%mm1, 8(%1) ;\n"
		452	" pxor 16(%5), %%mm2 ;\n"
		453	" pxor 24(%3), %%mm3 ;\n"
		454	" movq 32(%1), %%mm4 ;\n"
		455	" movq %%mm2, 16(%1) ;\n"
		456	" pxor 24(%4), %%mm3 ;\n"
		457	" pxor 32(%2), %%mm4 ;\n"
		458	" movq 40(%1), %%mm5 ;\n"
		459	" pxor 24(%5), %%mm3 ;\n"
		460	" pxor 32(%3), %%mm4 ;\n"
		461	" pxor 40(%2), %%mm5 ;\n"
		462	" movq %%mm3, 24(%1) ;\n"
		463	" pxor 32(%4), %%mm4 ;\n"
		464	" pxor 40(%3), %%mm5 ;\n"
		465	" movq 48(%1), %%mm6 ;\n"
		466	" movq 56(%1), %%mm7 ;\n"
		467	" pxor 32(%5), %%mm4 ;\n"
		468	" pxor 40(%4), %%mm5 ;\n"
		469	" pxor 48(%2), %%mm6 ;\n"
		470	" pxor 56(%2), %%mm7 ;\n"
		471	" movq %%mm4, 32(%1) ;\n"
		472	" pxor 48(%3), %%mm6 ;\n"
		473	" pxor 56(%3), %%mm7 ;\n"
		474	" pxor 40(%5), %%mm5 ;\n"
		475	" pxor 48(%4), %%mm6 ;\n"
		476	" pxor 56(%4), %%mm7 ;\n"
		477	" movq %%mm5, 40(%1) ;\n"
		478	" pxor 48(%5), %%mm6 ;\n"
		479	" pxor 56(%5), %%mm7 ;\n"
		480	" movq %%mm6, 48(%1) ;\n"
		481	" movq %%mm7, 56(%1) ;\n"
		482
		483	" addl $64, %1 ;\n"
		484	" addl $64, %2 ;\n"
		485	" addl $64, %3 ;\n"
		486	" addl $64, %4 ;\n"
		487	" addl $64, %5 ;\n"
		488	" decl %0 ;\n"
		489	" jnz 1b ;\n"
		490	" popl %5\n"
		491	" popl %4\n"
		492	: "+g" (lines),
		493	"+r" (p1), "+r" (p2), "+r" (p3)
		494	: "r" (p4), "r" (p5)
		495	: "memory");
		496
		497	kernel_fpu_end();
		498	}
		499
		500	static struct xor_block_template xor_block_pII_mmx = {
		501	.name = "pII_mmx",
		502	.do_2 = xor_pII_mmx_2,
		503	.do_3 = xor_pII_mmx_3,
		504	.do_4 = xor_pII_mmx_4,
		505	.do_5 = xor_pII_mmx_5,
		506	};
		507
		508	static struct xor_block_template xor_block_p5_mmx = {
		509	.name = "p5_mmx",
		510	.do_2 = xor_p5_mmx_2,
		511	.do_3 = xor_p5_mmx_3,
		512	.do_4 = xor_p5_mmx_4,
		513	.do_5 = xor_p5_mmx_5,
		514	};
		515
		516	/*
		517	* Cache avoiding checksumming functions utilizing KNI instructions
		518	* Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
		519	*/
		520
		521	#define XMMS_SAVE do { \
		522	preempt_disable(); \
		523	__asm__ __volatile__ ( \
		524	"movl %%cr0,%0 ;\n\t" \
		525	"clts ;\n\t" \
		526	"movups %%xmm0,(%1) ;\n\t" \
		527	"movups %%xmm1,0x10(%1) ;\n\t" \
		528	"movups %%xmm2,0x20(%1) ;\n\t" \
		529	"movups %%xmm3,0x30(%1) ;\n\t" \
		530	: "=&r" (cr0) \
		531	: "r" (xmm_save) \
		532	: "memory"); \
		533	} while(0)
		534
		535	#define XMMS_RESTORE do { \
		536	__asm__ __volatile__ ( \
		537	"sfence ;\n\t" \
		538	"movups (%1),%%xmm0 ;\n\t" \
		539	"movups 0x10(%1),%%xmm1 ;\n\t" \
		540	"movups 0x20(%1),%%xmm2 ;\n\t" \
		541	"movups 0x30(%1),%%xmm3 ;\n\t" \
		542	"movl %0,%%cr0 ;\n\t" \
		543	: \
		544	: "r" (cr0), "r" (xmm_save) \
		545	: "memory"); \
		546	preempt_enable(); \
		547	} while(0)
		548
		549	#define ALIGN16 __attribute__((aligned(16)))
		550
		551	#define OFFS(x) "16*("#x")"
		552	#define PF_OFFS(x) "256+16*("#x")"
		553	#define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n"
		554	#define LD(x,y) " movaps "OFFS(x)"(%1), %%xmm"#y" ;\n"
		555	#define ST(x,y) " movaps %%xmm"#y", "OFFS(x)"(%1) ;\n"
		556	#define PF1(x) " prefetchnta "PF_OFFS(x)"(%2) ;\n"
		557	#define PF2(x) " prefetchnta "PF_OFFS(x)"(%3) ;\n"
		558	#define PF3(x) " prefetchnta "PF_OFFS(x)"(%4) ;\n"
		559	#define PF4(x) " prefetchnta "PF_OFFS(x)"(%5) ;\n"
		560	#define PF5(x) " prefetchnta "PF_OFFS(x)"(%6) ;\n"
		561	#define XO1(x,y) " xorps "OFFS(x)"(%2), %%xmm"#y" ;\n"
		562	#define XO2(x,y) " xorps "OFFS(x)"(%3), %%xmm"#y" ;\n"
		563	#define XO3(x,y) " xorps "OFFS(x)"(%4), %%xmm"#y" ;\n"
		564	#define XO4(x,y) " xorps "OFFS(x)"(%5), %%xmm"#y" ;\n"
		565	#define XO5(x,y) " xorps "OFFS(x)"(%6), %%xmm"#y" ;\n"
		566
		567
		568	static void
		569	xor_sse_2(unsigned long bytes, unsigned long p1, unsigned long p2)
		570	{
		571	unsigned long lines = bytes >> 8;
		572	char xmm_save[16*4] ALIGN16;
		573	int cr0;
		574
		575	XMMS_SAVE;
		576
		577	__asm__ __volatile__ (
		578	#undef BLOCK
		579	#define BLOCK(i) \
		580	LD(i,0) \
		581	LD(i+1,1) \
		582	PF1(i) \
		583	PF1(i+2) \
		584	LD(i+2,2) \
		585	LD(i+3,3) \
		586	PF0(i+4) \
		587	PF0(i+6) \
		588	XO1(i,0) \
		589	XO1(i+1,1) \
		590	XO1(i+2,2) \
		591	XO1(i+3,3) \
		592	ST(i,0) \
		593	ST(i+1,1) \
		594	ST(i+2,2) \
		595	ST(i+3,3) \
		596
		597
		598	PF0(0)
		599	PF0(2)
		600
		601	" .align 32 ;\n"
		602	" 1: ;\n"
		603
		604	BLOCK(0)
		605	BLOCK(4)
		606	BLOCK(8)
		607	BLOCK(12)
		608
		609	" addl $256, %1 ;\n"
		610	" addl $256, %2 ;\n"
		611	" decl %0 ;\n"
		612	" jnz 1b ;\n"
		613	: "+r" (lines),
		614	"+r" (p1), "+r" (p2)
		615	:
		616	: "memory");
		617
		618	XMMS_RESTORE;
		619	}
		620
		621	static void
		622	xor_sse_3(unsigned long bytes, unsigned long p1, unsigned long p2,
		623	unsigned long *p3)
		624	{
		625	unsigned long lines = bytes >> 8;
		626	char xmm_save[16*4] ALIGN16;
		627	int cr0;
		628
		629	XMMS_SAVE;
		630
		631	__asm__ __volatile__ (
		632	#undef BLOCK
		633	#define BLOCK(i) \
		634	PF1(i) \
		635	PF1(i+2) \
		636	LD(i,0) \
		637	LD(i+1,1) \
		638	LD(i+2,2) \
		639	LD(i+3,3) \
		640	PF2(i) \
		641	PF2(i+2) \
		642	PF0(i+4) \
		643	PF0(i+6) \
		644	XO1(i,0) \
		645	XO1(i+1,1) \
		646	XO1(i+2,2) \
		647	XO1(i+3,3) \
		648	XO2(i,0) \
		649	XO2(i+1,1) \
		650	XO2(i+2,2) \
		651	XO2(i+3,3) \
		652	ST(i,0) \
		653	ST(i+1,1) \
		654	ST(i+2,2) \
		655	ST(i+3,3) \
		656
		657
		658	PF0(0)
		659	PF0(2)
		660
		661	" .align 32 ;\n"
		662	" 1: ;\n"
		663
		664	BLOCK(0)
		665	BLOCK(4)
		666	BLOCK(8)
		667	BLOCK(12)
		668
		669	" addl $256, %1 ;\n"
		670	" addl $256, %2 ;\n"
		671	" addl $256, %3 ;\n"
		672	" decl %0 ;\n"
		673	" jnz 1b ;\n"
		674	: "+r" (lines),
		675	"+r" (p1), "+r"(p2), "+r"(p3)
		676	:
		677	: "memory" );
		678
		679	XMMS_RESTORE;
		680	}
		681
		682	static void
		683	xor_sse_4(unsigned long bytes, unsigned long p1, unsigned long p2,
		684	unsigned long p3, unsigned long p4)
		685	{
		686	unsigned long lines = bytes >> 8;
		687	char xmm_save[16*4] ALIGN16;
		688	int cr0;
		689
		690	XMMS_SAVE;
		691
		692	__asm__ __volatile__ (
		693	#undef BLOCK
		694	#define BLOCK(i) \
		695	PF1(i) \
		696	PF1(i+2) \
		697	LD(i,0) \
		698	LD(i+1,1) \
		699	LD(i+2,2) \
		700	LD(i+3,3) \
		701	PF2(i) \
		702	PF2(i+2) \
		703	XO1(i,0) \
		704	XO1(i+1,1) \
		705	XO1(i+2,2) \
		706	XO1(i+3,3) \
		707	PF3(i) \
		708	PF3(i+2) \
		709	PF0(i+4) \
		710	PF0(i+6) \
		711	XO2(i,0) \
		712	XO2(i+1,1) \
		713	XO2(i+2,2) \
		714	XO2(i+3,3) \
		715	XO3(i,0) \
		716	XO3(i+1,1) \
		717	XO3(i+2,2) \
		718	XO3(i+3,3) \
		719	ST(i,0) \
		720	ST(i+1,1) \
		721	ST(i+2,2) \
		722	ST(i+3,3) \
		723
		724
		725	PF0(0)
		726	PF0(2)
		727
		728	" .align 32 ;\n"
		729	" 1: ;\n"
		730
		731	BLOCK(0)
		732	BLOCK(4)
		733	BLOCK(8)
		734	BLOCK(12)
		735
		736	" addl $256, %1 ;\n"
		737	" addl $256, %2 ;\n"
		738	" addl $256, %3 ;\n"
		739	" addl $256, %4 ;\n"
		740	" decl %0 ;\n"
		741	" jnz 1b ;\n"
		742	: "+r" (lines),
		743	"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
		744	:
		745	: "memory" );
		746
		747	XMMS_RESTORE;
		748	}
		749
		750	static void
		751	xor_sse_5(unsigned long bytes, unsigned long p1, unsigned long p2,
		752	unsigned long p3, unsigned long p4, unsigned long *p5)
		753	{
		754	unsigned long lines = bytes >> 8;
		755	char xmm_save[16*4] ALIGN16;
		756	int cr0;
		757
		758	XMMS_SAVE;
		759
		760	/* need to save p4/p5 manually to not exceed gcc's 10 argument limit */
		761	__asm__ __volatile__ (
		762	" pushl %4\n"
		763	" pushl %5\n"
		764	#undef BLOCK
		765	#define BLOCK(i) \
		766	PF1(i) \
		767	PF1(i+2) \
		768	LD(i,0) \
		769	LD(i+1,1) \
		770	LD(i+2,2) \
		771	LD(i+3,3) \
		772	PF2(i) \
		773	PF2(i+2) \
		774	XO1(i,0) \
		775	XO1(i+1,1) \
		776	XO1(i+2,2) \
		777	XO1(i+3,3) \
		778	PF3(i) \
		779	PF3(i+2) \
		780	XO2(i,0) \
		781	XO2(i+1,1) \
		782	XO2(i+2,2) \
		783	XO2(i+3,3) \
		784	PF4(i) \
		785	PF4(i+2) \
		786	PF0(i+4) \
		787	PF0(i+6) \
		788	XO3(i,0) \
		789	XO3(i+1,1) \
		790	XO3(i+2,2) \
		791	XO3(i+3,3) \
		792	XO4(i,0) \
		793	XO4(i+1,1) \
		794	XO4(i+2,2) \
		795	XO4(i+3,3) \
		796	ST(i,0) \
		797	ST(i+1,1) \
		798	ST(i+2,2) \
		799	ST(i+3,3) \
		800
		801
		802	PF0(0)
		803	PF0(2)
		804
		805	" .align 32 ;\n"
		806	" 1: ;\n"
		807
		808	BLOCK(0)
		809	BLOCK(4)
		810	BLOCK(8)
		811	BLOCK(12)
		812
		813	" addl $256, %1 ;\n"
		814	" addl $256, %2 ;\n"
		815	" addl $256, %3 ;\n"
		816	" addl $256, %4 ;\n"
		817	" addl $256, %5 ;\n"
		818	" decl %0 ;\n"
		819	" jnz 1b ;\n"
		820	" popl %5\n"
		821	" popl %4\n"
		822	: "+r" (lines),
		823	"+r" (p1), "+r" (p2), "+r" (p3)
		824	: "r" (p4), "r" (p5)
		825	: "memory");
		826
		827	XMMS_RESTORE;
		828	}
		829
		830	static struct xor_block_template xor_block_pIII_sse = {
		831	.name = "pIII_sse",
		832	.do_2 = xor_sse_2,
		833	.do_3 = xor_sse_3,
		834	.do_4 = xor_sse_4,
		835	.do_5 = xor_sse_5,
		836	};
		837
		838	/* Also try the generic routines. */
		839	#include <asm-generic/xor.h>
		840
		841	#undef XOR_TRY_TEMPLATES
		842	#define XOR_TRY_TEMPLATES \
		843	do { \
		844	xor_speed(&xor_block_8regs); \
		845	xor_speed(&xor_block_8regs_p); \
		846	xor_speed(&xor_block_32regs); \
		847	xor_speed(&xor_block_32regs_p); \
		848	if (cpu_has_xmm) \
		849	xor_speed(&xor_block_pIII_sse); \
		850	if (cpu_has_mmx) { \
		851	xor_speed(&xor_block_pII_mmx); \
		852	xor_speed(&xor_block_p5_mmx); \
		853	} \
		854	} while (0)
		855
		856	/* We force the use of the SSE xor block because it can write around L2.
		857	We may also be able to load into the L1 only depending on how the cpu
		858	deals with a load to a line that is being prefetched. */
		859	#define XOR_SELECT_TEMPLATE(FASTEST) \
		860	(cpu_has_xmm ? &xor_block_pIII_sse : FASTEST)

Subversion Repositories shark

(root)/shark/branches/xen/drivers/linuxc26/include/asm/xor.h - Rev 1618