x86/rgb2rgb_template.c
Go to the documentation of this file.
1 /*
2  * software RGB to RGB converter
3  * pluralize by software PAL8 to RGB converter
4  * software YUV to YUV converter
5  * software YUV to RGB converter
6  * Written by Nick Kurshev.
7  * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
8  * lot of big-endian byte order fixes by Alex Beregszaszi
9  *
10  * This file is part of FFmpeg.
11  *
12  * FFmpeg is free software; you can redistribute it and/or
13  * modify it under the terms of the GNU Lesser General Public
14  * License as published by the Free Software Foundation; either
15  * version 2.1 of the License, or (at your option) any later version.
16  *
17  * FFmpeg is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20  * Lesser General Public License for more details.
21  *
22  * You should have received a copy of the GNU Lesser General Public
23  * License along with FFmpeg; if not, write to the Free Software
24  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25  */
26 
27 #include <stddef.h>
28 
29 #undef PREFETCH
30 #undef MOVNTQ
31 #undef EMMS
32 #undef SFENCE
33 #undef PAVGB
34 
35 #if COMPILE_TEMPLATE_AMD3DNOW
36 #define PREFETCH "prefetch"
37 #define PAVGB "pavgusb"
38 #elif COMPILE_TEMPLATE_MMXEXT
39 #define PREFETCH "prefetchnta"
40 #define PAVGB "pavgb"
41 #else
42 #define PREFETCH " # nop"
43 #endif
44 
45 #if COMPILE_TEMPLATE_AMD3DNOW
46 /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
47 #define EMMS "femms"
48 #else
49 #define EMMS "emms"
50 #endif
51 
52 #if COMPILE_TEMPLATE_MMXEXT
53 #define MOVNTQ "movntq"
54 #define SFENCE "sfence"
55 #else
56 #define MOVNTQ "movq"
57 #define SFENCE " # nop"
58 #endif
59 
60 #if !COMPILE_TEMPLATE_SSE2
61 
62 #if !COMPILE_TEMPLATE_AMD3DNOW
63 
64 static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, int src_size)
65 {
66  uint8_t *dest = dst;
67  const uint8_t *s = src;
68  const uint8_t *end;
69  const uint8_t *mm_end;
70  end = s + src_size;
71  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
72  mm_end = end - 23;
73  __asm__ volatile("movq %0, %%mm7"::"m"(mask32a):"memory");
74  while (s < mm_end) {
75  __asm__ volatile(
76  PREFETCH" 32(%1) \n\t"
77  "movd (%1), %%mm0 \n\t"
78  "punpckldq 3(%1), %%mm0 \n\t"
79  "movd 6(%1), %%mm1 \n\t"
80  "punpckldq 9(%1), %%mm1 \n\t"
81  "movd 12(%1), %%mm2 \n\t"
82  "punpckldq 15(%1), %%mm2 \n\t"
83  "movd 18(%1), %%mm3 \n\t"
84  "punpckldq 21(%1), %%mm3 \n\t"
85  "por %%mm7, %%mm0 \n\t"
86  "por %%mm7, %%mm1 \n\t"
87  "por %%mm7, %%mm2 \n\t"
88  "por %%mm7, %%mm3 \n\t"
89  MOVNTQ" %%mm0, (%0) \n\t"
90  MOVNTQ" %%mm1, 8(%0) \n\t"
91  MOVNTQ" %%mm2, 16(%0) \n\t"
92  MOVNTQ" %%mm3, 24(%0)"
93  :: "r"(dest), "r"(s)
94  :"memory");
95  dest += 32;
96  s += 24;
97  }
98  __asm__ volatile(SFENCE:::"memory");
99  __asm__ volatile(EMMS:::"memory");
100  while (s < end) {
101  *dest++ = *s++;
102  *dest++ = *s++;
103  *dest++ = *s++;
104  *dest++ = 255;
105  }
106 }
107 
108 #define STORE_BGR24_MMX \
109  "psrlq $8, %%mm2 \n\t" \
110  "psrlq $8, %%mm3 \n\t" \
111  "psrlq $8, %%mm6 \n\t" \
112  "psrlq $8, %%mm7 \n\t" \
113  "pand "MANGLE(mask24l)", %%mm0\n\t" \
114  "pand "MANGLE(mask24l)", %%mm1\n\t" \
115  "pand "MANGLE(mask24l)", %%mm4\n\t" \
116  "pand "MANGLE(mask24l)", %%mm5\n\t" \
117  "pand "MANGLE(mask24h)", %%mm2\n\t" \
118  "pand "MANGLE(mask24h)", %%mm3\n\t" \
119  "pand "MANGLE(mask24h)", %%mm6\n\t" \
120  "pand "MANGLE(mask24h)", %%mm7\n\t" \
121  "por %%mm2, %%mm0 \n\t" \
122  "por %%mm3, %%mm1 \n\t" \
123  "por %%mm6, %%mm4 \n\t" \
124  "por %%mm7, %%mm5 \n\t" \
125  \
126  "movq %%mm1, %%mm2 \n\t" \
127  "movq %%mm4, %%mm3 \n\t" \
128  "psllq $48, %%mm2 \n\t" \
129  "psllq $32, %%mm3 \n\t" \
130  "por %%mm2, %%mm0 \n\t" \
131  "psrlq $16, %%mm1 \n\t" \
132  "psrlq $32, %%mm4 \n\t" \
133  "psllq $16, %%mm5 \n\t" \
134  "por %%mm3, %%mm1 \n\t" \
135  "por %%mm5, %%mm4 \n\t" \
136  \
137  MOVNTQ" %%mm0, (%0) \n\t" \
138  MOVNTQ" %%mm1, 8(%0) \n\t" \
139  MOVNTQ" %%mm4, 16(%0)"
140 
141 
142 static inline void RENAME(rgb32tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
143 {
144  uint8_t *dest = dst;
145  const uint8_t *s = src;
146  const uint8_t *end;
147  const uint8_t *mm_end;
148  end = s + src_size;
149  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
150  mm_end = end - 31;
151  while (s < mm_end) {
152  __asm__ volatile(
153  PREFETCH" 32(%1) \n\t"
154  "movq (%1), %%mm0 \n\t"
155  "movq 8(%1), %%mm1 \n\t"
156  "movq 16(%1), %%mm4 \n\t"
157  "movq 24(%1), %%mm5 \n\t"
158  "movq %%mm0, %%mm2 \n\t"
159  "movq %%mm1, %%mm3 \n\t"
160  "movq %%mm4, %%mm6 \n\t"
161  "movq %%mm5, %%mm7 \n\t"
162  STORE_BGR24_MMX
163  :: "r"(dest), "r"(s)
164  :"memory");
165  dest += 24;
166  s += 32;
167  }
168  __asm__ volatile(SFENCE:::"memory");
169  __asm__ volatile(EMMS:::"memory");
170  while (s < end) {
171  *dest++ = *s++;
172  *dest++ = *s++;
173  *dest++ = *s++;
174  s++;
175  }
176 }
177 
178 /*
179  original by Strepto/Astral
180  ported to gcc & bugfixed: A'rpi
181  MMXEXT, 3DNOW optimization by Nick Kurshev
182  32-bit C version, and and&add trick by Michael Niedermayer
183 */
184 static inline void RENAME(rgb15to16)(const uint8_t *src, uint8_t *dst, int src_size)
185 {
186  register const uint8_t* s=src;
187  register uint8_t* d=dst;
188  register const uint8_t *end;
189  const uint8_t *mm_end;
190  end = s + src_size;
191  __asm__ volatile(PREFETCH" %0"::"m"(*s));
192  __asm__ volatile("movq %0, %%mm4"::"m"(mask15s));
193  mm_end = end - 15;
194  while (s<mm_end) {
195  __asm__ volatile(
196  PREFETCH" 32(%1) \n\t"
197  "movq (%1), %%mm0 \n\t"
198  "movq 8(%1), %%mm2 \n\t"
199  "movq %%mm0, %%mm1 \n\t"
200  "movq %%mm2, %%mm3 \n\t"
201  "pand %%mm4, %%mm0 \n\t"
202  "pand %%mm4, %%mm2 \n\t"
203  "paddw %%mm1, %%mm0 \n\t"
204  "paddw %%mm3, %%mm2 \n\t"
205  MOVNTQ" %%mm0, (%0) \n\t"
206  MOVNTQ" %%mm2, 8(%0)"
207  :: "r"(d), "r"(s)
208  );
209  d+=16;
210  s+=16;
211  }
212  __asm__ volatile(SFENCE:::"memory");
213  __asm__ volatile(EMMS:::"memory");
214  mm_end = end - 3;
215  while (s < mm_end) {
216  register unsigned x= *((const uint32_t *)s);
217  *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
218  d+=4;
219  s+=4;
220  }
221  if (s < end) {
222  register unsigned short x= *((const uint16_t *)s);
223  *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
224  }
225 }
226 
227 static inline void RENAME(rgb16to15)(const uint8_t *src, uint8_t *dst, int src_size)
228 {
229  register const uint8_t* s=src;
230  register uint8_t* d=dst;
231  register const uint8_t *end;
232  const uint8_t *mm_end;
233  end = s + src_size;
234  __asm__ volatile(PREFETCH" %0"::"m"(*s));
235  __asm__ volatile("movq %0, %%mm7"::"m"(mask15rg));
236  __asm__ volatile("movq %0, %%mm6"::"m"(mask15b));
237  mm_end = end - 15;
238  while (s<mm_end) {
239  __asm__ volatile(
240  PREFETCH" 32(%1) \n\t"
241  "movq (%1), %%mm0 \n\t"
242  "movq 8(%1), %%mm2 \n\t"
243  "movq %%mm0, %%mm1 \n\t"
244  "movq %%mm2, %%mm3 \n\t"
245  "psrlq $1, %%mm0 \n\t"
246  "psrlq $1, %%mm2 \n\t"
247  "pand %%mm7, %%mm0 \n\t"
248  "pand %%mm7, %%mm2 \n\t"
249  "pand %%mm6, %%mm1 \n\t"
250  "pand %%mm6, %%mm3 \n\t"
251  "por %%mm1, %%mm0 \n\t"
252  "por %%mm3, %%mm2 \n\t"
253  MOVNTQ" %%mm0, (%0) \n\t"
254  MOVNTQ" %%mm2, 8(%0)"
255  :: "r"(d), "r"(s)
256  );
257  d+=16;
258  s+=16;
259  }
260  __asm__ volatile(SFENCE:::"memory");
261  __asm__ volatile(EMMS:::"memory");
262  mm_end = end - 3;
263  while (s < mm_end) {
264  register uint32_t x= *((const uint32_t*)s);
265  *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
266  s+=4;
267  d+=4;
268  }
269  if (s < end) {
270  register uint16_t x= *((const uint16_t*)s);
271  *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
272  }
273 }
274 
275 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, int src_size)
276 {
277  const uint8_t *s = src;
278  const uint8_t *end;
279  const uint8_t *mm_end;
280  uint16_t *d = (uint16_t *)dst;
281  end = s + src_size;
282  mm_end = end - 15;
283  __asm__ volatile(
284  "movq %3, %%mm5 \n\t"
285  "movq %4, %%mm6 \n\t"
286  "movq %5, %%mm7 \n\t"
287  "jmp 2f \n\t"
288  ".p2align 4 \n\t"
289  "1: \n\t"
290  PREFETCH" 32(%1) \n\t"
291  "movd (%1), %%mm0 \n\t"
292  "movd 4(%1), %%mm3 \n\t"
293  "punpckldq 8(%1), %%mm0 \n\t"
294  "punpckldq 12(%1), %%mm3 \n\t"
295  "movq %%mm0, %%mm1 \n\t"
296  "movq %%mm3, %%mm4 \n\t"
297  "pand %%mm6, %%mm0 \n\t"
298  "pand %%mm6, %%mm3 \n\t"
299  "pmaddwd %%mm7, %%mm0 \n\t"
300  "pmaddwd %%mm7, %%mm3 \n\t"
301  "pand %%mm5, %%mm1 \n\t"
302  "pand %%mm5, %%mm4 \n\t"
303  "por %%mm1, %%mm0 \n\t"
304  "por %%mm4, %%mm3 \n\t"
305  "psrld $5, %%mm0 \n\t"
306  "pslld $11, %%mm3 \n\t"
307  "por %%mm3, %%mm0 \n\t"
308  MOVNTQ" %%mm0, (%0) \n\t"
309  "add $16, %1 \n\t"
310  "add $8, %0 \n\t"
311  "2: \n\t"
312  "cmp %2, %1 \n\t"
313  " jb 1b \n\t"
314  : "+r" (d), "+r"(s)
315  : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
316  );
317  __asm__ volatile(SFENCE:::"memory");
318  __asm__ volatile(EMMS:::"memory");
319  while (s < end) {
320  register int rgb = *(const uint32_t*)s; s += 4;
321  *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
322  }
323 }
324 
325 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
326 {
327  const uint8_t *s = src;
328  const uint8_t *end;
329  const uint8_t *mm_end;
330  uint16_t *d = (uint16_t *)dst;
331  end = s + src_size;
332  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
333  __asm__ volatile(
334  "movq %0, %%mm7 \n\t"
335  "movq %1, %%mm6 \n\t"
336  ::"m"(red_16mask),"m"(green_16mask));
337  mm_end = end - 15;
338  while (s < mm_end) {
339  __asm__ volatile(
340  PREFETCH" 32(%1) \n\t"
341  "movd (%1), %%mm0 \n\t"
342  "movd 4(%1), %%mm3 \n\t"
343  "punpckldq 8(%1), %%mm0 \n\t"
344  "punpckldq 12(%1), %%mm3 \n\t"
345  "movq %%mm0, %%mm1 \n\t"
346  "movq %%mm0, %%mm2 \n\t"
347  "movq %%mm3, %%mm4 \n\t"
348  "movq %%mm3, %%mm5 \n\t"
349  "psllq $8, %%mm0 \n\t"
350  "psllq $8, %%mm3 \n\t"
351  "pand %%mm7, %%mm0 \n\t"
352  "pand %%mm7, %%mm3 \n\t"
353  "psrlq $5, %%mm1 \n\t"
354  "psrlq $5, %%mm4 \n\t"
355  "pand %%mm6, %%mm1 \n\t"
356  "pand %%mm6, %%mm4 \n\t"
357  "psrlq $19, %%mm2 \n\t"
358  "psrlq $19, %%mm5 \n\t"
359  "pand %2, %%mm2 \n\t"
360  "pand %2, %%mm5 \n\t"
361  "por %%mm1, %%mm0 \n\t"
362  "por %%mm4, %%mm3 \n\t"
363  "por %%mm2, %%mm0 \n\t"
364  "por %%mm5, %%mm3 \n\t"
365  "psllq $16, %%mm3 \n\t"
366  "por %%mm3, %%mm0 \n\t"
367  MOVNTQ" %%mm0, (%0) \n\t"
368  :: "r"(d),"r"(s),"m"(blue_16mask):"memory");
369  d += 4;
370  s += 16;
371  }
372  __asm__ volatile(SFENCE:::"memory");
373  __asm__ volatile(EMMS:::"memory");
374  while (s < end) {
375  register int rgb = *(const uint32_t*)s; s += 4;
376  *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
377  }
378 }
379 
380 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, int src_size)
381 {
382  const uint8_t *s = src;
383  const uint8_t *end;
384  const uint8_t *mm_end;
385  uint16_t *d = (uint16_t *)dst;
386  end = s + src_size;
387  mm_end = end - 15;
388  __asm__ volatile(
389  "movq %3, %%mm5 \n\t"
390  "movq %4, %%mm6 \n\t"
391  "movq %5, %%mm7 \n\t"
392  "jmp 2f \n\t"
393  ".p2align 4 \n\t"
394  "1: \n\t"
395  PREFETCH" 32(%1) \n\t"
396  "movd (%1), %%mm0 \n\t"
397  "movd 4(%1), %%mm3 \n\t"
398  "punpckldq 8(%1), %%mm0 \n\t"
399  "punpckldq 12(%1), %%mm3 \n\t"
400  "movq %%mm0, %%mm1 \n\t"
401  "movq %%mm3, %%mm4 \n\t"
402  "pand %%mm6, %%mm0 \n\t"
403  "pand %%mm6, %%mm3 \n\t"
404  "pmaddwd %%mm7, %%mm0 \n\t"
405  "pmaddwd %%mm7, %%mm3 \n\t"
406  "pand %%mm5, %%mm1 \n\t"
407  "pand %%mm5, %%mm4 \n\t"
408  "por %%mm1, %%mm0 \n\t"
409  "por %%mm4, %%mm3 \n\t"
410  "psrld $6, %%mm0 \n\t"
411  "pslld $10, %%mm3 \n\t"
412  "por %%mm3, %%mm0 \n\t"
413  MOVNTQ" %%mm0, (%0) \n\t"
414  "add $16, %1 \n\t"
415  "add $8, %0 \n\t"
416  "2: \n\t"
417  "cmp %2, %1 \n\t"
418  " jb 1b \n\t"
419  : "+r" (d), "+r"(s)
420  : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
421  );
422  __asm__ volatile(SFENCE:::"memory");
423  __asm__ volatile(EMMS:::"memory");
424  while (s < end) {
425  register int rgb = *(const uint32_t*)s; s += 4;
426  *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
427  }
428 }
429 
430 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
431 {
432  const uint8_t *s = src;
433  const uint8_t *end;
434  const uint8_t *mm_end;
435  uint16_t *d = (uint16_t *)dst;
436  end = s + src_size;
437  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
438  __asm__ volatile(
439  "movq %0, %%mm7 \n\t"
440  "movq %1, %%mm6 \n\t"
441  ::"m"(red_15mask),"m"(green_15mask));
442  mm_end = end - 15;
443  while (s < mm_end) {
444  __asm__ volatile(
445  PREFETCH" 32(%1) \n\t"
446  "movd (%1), %%mm0 \n\t"
447  "movd 4(%1), %%mm3 \n\t"
448  "punpckldq 8(%1), %%mm0 \n\t"
449  "punpckldq 12(%1), %%mm3 \n\t"
450  "movq %%mm0, %%mm1 \n\t"
451  "movq %%mm0, %%mm2 \n\t"
452  "movq %%mm3, %%mm4 \n\t"
453  "movq %%mm3, %%mm5 \n\t"
454  "psllq $7, %%mm0 \n\t"
455  "psllq $7, %%mm3 \n\t"
456  "pand %%mm7, %%mm0 \n\t"
457  "pand %%mm7, %%mm3 \n\t"
458  "psrlq $6, %%mm1 \n\t"
459  "psrlq $6, %%mm4 \n\t"
460  "pand %%mm6, %%mm1 \n\t"
461  "pand %%mm6, %%mm4 \n\t"
462  "psrlq $19, %%mm2 \n\t"
463  "psrlq $19, %%mm5 \n\t"
464  "pand %2, %%mm2 \n\t"
465  "pand %2, %%mm5 \n\t"
466  "por %%mm1, %%mm0 \n\t"
467  "por %%mm4, %%mm3 \n\t"
468  "por %%mm2, %%mm0 \n\t"
469  "por %%mm5, %%mm3 \n\t"
470  "psllq $16, %%mm3 \n\t"
471  "por %%mm3, %%mm0 \n\t"
472  MOVNTQ" %%mm0, (%0) \n\t"
473  ::"r"(d),"r"(s),"m"(blue_15mask):"memory");
474  d += 4;
475  s += 16;
476  }
477  __asm__ volatile(SFENCE:::"memory");
478  __asm__ volatile(EMMS:::"memory");
479  while (s < end) {
480  register int rgb = *(const uint32_t*)s; s += 4;
481  *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
482  }
483 }
484 
485 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
486 {
487  const uint8_t *s = src;
488  const uint8_t *end;
489  const uint8_t *mm_end;
490  uint16_t *d = (uint16_t *)dst;
491  end = s + src_size;
492  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
493  __asm__ volatile(
494  "movq %0, %%mm7 \n\t"
495  "movq %1, %%mm6 \n\t"
496  ::"m"(red_16mask),"m"(green_16mask));
497  mm_end = end - 11;
498  while (s < mm_end) {
499  __asm__ volatile(
500  PREFETCH" 32(%1) \n\t"
501  "movd (%1), %%mm0 \n\t"
502  "movd 3(%1), %%mm3 \n\t"
503  "punpckldq 6(%1), %%mm0 \n\t"
504  "punpckldq 9(%1), %%mm3 \n\t"
505  "movq %%mm0, %%mm1 \n\t"
506  "movq %%mm0, %%mm2 \n\t"
507  "movq %%mm3, %%mm4 \n\t"
508  "movq %%mm3, %%mm5 \n\t"
509  "psrlq $3, %%mm0 \n\t"
510  "psrlq $3, %%mm3 \n\t"
511  "pand %2, %%mm0 \n\t"
512  "pand %2, %%mm3 \n\t"
513  "psrlq $5, %%mm1 \n\t"
514  "psrlq $5, %%mm4 \n\t"
515  "pand %%mm6, %%mm1 \n\t"
516  "pand %%mm6, %%mm4 \n\t"
517  "psrlq $8, %%mm2 \n\t"
518  "psrlq $8, %%mm5 \n\t"
519  "pand %%mm7, %%mm2 \n\t"
520  "pand %%mm7, %%mm5 \n\t"
521  "por %%mm1, %%mm0 \n\t"
522  "por %%mm4, %%mm3 \n\t"
523  "por %%mm2, %%mm0 \n\t"
524  "por %%mm5, %%mm3 \n\t"
525  "psllq $16, %%mm3 \n\t"
526  "por %%mm3, %%mm0 \n\t"
527  MOVNTQ" %%mm0, (%0) \n\t"
528  ::"r"(d),"r"(s),"m"(blue_16mask):"memory");
529  d += 4;
530  s += 12;
531  }
532  __asm__ volatile(SFENCE:::"memory");
533  __asm__ volatile(EMMS:::"memory");
534  while (s < end) {
535  const int b = *s++;
536  const int g = *s++;
537  const int r = *s++;
538  *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
539  }
540 }
541 
542 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, int src_size)
543 {
544  const uint8_t *s = src;
545  const uint8_t *end;
546  const uint8_t *mm_end;
547  uint16_t *d = (uint16_t *)dst;
548  end = s + src_size;
549  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
550  __asm__ volatile(
551  "movq %0, %%mm7 \n\t"
552  "movq %1, %%mm6 \n\t"
553  ::"m"(red_16mask),"m"(green_16mask));
554  mm_end = end - 15;
555  while (s < mm_end) {
556  __asm__ volatile(
557  PREFETCH" 32(%1) \n\t"
558  "movd (%1), %%mm0 \n\t"
559  "movd 3(%1), %%mm3 \n\t"
560  "punpckldq 6(%1), %%mm0 \n\t"
561  "punpckldq 9(%1), %%mm3 \n\t"
562  "movq %%mm0, %%mm1 \n\t"
563  "movq %%mm0, %%mm2 \n\t"
564  "movq %%mm3, %%mm4 \n\t"
565  "movq %%mm3, %%mm5 \n\t"
566  "psllq $8, %%mm0 \n\t"
567  "psllq $8, %%mm3 \n\t"
568  "pand %%mm7, %%mm0 \n\t"
569  "pand %%mm7, %%mm3 \n\t"
570  "psrlq $5, %%mm1 \n\t"
571  "psrlq $5, %%mm4 \n\t"
572  "pand %%mm6, %%mm1 \n\t"
573  "pand %%mm6, %%mm4 \n\t"
574  "psrlq $19, %%mm2 \n\t"
575  "psrlq $19, %%mm5 \n\t"
576  "pand %2, %%mm2 \n\t"
577  "pand %2, %%mm5 \n\t"
578  "por %%mm1, %%mm0 \n\t"
579  "por %%mm4, %%mm3 \n\t"
580  "por %%mm2, %%mm0 \n\t"
581  "por %%mm5, %%mm3 \n\t"
582  "psllq $16, %%mm3 \n\t"
583  "por %%mm3, %%mm0 \n\t"
584  MOVNTQ" %%mm0, (%0) \n\t"
585  ::"r"(d),"r"(s),"m"(blue_16mask):"memory");
586  d += 4;
587  s += 12;
588  }
589  __asm__ volatile(SFENCE:::"memory");
590  __asm__ volatile(EMMS:::"memory");
591  while (s < end) {
592  const int r = *s++;
593  const int g = *s++;
594  const int b = *s++;
595  *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
596  }
597 }
598 
599 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
600 {
601  const uint8_t *s = src;
602  const uint8_t *end;
603  const uint8_t *mm_end;
604  uint16_t *d = (uint16_t *)dst;
605  end = s + src_size;
606  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
607  __asm__ volatile(
608  "movq %0, %%mm7 \n\t"
609  "movq %1, %%mm6 \n\t"
610  ::"m"(red_15mask),"m"(green_15mask));
611  mm_end = end - 11;
612  while (s < mm_end) {
613  __asm__ volatile(
614  PREFETCH" 32(%1) \n\t"
615  "movd (%1), %%mm0 \n\t"
616  "movd 3(%1), %%mm3 \n\t"
617  "punpckldq 6(%1), %%mm0 \n\t"
618  "punpckldq 9(%1), %%mm3 \n\t"
619  "movq %%mm0, %%mm1 \n\t"
620  "movq %%mm0, %%mm2 \n\t"
621  "movq %%mm3, %%mm4 \n\t"
622  "movq %%mm3, %%mm5 \n\t"
623  "psrlq $3, %%mm0 \n\t"
624  "psrlq $3, %%mm3 \n\t"
625  "pand %2, %%mm0 \n\t"
626  "pand %2, %%mm3 \n\t"
627  "psrlq $6, %%mm1 \n\t"
628  "psrlq $6, %%mm4 \n\t"
629  "pand %%mm6, %%mm1 \n\t"
630  "pand %%mm6, %%mm4 \n\t"
631  "psrlq $9, %%mm2 \n\t"
632  "psrlq $9, %%mm5 \n\t"
633  "pand %%mm7, %%mm2 \n\t"
634  "pand %%mm7, %%mm5 \n\t"
635  "por %%mm1, %%mm0 \n\t"
636  "por %%mm4, %%mm3 \n\t"
637  "por %%mm2, %%mm0 \n\t"
638  "por %%mm5, %%mm3 \n\t"
639  "psllq $16, %%mm3 \n\t"
640  "por %%mm3, %%mm0 \n\t"
641  MOVNTQ" %%mm0, (%0) \n\t"
642  ::"r"(d),"r"(s),"m"(blue_15mask):"memory");
643  d += 4;
644  s += 12;
645  }
646  __asm__ volatile(SFENCE:::"memory");
647  __asm__ volatile(EMMS:::"memory");
648  while (s < end) {
649  const int b = *s++;
650  const int g = *s++;
651  const int r = *s++;
652  *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
653  }
654 }
655 
656 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, int src_size)
657 {
658  const uint8_t *s = src;
659  const uint8_t *end;
660  const uint8_t *mm_end;
661  uint16_t *d = (uint16_t *)dst;
662  end = s + src_size;
663  __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory");
664  __asm__ volatile(
665  "movq %0, %%mm7 \n\t"
666  "movq %1, %%mm6 \n\t"
667  ::"m"(red_15mask),"m"(green_15mask));
668  mm_end = end - 15;
669  while (s < mm_end) {
670  __asm__ volatile(
671  PREFETCH" 32(%1) \n\t"
672  "movd (%1), %%mm0 \n\t"
673  "movd 3(%1), %%mm3 \n\t"
674  "punpckldq 6(%1), %%mm0 \n\t"
675  "punpckldq 9(%1), %%mm3 \n\t"
676  "movq %%mm0, %%mm1 \n\t"
677  "movq %%mm0, %%mm2 \n\t"
678  "movq %%mm3, %%mm4 \n\t"
679  "movq %%mm3, %%mm5 \n\t"
680  "psllq $7, %%mm0 \n\t"
681  "psllq $7, %%mm3 \n\t"
682  "pand %%mm7, %%mm0 \n\t"
683  "pand %%mm7, %%mm3 \n\t"
684  "psrlq $6, %%mm1 \n\t"
685  "psrlq $6, %%mm4 \n\t"
686  "pand %%mm6, %%mm1 \n\t"
687  "pand %%mm6, %%mm4 \n\t"
688  "psrlq $19, %%mm2 \n\t"
689  "psrlq $19, %%mm5 \n\t"
690  "pand %2, %%mm2 \n\t"
691  "pand %2, %%mm5 \n\t"
692  "por %%mm1, %%mm0 \n\t"
693  "por %%mm4, %%mm3 \n\t"
694  "por %%mm2, %%mm0 \n\t"
695  "por %%mm5, %%mm3 \n\t"
696  "psllq $16, %%mm3 \n\t"
697  "por %%mm3, %%mm0 \n\t"
698  MOVNTQ" %%mm0, (%0) \n\t"
699  ::"r"(d),"r"(s),"m"(blue_15mask):"memory");
700  d += 4;
701  s += 12;
702  }
703  __asm__ volatile(SFENCE:::"memory");
704  __asm__ volatile(EMMS:::"memory");
705  while (s < end) {
706  const int r = *s++;
707  const int g = *s++;
708  const int b = *s++;
709  *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
710  }
711 }
712 
713 static inline void RENAME(rgb15tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
714 {
715  const uint16_t *end;
716  const uint16_t *mm_end;
717  uint8_t *d = dst;
718  const uint16_t *s = (const uint16_t*)src;
719  end = s + src_size/2;
720  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
721  mm_end = end - 7;
722  while (s < mm_end) {
723  __asm__ volatile(
724  PREFETCH" 32(%1) \n\t"
725  "movq (%1), %%mm0 \n\t"
726  "movq (%1), %%mm1 \n\t"
727  "movq (%1), %%mm2 \n\t"
728  "pand %2, %%mm0 \n\t"
729  "pand %3, %%mm1 \n\t"
730  "pand %4, %%mm2 \n\t"
731  "psllq $5, %%mm0 \n\t"
732  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
733  "pmulhw "MANGLE(mul15_mid)", %%mm1 \n\t"
734  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
735  "movq %%mm0, %%mm3 \n\t"
736  "movq %%mm1, %%mm4 \n\t"
737  "movq %%mm2, %%mm5 \n\t"
738  "punpcklwd %5, %%mm0 \n\t"
739  "punpcklwd %5, %%mm1 \n\t"
740  "punpcklwd %5, %%mm2 \n\t"
741  "punpckhwd %5, %%mm3 \n\t"
742  "punpckhwd %5, %%mm4 \n\t"
743  "punpckhwd %5, %%mm5 \n\t"
744  "psllq $8, %%mm1 \n\t"
745  "psllq $16, %%mm2 \n\t"
746  "por %%mm1, %%mm0 \n\t"
747  "por %%mm2, %%mm0 \n\t"
748  "psllq $8, %%mm4 \n\t"
749  "psllq $16, %%mm5 \n\t"
750  "por %%mm4, %%mm3 \n\t"
751  "por %%mm5, %%mm3 \n\t"
752 
753  "movq %%mm0, %%mm6 \n\t"
754  "movq %%mm3, %%mm7 \n\t"
755 
756  "movq 8(%1), %%mm0 \n\t"
757  "movq 8(%1), %%mm1 \n\t"
758  "movq 8(%1), %%mm2 \n\t"
759  "pand %2, %%mm0 \n\t"
760  "pand %3, %%mm1 \n\t"
761  "pand %4, %%mm2 \n\t"
762  "psllq $5, %%mm0 \n\t"
763  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
764  "pmulhw "MANGLE(mul15_mid)", %%mm1 \n\t"
765  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
766  "movq %%mm0, %%mm3 \n\t"
767  "movq %%mm1, %%mm4 \n\t"
768  "movq %%mm2, %%mm5 \n\t"
769  "punpcklwd %5, %%mm0 \n\t"
770  "punpcklwd %5, %%mm1 \n\t"
771  "punpcklwd %5, %%mm2 \n\t"
772  "punpckhwd %5, %%mm3 \n\t"
773  "punpckhwd %5, %%mm4 \n\t"
774  "punpckhwd %5, %%mm5 \n\t"
775  "psllq $8, %%mm1 \n\t"
776  "psllq $16, %%mm2 \n\t"
777  "por %%mm1, %%mm0 \n\t"
778  "por %%mm2, %%mm0 \n\t"
779  "psllq $8, %%mm4 \n\t"
780  "psllq $16, %%mm5 \n\t"
781  "por %%mm4, %%mm3 \n\t"
782  "por %%mm5, %%mm3 \n\t"
783 
784  :"=m"(*d)
785  :"r"(s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
786  :"memory");
787  /* borrowed 32 to 24 */
788  __asm__ volatile(
789  "movq %%mm0, %%mm4 \n\t"
790  "movq %%mm3, %%mm5 \n\t"
791  "movq %%mm6, %%mm0 \n\t"
792  "movq %%mm7, %%mm1 \n\t"
793 
794  "movq %%mm4, %%mm6 \n\t"
795  "movq %%mm5, %%mm7 \n\t"
796  "movq %%mm0, %%mm2 \n\t"
797  "movq %%mm1, %%mm3 \n\t"
798 
799  STORE_BGR24_MMX
800 
801  :: "r"(d), "m"(*s)
802  :"memory");
803  d += 24;
804  s += 8;
805  }
806  __asm__ volatile(SFENCE:::"memory");
807  __asm__ volatile(EMMS:::"memory");
808  while (s < end) {
809  register uint16_t bgr;
810  bgr = *s++;
811  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
812  *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
813  *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
814  }
815 }
816 
817 static inline void RENAME(rgb16tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
818 {
819  const uint16_t *end;
820  const uint16_t *mm_end;
821  uint8_t *d = (uint8_t *)dst;
822  const uint16_t *s = (const uint16_t *)src;
823  end = s + src_size/2;
824  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
825  mm_end = end - 7;
826  while (s < mm_end) {
827  __asm__ volatile(
828  PREFETCH" 32(%1) \n\t"
829  "movq (%1), %%mm0 \n\t"
830  "movq (%1), %%mm1 \n\t"
831  "movq (%1), %%mm2 \n\t"
832  "pand %2, %%mm0 \n\t"
833  "pand %3, %%mm1 \n\t"
834  "pand %4, %%mm2 \n\t"
835  "psllq $5, %%mm0 \n\t"
836  "psrlq $1, %%mm2 \n\t"
837  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
838  "pmulhw "MANGLE(mul16_mid)", %%mm1 \n\t"
839  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
840  "movq %%mm0, %%mm3 \n\t"
841  "movq %%mm1, %%mm4 \n\t"
842  "movq %%mm2, %%mm5 \n\t"
843  "punpcklwd %5, %%mm0 \n\t"
844  "punpcklwd %5, %%mm1 \n\t"
845  "punpcklwd %5, %%mm2 \n\t"
846  "punpckhwd %5, %%mm3 \n\t"
847  "punpckhwd %5, %%mm4 \n\t"
848  "punpckhwd %5, %%mm5 \n\t"
849  "psllq $8, %%mm1 \n\t"
850  "psllq $16, %%mm2 \n\t"
851  "por %%mm1, %%mm0 \n\t"
852  "por %%mm2, %%mm0 \n\t"
853  "psllq $8, %%mm4 \n\t"
854  "psllq $16, %%mm5 \n\t"
855  "por %%mm4, %%mm3 \n\t"
856  "por %%mm5, %%mm3 \n\t"
857 
858  "movq %%mm0, %%mm6 \n\t"
859  "movq %%mm3, %%mm7 \n\t"
860 
861  "movq 8(%1), %%mm0 \n\t"
862  "movq 8(%1), %%mm1 \n\t"
863  "movq 8(%1), %%mm2 \n\t"
864  "pand %2, %%mm0 \n\t"
865  "pand %3, %%mm1 \n\t"
866  "pand %4, %%mm2 \n\t"
867  "psllq $5, %%mm0 \n\t"
868  "psrlq $1, %%mm2 \n\t"
869  "pmulhw "MANGLE(mul15_mid)", %%mm0 \n\t"
870  "pmulhw "MANGLE(mul16_mid)", %%mm1 \n\t"
871  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
872  "movq %%mm0, %%mm3 \n\t"
873  "movq %%mm1, %%mm4 \n\t"
874  "movq %%mm2, %%mm5 \n\t"
875  "punpcklwd %5, %%mm0 \n\t"
876  "punpcklwd %5, %%mm1 \n\t"
877  "punpcklwd %5, %%mm2 \n\t"
878  "punpckhwd %5, %%mm3 \n\t"
879  "punpckhwd %5, %%mm4 \n\t"
880  "punpckhwd %5, %%mm5 \n\t"
881  "psllq $8, %%mm1 \n\t"
882  "psllq $16, %%mm2 \n\t"
883  "por %%mm1, %%mm0 \n\t"
884  "por %%mm2, %%mm0 \n\t"
885  "psllq $8, %%mm4 \n\t"
886  "psllq $16, %%mm5 \n\t"
887  "por %%mm4, %%mm3 \n\t"
888  "por %%mm5, %%mm3 \n\t"
889  :"=m"(*d)
890  :"r"(s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)
891  :"memory");
892  /* borrowed 32 to 24 */
893  __asm__ volatile(
894  "movq %%mm0, %%mm4 \n\t"
895  "movq %%mm3, %%mm5 \n\t"
896  "movq %%mm6, %%mm0 \n\t"
897  "movq %%mm7, %%mm1 \n\t"
898 
899  "movq %%mm4, %%mm6 \n\t"
900  "movq %%mm5, %%mm7 \n\t"
901  "movq %%mm0, %%mm2 \n\t"
902  "movq %%mm1, %%mm3 \n\t"
903 
904  STORE_BGR24_MMX
905 
906  :: "r"(d), "m"(*s)
907  :"memory");
908  d += 24;
909  s += 8;
910  }
911  __asm__ volatile(SFENCE:::"memory");
912  __asm__ volatile(EMMS:::"memory");
913  while (s < end) {
914  register uint16_t bgr;
915  bgr = *s++;
916  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
917  *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
918  *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
919  }
920 }
921 
922 /*
923  * mm0 = 00 B3 00 B2 00 B1 00 B0
924  * mm1 = 00 G3 00 G2 00 G1 00 G0
925  * mm2 = 00 R3 00 R2 00 R1 00 R0
926  * mm6 = FF FF FF FF FF FF FF FF
927  * mm7 = 00 00 00 00 00 00 00 00
928  */
929 #define PACK_RGB32 \
930  "packuswb %%mm7, %%mm0 \n\t" /* 00 00 00 00 B3 B2 B1 B0 */ \
931  "packuswb %%mm7, %%mm1 \n\t" /* 00 00 00 00 G3 G2 G1 G0 */ \
932  "packuswb %%mm7, %%mm2 \n\t" /* 00 00 00 00 R3 R2 R1 R0 */ \
933  "punpcklbw %%mm1, %%mm0 \n\t" /* G3 B3 G2 B2 G1 B1 G0 B0 */ \
934  "punpcklbw %%mm6, %%mm2 \n\t" /* FF R3 FF R2 FF R1 FF R0 */ \
935  "movq %%mm0, %%mm3 \n\t" \
936  "punpcklwd %%mm2, %%mm0 \n\t" /* FF R1 G1 B1 FF R0 G0 B0 */ \
937  "punpckhwd %%mm2, %%mm3 \n\t" /* FF R3 G3 B3 FF R2 G2 B2 */ \
938  MOVNTQ" %%mm0, (%0) \n\t" \
939  MOVNTQ" %%mm3, 8(%0) \n\t" \
940 
941 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, int src_size)
942 {
943  const uint16_t *end;
944  const uint16_t *mm_end;
945  uint8_t *d = dst;
946  const uint16_t *s = (const uint16_t *)src;
947  end = s + src_size/2;
948  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
949  __asm__ volatile("pxor %%mm7,%%mm7 \n\t":::"memory");
950  __asm__ volatile("pcmpeqd %%mm6,%%mm6 \n\t":::"memory");
951  mm_end = end - 3;
952  while (s < mm_end) {
953  __asm__ volatile(
954  PREFETCH" 32(%1) \n\t"
955  "movq (%1), %%mm0 \n\t"
956  "movq (%1), %%mm1 \n\t"
957  "movq (%1), %%mm2 \n\t"
958  "pand %2, %%mm0 \n\t"
959  "pand %3, %%mm1 \n\t"
960  "pand %4, %%mm2 \n\t"
961  "psllq $5, %%mm0 \n\t"
962  "pmulhw %5, %%mm0 \n\t"
963  "pmulhw %5, %%mm1 \n\t"
964  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
965  PACK_RGB32
966  ::"r"(d),"r"(s),"m"(mask15b),"m"(mask15g),"m"(mask15r) ,"m"(mul15_mid)
967  :"memory");
968  d += 16;
969  s += 4;
970  }
971  __asm__ volatile(SFENCE:::"memory");
972  __asm__ volatile(EMMS:::"memory");
973  while (s < end) {
974  register uint16_t bgr;
975  bgr = *s++;
976  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
977  *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
978  *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
979  *d++ = 255;
980  }
981 }
982 
983 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, int src_size)
984 {
985  const uint16_t *end;
986  const uint16_t *mm_end;
987  uint8_t *d = dst;
988  const uint16_t *s = (const uint16_t*)src;
989  end = s + src_size/2;
990  __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory");
991  __asm__ volatile("pxor %%mm7,%%mm7 \n\t":::"memory");
992  __asm__ volatile("pcmpeqd %%mm6,%%mm6 \n\t":::"memory");
993  mm_end = end - 3;
994  while (s < mm_end) {
995  __asm__ volatile(
996  PREFETCH" 32(%1) \n\t"
997  "movq (%1), %%mm0 \n\t"
998  "movq (%1), %%mm1 \n\t"
999  "movq (%1), %%mm2 \n\t"
1000  "pand %2, %%mm0 \n\t"
1001  "pand %3, %%mm1 \n\t"
1002  "pand %4, %%mm2 \n\t"
1003  "psllq $5, %%mm0 \n\t"
1004  "psrlq $1, %%mm2 \n\t"
1005  "pmulhw %5, %%mm0 \n\t"
1006  "pmulhw "MANGLE(mul16_mid)", %%mm1 \n\t"
1007  "pmulhw "MANGLE(mul15_hi)", %%mm2 \n\t"
1008  PACK_RGB32
1009  ::"r"(d),"r"(s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mul15_mid)
1010  :"memory");
1011  d += 16;
1012  s += 4;
1013  }
1014  __asm__ volatile(SFENCE:::"memory");
1015  __asm__ volatile(EMMS:::"memory");
1016  while (s < end) {
1017  register uint16_t bgr;
1018  bgr = *s++;
1019  *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
1020  *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
1021  *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
1022  *d++ = 255;
1023  }
1024 }
1025 
1026 static inline void RENAME(shuffle_bytes_2103)(const uint8_t *src, uint8_t *dst, int src_size)
1027 {
1028  x86_reg idx = 15 - src_size;
1029  const uint8_t *s = src-idx;
1030  uint8_t *d = dst-idx;
1031  __asm__ volatile(
1032  "test %0, %0 \n\t"
1033  "jns 2f \n\t"
1034  PREFETCH" (%1, %0) \n\t"
1035  "movq %3, %%mm7 \n\t"
1036  "pxor %4, %%mm7 \n\t"
1037  "movq %%mm7, %%mm6 \n\t"
1038  "pxor %5, %%mm7 \n\t"
1039  ".p2align 4 \n\t"
1040  "1: \n\t"
1041  PREFETCH" 32(%1, %0) \n\t"
1042  "movq (%1, %0), %%mm0 \n\t"
1043  "movq 8(%1, %0), %%mm1 \n\t"
1044 # if COMPILE_TEMPLATE_MMXEXT
1045  "pshufw $177, %%mm0, %%mm3 \n\t"
1046  "pshufw $177, %%mm1, %%mm5 \n\t"
1047  "pand %%mm7, %%mm0 \n\t"
1048  "pand %%mm6, %%mm3 \n\t"
1049  "pand %%mm7, %%mm1 \n\t"
1050  "pand %%mm6, %%mm5 \n\t"
1051  "por %%mm3, %%mm0 \n\t"
1052  "por %%mm5, %%mm1 \n\t"
1053 # else
1054  "movq %%mm0, %%mm2 \n\t"
1055  "movq %%mm1, %%mm4 \n\t"
1056  "pand %%mm7, %%mm0 \n\t"
1057  "pand %%mm6, %%mm2 \n\t"
1058  "pand %%mm7, %%mm1 \n\t"
1059  "pand %%mm6, %%mm4 \n\t"
1060  "movq %%mm2, %%mm3 \n\t"
1061  "movq %%mm4, %%mm5 \n\t"
1062  "pslld $16, %%mm2 \n\t"
1063  "psrld $16, %%mm3 \n\t"
1064  "pslld $16, %%mm4 \n\t"
1065  "psrld $16, %%mm5 \n\t"
1066  "por %%mm2, %%mm0 \n\t"
1067  "por %%mm4, %%mm1 \n\t"
1068  "por %%mm3, %%mm0 \n\t"
1069  "por %%mm5, %%mm1 \n\t"
1070 # endif
1071  MOVNTQ" %%mm0, (%2, %0) \n\t"
1072  MOVNTQ" %%mm1, 8(%2, %0) \n\t"
1073  "add $16, %0 \n\t"
1074  "js 1b \n\t"
1075  SFENCE" \n\t"
1076  EMMS" \n\t"
1077  "2: \n\t"
1078  : "+&r"(idx)
1079  : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
1080  : "memory");
1081  for (; idx<15; idx+=4) {
1082  register int v = *(const uint32_t *)&s[idx], g = v & 0xff00ff00;
1083  v &= 0xff00ff;
1084  *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);
1085  }
1086 }
1087 
1088 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
1089 {
1090  unsigned i;
1091  x86_reg mmx_size= 23 - src_size;
1092  __asm__ volatile (
1093  "test %%"REG_a", %%"REG_a" \n\t"
1094  "jns 2f \n\t"
1095  "movq "MANGLE(mask24r)", %%mm5 \n\t"
1096  "movq "MANGLE(mask24g)", %%mm6 \n\t"
1097  "movq "MANGLE(mask24b)", %%mm7 \n\t"
1098  ".p2align 4 \n\t"
1099  "1: \n\t"
1100  PREFETCH" 32(%1, %%"REG_a") \n\t"
1101  "movq (%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1102  "movq (%1, %%"REG_a"), %%mm1 \n\t" // BGR BGR BG
1103  "movq 2(%1, %%"REG_a"), %%mm2 \n\t" // R BGR BGR B
1104  "psllq $16, %%mm0 \n\t" // 00 BGR BGR
1105  "pand %%mm5, %%mm0 \n\t"
1106  "pand %%mm6, %%mm1 \n\t"
1107  "pand %%mm7, %%mm2 \n\t"
1108  "por %%mm0, %%mm1 \n\t"
1109  "por %%mm2, %%mm1 \n\t"
1110  "movq 6(%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1111  MOVNTQ" %%mm1, (%2, %%"REG_a") \n\t" // RGB RGB RG
1112  "movq 8(%1, %%"REG_a"), %%mm1 \n\t" // R BGR BGR B
1113  "movq 10(%1, %%"REG_a"), %%mm2 \n\t" // GR BGR BGR
1114  "pand %%mm7, %%mm0 \n\t"
1115  "pand %%mm5, %%mm1 \n\t"
1116  "pand %%mm6, %%mm2 \n\t"
1117  "por %%mm0, %%mm1 \n\t"
1118  "por %%mm2, %%mm1 \n\t"
1119  "movq 14(%1, %%"REG_a"), %%mm0 \n\t" // R BGR BGR B
1120  MOVNTQ" %%mm1, 8(%2, %%"REG_a") \n\t" // B RGB RGB R
1121  "movq 16(%1, %%"REG_a"), %%mm1 \n\t" // GR BGR BGR
1122  "movq 18(%1, %%"REG_a"), %%mm2 \n\t" // BGR BGR BG
1123  "pand %%mm6, %%mm0 \n\t"
1124  "pand %%mm7, %%mm1 \n\t"
1125  "pand %%mm5, %%mm2 \n\t"
1126  "por %%mm0, %%mm1 \n\t"
1127  "por %%mm2, %%mm1 \n\t"
1128  MOVNTQ" %%mm1, 16(%2, %%"REG_a") \n\t"
1129  "add $24, %%"REG_a" \n\t"
1130  " js 1b \n\t"
1131  "2: \n\t"
1132  : "+a" (mmx_size)
1133  : "r" (src-mmx_size), "r"(dst-mmx_size)
1134  );
1135 
1136  __asm__ volatile(SFENCE:::"memory");
1137  __asm__ volatile(EMMS:::"memory");
1138 
1139  if (mmx_size==23) return; //finished, was multiple of 8
1140 
1141  src+= src_size;
1142  dst+= src_size;
1143  src_size= 23-mmx_size;
1144  src-= src_size;
1145  dst-= src_size;
1146  for (i=0; i<src_size; i+=3) {
1147  register uint8_t x;
1148  x = src[i + 2];
1149  dst[i + 1] = src[i + 1];
1150  dst[i + 2] = src[i + 0];
1151  dst[i + 0] = x;
1152  }
1153 }
1154 
1155 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1156  int width, int height,
1157  int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
1158 {
1159  int y;
1160  const x86_reg chromWidth= width>>1;
1161  for (y=0; y<height; y++) {
1162  //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
1163  __asm__ volatile(
1164  "xor %%"REG_a", %%"REG_a" \n\t"
1165  ".p2align 4 \n\t"
1166  "1: \n\t"
1167  PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1168  PREFETCH" 32(%2, %%"REG_a") \n\t"
1169  PREFETCH" 32(%3, %%"REG_a") \n\t"
1170  "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1171  "movq %%mm0, %%mm2 \n\t" // U(0)
1172  "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1173  "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1174  "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1175 
1176  "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1177  "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1178  "movq %%mm3, %%mm4 \n\t" // Y(0)
1179  "movq %%mm5, %%mm6 \n\t" // Y(8)
1180  "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
1181  "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
1182  "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
1183  "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
1184 
1185  MOVNTQ" %%mm3, (%0, %%"REG_a", 4) \n\t"
1186  MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4) \n\t"
1187  MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4) \n\t"
1188  MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4) \n\t"
1189 
1190  "add $8, %%"REG_a" \n\t"
1191  "cmp %4, %%"REG_a" \n\t"
1192  " jb 1b \n\t"
1193  ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1194  : "%"REG_a
1195  );
1196  if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
1197  usrc += chromStride;
1198  vsrc += chromStride;
1199  }
1200  ysrc += lumStride;
1201  dst += dstStride;
1202  }
1203  __asm__(EMMS" \n\t"
1204  SFENCE" \n\t"
1205  :::"memory");
1206 }
1207 
1208 /**
1209  * Height should be a multiple of 2 and width should be a multiple of 16.
1210  * (If this is a problem for anyone then tell me, and I will fix it.)
1211  */
1212 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1213  int width, int height,
1214  int lumStride, int chromStride, int dstStride)
1215 {
1216  //FIXME interpolate chroma
1217  RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1218 }
1219 
1220 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1221  int width, int height,
1222  int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
1223 {
1224  int y;
1225  const x86_reg chromWidth= width>>1;
1226  for (y=0; y<height; y++) {
1227  //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
1228  __asm__ volatile(
1229  "xor %%"REG_a", %%"REG_a" \n\t"
1230  ".p2align 4 \n\t"
1231  "1: \n\t"
1232  PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1233  PREFETCH" 32(%2, %%"REG_a") \n\t"
1234  PREFETCH" 32(%3, %%"REG_a") \n\t"
1235  "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1236  "movq %%mm0, %%mm2 \n\t" // U(0)
1237  "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1238  "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1239  "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1240 
1241  "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1242  "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1243  "movq %%mm0, %%mm4 \n\t" // Y(0)
1244  "movq %%mm2, %%mm6 \n\t" // Y(8)
1245  "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
1246  "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
1247  "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
1248  "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
1249 
1250  MOVNTQ" %%mm0, (%0, %%"REG_a", 4) \n\t"
1251  MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4) \n\t"
1252  MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4) \n\t"
1253  MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4) \n\t"
1254 
1255  "add $8, %%"REG_a" \n\t"
1256  "cmp %4, %%"REG_a" \n\t"
1257  " jb 1b \n\t"
1258  ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1259  : "%"REG_a
1260  );
1261  if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
1262  usrc += chromStride;
1263  vsrc += chromStride;
1264  }
1265  ysrc += lumStride;
1266  dst += dstStride;
1267  }
1268  __asm__(EMMS" \n\t"
1269  SFENCE" \n\t"
1270  :::"memory");
1271 }
1272 
1273 /**
1274  * Height should be a multiple of 2 and width should be a multiple of 16
1275  * (If this is a problem for anyone then tell me, and I will fix it.)
1276  */
1277 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1278  int width, int height,
1279  int lumStride, int chromStride, int dstStride)
1280 {
1281  //FIXME interpolate chroma
1282  RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1283 }
1284 
1285 /**
1286  * Width should be a multiple of 16.
1287  */
1288 static inline void RENAME(yuv422ptouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1289  int width, int height,
1290  int lumStride, int chromStride, int dstStride)
1291 {
1292  RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1293 }
1294 
1295 /**
1296  * Width should be a multiple of 16.
1297  */
1298 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1299  int width, int height,
1300  int lumStride, int chromStride, int dstStride)
1301 {
1302  RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1303 }
1304 
1305 /**
1306  * Height should be a multiple of 2 and width should be a multiple of 16.
1307  * (If this is a problem for anyone then tell me, and I will fix it.)
1308  */
1309 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1310  int width, int height,
1311  int lumStride, int chromStride, int srcStride)
1312 {
1313  int y;
1314  const x86_reg chromWidth= width>>1;
1315  for (y=0; y<height; y+=2) {
1316  __asm__ volatile(
1317  "xor %%"REG_a", %%"REG_a" \n\t"
1318  "pcmpeqw %%mm7, %%mm7 \n\t"
1319  "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1320  ".p2align 4 \n\t"
1321  "1: \n\t"
1322  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1323  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1324  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(4)
1325  "movq %%mm0, %%mm2 \n\t" // YUYV YUYV(0)
1326  "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(4)
1327  "psrlw $8, %%mm0 \n\t" // U0V0 U0V0(0)
1328  "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(4)
1329  "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1330  "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1331  "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1332  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1333 
1334  MOVNTQ" %%mm2, (%1, %%"REG_a", 2) \n\t"
1335 
1336  "movq 16(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(8)
1337  "movq 24(%0, %%"REG_a", 4), %%mm2 \n\t" // YUYV YUYV(12)
1338  "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(8)
1339  "movq %%mm2, %%mm4 \n\t" // YUYV YUYV(12)
1340  "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(8)
1341  "psrlw $8, %%mm2 \n\t" // U0V0 U0V0(12)
1342  "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1343  "pand %%mm7, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1344  "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1345  "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1346 
1347  MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2) \n\t"
1348 
1349  "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1350  "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1351  "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1352  "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1353  "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1354  "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1355  "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1356  "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1357 
1358  MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1359  MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1360 
1361  "add $8, %%"REG_a" \n\t"
1362  "cmp %4, %%"REG_a" \n\t"
1363  " jb 1b \n\t"
1364  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1365  : "memory", "%"REG_a
1366  );
1367 
1368  ydst += lumStride;
1369  src += srcStride;
1370 
1371  __asm__ volatile(
1372  "xor %%"REG_a", %%"REG_a" \n\t"
1373  ".p2align 4 \n\t"
1374  "1: \n\t"
1375  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1376  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1377  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(4)
1378  "movq 16(%0, %%"REG_a", 4), %%mm2 \n\t" // YUYV YUYV(8)
1379  "movq 24(%0, %%"REG_a", 4), %%mm3 \n\t" // YUYV YUYV(12)
1380  "pand %%mm7, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1381  "pand %%mm7, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1382  "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1383  "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1384  "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1385  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1386 
1387  MOVNTQ" %%mm0, (%1, %%"REG_a", 2) \n\t"
1388  MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2) \n\t"
1389 
1390  "add $8, %%"REG_a" \n\t"
1391  "cmp %4, %%"REG_a" \n\t"
1392  " jb 1b \n\t"
1393 
1394  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1395  : "memory", "%"REG_a
1396  );
1397  udst += chromStride;
1398  vdst += chromStride;
1399  ydst += lumStride;
1400  src += srcStride;
1401  }
1402  __asm__ volatile(EMMS" \n\t"
1403  SFENCE" \n\t"
1404  :::"memory");
1405 }
1406 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
1407 
1408 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
1409 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, int srcWidth, int srcHeight, int srcStride, int dstStride)
1410 {
1411  int x,y;
1412 
1413  dst[0]= src[0];
1414 
1415  // first line
1416  for (x=0; x<srcWidth-1; x++) {
1417  dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1418  dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1419  }
1420  dst[2*srcWidth-1]= src[srcWidth-1];
1421 
1422  dst+= dstStride;
1423 
1424  for (y=1; y<srcHeight; y++) {
1425  const x86_reg mmxSize= srcWidth&~15;
1426  __asm__ volatile(
1427  "mov %4, %%"REG_a" \n\t"
1428  "movq "MANGLE(mmx_ff)", %%mm0 \n\t"
1429  "movq (%0, %%"REG_a"), %%mm4 \n\t"
1430  "movq %%mm4, %%mm2 \n\t"
1431  "psllq $8, %%mm4 \n\t"
1432  "pand %%mm0, %%mm2 \n\t"
1433  "por %%mm2, %%mm4 \n\t"
1434  "movq (%1, %%"REG_a"), %%mm5 \n\t"
1435  "movq %%mm5, %%mm3 \n\t"
1436  "psllq $8, %%mm5 \n\t"
1437  "pand %%mm0, %%mm3 \n\t"
1438  "por %%mm3, %%mm5 \n\t"
1439  "1: \n\t"
1440  "movq (%0, %%"REG_a"), %%mm0 \n\t"
1441  "movq (%1, %%"REG_a"), %%mm1 \n\t"
1442  "movq 1(%0, %%"REG_a"), %%mm2 \n\t"
1443  "movq 1(%1, %%"REG_a"), %%mm3 \n\t"
1444  PAVGB" %%mm0, %%mm5 \n\t"
1445  PAVGB" %%mm0, %%mm3 \n\t"
1446  PAVGB" %%mm0, %%mm5 \n\t"
1447  PAVGB" %%mm0, %%mm3 \n\t"
1448  PAVGB" %%mm1, %%mm4 \n\t"
1449  PAVGB" %%mm1, %%mm2 \n\t"
1450  PAVGB" %%mm1, %%mm4 \n\t"
1451  PAVGB" %%mm1, %%mm2 \n\t"
1452  "movq %%mm5, %%mm7 \n\t"
1453  "movq %%mm4, %%mm6 \n\t"
1454  "punpcklbw %%mm3, %%mm5 \n\t"
1455  "punpckhbw %%mm3, %%mm7 \n\t"
1456  "punpcklbw %%mm2, %%mm4 \n\t"
1457  "punpckhbw %%mm2, %%mm6 \n\t"
1458  MOVNTQ" %%mm5, (%2, %%"REG_a", 2) \n\t"
1459  MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2) \n\t"
1460  MOVNTQ" %%mm4, (%3, %%"REG_a", 2) \n\t"
1461  MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2) \n\t"
1462  "add $8, %%"REG_a" \n\t"
1463  "movq -1(%0, %%"REG_a"), %%mm4 \n\t"
1464  "movq -1(%1, %%"REG_a"), %%mm5 \n\t"
1465  " js 1b \n\t"
1466  :: "r" (src + mmxSize ), "r" (src + srcStride + mmxSize ),
1467  "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1468  "g" (-mmxSize)
1469  : "%"REG_a
1470  );
1471 
1472  for (x=mmxSize-1; x<srcWidth-1; x++) {
1473  dst[2*x +1]= (3*src[x+0] + src[x+srcStride+1])>>2;
1474  dst[2*x+dstStride+2]= ( src[x+0] + 3*src[x+srcStride+1])>>2;
1475  dst[2*x+dstStride+1]= ( src[x+1] + 3*src[x+srcStride ])>>2;
1476  dst[2*x +2]= (3*src[x+1] + src[x+srcStride ])>>2;
1477  }
1478  dst[srcWidth*2 -1 ]= (3*src[srcWidth-1] + src[srcWidth-1 + srcStride])>>2;
1479  dst[srcWidth*2 -1 + dstStride]= ( src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1480 
1481  dst+=dstStride*2;
1482  src+=srcStride;
1483  }
1484 
1485  // last line
1486  dst[0]= src[0];
1487 
1488  for (x=0; x<srcWidth-1; x++) {
1489  dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1490  dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1491  }
1492  dst[2*srcWidth-1]= src[srcWidth-1];
1493 
1494  __asm__ volatile(EMMS" \n\t"
1495  SFENCE" \n\t"
1496  :::"memory");
1497 }
1498 #endif /* COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW */
1499 
1500 #if !COMPILE_TEMPLATE_AMD3DNOW
1501 /**
1502  * Height should be a multiple of 2 and width should be a multiple of 16.
1503  * (If this is a problem for anyone then tell me, and I will fix it.)
1504  * Chrominance data is only taken from every second line, others are ignored.
1505  * FIXME: Write HQ version.
1506  */
1507 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1508  int width, int height,
1509  int lumStride, int chromStride, int srcStride)
1510 {
1511  int y;
1512  const x86_reg chromWidth= width>>1;
1513  for (y=0; y<height; y+=2) {
1514  __asm__ volatile(
1515  "xor %%"REG_a", %%"REG_a" \n\t"
1516  "pcmpeqw %%mm7, %%mm7 \n\t"
1517  "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1518  ".p2align 4 \n\t"
1519  "1: \n\t"
1520  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1521  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // UYVY UYVY(0)
1522  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // UYVY UYVY(4)
1523  "movq %%mm0, %%mm2 \n\t" // UYVY UYVY(0)
1524  "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(4)
1525  "pand %%mm7, %%mm0 \n\t" // U0V0 U0V0(0)
1526  "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(4)
1527  "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1528  "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1529  "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1530  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1531 
1532  MOVNTQ" %%mm2, (%1, %%"REG_a", 2) \n\t"
1533 
1534  "movq 16(%0, %%"REG_a", 4), %%mm1 \n\t" // UYVY UYVY(8)
1535  "movq 24(%0, %%"REG_a", 4), %%mm2 \n\t" // UYVY UYVY(12)
1536  "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(8)
1537  "movq %%mm2, %%mm4 \n\t" // UYVY UYVY(12)
1538  "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(8)
1539  "pand %%mm7, %%mm2 \n\t" // U0V0 U0V0(12)
1540  "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1541  "psrlw $8, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1542  "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1543  "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1544 
1545  MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2) \n\t"
1546 
1547  "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1548  "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1549  "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1550  "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1551  "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1552  "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1553  "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1554  "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1555 
1556  MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1557  MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1558 
1559  "add $8, %%"REG_a" \n\t"
1560  "cmp %4, %%"REG_a" \n\t"
1561  " jb 1b \n\t"
1562  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1563  : "memory", "%"REG_a
1564  );
1565 
1566  ydst += lumStride;
1567  src += srcStride;
1568 
1569  __asm__ volatile(
1570  "xor %%"REG_a", %%"REG_a" \n\t"
1571  ".p2align 4 \n\t"
1572  "1: \n\t"
1573  PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1574  "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1575  "movq 8(%0, %%"REG_a", 4), %%mm1 \n\t" // YUYV YUYV(4)
1576  "movq 16(%0, %%"REG_a", 4), %%mm2 \n\t" // YUYV YUYV(8)
1577  "movq 24(%0, %%"REG_a", 4), %%mm3 \n\t" // YUYV YUYV(12)
1578  "psrlw $8, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1579  "psrlw $8, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1580  "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1581  "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1582  "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1583  "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1584 
1585  MOVNTQ" %%mm0, (%1, %%"REG_a", 2) \n\t"
1586  MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2) \n\t"
1587 
1588  "add $8, %%"REG_a" \n\t"
1589  "cmp %4, %%"REG_a" \n\t"
1590  " jb 1b \n\t"
1591 
1592  ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1593  : "memory", "%"REG_a
1594  );
1595  udst += chromStride;
1596  vdst += chromStride;
1597  ydst += lumStride;
1598  src += srcStride;
1599  }
1600  __asm__ volatile(EMMS" \n\t"
1601  SFENCE" \n\t"
1602  :::"memory");
1603 }
1604 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
1605 
1606 /**
1607  * Height should be a multiple of 2 and width should be a multiple of 2.
1608  * (If this is a problem for anyone then tell me, and I will fix it.)
1609  * Chrominance data is only taken from every second line,
1610  * others are ignored in the C version.
1611  * FIXME: Write HQ version.
1612  */
1613 #if HAVE_7REGS
1614 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1615  int width, int height,
1616  int lumStride, int chromStride, int srcStride,
1617  int32_t *rgb2yuv)
1618 {
1619 #define BGR2Y_IDX "16*4+16*32"
1620 #define BGR2U_IDX "16*4+16*33"
1621 #define BGR2V_IDX "16*4+16*34"
1622  int y;
1623  const x86_reg chromWidth= width>>1;
1624  for (y=0; y<height-2; y+=2) {
1625  int i;
1626  for (i=0; i<2; i++) {
1627  __asm__ volatile(
1628  "mov %2, %%"REG_a" \n\t"
1629  "movq "BGR2Y_IDX"(%3), %%mm6 \n\t"
1630  "movq "MANGLE(ff_w1111)", %%mm5 \n\t"
1631  "pxor %%mm7, %%mm7 \n\t"
1632  "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d" \n\t"
1633  ".p2align 4 \n\t"
1634  "1: \n\t"
1635  PREFETCH" 64(%0, %%"REG_d") \n\t"
1636  "movd (%0, %%"REG_d"), %%mm0 \n\t"
1637  "movd 3(%0, %%"REG_d"), %%mm1 \n\t"
1638  "punpcklbw %%mm7, %%mm0 \n\t"
1639  "punpcklbw %%mm7, %%mm1 \n\t"
1640  "movd 6(%0, %%"REG_d"), %%mm2 \n\t"
1641  "movd 9(%0, %%"REG_d"), %%mm3 \n\t"
1642  "punpcklbw %%mm7, %%mm2 \n\t"
1643  "punpcklbw %%mm7, %%mm3 \n\t"
1644  "pmaddwd %%mm6, %%mm0 \n\t"
1645  "pmaddwd %%mm6, %%mm1 \n\t"
1646  "pmaddwd %%mm6, %%mm2 \n\t"
1647  "pmaddwd %%mm6, %%mm3 \n\t"
1648  "psrad $8, %%mm0 \n\t"
1649  "psrad $8, %%mm1 \n\t"
1650  "psrad $8, %%mm2 \n\t"
1651  "psrad $8, %%mm3 \n\t"
1652  "packssdw %%mm1, %%mm0 \n\t"
1653  "packssdw %%mm3, %%mm2 \n\t"
1654  "pmaddwd %%mm5, %%mm0 \n\t"
1655  "pmaddwd %%mm5, %%mm2 \n\t"
1656  "packssdw %%mm2, %%mm0 \n\t"
1657  "psraw $7, %%mm0 \n\t"
1658 
1659  "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
1660  "movd 15(%0, %%"REG_d"), %%mm1 \n\t"
1661  "punpcklbw %%mm7, %%mm4 \n\t"
1662  "punpcklbw %%mm7, %%mm1 \n\t"
1663  "movd 18(%0, %%"REG_d"), %%mm2 \n\t"
1664  "movd 21(%0, %%"REG_d"), %%mm3 \n\t"
1665  "punpcklbw %%mm7, %%mm2 \n\t"
1666  "punpcklbw %%mm7, %%mm3 \n\t"
1667  "pmaddwd %%mm6, %%mm4 \n\t"
1668  "pmaddwd %%mm6, %%mm1 \n\t"
1669  "pmaddwd %%mm6, %%mm2 \n\t"
1670  "pmaddwd %%mm6, %%mm3 \n\t"
1671  "psrad $8, %%mm4 \n\t"
1672  "psrad $8, %%mm1 \n\t"
1673  "psrad $8, %%mm2 \n\t"
1674  "psrad $8, %%mm3 \n\t"
1675  "packssdw %%mm1, %%mm4 \n\t"
1676  "packssdw %%mm3, %%mm2 \n\t"
1677  "pmaddwd %%mm5, %%mm4 \n\t"
1678  "pmaddwd %%mm5, %%mm2 \n\t"
1679  "add $24, %%"REG_d" \n\t"
1680  "packssdw %%mm2, %%mm4 \n\t"
1681  "psraw $7, %%mm4 \n\t"
1682 
1683  "packuswb %%mm4, %%mm0 \n\t"
1684  "paddusb "MANGLE(ff_bgr2YOffset)", %%mm0 \n\t"
1685 
1686  MOVNTQ" %%mm0, (%1, %%"REG_a") \n\t"
1687  "add $8, %%"REG_a" \n\t"
1688  " js 1b \n\t"
1689  : : "r" (src+width*3), "r" (ydst+width), "g" ((x86_reg)-width), "r"(rgb2yuv)
1690  : "%"REG_a, "%"REG_d
1691  );
1692  ydst += lumStride;
1693  src += srcStride;
1694  }
1695  src -= srcStride*2;
1696  __asm__ volatile(
1697  "mov %4, %%"REG_a" \n\t"
1698  "movq "MANGLE(ff_w1111)", %%mm5 \n\t"
1699  "movq "BGR2U_IDX"(%5), %%mm6 \n\t"
1700  "pxor %%mm7, %%mm7 \n\t"
1701  "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d" \n\t"
1702  "add %%"REG_d", %%"REG_d" \n\t"
1703  ".p2align 4 \n\t"
1704  "1: \n\t"
1705  PREFETCH" 64(%0, %%"REG_d") \n\t"
1706  PREFETCH" 64(%1, %%"REG_d") \n\t"
1707 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
1708  "movq (%0, %%"REG_d"), %%mm0 \n\t"
1709  "movq (%1, %%"REG_d"), %%mm1 \n\t"
1710  "movq 6(%0, %%"REG_d"), %%mm2 \n\t"
1711  "movq 6(%1, %%"REG_d"), %%mm3 \n\t"
1712  PAVGB" %%mm1, %%mm0 \n\t"
1713  PAVGB" %%mm3, %%mm2 \n\t"
1714  "movq %%mm0, %%mm1 \n\t"
1715  "movq %%mm2, %%mm3 \n\t"
1716  "psrlq $24, %%mm0 \n\t"
1717  "psrlq $24, %%mm2 \n\t"
1718  PAVGB" %%mm1, %%mm0 \n\t"
1719  PAVGB" %%mm3, %%mm2 \n\t"
1720  "punpcklbw %%mm7, %%mm0 \n\t"
1721  "punpcklbw %%mm7, %%mm2 \n\t"
1722 #else
1723  "movd (%0, %%"REG_d"), %%mm0 \n\t"
1724  "movd (%1, %%"REG_d"), %%mm1 \n\t"
1725  "movd 3(%0, %%"REG_d"), %%mm2 \n\t"
1726  "movd 3(%1, %%"REG_d"), %%mm3 \n\t"
1727  "punpcklbw %%mm7, %%mm0 \n\t"
1728  "punpcklbw %%mm7, %%mm1 \n\t"
1729  "punpcklbw %%mm7, %%mm2 \n\t"
1730  "punpcklbw %%mm7, %%mm3 \n\t"
1731  "paddw %%mm1, %%mm0 \n\t"
1732  "paddw %%mm3, %%mm2 \n\t"
1733  "paddw %%mm2, %%mm0 \n\t"
1734  "movd 6(%0, %%"REG_d"), %%mm4 \n\t"
1735  "movd 6(%1, %%"REG_d"), %%mm1 \n\t"
1736  "movd 9(%0, %%"REG_d"), %%mm2 \n\t"
1737  "movd 9(%1, %%"REG_d"), %%mm3 \n\t"
1738  "punpcklbw %%mm7, %%mm4 \n\t"
1739  "punpcklbw %%mm7, %%mm1 \n\t"
1740  "punpcklbw %%mm7, %%mm2 \n\t"
1741  "punpcklbw %%mm7, %%mm3 \n\t"
1742  "paddw %%mm1, %%mm4 \n\t"
1743  "paddw %%mm3, %%mm2 \n\t"
1744  "paddw %%mm4, %%mm2 \n\t"
1745  "psrlw $2, %%mm0 \n\t"
1746  "psrlw $2, %%mm2 \n\t"
1747 #endif
1748  "movq "BGR2V_IDX"(%5), %%mm1 \n\t"
1749  "movq "BGR2V_IDX"(%5), %%mm3 \n\t"
1750 
1751  "pmaddwd %%mm0, %%mm1 \n\t"
1752  "pmaddwd %%mm2, %%mm3 \n\t"
1753  "pmaddwd %%mm6, %%mm0 \n\t"
1754  "pmaddwd %%mm6, %%mm2 \n\t"
1755  "psrad $8, %%mm0 \n\t"
1756  "psrad $8, %%mm1 \n\t"
1757  "psrad $8, %%mm2 \n\t"
1758  "psrad $8, %%mm3 \n\t"
1759  "packssdw %%mm2, %%mm0 \n\t"
1760  "packssdw %%mm3, %%mm1 \n\t"
1761  "pmaddwd %%mm5, %%mm0 \n\t"
1762  "pmaddwd %%mm5, %%mm1 \n\t"
1763  "packssdw %%mm1, %%mm0 \n\t" // V1 V0 U1 U0
1764  "psraw $7, %%mm0 \n\t"
1765 
1766 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
1767  "movq 12(%0, %%"REG_d"), %%mm4 \n\t"
1768  "movq 12(%1, %%"REG_d"), %%mm1 \n\t"
1769  "movq 18(%0, %%"REG_d"), %%mm2 \n\t"
1770  "movq 18(%1, %%"REG_d"), %%mm3 \n\t"
1771  PAVGB" %%mm1, %%mm4 \n\t"
1772  PAVGB" %%mm3, %%mm2 \n\t"
1773  "movq %%mm4, %%mm1 \n\t"
1774  "movq %%mm2, %%mm3 \n\t"
1775  "psrlq $24, %%mm4 \n\t"
1776  "psrlq $24, %%mm2 \n\t"
1777  PAVGB" %%mm1, %%mm4 \n\t"
1778  PAVGB" %%mm3, %%mm2 \n\t"
1779  "punpcklbw %%mm7, %%mm4 \n\t"
1780  "punpcklbw %%mm7, %%mm2 \n\t"
1781 #else
1782  "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
1783  "movd 12(%1, %%"REG_d"), %%mm1 \n\t"
1784  "movd 15(%0, %%"REG_d"), %%mm2 \n\t"
1785  "movd 15(%1, %%"REG_d"), %%mm3 \n\t"
1786  "punpcklbw %%mm7, %%mm4 \n\t"
1787  "punpcklbw %%mm7, %%mm1 \n\t"
1788  "punpcklbw %%mm7, %%mm2 \n\t"
1789  "punpcklbw %%mm7, %%mm3 \n\t"
1790  "paddw %%mm1, %%mm4 \n\t"
1791  "paddw %%mm3, %%mm2 \n\t"
1792  "paddw %%mm2, %%mm4 \n\t"
1793  "movd 18(%0, %%"REG_d"), %%mm5 \n\t"
1794  "movd 18(%1, %%"REG_d"), %%mm1 \n\t"
1795  "movd 21(%0, %%"REG_d"), %%mm2 \n\t"
1796  "movd 21(%1, %%"REG_d"), %%mm3 \n\t"
1797  "punpcklbw %%mm7, %%mm5 \n\t"
1798  "punpcklbw %%mm7, %%mm1 \n\t"
1799  "punpcklbw %%mm7, %%mm2 \n\t"
1800  "punpcklbw %%mm7, %%mm3 \n\t"
1801  "paddw %%mm1, %%mm5 \n\t"
1802  "paddw %%mm3, %%mm2 \n\t"
1803  "paddw %%mm5, %%mm2 \n\t"
1804  "movq "MANGLE(ff_w1111)", %%mm5 \n\t"
1805  "psrlw $2, %%mm4 \n\t"
1806  "psrlw $2, %%mm2 \n\t"
1807 #endif
1808  "movq "BGR2V_IDX"(%5), %%mm1 \n\t"
1809  "movq "BGR2V_IDX"(%5), %%mm3 \n\t"
1810 
1811  "pmaddwd %%mm4, %%mm1 \n\t"
1812  "pmaddwd %%mm2, %%mm3 \n\t"
1813  "pmaddwd %%mm6, %%mm4 \n\t"
1814  "pmaddwd %%mm6, %%mm2 \n\t"
1815  "psrad $8, %%mm4 \n\t"
1816  "psrad $8, %%mm1 \n\t"
1817  "psrad $8, %%mm2 \n\t"
1818  "psrad $8, %%mm3 \n\t"
1819  "packssdw %%mm2, %%mm4 \n\t"
1820  "packssdw %%mm3, %%mm1 \n\t"
1821  "pmaddwd %%mm5, %%mm4 \n\t"
1822  "pmaddwd %%mm5, %%mm1 \n\t"
1823  "add $24, %%"REG_d" \n\t"
1824  "packssdw %%mm1, %%mm4 \n\t" // V3 V2 U3 U2
1825  "psraw $7, %%mm4 \n\t"
1826 
1827  "movq %%mm0, %%mm1 \n\t"
1828  "punpckldq %%mm4, %%mm0 \n\t"
1829  "punpckhdq %%mm4, %%mm1 \n\t"
1830  "packsswb %%mm1, %%mm0 \n\t"
1831  "paddb "MANGLE(ff_bgr2UVOffset)", %%mm0 \n\t"
1832  "movd %%mm0, (%2, %%"REG_a") \n\t"
1833  "punpckhdq %%mm0, %%mm0 \n\t"
1834  "movd %%mm0, (%3, %%"REG_a") \n\t"
1835  "add $4, %%"REG_a" \n\t"
1836  " js 1b \n\t"
1837  : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth), "r"(rgb2yuv)
1838  : "%"REG_a, "%"REG_d
1839  );
1840 
1841  udst += chromStride;
1842  vdst += chromStride;
1843  src += srcStride*2;
1844  }
1845 
1846  __asm__ volatile(EMMS" \n\t"
1847  SFENCE" \n\t"
1848  :::"memory");
1849 
1850  ff_rgb24toyv12_c(src, ydst, udst, vdst, width, height-y, lumStride, chromStride, srcStride, rgb2yuv);
1851 }
1852 #endif /* HAVE_7REGS */
1853 #endif /* !COMPILE_TEMPLATE_SSE2 */
1854 
1855 #if !COMPILE_TEMPLATE_AMD3DNOW
1856 static void RENAME(interleaveBytes)(const uint8_t *src1, const uint8_t *src2, uint8_t *dest,
1857  int width, int height, int src1Stride,
1858  int src2Stride, int dstStride)
1859 {
1860  int h;
1861 
1862  for (h=0; h < height; h++) {
1863  int w;
1864 
1865 #if COMPILE_TEMPLATE_SSE2
1866  __asm__(
1867  "xor %%"REG_a", %%"REG_a" \n\t"
1868  "1: \n\t"
1869  PREFETCH" 64(%1, %%"REG_a") \n\t"
1870  PREFETCH" 64(%2, %%"REG_a") \n\t"
1871  "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
1872  "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
1873  "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
1874  "punpcklbw %%xmm2, %%xmm0 \n\t"
1875  "punpckhbw %%xmm2, %%xmm1 \n\t"
1876  "movntdq %%xmm0, (%0, %%"REG_a", 2) \n\t"
1877  "movntdq %%xmm1, 16(%0, %%"REG_a", 2) \n\t"
1878  "add $16, %%"REG_a" \n\t"
1879  "cmp %3, %%"REG_a" \n\t"
1880  " jb 1b \n\t"
1881  ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)
1882  : "memory", "%"REG_a""
1883  );
1884 #else
1885  __asm__(
1886  "xor %%"REG_a", %%"REG_a" \n\t"
1887  "1: \n\t"
1888  PREFETCH" 64(%1, %%"REG_a") \n\t"
1889  PREFETCH" 64(%2, %%"REG_a") \n\t"
1890  "movq (%1, %%"REG_a"), %%mm0 \n\t"
1891  "movq 8(%1, %%"REG_a"), %%mm2 \n\t"
1892  "movq %%mm0, %%mm1 \n\t"
1893  "movq %%mm2, %%mm3 \n\t"
1894  "movq (%2, %%"REG_a"), %%mm4 \n\t"
1895  "movq 8(%2, %%"REG_a"), %%mm5 \n\t"
1896  "punpcklbw %%mm4, %%mm0 \n\t"
1897  "punpckhbw %%mm4, %%mm1 \n\t"
1898  "punpcklbw %%mm5, %%mm2 \n\t"
1899  "punpckhbw %%mm5, %%mm3 \n\t"
1900  MOVNTQ" %%mm0, (%0, %%"REG_a", 2) \n\t"
1901  MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2) \n\t"
1902  MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2) \n\t"
1903  MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2) \n\t"
1904  "add $16, %%"REG_a" \n\t"
1905  "cmp %3, %%"REG_a" \n\t"
1906  " jb 1b \n\t"
1907  ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)
1908  : "memory", "%"REG_a
1909  );
1910 #endif
1911  for (w= (width&(~15)); w < width; w++) {
1912  dest[2*w+0] = src1[w];
1913  dest[2*w+1] = src2[w];
1914  }
1915  dest += dstStride;
1916  src1 += src1Stride;
1917  src2 += src2Stride;
1918  }
1919  __asm__(
1920  EMMS" \n\t"
1921  SFENCE" \n\t"
1922  ::: "memory"
1923  );
1924 }
1925 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
1926 
1927 #if !COMPILE_TEMPLATE_SSE2
1928 #if !COMPILE_TEMPLATE_AMD3DNOW
1929 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
1930  uint8_t *dst1, uint8_t *dst2,
1931  int width, int height,
1932  int srcStride1, int srcStride2,
1933  int dstStride1, int dstStride2)
1934 {
1935  x86_reg x, y;
1936  int w,h;
1937  w=width/2; h=height/2;
1938  __asm__ volatile(
1939  PREFETCH" %0 \n\t"
1940  PREFETCH" %1 \n\t"
1941  ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
1942  for (y=0;y<h;y++) {
1943  const uint8_t* s1=src1+srcStride1*(y>>1);
1944  uint8_t* d=dst1+dstStride1*y;
1945  x=0;
1946  for (;x<w-31;x+=32) {
1947  __asm__ volatile(
1948  PREFETCH" 32(%1,%2) \n\t"
1949  "movq (%1,%2), %%mm0 \n\t"
1950  "movq 8(%1,%2), %%mm2 \n\t"
1951  "movq 16(%1,%2), %%mm4 \n\t"
1952  "movq 24(%1,%2), %%mm6 \n\t"
1953  "movq %%mm0, %%mm1 \n\t"
1954  "movq %%mm2, %%mm3 \n\t"
1955  "movq %%mm4, %%mm5 \n\t"
1956  "movq %%mm6, %%mm7 \n\t"
1957  "punpcklbw %%mm0, %%mm0 \n\t"
1958  "punpckhbw %%mm1, %%mm1 \n\t"
1959  "punpcklbw %%mm2, %%mm2 \n\t"
1960  "punpckhbw %%mm3, %%mm3 \n\t"
1961  "punpcklbw %%mm4, %%mm4 \n\t"
1962  "punpckhbw %%mm5, %%mm5 \n\t"
1963  "punpcklbw %%mm6, %%mm6 \n\t"
1964  "punpckhbw %%mm7, %%mm7 \n\t"
1965  MOVNTQ" %%mm0, (%0,%2,2) \n\t"
1966  MOVNTQ" %%mm1, 8(%0,%2,2) \n\t"
1967  MOVNTQ" %%mm2, 16(%0,%2,2) \n\t"
1968  MOVNTQ" %%mm3, 24(%0,%2,2) \n\t"
1969  MOVNTQ" %%mm4, 32(%0,%2,2) \n\t"
1970  MOVNTQ" %%mm5, 40(%0,%2,2) \n\t"
1971  MOVNTQ" %%mm6, 48(%0,%2,2) \n\t"
1972  MOVNTQ" %%mm7, 56(%0,%2,2)"
1973  :: "r"(d), "r"(s1), "r"(x)
1974  :"memory");
1975  }
1976  for (;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
1977  }
1978  for (y=0;y<h;y++) {
1979  const uint8_t* s2=src2+srcStride2*(y>>1);
1980  uint8_t* d=dst2+dstStride2*y;
1981  x=0;
1982  for (;x<w-31;x+=32) {
1983  __asm__ volatile(
1984  PREFETCH" 32(%1,%2) \n\t"
1985  "movq (%1,%2), %%mm0 \n\t"
1986  "movq 8(%1,%2), %%mm2 \n\t"
1987  "movq 16(%1,%2), %%mm4 \n\t"
1988  "movq 24(%1,%2), %%mm6 \n\t"
1989  "movq %%mm0, %%mm1 \n\t"
1990  "movq %%mm2, %%mm3 \n\t"
1991  "movq %%mm4, %%mm5 \n\t"
1992  "movq %%mm6, %%mm7 \n\t"
1993  "punpcklbw %%mm0, %%mm0 \n\t"
1994  "punpckhbw %%mm1, %%mm1 \n\t"
1995  "punpcklbw %%mm2, %%mm2 \n\t"
1996  "punpckhbw %%mm3, %%mm3 \n\t"
1997  "punpcklbw %%mm4, %%mm4 \n\t"
1998  "punpckhbw %%mm5, %%mm5 \n\t"
1999  "punpcklbw %%mm6, %%mm6 \n\t"
2000  "punpckhbw %%mm7, %%mm7 \n\t"
2001  MOVNTQ" %%mm0, (%0,%2,2) \n\t"
2002  MOVNTQ" %%mm1, 8(%0,%2,2) \n\t"
2003  MOVNTQ" %%mm2, 16(%0,%2,2) \n\t"
2004  MOVNTQ" %%mm3, 24(%0,%2,2) \n\t"
2005  MOVNTQ" %%mm4, 32(%0,%2,2) \n\t"
2006  MOVNTQ" %%mm5, 40(%0,%2,2) \n\t"
2007  MOVNTQ" %%mm6, 48(%0,%2,2) \n\t"
2008  MOVNTQ" %%mm7, 56(%0,%2,2)"
2009  :: "r"(d), "r"(s2), "r"(x)
2010  :"memory");
2011  }
2012  for (;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2013  }
2014  __asm__(
2015  EMMS" \n\t"
2016  SFENCE" \n\t"
2017  ::: "memory"
2018  );
2019 }
2020 
2021 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2022  uint8_t *dst,
2023  int width, int height,
2024  int srcStride1, int srcStride2,
2025  int srcStride3, int dstStride)
2026 {
2027  x86_reg x;
2028  int y,w,h;
2029  w=width/2; h=height;
2030  for (y=0;y<h;y++) {
2031  const uint8_t* yp=src1+srcStride1*y;
2032  const uint8_t* up=src2+srcStride2*(y>>2);
2033  const uint8_t* vp=src3+srcStride3*(y>>2);
2034  uint8_t* d=dst+dstStride*y;
2035  x=0;
2036  for (;x<w-7;x+=8) {
2037  __asm__ volatile(
2038  PREFETCH" 32(%1, %0) \n\t"
2039  PREFETCH" 32(%2, %0) \n\t"
2040  PREFETCH" 32(%3, %0) \n\t"
2041  "movq (%1, %0, 4), %%mm0 \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2042  "movq (%2, %0), %%mm1 \n\t" /* U0U1U2U3U4U5U6U7 */
2043  "movq (%3, %0), %%mm2 \n\t" /* V0V1V2V3V4V5V6V7 */
2044  "movq %%mm0, %%mm3 \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2045  "movq %%mm1, %%mm4 \n\t" /* U0U1U2U3U4U5U6U7 */
2046  "movq %%mm2, %%mm5 \n\t" /* V0V1V2V3V4V5V6V7 */
2047  "punpcklbw %%mm1, %%mm1 \n\t" /* U0U0 U1U1 U2U2 U3U3 */
2048  "punpcklbw %%mm2, %%mm2 \n\t" /* V0V0 V1V1 V2V2 V3V3 */
2049  "punpckhbw %%mm4, %%mm4 \n\t" /* U4U4 U5U5 U6U6 U7U7 */
2050  "punpckhbw %%mm5, %%mm5 \n\t" /* V4V4 V5V5 V6V6 V7V7 */
2051 
2052  "movq %%mm1, %%mm6 \n\t"
2053  "punpcklbw %%mm2, %%mm1 \n\t" /* U0V0 U0V0 U1V1 U1V1*/
2054  "punpcklbw %%mm1, %%mm0 \n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2055  "punpckhbw %%mm1, %%mm3 \n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2056  MOVNTQ" %%mm0, (%4, %0, 8) \n\t"
2057  MOVNTQ" %%mm3, 8(%4, %0, 8) \n\t"
2058 
2059  "punpckhbw %%mm2, %%mm6 \n\t" /* U2V2 U2V2 U3V3 U3V3*/
2060  "movq 8(%1, %0, 4), %%mm0 \n\t"
2061  "movq %%mm0, %%mm3 \n\t"
2062  "punpcklbw %%mm6, %%mm0 \n\t" /* Y U2 Y V2 Y U2 Y V2*/
2063  "punpckhbw %%mm6, %%mm3 \n\t" /* Y U3 Y V3 Y U3 Y V3*/
2064  MOVNTQ" %%mm0, 16(%4, %0, 8) \n\t"
2065  MOVNTQ" %%mm3, 24(%4, %0, 8) \n\t"
2066 
2067  "movq %%mm4, %%mm6 \n\t"
2068  "movq 16(%1, %0, 4), %%mm0 \n\t"
2069  "movq %%mm0, %%mm3 \n\t"
2070  "punpcklbw %%mm5, %%mm4 \n\t"
2071  "punpcklbw %%mm4, %%mm0 \n\t" /* Y U4 Y V4 Y U4 Y V4*/
2072  "punpckhbw %%mm4, %%mm3 \n\t" /* Y U5 Y V5 Y U5 Y V5*/
2073  MOVNTQ" %%mm0, 32(%4, %0, 8) \n\t"
2074  MOVNTQ" %%mm3, 40(%4, %0, 8) \n\t"
2075 
2076  "punpckhbw %%mm5, %%mm6 \n\t"
2077  "movq 24(%1, %0, 4), %%mm0 \n\t"
2078  "movq %%mm0, %%mm3 \n\t"
2079  "punpcklbw %%mm6, %%mm0 \n\t" /* Y U6 Y V6 Y U6 Y V6*/
2080  "punpckhbw %%mm6, %%mm3 \n\t" /* Y U7 Y V7 Y U7 Y V7*/
2081  MOVNTQ" %%mm0, 48(%4, %0, 8) \n\t"
2082  MOVNTQ" %%mm3, 56(%4, %0, 8) \n\t"
2083 
2084  : "+r" (x)
2085  : "r"(yp), "r" (up), "r"(vp), "r"(d)
2086  :"memory");
2087  }
2088  for (; x<w; x++) {
2089  const int x2 = x<<2;
2090  d[8*x+0] = yp[x2];
2091  d[8*x+1] = up[x];
2092  d[8*x+2] = yp[x2+1];
2093  d[8*x+3] = vp[x];
2094  d[8*x+4] = yp[x2+2];
2095  d[8*x+5] = up[x];
2096  d[8*x+6] = yp[x2+3];
2097  d[8*x+7] = vp[x];
2098  }
2099  }
2100  __asm__(
2101  EMMS" \n\t"
2102  SFENCE" \n\t"
2103  ::: "memory"
2104  );
2105 }
2106 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2107 
2108 static void RENAME(extract_even)(const uint8_t *src, uint8_t *dst, x86_reg count)
2109 {
2110  dst += count;
2111  src += 2*count;
2112  count= - count;
2113 
2114  if(count <= -16) {
2115  count += 15;
2116  __asm__ volatile(
2117  "pcmpeqw %%mm7, %%mm7 \n\t"
2118  "psrlw $8, %%mm7 \n\t"
2119  "1: \n\t"
2120  "movq -30(%1, %0, 2), %%mm0 \n\t"
2121  "movq -22(%1, %0, 2), %%mm1 \n\t"
2122  "movq -14(%1, %0, 2), %%mm2 \n\t"
2123  "movq -6(%1, %0, 2), %%mm3 \n\t"
2124  "pand %%mm7, %%mm0 \n\t"
2125  "pand %%mm7, %%mm1 \n\t"
2126  "pand %%mm7, %%mm2 \n\t"
2127  "pand %%mm7, %%mm3 \n\t"
2128  "packuswb %%mm1, %%mm0 \n\t"
2129  "packuswb %%mm3, %%mm2 \n\t"
2130  MOVNTQ" %%mm0,-15(%2, %0) \n\t"
2131  MOVNTQ" %%mm2,- 7(%2, %0) \n\t"
2132  "add $16, %0 \n\t"
2133  " js 1b \n\t"
2134  : "+r"(count)
2135  : "r"(src), "r"(dst)
2136  );
2137  count -= 15;
2138  }
2139  while(count<0) {
2140  dst[count]= src[2*count];
2141  count++;
2142  }
2143 }
2144 
2145 #if !COMPILE_TEMPLATE_AMD3DNOW
2146 static void RENAME(extract_even2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2147 {
2148  dst0+= count;
2149  dst1+= count;
2150  src += 4*count;
2151  count= - count;
2152  if(count <= -8) {
2153  count += 7;
2154  __asm__ volatile(
2155  "pcmpeqw %%mm7, %%mm7 \n\t"
2156  "psrlw $8, %%mm7 \n\t"
2157  "1: \n\t"
2158  "movq -28(%1, %0, 4), %%mm0 \n\t"
2159  "movq -20(%1, %0, 4), %%mm1 \n\t"
2160  "movq -12(%1, %0, 4), %%mm2 \n\t"
2161  "movq -4(%1, %0, 4), %%mm3 \n\t"
2162  "pand %%mm7, %%mm0 \n\t"
2163  "pand %%mm7, %%mm1 \n\t"
2164  "pand %%mm7, %%mm2 \n\t"
2165  "pand %%mm7, %%mm3 \n\t"
2166  "packuswb %%mm1, %%mm0 \n\t"
2167  "packuswb %%mm3, %%mm2 \n\t"
2168  "movq %%mm0, %%mm1 \n\t"
2169  "movq %%mm2, %%mm3 \n\t"
2170  "psrlw $8, %%mm0 \n\t"
2171  "psrlw $8, %%mm2 \n\t"
2172  "pand %%mm7, %%mm1 \n\t"
2173  "pand %%mm7, %%mm3 \n\t"
2174  "packuswb %%mm2, %%mm0 \n\t"
2175  "packuswb %%mm3, %%mm1 \n\t"
2176  MOVNTQ" %%mm0,- 7(%3, %0) \n\t"
2177  MOVNTQ" %%mm1,- 7(%2, %0) \n\t"
2178  "add $8, %0 \n\t"
2179  " js 1b \n\t"
2180  : "+r"(count)
2181  : "r"(src), "r"(dst0), "r"(dst1)
2182  );
2183  count -= 7;
2184  }
2185  while(count<0) {
2186  dst0[count]= src[4*count+0];
2187  dst1[count]= src[4*count+2];
2188  count++;
2189  }
2190 }
2191 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2192 
2193 static void RENAME(extract_even2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2194 {
2195  dst0 += count;
2196  dst1 += count;
2197  src0 += 4*count;
2198  src1 += 4*count;
2199  count= - count;
2200 #ifdef PAVGB
2201  if(count <= -8) {
2202  count += 7;
2203  __asm__ volatile(
2204  "pcmpeqw %%mm7, %%mm7 \n\t"
2205  "psrlw $8, %%mm7 \n\t"
2206  "1: \n\t"
2207  "movq -28(%1, %0, 4), %%mm0 \n\t"
2208  "movq -20(%1, %0, 4), %%mm1 \n\t"
2209  "movq -12(%1, %0, 4), %%mm2 \n\t"
2210  "movq -4(%1, %0, 4), %%mm3 \n\t"
2211  PAVGB" -28(%2, %0, 4), %%mm0 \n\t"
2212  PAVGB" -20(%2, %0, 4), %%mm1 \n\t"
2213  PAVGB" -12(%2, %0, 4), %%mm2 \n\t"
2214  PAVGB" - 4(%2, %0, 4), %%mm3 \n\t"
2215  "pand %%mm7, %%mm0 \n\t"
2216  "pand %%mm7, %%mm1 \n\t"
2217  "pand %%mm7, %%mm2 \n\t"
2218  "pand %%mm7, %%mm3 \n\t"
2219  "packuswb %%mm1, %%mm0 \n\t"
2220  "packuswb %%mm3, %%mm2 \n\t"
2221  "movq %%mm0, %%mm1 \n\t"
2222  "movq %%mm2, %%mm3 \n\t"
2223  "psrlw $8, %%mm0 \n\t"
2224  "psrlw $8, %%mm2 \n\t"
2225  "pand %%mm7, %%mm1 \n\t"
2226  "pand %%mm7, %%mm3 \n\t"
2227  "packuswb %%mm2, %%mm0 \n\t"
2228  "packuswb %%mm3, %%mm1 \n\t"
2229  MOVNTQ" %%mm0,- 7(%4, %0) \n\t"
2230  MOVNTQ" %%mm1,- 7(%3, %0) \n\t"
2231  "add $8, %0 \n\t"
2232  " js 1b \n\t"
2233  : "+r"(count)
2234  : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)
2235  );
2236  count -= 7;
2237  }
2238 #endif
2239  while(count<0) {
2240  dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;
2241  dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;
2242  count++;
2243  }
2244 }
2245 
2246 #if !COMPILE_TEMPLATE_AMD3DNOW
2247 static void RENAME(extract_odd2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2248 {
2249  dst0+= count;
2250  dst1+= count;
2251  src += 4*count;
2252  count= - count;
2253  if(count <= -8) {
2254  count += 7;
2255  __asm__ volatile(
2256  "pcmpeqw %%mm7, %%mm7 \n\t"
2257  "psrlw $8, %%mm7 \n\t"
2258  "1: \n\t"
2259  "movq -28(%1, %0, 4), %%mm0 \n\t"
2260  "movq -20(%1, %0, 4), %%mm1 \n\t"
2261  "movq -12(%1, %0, 4), %%mm2 \n\t"
2262  "movq -4(%1, %0, 4), %%mm3 \n\t"
2263  "psrlw $8, %%mm0 \n\t"
2264  "psrlw $8, %%mm1 \n\t"
2265  "psrlw $8, %%mm2 \n\t"
2266  "psrlw $8, %%mm3 \n\t"
2267  "packuswb %%mm1, %%mm0 \n\t"
2268  "packuswb %%mm3, %%mm2 \n\t"
2269  "movq %%mm0, %%mm1 \n\t"
2270  "movq %%mm2, %%mm3 \n\t"
2271  "psrlw $8, %%mm0 \n\t"
2272  "psrlw $8, %%mm2 \n\t"
2273  "pand %%mm7, %%mm1 \n\t"
2274  "pand %%mm7, %%mm3 \n\t"
2275  "packuswb %%mm2, %%mm0 \n\t"
2276  "packuswb %%mm3, %%mm1 \n\t"
2277  MOVNTQ" %%mm0,- 7(%3, %0) \n\t"
2278  MOVNTQ" %%mm1,- 7(%2, %0) \n\t"
2279  "add $8, %0 \n\t"
2280  " js 1b \n\t"
2281  : "+r"(count)
2282  : "r"(src), "r"(dst0), "r"(dst1)
2283  );
2284  count -= 7;
2285  }
2286  src++;
2287  while(count<0) {
2288  dst0[count]= src[4*count+0];
2289  dst1[count]= src[4*count+2];
2290  count++;
2291  }
2292 }
2293 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2294 
2295 static void RENAME(extract_odd2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
2296 {
2297  dst0 += count;
2298  dst1 += count;
2299  src0 += 4*count;
2300  src1 += 4*count;
2301  count= - count;
2302 #ifdef PAVGB
2303  if(count <= -8) {
2304  count += 7;
2305  __asm__ volatile(
2306  "pcmpeqw %%mm7, %%mm7 \n\t"
2307  "psrlw $8, %%mm7 \n\t"
2308  "1: \n\t"
2309  "movq -28(%1, %0, 4), %%mm0 \n\t"
2310  "movq -20(%1, %0, 4), %%mm1 \n\t"
2311  "movq -12(%1, %0, 4), %%mm2 \n\t"
2312  "movq -4(%1, %0, 4), %%mm3 \n\t"
2313  PAVGB" -28(%2, %0, 4), %%mm0 \n\t"
2314  PAVGB" -20(%2, %0, 4), %%mm1 \n\t"
2315  PAVGB" -12(%2, %0, 4), %%mm2 \n\t"
2316  PAVGB" - 4(%2, %0, 4), %%mm3 \n\t"
2317  "psrlw $8, %%mm0 \n\t"
2318  "psrlw $8, %%mm1 \n\t"
2319  "psrlw $8, %%mm2 \n\t"
2320  "psrlw $8, %%mm3 \n\t"
2321  "packuswb %%mm1, %%mm0 \n\t"
2322  "packuswb %%mm3, %%mm2 \n\t"
2323  "movq %%mm0, %%mm1 \n\t"
2324  "movq %%mm2, %%mm3 \n\t"
2325  "psrlw $8, %%mm0 \n\t"
2326  "psrlw $8, %%mm2 \n\t"
2327  "pand %%mm7, %%mm1 \n\t"
2328  "pand %%mm7, %%mm3 \n\t"
2329  "packuswb %%mm2, %%mm0 \n\t"
2330  "packuswb %%mm3, %%mm1 \n\t"
2331  MOVNTQ" %%mm0,- 7(%4, %0) \n\t"
2332  MOVNTQ" %%mm1,- 7(%3, %0) \n\t"
2333  "add $8, %0 \n\t"
2334  " js 1b \n\t"
2335  : "+r"(count)
2336  : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)
2337  );
2338  count -= 7;
2339  }
2340 #endif
2341  src0++;
2342  src1++;
2343  while(count<0) {
2344  dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;
2345  dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;
2346  count++;
2347  }
2348 }
2349 
2350 static void RENAME(yuyvtoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2351  int width, int height,
2352  int lumStride, int chromStride, int srcStride)
2353 {
2354  int y;
2355  const int chromWidth= -((-width)>>1);
2356 
2357  for (y=0; y<height; y++) {
2358  RENAME(extract_even)(src, ydst, width);
2359  if(y&1) {
2360  RENAME(extract_odd2avg)(src-srcStride, src, udst, vdst, chromWidth);
2361  udst+= chromStride;
2362  vdst+= chromStride;
2363  }
2364 
2365  src += srcStride;
2366  ydst+= lumStride;
2367  }
2368  __asm__(
2369  EMMS" \n\t"
2370  SFENCE" \n\t"
2371  ::: "memory"
2372  );
2373 }
2374 
2375 #if !COMPILE_TEMPLATE_AMD3DNOW
2376 static void RENAME(yuyvtoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2377  int width, int height,
2378  int lumStride, int chromStride, int srcStride)
2379 {
2380  int y;
2381  const int chromWidth= -((-width)>>1);
2382 
2383  for (y=0; y<height; y++) {
2384  RENAME(extract_even)(src, ydst, width);
2385  RENAME(extract_odd2)(src, udst, vdst, chromWidth);
2386 
2387  src += srcStride;
2388  ydst+= lumStride;
2389  udst+= chromStride;
2390  vdst+= chromStride;
2391  }
2392  __asm__(
2393  EMMS" \n\t"
2394  SFENCE" \n\t"
2395  ::: "memory"
2396  );
2397 }
2398 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2399 
2400 static void RENAME(uyvytoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2401  int width, int height,
2402  int lumStride, int chromStride, int srcStride)
2403 {
2404  int y;
2405  const int chromWidth= -((-width)>>1);
2406 
2407  for (y=0; y<height; y++) {
2408  RENAME(extract_even)(src+1, ydst, width);
2409  if(y&1) {
2410  RENAME(extract_even2avg)(src-srcStride, src, udst, vdst, chromWidth);
2411  udst+= chromStride;
2412  vdst+= chromStride;
2413  }
2414 
2415  src += srcStride;
2416  ydst+= lumStride;
2417  }
2418  __asm__(
2419  EMMS" \n\t"
2420  SFENCE" \n\t"
2421  ::: "memory"
2422  );
2423 }
2424 
2425 #if !COMPILE_TEMPLATE_AMD3DNOW
2426 static void RENAME(uyvytoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
2427  int width, int height,
2428  int lumStride, int chromStride, int srcStride)
2429 {
2430  int y;
2431  const int chromWidth= -((-width)>>1);
2432 
2433  for (y=0; y<height; y++) {
2434  RENAME(extract_even)(src+1, ydst, width);
2435  RENAME(extract_even2)(src, udst, vdst, chromWidth);
2436 
2437  src += srcStride;
2438  ydst+= lumStride;
2439  udst+= chromStride;
2440  vdst+= chromStride;
2441  }
2442  __asm__(
2443  EMMS" \n\t"
2444  SFENCE" \n\t"
2445  ::: "memory"
2446  );
2447 }
2448 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2449 #endif /* !COMPILE_TEMPLATE_SSE2 */
2450 
2451 static inline void RENAME(rgb2rgb_init)(void)
2452 {
2453 #if !COMPILE_TEMPLATE_SSE2
2454 #if !COMPILE_TEMPLATE_AMD3DNOW
2455  rgb15to16 = RENAME(rgb15to16);
2456  rgb15tobgr24 = RENAME(rgb15tobgr24);
2457  rgb15to32 = RENAME(rgb15to32);
2458  rgb16tobgr24 = RENAME(rgb16tobgr24);
2459  rgb16to32 = RENAME(rgb16to32);
2460  rgb16to15 = RENAME(rgb16to15);
2461  rgb24tobgr16 = RENAME(rgb24tobgr16);
2462  rgb24tobgr15 = RENAME(rgb24tobgr15);
2463  rgb24tobgr32 = RENAME(rgb24tobgr32);
2464  rgb32to16 = RENAME(rgb32to16);
2465  rgb32to15 = RENAME(rgb32to15);
2466  rgb32tobgr24 = RENAME(rgb32tobgr24);
2467  rgb24to15 = RENAME(rgb24to15);
2468  rgb24to16 = RENAME(rgb24to16);
2469  rgb24tobgr24 = RENAME(rgb24tobgr24);
2470  shuffle_bytes_2103 = RENAME(shuffle_bytes_2103);
2471  rgb32tobgr16 = RENAME(rgb32tobgr16);
2472  rgb32tobgr15 = RENAME(rgb32tobgr15);
2473  yv12toyuy2 = RENAME(yv12toyuy2);
2474  yv12touyvy = RENAME(yv12touyvy);
2475  yuv422ptoyuy2 = RENAME(yuv422ptoyuy2);
2476  yuv422ptouyvy = RENAME(yuv422ptouyvy);
2477  yuy2toyv12 = RENAME(yuy2toyv12);
2478  vu9_to_vu12 = RENAME(vu9_to_vu12);
2479  yvu9_to_yuy2 = RENAME(yvu9_to_yuy2);
2480  uyvytoyuv422 = RENAME(uyvytoyuv422);
2481  yuyvtoyuv422 = RENAME(yuyvtoyuv422);
2482 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2483 
2484 #if COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW
2485  planar2x = RENAME(planar2x);
2486 #endif /* COMPILE_TEMPLATE_MMXEXT || COMPILE_TEMPLATE_AMD3DNOW */
2487 #if HAVE_7REGS
2488  ff_rgb24toyv12 = RENAME(rgb24toyv12);
2489 #endif /* HAVE_7REGS */
2490 
2491  yuyvtoyuv420 = RENAME(yuyvtoyuv420);
2492  uyvytoyuv420 = RENAME(uyvytoyuv420);
2493 #endif /* !COMPILE_TEMPLATE_SSE2 */
2494 
2495 #if !COMPILE_TEMPLATE_AMD3DNOW
2496  interleaveBytes = RENAME(interleaveBytes);
2497 #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
2498 }
rgb32tobgr15
static void RENAME() rgb32tobgr15(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:430
yuyvtoyuv422
static void RENAME() yuyvtoyuv422(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src, int width, int height, int lumStride, int chromStride, int srcStride)
Definition: x86/rgb2rgb_template.c:2376
vu9_to_vu12
static void RENAME() vu9_to_vu12(const uint8_t *src1, const uint8_t *src2, uint8_t *dst1, uint8_t *dst2, int width, int height, int srcStride1, int srcStride2, int dstStride1, int dstStride2)
Definition: x86/rgb2rgb_template.c:1929
MANGLE
#define MANGLE(a)
Definition: libavutil/x86/asm.h:110
uyvytoyuv422
static void RENAME() uyvytoyuv422(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src, int width, int height, int lumStride, int chromStride, int srcStride)
Definition: x86/rgb2rgb_template.c:2426
v
float v
Definition: avf_showspectrum.c:88
s
const char * s
Definition: avisynth_c.h:668
ff_rgb24toyv12_c
void ff_rgb24toyv12_c(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst, int width, int height, int lumStride, int chromStride, int srcStride, int32_t *rgb2yuv)
Height should be a multiple of 2 and width should be a multiple of 2.
Definition: rgb2rgb_template.c:615
rgb16tobgr24
static void RENAME() rgb16tobgr24(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:817
rgb32tobgr16
static void RENAME() rgb32tobgr16(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:325
rgb24tobgr24
static void RENAME() rgb24tobgr24(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:1088
yuv422ptoyuy2
static void RENAME() yuv422ptoyuy2(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, int width, int height, int lumStride, int chromStride, int dstStride)
Width should be a multiple of 16.
Definition: x86/rgb2rgb_template.c:1298
extract_even2
static void RENAME() extract_even2(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
Definition: x86/rgb2rgb_template.c:2146
rgb15to32
static void RENAME() rgb15to32(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:941
uyvytoyv12
static void RENAME() uyvytoyv12(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst, int width, int height, int lumStride, int chromStride, int srcStride)
Height should be a multiple of 2 and width should be a multiple of 16.
Definition: x86/rgb2rgb_template.c:1507
w
output residual component w
Definition: code Kat/hpsmodel.m:23
d
set threshold d
Definition: extra/f0detectionyin.m:6
uint8_t
uint8_t
Definition: audio_convert.c:194
yuy2toyv12
static void RENAME() yuy2toyv12(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst, int width, int height, int lumStride, int chromStride, int srcStride)
Height should be a multiple of 2 and width should be a multiple of 16.
Definition: x86/rgb2rgb_template.c:1309
b
#define b
Definition: input.c:42
end
end end
Definition: extra/f0detection.m:15
rgb24tobgr15
static void RENAME() rgb24tobgr15(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:599
extract_odd2avg
static void RENAME() extract_odd2avg(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
Definition: x86/rgb2rgb_template.c:2295
PREFETCH
#define PREFETCH
Definition: x86/rgb2rgb_template.c:42
rgb24tobgr16
static void RENAME() rgb24tobgr16(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:485
x
Discrete Time axis x
Definition: FFT of a Simple Sinusoid.m:10
s2
#define s2
Definition: regdef.h:39
uyvytoyuv420
static void RENAME() uyvytoyuv420(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src, int width, int height, int lumStride, int chromStride, int srcStride)
Definition: x86/rgb2rgb_template.c:2400
shuffle_bytes_2103
static void RENAME() shuffle_bytes_2103(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:1026
yv12touyvy
static void RENAME() yv12touyvy(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, int width, int height, int lumStride, int chromStride, int dstStride)
Height should be a multiple of 2 and width should be a multiple of 16 (If this is a problem for anyon...
Definition: x86/rgb2rgb_template.c:1277
r
const char * r
Definition: vf_curves.c:94
yuvPlanartouyvy
static void RENAME() yuvPlanartouyvy(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, int width, int height, int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
Definition: x86/rgb2rgb_template.c:1220
PACK_RGB32
#define PACK_RGB32
Definition: x86/rgb2rgb_template.c:929
MOVNTQ
#define MOVNTQ
Definition: x86/rgb2rgb_template.c:56
ff_rgb24toyv12
void(* ff_rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst, int width, int height, int lumStride, int chromStride, int srcStride, int32_t *rgb2yuv)
Height should be a multiple of 2 and width should be a multiple of 2.
Definition: rgb2rgb.c:76
g
FFT buffer for g
Definition: stft_peak.m:17
rgb24to15
static void RENAME() rgb24to15(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:656
rgb15to16
static void RENAME() rgb15to16(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:184
rgb16to15
static void RENAME() rgb16to15(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:227
int32_t
int32_t
Definition: audio_convert.c:194
rgb32to16
static void RENAME() rgb32to16(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:275
extract_even2avg
static void RENAME() extract_even2avg(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
Definition: x86/rgb2rgb_template.c:2193
rgb2rgb_init
static void RENAME() rgb2rgb_init(void)
Definition: x86/rgb2rgb_template.c:2451
yuv422ptouyvy
static void RENAME() yuv422ptouyvy(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, int width, int height, int lumStride, int chromStride, int dstStride)
Width should be a multiple of 16.
Definition: x86/rgb2rgb_template.c:1288
width
static int width
Definition: tests/utils.c:158
start.dest
dest
Definition: start.py:60
src
AVS_Value src
Definition: avisynth_c.h:523
void
typedef void(RENAME(mix_any_func_type))
Definition: rematrix_template.c:41
yuvPlanartoyuy2
static void RENAME() yuvPlanartoyuy2(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, int width, int height, int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
Definition: x86/rgb2rgb_template.c:1155
rgb16to32
static void RENAME() rgb16to32(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:983
RENAME
#define RENAME(a)
Definition: mpegaudiodec.c:109
rgb15tobgr24
static void RENAME() rgb15tobgr24(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:713
extract_even
static void RENAME() extract_even(const uint8_t *src, uint8_t *dst, x86_reg count)
Definition: x86/rgb2rgb_template.c:2108
height
BYTE int const BYTE int int int height
Definition: avisynth_c.h:713
x2
x2
Definition: genspecsines3.m:8
i
synthesis window for stochastic i
Definition: code Kat/hpsmodel.m:34
rgb32tobgr24
static void RENAME() rgb32tobgr24(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:142
s1
#define s1
Definition: regdef.h:38
yuyvtoyuv420
static void RENAME() yuyvtoyuv420(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src, int width, int height, int lumStride, int chromStride, int srcStride)
Definition: x86/rgb2rgb_template.c:2350
EMMS
#define EMMS
Definition: x86/rgb2rgb_template.c:49
yvu9_to_yuy2
static void RENAME() yvu9_to_yuy2(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3, uint8_t *dst, int width, int height, int srcStride1, int srcStride2, int srcStride3, int dstStride)
Definition: x86/rgb2rgb_template.c:2021
extract_odd2
static void RENAME() extract_odd2(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
Definition: x86/rgb2rgb_template.c:2247
rgb24tobgr32
static void RENAME() rgb24tobgr32(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:64
y
function y
Definition: D.m:1
yv12toyuy2
static void RENAME() yv12toyuy2(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, int width, int height, int lumStride, int chromStride, int dstStride)
Height should be a multiple of 2 and width should be a multiple of 16.
Definition: x86/rgb2rgb_template.c:1212
PAVGB
#define PAVGB(a, b)
Definition: postprocess_template.c:87
x86_reg
int x86_reg
Definition: libavutil/x86/asm.h:70
STORE_BGR24_MMX
#define STORE_BGR24_MMX
Definition: x86/rgb2rgb_template.c:108
dst
else dst[i][x+y *dst_stride[i]]
Definition: vf_mcdeint.c:160
SFENCE
#define SFENCE
Definition: x86/rgb2rgb_template.c:57
planar2x
void(* planar2x)(const uint8_t *src, uint8_t *dst, int width, int height, int srcStride, int dstStride)
Definition: rgb2rgb.c:81
rgb24to16
static void RENAME() rgb24to16(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:542
count
void INT64 INT64 count
Definition: avisynth_c.h:594
rgb32to15
static void RENAME() rgb32to15(const uint8_t *src, uint8_t *dst, int src_size)
Definition: x86/rgb2rgb_template.c:380
interleaveBytes
static void RENAME() interleaveBytes(const uint8_t *src1, const uint8_t *src2, uint8_t *dest, int width, int height, int src1Stride, int src2Stride, int dstStride)
Height should be a multiple of 2 and width should be a multiple of 2.
Definition: x86/rgb2rgb_template.c:1856
Generated on Fri Dec 20 2024 06:56:06 for FFmpeg by   doxygen 1.8.11