00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024 #undef MOVNTQ
00025 #undef EMMS
00026 #undef SFENCE
00027
00028 #if HAVE_AMD3DNOW
00029
00030 #define EMMS "femms"
00031 #else
00032 #define EMMS "emms"
00033 #endif
00034
00035 #if HAVE_MMX2
00036 #define MOVNTQ "movntq"
00037 #define SFENCE "sfence"
00038 #else
00039 #define MOVNTQ "movq"
00040 #define SFENCE " # nop"
00041 #endif
00042
00043 #define REG_BLUE "0"
00044 #define REG_RED "1"
00045 #define REG_GREEN "2"
00046 #define REG_ALPHA "3"
00047
00048 #define YUV2RGB_LOOP(depth) \
00049 h_size = (c->dstW + 7) & ~7; \
00050 if (h_size * depth > FFABS(dstStride[0])) \
00051 h_size -= 8; \
00052 \
00053 if (c->srcFormat == PIX_FMT_YUV422P) { \
00054 srcStride[1] *= 2; \
00055 srcStride[2] *= 2; \
00056 } \
00057 \
00058 __asm__ volatile ("pxor %mm4, %mm4\n\t"); \
00059 for (y = 0; y < srcSliceH; y++) { \
00060 uint8_t *image = dst[0] + (y + srcSliceY) * dstStride[0]; \
00061 const uint8_t *py = src[0] + y * srcStride[0]; \
00062 const uint8_t *pu = src[1] + (y >> 1) * srcStride[1]; \
00063 const uint8_t *pv = src[2] + (y >> 1) * srcStride[2]; \
00064 x86_reg index = -h_size / 2; \
00065
00066 #define YUV2RGB_INITIAL_LOAD \
00067 __asm__ volatile ( \
00068 "movq (%5, %0, 2), %%mm6\n\t" \
00069 "movd (%2, %0), %%mm0\n\t" \
00070 "movd (%3, %0), %%mm1\n\t" \
00071 "1: \n\t" \
00072
00073
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085
00086
00087
00088
00089 #define YUV2RGB \
00090 \
00091 "movq %%mm6, %%mm7\n\t" \
00092 "punpcklbw %%mm4, %%mm0\n\t" \
00093 "punpcklbw %%mm4, %%mm1\n\t" \
00094 "pand "MANGLE(mmx_00ffw)", %%mm6\n\t" \
00095 "psrlw $8, %%mm7\n\t" \
00096 "psllw $3, %%mm0\n\t" \
00097 "psllw $3, %%mm1\n\t" \
00098 "psllw $3, %%mm6\n\t" \
00099 "psllw $3, %%mm7\n\t" \
00100 "psubsw "U_OFFSET"(%4), %%mm0\n\t" \
00101 "psubsw "V_OFFSET"(%4), %%mm1\n\t" \
00102 "psubw "Y_OFFSET"(%4), %%mm6\n\t" \
00103 "psubw "Y_OFFSET"(%4), %%mm7\n\t" \
00104 \
00105 \
00106 "movq %%mm0, %%mm2\n\t" \
00107 "movq %%mm1, %%mm3\n\t" \
00108 "pmulhw "UG_COEFF"(%4), %%mm2\n\t" \
00109 "pmulhw "VG_COEFF"(%4), %%mm3\n\t" \
00110 "pmulhw "Y_COEFF" (%4), %%mm6\n\t" \
00111 "pmulhw "Y_COEFF" (%4), %%mm7\n\t" \
00112 "pmulhw "UB_COEFF"(%4), %%mm0\n\t" \
00113 "pmulhw "VR_COEFF"(%4), %%mm1\n\t" \
00114 "paddsw %%mm3, %%mm2\n\t" \
00115 \
00116 \
00117 \
00118 \
00119 "movq %%mm7, %%mm3\n\t" \
00120 "movq %%mm7, %%mm5\n\t" \
00121 "paddsw %%mm0, %%mm3\n\t" \
00122 "paddsw %%mm1, %%mm5\n\t" \
00123 "paddsw %%mm2, %%mm7\n\t" \
00124 "paddsw %%mm6, %%mm0\n\t" \
00125 "paddsw %%mm6, %%mm1\n\t" \
00126 "paddsw %%mm6, %%mm2\n\t" \
00127
00128 #define RGB_PACK_INTERLEAVE \
00129 \
00130 "packuswb %%mm1, %%mm0\n\t" \
00131 "packuswb %%mm5, %%mm3\n\t" \
00132 "packuswb %%mm2, %%mm2\n\t" \
00133 "movq %%mm0, %%mm1\n\n" \
00134 "packuswb %%mm7, %%mm7\n\t" \
00135 "punpcklbw %%mm3, %%mm0\n\t" \
00136 "punpckhbw %%mm3, %%mm1\n\t" \
00137 "punpcklbw %%mm7, %%mm2\n\t" \
00138
00139 #define YUV2RGB_ENDLOOP(depth) \
00140 "movq 8 (%5, %0, 2), %%mm6\n\t" \
00141 "movd 4 (%3, %0), %%mm1\n\t" \
00142 "movd 4 (%2, %0), %%mm0\n\t" \
00143 "add $"AV_STRINGIFY(depth * 8)", %1\n\t" \
00144 "add $4, %0\n\t" \
00145 "js 1b\n\t" \
00146
00147 #define YUV2RGB_OPERANDS \
00148 : "+r" (index), "+r" (image) \
00149 : "r" (pu - index), "r" (pv - index), "r"(&c->redDither), \
00150 "r" (py - 2*index) \
00151 ); \
00152 } \
00153
00154 #define YUV2RGB_OPERANDS_ALPHA \
00155 : "+r" (index), "+r" (image) \
00156 : "r" (pu - index), "r" (pv - index), "r"(&c->redDither), \
00157 "r" (py - 2*index), "r" (pa - 2*index) \
00158 ); \
00159 } \
00160
00161 #define YUV2RGB_ENDFUNC \
00162 __asm__ volatile (SFENCE"\n\t"EMMS); \
00163 return srcSliceH; \
00164
00165 #define IF0(x)
00166 #define IF1(x) x
00167
00168 #define RGB_PACK16(gmask, is15) \
00169 "pand "MANGLE(mmx_redmask)", %%mm0\n\t" \
00170 "pand "MANGLE(mmx_redmask)", %%mm1\n\t" \
00171 "movq %%mm2, %%mm3\n\t" \
00172 "psllw $"AV_STRINGIFY(3-is15)", %%mm2\n\t" \
00173 "psrlw $"AV_STRINGIFY(5+is15)", %%mm3\n\t" \
00174 "psrlw $3, %%mm0\n\t" \
00175 IF##is15("psrlw $1, %%mm1\n\t") \
00176 "pand "MANGLE(pb_e0)", %%mm2\n\t" \
00177 "pand "MANGLE(gmask)", %%mm3\n\t" \
00178 "por %%mm2, %%mm0\n\t" \
00179 "por %%mm3, %%mm1\n\t" \
00180 "movq %%mm0, %%mm2\n\t" \
00181 "punpcklbw %%mm1, %%mm0\n\t" \
00182 "punpckhbw %%mm1, %%mm2\n\t" \
00183 MOVNTQ " %%mm0, (%1)\n\t" \
00184 MOVNTQ " %%mm2, 8(%1)\n\t" \
00185
00186 #define DITHER_RGB \
00187 "paddusb "BLUE_DITHER"(%4), %%mm0\n\t" \
00188 "paddusb "GREEN_DITHER"(%4), %%mm2\n\t" \
00189 "paddusb "RED_DITHER"(%4), %%mm1\n\t" \
00190
00191 static inline int RENAME(yuv420_rgb15)(SwsContext *c, const uint8_t *src[],
00192 int srcStride[],
00193 int srcSliceY, int srcSliceH,
00194 uint8_t *dst[], int dstStride[])
00195 {
00196 int y, h_size;
00197
00198 YUV2RGB_LOOP(2)
00199
00200 #ifdef DITHER1XBPP
00201 c->blueDither = ff_dither8[y & 1];
00202 c->greenDither = ff_dither8[y & 1];
00203 c->redDither = ff_dither8[(y + 1) & 1];
00204 #endif
00205
00206 YUV2RGB_INITIAL_LOAD
00207 YUV2RGB
00208 RGB_PACK_INTERLEAVE
00209 #ifdef DITHER1XBPP
00210 DITHER_RGB
00211 #endif
00212 RGB_PACK16(pb_03, 1)
00213
00214 YUV2RGB_ENDLOOP(2)
00215 YUV2RGB_OPERANDS
00216 YUV2RGB_ENDFUNC
00217 }
00218
00219 static inline int RENAME(yuv420_rgb16)(SwsContext *c, const uint8_t *src[],
00220 int srcStride[],
00221 int srcSliceY, int srcSliceH,
00222 uint8_t *dst[], int dstStride[])
00223 {
00224 int y, h_size;
00225
00226 YUV2RGB_LOOP(2)
00227
00228 #ifdef DITHER1XBPP
00229 c->blueDither = ff_dither8[y & 1];
00230 c->greenDither = ff_dither4[y & 1];
00231 c->redDither = ff_dither8[(y + 1) & 1];
00232 #endif
00233
00234 YUV2RGB_INITIAL_LOAD
00235 YUV2RGB
00236 RGB_PACK_INTERLEAVE
00237 #ifdef DITHER1XBPP
00238 DITHER_RGB
00239 #endif
00240 RGB_PACK16(pb_07, 0)
00241
00242 YUV2RGB_ENDLOOP(2)
00243 YUV2RGB_OPERANDS
00244 YUV2RGB_ENDFUNC
00245 }
00246
00247 #define RGB_PACK24(blue, red)\
00248 "packuswb %%mm3, %%mm0 \n" \
00249 "packuswb %%mm5, %%mm1 \n" \
00250 "packuswb %%mm7, %%mm2 \n" \
00251 "movq %%mm"red", %%mm3 \n"\
00252 "movq %%mm"blue", %%mm6 \n"\
00253 "psrlq $32, %%mm"red" \n" \
00254 "punpcklbw %%mm2, %%mm3 \n" \
00255 "punpcklbw %%mm"red", %%mm6 \n" \
00256 "movq %%mm3, %%mm5 \n"\
00257 "punpckhbw %%mm"blue", %%mm2 \n" \
00258 "punpcklwd %%mm6, %%mm3 \n" \
00259 "punpckhwd %%mm6, %%mm5 \n" \
00260 RGB_PACK24_B
00261
00262 #if HAVE_MMX2
00263 DECLARE_ASM_CONST(8, int16_t, mask1101[4]) = {-1,-1, 0,-1};
00264 DECLARE_ASM_CONST(8, int16_t, mask0010[4]) = { 0, 0,-1, 0};
00265 DECLARE_ASM_CONST(8, int16_t, mask0110[4]) = { 0,-1,-1, 0};
00266 DECLARE_ASM_CONST(8, int16_t, mask1001[4]) = {-1, 0, 0,-1};
00267 DECLARE_ASM_CONST(8, int16_t, mask0100[4]) = { 0,-1, 0, 0};
00268 #undef RGB_PACK24_B
00269 #define RGB_PACK24_B\
00270 "pshufw $0xc6, %%mm2, %%mm1 \n"\
00271 "pshufw $0x84, %%mm3, %%mm6 \n"\
00272 "pshufw $0x38, %%mm5, %%mm7 \n"\
00273 "pand "MANGLE(mask1101)", %%mm6 \n" \
00274 "movq %%mm1, %%mm0 \n"\
00275 "pand "MANGLE(mask0110)", %%mm7 \n" \
00276 "movq %%mm1, %%mm2 \n"\
00277 "pand "MANGLE(mask0100)", %%mm1 \n" \
00278 "psrlq $48, %%mm3 \n" \
00279 "pand "MANGLE(mask0010)", %%mm0 \n" \
00280 "psllq $32, %%mm5 \n" \
00281 "pand "MANGLE(mask1001)", %%mm2 \n" \
00282 "por %%mm3, %%mm1 \n"\
00283 "por %%mm6, %%mm0 \n"\
00284 "por %%mm5, %%mm1 \n"\
00285 "por %%mm7, %%mm2 \n"\
00286 MOVNTQ" %%mm0, (%1) \n"\
00287 MOVNTQ" %%mm1, 8(%1) \n"\
00288 MOVNTQ" %%mm2, 16(%1) \n"\
00289
00290 #else
00291 #undef RGB_PACK24_B
00292 #define RGB_PACK24_B\
00293 "movd %%mm3, (%1) \n" \
00294 "movd %%mm2, 4(%1) \n" \
00295 "psrlq $32, %%mm3 \n"\
00296 "psrlq $16, %%mm2 \n"\
00297 "movd %%mm3, 6(%1) \n" \
00298 "movd %%mm2, 10(%1) \n" \
00299 "psrlq $16, %%mm2 \n"\
00300 "movd %%mm5, 12(%1) \n" \
00301 "movd %%mm2, 16(%1) \n" \
00302 "psrlq $32, %%mm5 \n"\
00303 "movd %%mm2, 20(%1) \n" \
00304 "movd %%mm5, 18(%1) \n" \
00305
00306 #endif
00307
00308 static inline int RENAME(yuv420_rgb24)(SwsContext *c, const uint8_t *src[],
00309 int srcStride[],
00310 int srcSliceY, int srcSliceH,
00311 uint8_t *dst[], int dstStride[])
00312 {
00313 int y, h_size;
00314
00315 YUV2RGB_LOOP(3)
00316
00317 YUV2RGB_INITIAL_LOAD
00318 YUV2RGB
00319 RGB_PACK24(REG_BLUE, REG_RED)
00320
00321 YUV2RGB_ENDLOOP(3)
00322 YUV2RGB_OPERANDS
00323 YUV2RGB_ENDFUNC
00324 }
00325
00326 static inline int RENAME(yuv420_bgr24)(SwsContext *c, const uint8_t *src[],
00327 int srcStride[],
00328 int srcSliceY, int srcSliceH,
00329 uint8_t *dst[], int dstStride[])
00330 {
00331 int y, h_size;
00332
00333 YUV2RGB_LOOP(3)
00334
00335 YUV2RGB_INITIAL_LOAD
00336 YUV2RGB
00337 RGB_PACK24(REG_RED, REG_BLUE)
00338
00339 YUV2RGB_ENDLOOP(3)
00340 YUV2RGB_OPERANDS
00341 YUV2RGB_ENDFUNC
00342 }
00343
00344
00345 #define SET_EMPTY_ALPHA \
00346 "pcmpeqd %%mm"REG_ALPHA", %%mm"REG_ALPHA"\n\t" \
00347
00348 #define LOAD_ALPHA \
00349 "movq (%6, %0, 2), %%mm"REG_ALPHA"\n\t" \
00350
00351 #define RGB_PACK32(red, green, blue, alpha) \
00352 "movq %%mm"blue", %%mm5\n\t" \
00353 "movq %%mm"red", %%mm6\n\t" \
00354 "punpckhbw %%mm"green", %%mm5\n\t" \
00355 "punpcklbw %%mm"green", %%mm"blue"\n\t" \
00356 "punpckhbw %%mm"alpha", %%mm6\n\t" \
00357 "punpcklbw %%mm"alpha", %%mm"red"\n\t" \
00358 "movq %%mm"blue", %%mm"green"\n\t" \
00359 "movq %%mm5, %%mm"alpha"\n\t" \
00360 "punpcklwd %%mm"red", %%mm"blue"\n\t" \
00361 "punpckhwd %%mm"red", %%mm"green"\n\t" \
00362 "punpcklwd %%mm6, %%mm5\n\t" \
00363 "punpckhwd %%mm6, %%mm"alpha"\n\t" \
00364 MOVNTQ " %%mm"blue", 0(%1)\n\t" \
00365 MOVNTQ " %%mm"green", 8(%1)\n\t" \
00366 MOVNTQ " %%mm5, 16(%1)\n\t" \
00367 MOVNTQ " %%mm"alpha", 24(%1)\n\t" \
00368
00369 static inline int RENAME(yuv420_rgb32)(SwsContext *c, const uint8_t *src[],
00370 int srcStride[],
00371 int srcSliceY, int srcSliceH,
00372 uint8_t *dst[], int dstStride[])
00373 {
00374 int y, h_size;
00375
00376 YUV2RGB_LOOP(4)
00377
00378 YUV2RGB_INITIAL_LOAD
00379 YUV2RGB
00380 RGB_PACK_INTERLEAVE
00381 SET_EMPTY_ALPHA
00382 RGB_PACK32(REG_RED, REG_GREEN, REG_BLUE, REG_ALPHA)
00383
00384 YUV2RGB_ENDLOOP(4)
00385 YUV2RGB_OPERANDS
00386 YUV2RGB_ENDFUNC
00387 }
00388
00389 static inline int RENAME(yuva420_rgb32)(SwsContext *c, const uint8_t *src[],
00390 int srcStride[],
00391 int srcSliceY, int srcSliceH,
00392 uint8_t *dst[], int dstStride[])
00393 {
00394 #if HAVE_7REGS
00395 int y, h_size;
00396
00397 YUV2RGB_LOOP(4)
00398
00399 const uint8_t *pa = src[3] + y * srcStride[3];
00400 YUV2RGB_INITIAL_LOAD
00401 YUV2RGB
00402 RGB_PACK_INTERLEAVE
00403 LOAD_ALPHA
00404 RGB_PACK32(REG_RED, REG_GREEN, REG_BLUE, REG_ALPHA)
00405
00406 YUV2RGB_ENDLOOP(4)
00407 YUV2RGB_OPERANDS_ALPHA
00408 YUV2RGB_ENDFUNC
00409 #endif
00410 }
00411
00412 static inline int RENAME(yuv420_bgr32)(SwsContext *c, const uint8_t *src[],
00413 int srcStride[],
00414 int srcSliceY, int srcSliceH,
00415 uint8_t *dst[], int dstStride[])
00416 {
00417 int y, h_size;
00418
00419 YUV2RGB_LOOP(4)
00420
00421 YUV2RGB_INITIAL_LOAD
00422 YUV2RGB
00423 RGB_PACK_INTERLEAVE
00424 SET_EMPTY_ALPHA
00425 RGB_PACK32(REG_BLUE, REG_GREEN, REG_RED, REG_ALPHA)
00426
00427 YUV2RGB_ENDLOOP(4)
00428 YUV2RGB_OPERANDS
00429 YUV2RGB_ENDFUNC
00430 }
00431
00432 static inline int RENAME(yuva420_bgr32)(SwsContext *c, const uint8_t *src[],
00433 int srcStride[],
00434 int srcSliceY, int srcSliceH,
00435 uint8_t *dst[], int dstStride[])
00436 {
00437 #if HAVE_7REGS
00438 int y, h_size;
00439
00440 YUV2RGB_LOOP(4)
00441
00442 const uint8_t *pa = src[3] + y * srcStride[3];
00443 YUV2RGB_INITIAL_LOAD
00444 YUV2RGB
00445 RGB_PACK_INTERLEAVE
00446 LOAD_ALPHA
00447 RGB_PACK32(REG_BLUE, REG_GREEN, REG_RED, REG_ALPHA)
00448
00449 YUV2RGB_ENDLOOP(4)
00450 YUV2RGB_OPERANDS_ALPHA
00451 YUV2RGB_ENDFUNC
00452 #endif
00453 }