libpng-1.2.8/pnggccrd.c

   1 /* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file
   2  *
   3  * For Intel x86 CPU (Pentium-MMX or later) and GNU C compiler.
   4  *
   5  *     See http://www.intel.com/drg/pentiumII/appnotes/916/916.htm
   6  *     and http://www.intel.com/drg/pentiumII/appnotes/923/923.htm
   7  *     for Intel's performance analysis of the MMX vs. non-MMX code.
   8  *
   9  * libpng version 1.2.8 - December 3, 2004
  10  * For conditions of distribution and use, see copyright notice in png.h
  11  * Copyright (c) 1998-2004 Glenn Randers-Pehrson
  12  * Copyright (c) 1998, Intel Corporation
  13  *
  14  * Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998.
  15  * Interface to libpng contributed by Gilles Vollant, 1999.
  16  * GNU C port by Greg Roelofs, 1999-2001.
  17  *
  18  * Lines 2350-4300 converted in place with intel2gas 1.3.1:
  19  *
  20  *   intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c
  21  *
  22  * and then cleaned up by hand.  See http://hermes.terminal.at/intel2gas/ .
  23  *
  24  * NOTE:  A sufficiently recent version of GNU as (or as.exe under DOS/Windows)
  25  *        is required to assemble the newer MMX instructions such as movq.
  26  *        For djgpp, see
  27  *
  28  *           ftp://ftp.simtel.net/pub/simtelnet/gnu/djgpp/v2gnu/bnu281b.zip
  29  *
  30  *        (or a later version in the same directory).  For Linux, check your
  31  *        distribution's web site(s) or try these links:
  32  *
  33  *           http://rufus.w3.org/linux/RPM/binutils.html
  34  *           http://www.debian.org/Packages/stable/devel/binutils.html
  35  *           ftp://ftp.slackware.com/pub/linux/slackware/slackware/slakware/d1/
  36  *             binutils.tgz
  37  *
  38  *        For other platforms, see the main GNU site:
  39  *
  40  *           ftp://ftp.gnu.org/pub/gnu/binutils/
  41  *
  42  *        Version 2.5.2l.15 is definitely too old...
  43  */
  44
  45 /*
  46  * TEMPORARY PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs)
  47  * =====================================
  48  *
  49  * 19991006:
  50  *  - fixed sign error in post-MMX cleanup code (16- & 32-bit cases)
  51  *
  52  * 19991007:
  53  *  - additional optimizations (possible or definite):
  54  *     x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested]
  55  *     - write MMX code for 48-bit case (pixel_bytes == 6)
  56  *     - figure out what's up with 24-bit case (pixel_bytes == 3):
  57  *        why subtract 8 from width_mmx in the pass 4/5 case?
  58  *        (only width_mmx case) (near line 1606)
  59  *     x [DONE] replace pixel_bytes within each block with the true
  60  *        constant value (or are compilers smart enough to do that?)
  61  *     - rewrite all MMX interlacing code so it's aligned with
  62  *        the *beginning* of the row buffer, not the end.  This
  63  *        would not only allow one to eliminate half of the memory
  64  *        writes for odd passes (that is, pass == odd), it may also
  65  *        eliminate some unaligned-data-access exceptions (assuming
  66  *        there's a penalty for not aligning 64-bit accesses on
  67  *        64-bit boundaries).  The only catch is that the "leftover"
  68  *        pixel(s) at the end of the row would have to be saved,
  69  *        but there are enough unused MMX registers in every case,
  70  *        so this is not a problem.  A further benefit is that the
  71  *        post-MMX cleanup code (C code) in at least some of the
  72  *        cases could be done within the assembler block.
  73  *  x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing,
  74  *     inconsistent, and don't match the MMX Programmer's Reference
  75  *     Manual conventions anyway.  They should be changed to
  76  *     "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that
  77  *     was lowest in memory (e.g., corresponding to a left pixel)
  78  *     and b7 is the byte that was highest (e.g., a right pixel).
  79  *
  80  * 19991016:
  81  *  - Brennan's Guide notwithstanding, gcc under Linux does *not*
  82  *     want globals prefixed by underscores when referencing them--
  83  *     i.e., if the variable is const4, then refer to it as const4,
  84  *     not _const4.  This seems to be a djgpp-specific requirement.
  85  *     Also, such variables apparently *must* be declared outside
  86  *     of functions; neither static nor automatic variables work if
  87  *     defined within the scope of a single function, but both
  88  *     static and truly global (multi-module) variables work fine.
  89  *
  90  * 19991023:
  91  *  - fixed png_combine_row() non-MMX replication bug (odd passes only?)
  92  *  - switched from string-concatenation-with-macros to cleaner method of
  93  *     renaming global variables for djgpp--i.e., always use prefixes in
  94  *     inlined assembler code (== strings) and conditionally rename the
  95  *     variables, not the other way around.  Hence _const4, _mask8_0, etc.
  96  *
  97  * 19991024:
  98  *  - fixed mmxsupport()/png_do_read_interlace() first-row bug
  99  *     This one was severely weird:  even though mmxsupport() doesn't touch
 100  *     ebx (where "row" pointer was stored), it nevertheless managed to zero
 101  *     the register (even in static/non-fPIC code--see below), which in turn
 102  *     caused png_do_read_interlace() to return prematurely on the first row of
 103  *     interlaced images (i.e., without expanding the interlaced pixels).
 104  *     Inspection of the generated assembly code didn't turn up any clues,
 105  *     although it did point at a minor optimization (i.e., get rid of
 106  *     mmx_supported_local variable and just use eax).  Possibly the CPUID
 107  *     instruction is more destructive than it looks?  (Not yet checked.)
 108  *  - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly
 109  *     listings...  Apparently register spillage has to do with ebx, since
 110  *     it's used to index the global offset table.  Commenting it out of the
 111  *     input-reg lists in png_combine_row() eliminated compiler barfage, so
 112  *     ifdef'd with __PIC__ macro:  if defined, use a global for unmask
 113  *
 114  * 19991107:
 115  *  - verified CPUID clobberage:  12-char string constant ("GenuineIntel",
 116  *     "AuthenticAMD", etc.) placed in ebx:ecx:edx.  Still need to polish.
 117  *
 118  * 19991120:
 119  *  - made "diff" variable (now "_dif") global to simplify conversion of
 120  *     filtering routines (running out of regs, sigh).  "diff" is still used
 121  *     in interlacing routines, however.
 122  *  - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX
 123  *     macro determines which is used); original not yet tested.
 124  *
 125  * 20000213:
 126  *  - when compiling with gcc, be sure to use  -fomit-frame-pointer
 127  *
 128  * 20000319:
 129  *  - fixed a register-name typo in png_do_read_interlace(), default (MMX) case,
 130  *     pass == 4 or 5, that caused visible corruption of interlaced images
 131  *
 132  * 20000623:
 133  *  - Various problems were reported with gcc 2.95.2 in the Cygwin environment,
 134  *     many of the form "forbidden register 0 (ax) was spilled for class AREG."
 135  *     This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and
 136  *     Chuck Wilson supplied a patch involving dummy output registers.  See
 137  *     http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624
 138  *     for the original (anonymous) SourceForge bug report.
 139  *
 140  * 20000706:
 141  *  - Chuck Wilson passed along these remaining gcc 2.95.2 errors:
 142  *       pnggccrd.c: In function `png_combine_row':
 143  *       pnggccrd.c:525: more than 10 operands in `asm'
 144  *       pnggccrd.c:669: more than 10 operands in `asm'
 145  *       pnggccrd.c:828: more than 10 operands in `asm'
 146  *       pnggccrd.c:994: more than 10 operands in `asm'
 147  *       pnggccrd.c:1177: more than 10 operands in `asm'
 148  *     They are all the same problem and can be worked around by using the
 149  *     global _unmask variable unconditionally, not just in the -fPIC case.
 150  *     Reportedly earlier versions of gcc also have the problem with more than
 151  *     10 operands; they just don't report it.  Much strangeness ensues, etc.
 152  *
 153  * 20000729:
 154  *  - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted
 155  *     MMX routine); began converting png_read_filter_row_mmx_sub()
 156  *  - to finish remaining sections:
 157  *     - clean up indentation and comments
 158  *     - preload local variables
 159  *     - add output and input regs (order of former determines numerical
 160  *        mapping of latter)
 161  *     - avoid all usage of ebx (including bx, bh, bl) register [20000823]
 162  *     - remove "$" from addressing of Shift and Mask variables [20000823]
 163  *
 164  * 20000731:
 165  *  - global union vars causing segfaults in png_read_filter_row_mmx_sub()?
 166  *
 167  * 20000822:
 168  *  - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with
 169  *     shared-library (-fPIC) version!  Code works just fine as part of static
 170  *     library.  Damn damn damn damn damn, should have tested that sooner.
 171  *     ebx is getting clobbered again (explicitly this time); need to save it
 172  *     on stack or rewrite asm code to avoid using it altogether.  Blargh!
 173  *
 174  * 20000823:
 175  *  - first section was trickiest; all remaining sections have ebx -> edx now.
 176  *     (-fPIC works again.)  Also added missing underscores to various Shift*
 177  *     and *Mask* globals and got rid of leading "$" signs.
 178  *
 179  * 20000826:
 180  *  - added visual separators to help navigate microscopic printed copies
 181  *     (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working
 182  *     on png_read_filter_row_mmx_avg()
 183  *
 184  * 20000828:
 185  *  - finished png_read_filter_row_mmx_avg():  only Paeth left! (930 lines...)
 186  *     What the hell, did png_read_filter_row_mmx_paeth(), too.  Comments not
 187  *     cleaned up/shortened in either routine, but functionality is complete
 188  *     and seems to be working fine.
 189  *
 190  * 20000829:
 191  *  - ahhh, figured out last(?) bit of gcc/gas asm-fu:  if register is listed
 192  *     as an input reg (with dummy output variables, etc.), then it *cannot*
 193  *     also appear in the clobber list or gcc 2.95.2 will barf.  The solution
 194  *     is simple enough...
 195  *
 196  * 20000914:
 197  *  - bug in png_read_filter_row_mmx_avg():  16-bit grayscale not handled
 198  *     correctly (but 48-bit RGB just fine)
 199  *
 200  * 20000916:
 201  *  - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors:
 202  *     - "_ShiftBpp.use = 24;"      should have been   "_ShiftBpp.use = 16;"
 203  *     - "_ShiftRem.use = 40;"      should have been   "_ShiftRem.use = 48;"
 204  *     - "psllq _ShiftRem, %%mm2"   should have been   "psrlq _ShiftRem, %%mm2"
 205  *
 206  * 20010101:
 207  *  - added new png_init_mmx_flags() function (here only because it needs to
 208  *     call mmxsupport(), which should probably become global png_mmxsupport());
 209  *     modified other MMX routines to run conditionally (png_ptr->asm_flags)
 210  *
 211  * 20010103:
 212  *  - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported,
 213  *     and made it public; moved png_init_mmx_flags() to png.c as internal func
 214  *
 215  * 20010104:
 216  *  - removed dependency on png_read_filter_row_c() (C code already duplicated
 217  *     within MMX version of png_read_filter_row()) so no longer necessary to
 218  *     compile it into pngrutil.o
 219  *
 220  * 20010310:
 221  *  - fixed buffer-overrun bug in png_combine_row() C code (non-MMX)
 222  *
 223  * 20020304:
 224  *  - eliminated incorrect use of width_mmx in pixel_bytes == 8 case
 225  *
 226  * 20040724:
 227  *   - more tinkering with clobber list at lines 4529 and 5033, to get
 228  *     it to compile on gcc-3.4.
 229  *
 230  * STILL TO DO:
 231  *     - test png_do_read_interlace() 64-bit case (pixel_bytes == 8)
 232  *     - write MMX code for 48-bit case (pixel_bytes == 6)
 233  *     - figure out what's up with 24-bit case (pixel_bytes == 3):
 234  *        why subtract 8 from width_mmx in the pass 4/5 case?
 235  *        (only width_mmx case) (near line 1606)
 236  *     - rewrite all MMX interlacing code so it's aligned with beginning
 237  *        of the row buffer, not the end (see 19991007 for details)
 238  *     x pick one version of mmxsupport() and get rid of the other
 239  *     - add error messages to any remaining bogus default cases
 240  *     - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed)
 241  *     x add support for runtime enable/disable/query of various MMX routines
 242  */
 243
 244 #define PNG_INTERNAL
 245 #include "png.h"
 246
 247 #if defined(PNG_USE_PNGGCCRD)
 248
 249 int PNGAPI png_mmx_support(void);
 250
 251 #ifdef PNG_USE_LOCAL_ARRAYS
 252 static const int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0};
 253 static const int FARDATA png_pass_inc[7]   = {8, 8, 4, 4, 2, 2, 1};
 254 static const int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1};
 255 #endif
 256
 257 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
 258 /* djgpp, Win32, and Cygwin add their own underscores to global variables,
 259  * so define them without: */
 260 #if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__)
 261 #  define _mmx_supported  mmx_supported
 262 #  define _const4         const4
 263 #  define _const6         const6
 264 #  define _mask8_0        mask8_0
 265 #  define _mask16_1       mask16_1
 266 #  define _mask16_0       mask16_0
 267 #  define _mask24_2       mask24_2
 268 #  define _mask24_1       mask24_1
 269 #  define _mask24_0       mask24_0
 270 #  define _mask32_3       mask32_3
 271 #  define _mask32_2       mask32_2
 272 #  define _mask32_1       mask32_1
 273 #  define _mask32_0       mask32_0
 274 #  define _mask48_5       mask48_5
 275 #  define _mask48_4       mask48_4
 276 #  define _mask48_3       mask48_3
 277 #  define _mask48_2       mask48_2
 278 #  define _mask48_1       mask48_1
 279 #  define _mask48_0       mask48_0
 280 #  define _LBCarryMask    LBCarryMask
 281 #  define _HBClearMask    HBClearMask
 282 #  define _ActiveMask     ActiveMask
 283 #  define _ActiveMask2    ActiveMask2
 284 #  define _ActiveMaskEnd  ActiveMaskEnd
 285 #  define _ShiftBpp       ShiftBpp
 286 #  define _ShiftRem       ShiftRem
 287 #ifdef PNG_THREAD_UNSAFE_OK
 288 #  define _unmask         unmask
 289 #  define _FullLength     FullLength
 290 #  define _MMXLength      MMXLength
 291 #  define _dif            dif
 292 #  define _patemp         patemp
 293 #  define _pbtemp         pbtemp
 294 #  define _pctemp         pctemp
 295 #endif
 296 #endif
 297
 298
 299 /* These constants are used in the inlined MMX assembly code.
 300    Ignore gcc's "At top level: defined but not used" warnings. */
 301
 302 /* GRR 20000706:  originally _unmask was needed only when compiling with -fPIC,
 303  *  since that case uses the %ebx register for indexing the Global Offset Table
 304  *  and there were no other registers available.  But gcc 2.95 and later emit
 305  *  "more than 10 operands in `asm'" errors when %ebx is used to preload unmask
 306  *  in the non-PIC case, so we'll just use the global unconditionally now.
 307  */
 308 #ifdef PNG_THREAD_UNSAFE_OK
 309 static int _unmask;
 310 #endif
 311
 312 static unsigned long long _mask8_0  = 0x0102040810204080LL;
 313
 314 static unsigned long long _mask16_1 = 0x0101020204040808LL;
 315 static unsigned long long _mask16_0 = 0x1010202040408080LL;
 316
 317 static unsigned long long _mask24_2 = 0x0101010202020404LL;
 318 static unsigned long long _mask24_1 = 0x0408080810101020LL;
 319 static unsigned long long _mask24_0 = 0x2020404040808080LL;
 320
 321 static unsigned long long _mask32_3 = 0x0101010102020202LL;
 322 static unsigned long long _mask32_2 = 0x0404040408080808LL;
 323 static unsigned long long _mask32_1 = 0x1010101020202020LL;
 324 static unsigned long long _mask32_0 = 0x4040404080808080LL;
 325
 326 static unsigned long long _mask48_5 = 0x0101010101010202LL;
 327 static unsigned long long _mask48_4 = 0x0202020204040404LL;
 328 static unsigned long long _mask48_3 = 0x0404080808080808LL;
 329 static unsigned long long _mask48_2 = 0x1010101010102020LL;
 330 static unsigned long long _mask48_1 = 0x2020202040404040LL;
 331 static unsigned long long _mask48_0 = 0x4040808080808080LL;
 332
 333 static unsigned long long _const4   = 0x0000000000FFFFFFLL;
 334 //static unsigned long long _const5 = 0x000000FFFFFF0000LL;     // NOT USED
 335 static unsigned long long _const6   = 0x00000000000000FFLL;
 336
 337 // These are used in the row-filter routines and should/would be local
 338 //  variables if not for gcc addressing limitations.
 339 // WARNING: Their presence probably defeats the thread safety of libpng.
 340
 341 #ifdef PNG_THREAD_UNSAFE_OK
 342 static png_uint_32  _FullLength;
 343 static png_uint_32  _MMXLength;
 344 static int          _dif;
 345 static int          _patemp; // temp variables for Paeth routine
 346 static int          _pbtemp;
 347 static int          _pctemp;
 348 #endif
 349
 350 void /* PRIVATE */
 351 png_squelch_warnings(void)
 352 {
 353 #ifdef PNG_THREAD_UNSAFE_OK
 354    _dif = _dif;
 355    _patemp = _patemp;
 356    _pbtemp = _pbtemp;
 357    _pctemp = _pctemp;
 358    _MMXLength = _MMXLength;
 359 #endif
 360    _const4  = _const4;
 361    _const6  = _const6;
 362    _mask8_0  = _mask8_0;
 363    _mask16_1 = _mask16_1;
 364    _mask16_0 = _mask16_0;
 365    _mask24_2 = _mask24_2;
 366    _mask24_1 = _mask24_1;
 367    _mask24_0 = _mask24_0;
 368    _mask32_3 = _mask32_3;
 369    _mask32_2 = _mask32_2;
 370    _mask32_1 = _mask32_1;
 371    _mask32_0 = _mask32_0;
 372    _mask48_5 = _mask48_5;
 373    _mask48_4 = _mask48_4;
 374    _mask48_3 = _mask48_3;
 375    _mask48_2 = _mask48_2;
 376    _mask48_1 = _mask48_1;
 377    _mask48_0 = _mask48_0;
 378 }
 379 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
 380
 381
 382 static int _mmx_supported = 2;
 383
 384 /*===========================================================================*/
 385 /*                                                                           */
 386 /*                       P N G _ C O M B I N E _ R O W                       */
 387 /*                                                                           */
 388 /*===========================================================================*/
 389
 390 #if defined(PNG_HAVE_ASSEMBLER_COMBINE_ROW)
 391
 392 #define BPP2  2
 393 #define BPP3  3 /* bytes per pixel (a.k.a. pixel_bytes) */
 394 #define BPP4  4
 395 #define BPP6  6 /* (defined only to help avoid cut-and-paste errors) */
 396 #define BPP8  8
 397
 398 /* Combines the row recently read in with the previous row.
 399    This routine takes care of alpha and transparency if requested.
 400    This routine also handles the two methods of progressive display
 401    of interlaced images, depending on the mask value.
 402    The mask value describes which pixels are to be combined with
 403    the row.  The pattern always repeats every 8 pixels, so just 8
 404    bits are needed.  A one indicates the pixel is to be combined; a
 405    zero indicates the pixel is to be skipped.  This is in addition
 406    to any alpha or transparency value associated with the pixel.
 407    If you want all pixels to be combined, pass 0xff (255) in mask. */
 408
 409 /* Use this routine for the x86 platform - it uses a faster MMX routine
 410    if the machine supports MMX. */
 411
 412 void /* PRIVATE */
 413 png_combine_row(png_structp png_ptr, png_bytep row, int mask)
 414 {
 415    png_debug(1, "in png_combine_row (pnggccrd.c)\n");
 416
 417 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
 418    if (_mmx_supported == 2) {
 419 #if !defined(PNG_1_0_X)
 420        /* this should have happened in png_init_mmx_flags() already */
 421        png_warning(png_ptr, "asm_flags may not have been initialized");
 422 #endif
 423        png_mmx_support();
 424    }
 425 #endif
 426
 427    if (mask == 0xff)
 428    {
 429       png_debug(2,"mask == 0xff:  doing single png_memcpy()\n");
 430       png_memcpy(row, png_ptr->row_buf + 1,
 431        (png_size_t)PNG_ROWBYTES(png_ptr->row_info.pixel_depth,png_ptr->width));
 432    }
 433    else   /* (png_combine_row() is never called with mask == 0) */
 434    {
 435       switch (png_ptr->row_info.pixel_depth)
 436       {
 437          case 1:        /* png_ptr->row_info.pixel_depth */
 438          {
 439             png_bytep sp;
 440             png_bytep dp;
 441             int s_inc, s_start, s_end;
 442             int m;
 443             int shift;
 444             png_uint_32 i;
 445
 446             sp = png_ptr->row_buf + 1;
 447             dp = row;
 448             m = 0x80;
 449 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
 450             if (png_ptr->transformations & PNG_PACKSWAP)
 451             {
 452                 s_start = 0;
 453                 s_end = 7;
 454                 s_inc = 1;
 455             }
 456             else
 457 #endif
 458             {
 459                 s_start = 7;
 460                 s_end = 0;
 461                 s_inc = -1;
 462             }
 463
 464             shift = s_start;
 465
 466             for (i = 0; i < png_ptr->width; i++)
 467             {
 468                if (m & mask)
 469                {
 470                   int value;
 471
 472                   value = (*sp >> shift) & 0x1;
 473                   *dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff);
 474                   *dp |= (png_byte)(value << shift);
 475                }
 476
 477                if (shift == s_end)
 478                {
 479                   shift = s_start;
 480                   sp++;
 481                   dp++;
 482                }
 483                else
 484                   shift += s_inc;
 485
 486                if (m == 1)
 487                   m = 0x80;
 488                else
 489                   m >>= 1;
 490             }
 491             break;
 492          }
 493
 494          case 2:        /* png_ptr->row_info.pixel_depth */
 495          {
 496             png_bytep sp;
 497             png_bytep dp;
 498             int s_start, s_end, s_inc;
 499             int m;
 500             int shift;
 501             png_uint_32 i;
 502             int value;
 503
 504             sp = png_ptr->row_buf + 1;
 505             dp = row;
 506             m = 0x80;
 507 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
 508             if (png_ptr->transformations & PNG_PACKSWAP)
 509             {
 510                s_start = 0;
 511                s_end = 6;
 512                s_inc = 2;
 513             }
 514             else
 515 #endif
 516             {
 517                s_start = 6;
 518                s_end = 0;
 519                s_inc = -2;
 520             }
 521
 522             shift = s_start;
 523
 524             for (i = 0; i < png_ptr->width; i++)
 525             {
 526                if (m & mask)
 527                {
 528                   value = (*sp >> shift) & 0x3;
 529                   *dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff);
 530                   *dp |= (png_byte)(value << shift);
 531                }
 532
 533                if (shift == s_end)
 534                {
 535                   shift = s_start;
 536                   sp++;
 537                   dp++;
 538                }
 539                else
 540                   shift += s_inc;
 541                if (m == 1)
 542                   m = 0x80;
 543                else
 544                   m >>= 1;
 545             }
 546             break;
 547          }
 548
 549          case 4:        /* png_ptr->row_info.pixel_depth */
 550          {
 551             png_bytep sp;
 552             png_bytep dp;
 553             int s_start, s_end, s_inc;
 554             int m;
 555             int shift;
 556             png_uint_32 i;
 557             int value;
 558
 559             sp = png_ptr->row_buf + 1;
 560             dp = row;
 561             m = 0x80;
 562 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
 563             if (png_ptr->transformations & PNG_PACKSWAP)
 564             {
 565                s_start = 0;
 566                s_end = 4;
 567                s_inc = 4;
 568             }
 569             else
 570 #endif
 571             {
 572                s_start = 4;
 573                s_end = 0;
 574                s_inc = -4;
 575             }
 576             shift = s_start;
 577
 578             for (i = 0; i < png_ptr->width; i++)
 579             {
 580                if (m & mask)
 581                {
 582                   value = (*sp >> shift) & 0xf;
 583                   *dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff);
 584                   *dp |= (png_byte)(value << shift);
 585                }
 586
 587                if (shift == s_end)
 588                {
 589                   shift = s_start;
 590                   sp++;
 591                   dp++;
 592                }
 593                else
 594                   shift += s_inc;
 595                if (m == 1)
 596                   m = 0x80;
 597                else
 598                   m >>= 1;
 599             }
 600             break;
 601          }
 602
 603          case 8:        /* png_ptr->row_info.pixel_depth */
 604          {
 605             png_bytep srcptr;
 606             png_bytep dstptr;
 607
 608 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
 609 #if !defined(PNG_1_0_X)
 610             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
 611                 /* && _mmx_supported */ )
 612 #else
 613             if (_mmx_supported)
 614 #endif
 615             {
 616                png_uint_32 len;
 617                int diff;
 618                int dummy_value_a;   // fix 'forbidden register spilled' error
 619                int dummy_value_d;
 620                int dummy_value_c;
 621                int dummy_value_S;
 622                int dummy_value_D;
 623                _unmask = ~mask;            // global variable for -fPIC version
 624                srcptr = png_ptr->row_buf + 1;
 625                dstptr = row;
 626                len  = png_ptr->width &~7;  // reduce to multiple of 8
 627                diff = (int) (png_ptr->width & 7);  // amount lost
 628
 629                __asm__ __volatile__ (
 630                   "movd      _unmask, %%mm7  \n\t" // load bit pattern
 631                   "psubb     %%mm6, %%mm6    \n\t" // zero mm6
 632                   "punpcklbw %%mm7, %%mm7    \n\t"
 633                   "punpcklwd %%mm7, %%mm7    \n\t"
 634                   "punpckldq %%mm7, %%mm7    \n\t" // fill reg with 8 masks
 635
 636                   "movq      _mask8_0, %%mm0 \n\t"
 637                   "pand      %%mm7, %%mm0    \n\t" // nonzero if keep byte
 638                   "pcmpeqb   %%mm6, %%mm0    \n\t" // zeros->1s, v versa
 639
 640 // preload        "movl      len, %%ecx      \n\t" // load length of line
 641 // preload        "movl      srcptr, %%esi   \n\t" // load source
 642 // preload        "movl      dstptr, %%edi   \n\t" // load dest
 643
 644                   "cmpl      $0, %%ecx       \n\t" // len == 0 ?
 645                   "je        mainloop8end    \n\t"
 646
 647                 "mainloop8:                  \n\t"
 648                   "movq      (%%esi), %%mm4  \n\t" // *srcptr
 649                   "pand      %%mm0, %%mm4    \n\t"
 650                   "movq      %%mm0, %%mm6    \n\t"
 651                   "pandn     (%%edi), %%mm6  \n\t" // *dstptr
 652                   "por       %%mm6, %%mm4    \n\t"
 653                   "movq      %%mm4, (%%edi)  \n\t"
 654                   "addl      $8, %%esi       \n\t" // inc by 8 bytes processed
 655                   "addl      $8, %%edi       \n\t"
 656                   "subl      $8, %%ecx       \n\t" // dec by 8 pixels processed
 657                   "ja        mainloop8       \n\t"
 658
 659                 "mainloop8end:               \n\t"
 660 // preload        "movl      diff, %%ecx     \n\t" // (diff is in eax)
 661                   "movl      %%eax, %%ecx    \n\t"
 662                   "cmpl      $0, %%ecx       \n\t"
 663                   "jz        end8            \n\t"
 664 // preload        "movl      mask, %%edx     \n\t"
 665                   "sall      $24, %%edx      \n\t" // make low byte, high byte
 666
 667                 "secondloop8:                \n\t"
 668                   "sall      %%edx           \n\t" // move high bit to CF
 669                   "jnc       skip8           \n\t" // if CF = 0
 670                   "movb      (%%esi), %%al   \n\t"
 671                   "movb      %%al, (%%edi)   \n\t"
 672
 673                 "skip8:                      \n\t"
 674                   "incl      %%esi           \n\t"
 675                   "incl      %%edi           \n\t"
 676                   "decl      %%ecx           \n\t"
 677                   "jnz       secondloop8     \n\t"
 678
 679                 "end8:                       \n\t"
 680                   "EMMS                      \n\t"  // DONE
 681
 682                   : "=a" (dummy_value_a),           // output regs (dummy)
 683                     "=d" (dummy_value_d),
 684                     "=c" (dummy_value_c),
 685                     "=S" (dummy_value_S),
 686                     "=D" (dummy_value_D)
 687
 688                   : "3" (srcptr),      // esi       // input regs
 689                     "4" (dstptr),      // edi
 690                     "0" (diff),        // eax
 691 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
 692                     "2" (len),         // ecx
 693                     "1" (mask)         // edx
 694
 695 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
 696                   : "%mm0", "%mm4", "%mm6", "%mm7"  // clobber list
 697 #endif
 698                );
 699             }
 700             else /* mmx _not supported - Use modified C routine */
 701 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
 702             {
 703                register png_uint_32 i;
 704                png_uint_32 initial_val = png_pass_start[png_ptr->pass];
 705                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
 706                register int stride = png_pass_inc[png_ptr->pass];
 707                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
 708                register int rep_bytes = png_pass_width[png_ptr->pass];
 709                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
 710                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
 711                int diff = (int) (png_ptr->width & 7); /* amount lost */
 712                register png_uint_32 final_val = len;  /* GRR bugfix */
 713
 714                srcptr = png_ptr->row_buf + 1 + initial_val;
 715                dstptr = row + initial_val;
 716
 717                for (i = initial_val; i < final_val; i += stride)
 718                {
 719                   png_memcpy(dstptr, srcptr, rep_bytes);
 720                   srcptr += stride;
 721                   dstptr += stride;
 722                }
 723                if (diff)  /* number of leftover pixels:  3 for pngtest */
 724                {
 725                   final_val+=diff /* *BPP1 */ ;
 726                   for (; i < final_val; i += stride)
 727                   {
 728                      if (rep_bytes > (int)(final_val-i))
 729                         rep_bytes = (int)(final_val-i);
 730                      png_memcpy(dstptr, srcptr, rep_bytes);
 731                      srcptr += stride;
 732                      dstptr += stride;
 733                   }
 734                }
 735
 736             } /* end of else (_mmx_supported) */
 737
 738             break;
 739          }       /* end 8 bpp */
 740
 741          case 16:       /* png_ptr->row_info.pixel_depth */
 742          {
 743             png_bytep srcptr;
 744             png_bytep dstptr;
 745
 746 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
 747 #if !defined(PNG_1_0_X)
 748             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
 749                 /* && _mmx_supported */ )
 750 #else
 751             if (_mmx_supported)
 752 #endif
 753             {
 754                png_uint_32 len;
 755                int diff;
 756                int dummy_value_a;   // fix 'forbidden register spilled' error
 757                int dummy_value_d;
 758                int dummy_value_c;
 759                int dummy_value_S;
 760                int dummy_value_D;
 761                _unmask = ~mask;            // global variable for -fPIC version
 762                srcptr = png_ptr->row_buf + 1;
 763                dstptr = row;
 764                len  = png_ptr->width &~7;  // reduce to multiple of 8
 765                diff = (int) (png_ptr->width & 7); // amount lost //
 766
 767                __asm__ __volatile__ (
 768                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
 769                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
 770                   "punpcklbw %%mm7, %%mm7     \n\t"
 771                   "punpcklwd %%mm7, %%mm7     \n\t"
 772                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
 773
 774                   "movq      _mask16_0, %%mm0 \n\t"
 775                   "movq      _mask16_1, %%mm1 \n\t"
 776
 777                   "pand      %%mm7, %%mm0     \n\t"
 778                   "pand      %%mm7, %%mm1     \n\t"
 779
 780                   "pcmpeqb   %%mm6, %%mm0     \n\t"
 781                   "pcmpeqb   %%mm6, %%mm1     \n\t"
 782
 783 // preload        "movl      len, %%ecx       \n\t" // load length of line
 784 // preload        "movl      srcptr, %%esi    \n\t" // load source
 785 // preload        "movl      dstptr, %%edi    \n\t" // load dest
 786
 787                   "cmpl      $0, %%ecx        \n\t"
 788                   "jz        mainloop16end    \n\t"
 789
 790                 "mainloop16:                  \n\t"
 791                   "movq      (%%esi), %%mm4   \n\t"
 792                   "pand      %%mm0, %%mm4     \n\t"
 793                   "movq      %%mm0, %%mm6     \n\t"
 794                   "movq      (%%edi), %%mm7   \n\t"
 795                   "pandn     %%mm7, %%mm6     \n\t"
 796                   "por       %%mm6, %%mm4     \n\t"
 797                   "movq      %%mm4, (%%edi)   \n\t"
 798
 799                   "movq      8(%%esi), %%mm5  \n\t"
 800                   "pand      %%mm1, %%mm5     \n\t"
 801                   "movq      %%mm1, %%mm7     \n\t"
 802                   "movq      8(%%edi), %%mm6  \n\t"
 803                   "pandn     %%mm6, %%mm7     \n\t"
 804                   "por       %%mm7, %%mm5     \n\t"
 805                   "movq      %%mm5, 8(%%edi)  \n\t"
 806
 807                   "addl      $16, %%esi       \n\t" // inc by 16 bytes processed
 808                   "addl      $16, %%edi       \n\t"
 809                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
 810                   "ja        mainloop16       \n\t"
 811
 812                 "mainloop16end:               \n\t"
 813 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
 814                   "movl      %%eax, %%ecx     \n\t"
 815                   "cmpl      $0, %%ecx        \n\t"
 816                   "jz        end16            \n\t"
 817 // preload        "movl      mask, %%edx      \n\t"
 818                   "sall      $24, %%edx       \n\t" // make low byte, high byte
 819
 820                 "secondloop16:                \n\t"
 821                   "sall      %%edx            \n\t" // move high bit to CF
 822                   "jnc       skip16           \n\t" // if CF = 0
 823                   "movw      (%%esi), %%ax    \n\t"
 824                   "movw      %%ax, (%%edi)    \n\t"
 825
 826                 "skip16:                      \n\t"
 827                   "addl      $2, %%esi        \n\t"
 828                   "addl      $2, %%edi        \n\t"
 829                   "decl      %%ecx            \n\t"
 830                   "jnz       secondloop16     \n\t"
 831
 832                 "end16:                       \n\t"
 833                   "EMMS                       \n\t" // DONE
 834
 835                   : "=a" (dummy_value_a),           // output regs (dummy)
 836                     "=c" (dummy_value_c),
 837                     "=d" (dummy_value_d),
 838                     "=S" (dummy_value_S),
 839                     "=D" (dummy_value_D)
 840
 841                   : "0" (diff),        // eax       // input regs
 842 // was (unmask)     " "    RESERVED    // ebx       // Global Offset Table idx
 843                     "1" (len),         // ecx
 844                     "2" (mask),        // edx
 845                     "3" (srcptr),      // esi
 846                     "4" (dstptr)       // edi
 847
 848 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
 849                   : "%mm0", "%mm1", "%mm4"          // clobber list
 850                   , "%mm5", "%mm6", "%mm7"
 851 #endif
 852                );
 853             }
 854             else /* mmx _not supported - Use modified C routine */
 855 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
 856             {
 857                register png_uint_32 i;
 858                png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass];
 859                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
 860                register int stride = BPP2 * png_pass_inc[png_ptr->pass];
 861                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
 862                register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass];
 863                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
 864                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
 865                int diff = (int) (png_ptr->width & 7); /* amount lost */
 866                register png_uint_32 final_val = BPP2 * len;   /* GRR bugfix */
 867
 868                srcptr = png_ptr->row_buf + 1 + initial_val;
 869                dstptr = row + initial_val;
 870
 871                for (i = initial_val; i < final_val; i += stride)
 872                {
 873                   png_memcpy(dstptr, srcptr, rep_bytes);
 874                   srcptr += stride;
 875                   dstptr += stride;
 876                }
 877                if (diff)  /* number of leftover pixels:  3 for pngtest */
 878                {
 879                   final_val+=diff*BPP2;
 880                   for (; i < final_val; i += stride)
 881                   {
 882                      if (rep_bytes > (int)(final_val-i))
 883                         rep_bytes = (int)(final_val-i);
 884                      png_memcpy(dstptr, srcptr, rep_bytes);
 885                      srcptr += stride;
 886                      dstptr += stride;
 887                   }
 888                }
 889             } /* end of else (_mmx_supported) */
 890
 891             break;
 892          }       /* end 16 bpp */
 893
 894          case 24:       /* png_ptr->row_info.pixel_depth */
 895          {
 896             png_bytep srcptr;
 897             png_bytep dstptr;
 898
 899 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
 900 #if !defined(PNG_1_0_X)
 901             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
 902                 /* && _mmx_supported */ )
 903 #else
 904             if (_mmx_supported)
 905 #endif
 906             {
 907                png_uint_32 len;
 908                int diff;
 909                int dummy_value_a;   // fix 'forbidden register spilled' error
 910                int dummy_value_d;
 911                int dummy_value_c;
 912                int dummy_value_S;
 913                int dummy_value_D;
 914                _unmask = ~mask;            // global variable for -fPIC version
 915                srcptr = png_ptr->row_buf + 1;
 916                dstptr = row;
 917                len  = png_ptr->width &~7;  // reduce to multiple of 8
 918                diff = (int) (png_ptr->width & 7); // amount lost //
 919
 920                __asm__ __volatile__ (
 921                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
 922                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
 923                   "punpcklbw %%mm7, %%mm7     \n\t"
 924                   "punpcklwd %%mm7, %%mm7     \n\t"
 925                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
 926
 927                   "movq      _mask24_0, %%mm0 \n\t"
 928                   "movq      _mask24_1, %%mm1 \n\t"
 929                   "movq      _mask24_2, %%mm2 \n\t"
 930
 931                   "pand      %%mm7, %%mm0     \n\t"
 932                   "pand      %%mm7, %%mm1     \n\t"
 933                   "pand      %%mm7, %%mm2     \n\t"
 934
 935                   "pcmpeqb   %%mm6, %%mm0     \n\t"
 936                   "pcmpeqb   %%mm6, %%mm1     \n\t"
 937                   "pcmpeqb   %%mm6, %%mm2     \n\t"
 938
 939 // preload        "movl      len, %%ecx       \n\t" // load length of line
 940 // preload        "movl      srcptr, %%esi    \n\t" // load source
 941 // preload        "movl      dstptr, %%edi    \n\t" // load dest
 942
 943                   "cmpl      $0, %%ecx        \n\t"
 944                   "jz        mainloop24end    \n\t"
 945
 946                 "mainloop24:                  \n\t"
 947                   "movq      (%%esi), %%mm4   \n\t"
 948                   "pand      %%mm0, %%mm4     \n\t"
 949                   "movq      %%mm0, %%mm6     \n\t"
 950                   "movq      (%%edi), %%mm7   \n\t"
 951                   "pandn     %%mm7, %%mm6     \n\t"
 952                   "por       %%mm6, %%mm4     \n\t"
 953                   "movq      %%mm4, (%%edi)   \n\t"
 954
 955                   "movq      8(%%esi), %%mm5  \n\t"
 956                   "pand      %%mm1, %%mm5     \n\t"
 957                   "movq      %%mm1, %%mm7     \n\t"
 958                   "movq      8(%%edi), %%mm6  \n\t"
 959                   "pandn     %%mm6, %%mm7     \n\t"
 960                   "por       %%mm7, %%mm5     \n\t"
 961                   "movq      %%mm5, 8(%%edi)  \n\t"
 962
 963                   "movq      16(%%esi), %%mm6 \n\t"
 964                   "pand      %%mm2, %%mm6     \n\t"
 965                   "movq      %%mm2, %%mm4     \n\t"
 966                   "movq      16(%%edi), %%mm7 \n\t"
 967                   "pandn     %%mm7, %%mm4     \n\t"
 968                   "por       %%mm4, %%mm6     \n\t"
 969                   "movq      %%mm6, 16(%%edi) \n\t"
 970
 971                   "addl      $24, %%esi       \n\t" // inc by 24 bytes processed
 972                   "addl      $24, %%edi       \n\t"
 973                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
 974
 975                   "ja        mainloop24       \n\t"
 976
 977                 "mainloop24end:               \n\t"
 978 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
 979                   "movl      %%eax, %%ecx     \n\t"
 980                   "cmpl      $0, %%ecx        \n\t"
 981                   "jz        end24            \n\t"
 982 // preload        "movl      mask, %%edx      \n\t"
 983                   "sall      $24, %%edx       \n\t" // make low byte, high byte
 984
 985                 "secondloop24:                \n\t"
 986                   "sall      %%edx            \n\t" // move high bit to CF
 987                   "jnc       skip24           \n\t" // if CF = 0
 988                   "movw      (%%esi), %%ax    \n\t"
 989                   "movw      %%ax, (%%edi)    \n\t"
 990                   "xorl      %%eax, %%eax     \n\t"
 991                   "movb      2(%%esi), %%al   \n\t"
 992                   "movb      %%al, 2(%%edi)   \n\t"
 993
 994                 "skip24:                      \n\t"
 995                   "addl      $3, %%esi        \n\t"
 996                   "addl      $3, %%edi        \n\t"
 997                   "decl      %%ecx            \n\t"
 998                   "jnz       secondloop24     \n\t"
 999
1000                 "end24:                       \n\t"
1001                   "EMMS                       \n\t" // DONE
1002
1003                   : "=a" (dummy_value_a),           // output regs (dummy)
1004                     "=d" (dummy_value_d),
1005                     "=c" (dummy_value_c),
1006                     "=S" (dummy_value_S),
1007                     "=D" (dummy_value_D)
1008
1009                   : "3" (srcptr),      // esi       // input regs
1010                     "4" (dstptr),      // edi
1011                     "0" (diff),        // eax
1012 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
1013                     "2" (len),         // ecx
1014                     "1" (mask)         // edx
1015
1016 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1017                   : "%mm0", "%mm1", "%mm2"          // clobber list
1018                   , "%mm4", "%mm5", "%mm6", "%mm7"
1019 #endif
1020                );
1021             }
1022             else /* mmx _not supported - Use modified C routine */
1023 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1024             {
1025                register png_uint_32 i;
1026                png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass];
1027                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1028                register int stride = BPP3 * png_pass_inc[png_ptr->pass];
1029                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1030                register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass];
1031                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1032                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1033                int diff = (int) (png_ptr->width & 7); /* amount lost */
1034                register png_uint_32 final_val = BPP3 * len;   /* GRR bugfix */
1035
1036                srcptr = png_ptr->row_buf + 1 + initial_val;
1037                dstptr = row + initial_val;
1038
1039                for (i = initial_val; i < final_val; i += stride)
1040                {
1041                   png_memcpy(dstptr, srcptr, rep_bytes);
1042                   srcptr += stride;
1043                   dstptr += stride;
1044                }
1045                if (diff)  /* number of leftover pixels:  3 for pngtest */
1046                {
1047                   final_val+=diff*BPP3;
1048                   for (; i < final_val; i += stride)
1049                   {
1050                      if (rep_bytes > (int)(final_val-i))
1051                         rep_bytes = (int)(final_val-i);
1052                      png_memcpy(dstptr, srcptr, rep_bytes);
1053                      srcptr += stride;
1054                      dstptr += stride;
1055                   }
1056                }
1057             } /* end of else (_mmx_supported) */
1058
1059             break;
1060          }       /* end 24 bpp */
1061
1062          case 32:       /* png_ptr->row_info.pixel_depth */
1063          {
1064             png_bytep srcptr;
1065             png_bytep dstptr;
1066
1067 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
1068 #if !defined(PNG_1_0_X)
1069             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1070                 /* && _mmx_supported */ )
1071 #else
1072             if (_mmx_supported)
1073 #endif
1074             {
1075                png_uint_32 len;
1076                int diff;
1077                int dummy_value_a;   // fix 'forbidden register spilled' error
1078                int dummy_value_d;
1079                int dummy_value_c;
1080                int dummy_value_S;
1081                int dummy_value_D;
1082                _unmask = ~mask;            // global variable for -fPIC version
1083                srcptr = png_ptr->row_buf + 1;
1084                dstptr = row;
1085                len  = png_ptr->width &~7;  // reduce to multiple of 8
1086                diff = (int) (png_ptr->width & 7); // amount lost //
1087
1088                __asm__ __volatile__ (
1089                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
1090                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
1091                   "punpcklbw %%mm7, %%mm7     \n\t"
1092                   "punpcklwd %%mm7, %%mm7     \n\t"
1093                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
1094
1095                   "movq      _mask32_0, %%mm0 \n\t"
1096                   "movq      _mask32_1, %%mm1 \n\t"
1097                   "movq      _mask32_2, %%mm2 \n\t"
1098                   "movq      _mask32_3, %%mm3 \n\t"
1099
1100                   "pand      %%mm7, %%mm0     \n\t"
1101                   "pand      %%mm7, %%mm1     \n\t"
1102                   "pand      %%mm7, %%mm2     \n\t"
1103                   "pand      %%mm7, %%mm3     \n\t"
1104
1105                   "pcmpeqb   %%mm6, %%mm0     \n\t"
1106                   "pcmpeqb   %%mm6, %%mm1     \n\t"
1107                   "pcmpeqb   %%mm6, %%mm2     \n\t"
1108                   "pcmpeqb   %%mm6, %%mm3     \n\t"
1109
1110 // preload        "movl      len, %%ecx       \n\t" // load length of line
1111 // preload        "movl      srcptr, %%esi    \n\t" // load source
1112 // preload        "movl      dstptr, %%edi    \n\t" // load dest
1113
1114                   "cmpl      $0, %%ecx        \n\t" // lcr
1115                   "jz        mainloop32end    \n\t"
1116
1117                 "mainloop32:                  \n\t"
1118                   "movq      (%%esi), %%mm4   \n\t"
1119                   "pand      %%mm0, %%mm4     \n\t"
1120                   "movq      %%mm0, %%mm6     \n\t"
1121                   "movq      (%%edi), %%mm7   \n\t"
1122                   "pandn     %%mm7, %%mm6     \n\t"
1123                   "por       %%mm6, %%mm4     \n\t"
1124                   "movq      %%mm4, (%%edi)   \n\t"
1125
1126                   "movq      8(%%esi), %%mm5  \n\t"
1127                   "pand      %%mm1, %%mm5     \n\t"
1128                   "movq      %%mm1, %%mm7     \n\t"
1129                   "movq      8(%%edi), %%mm6  \n\t"
1130                   "pandn     %%mm6, %%mm7     \n\t"
1131                   "por       %%mm7, %%mm5     \n\t"
1132                   "movq      %%mm5, 8(%%edi)  \n\t"
1133
1134                   "movq      16(%%esi), %%mm6 \n\t"
1135                   "pand      %%mm2, %%mm6     \n\t"
1136                   "movq      %%mm2, %%mm4     \n\t"
1137                   "movq      16(%%edi), %%mm7 \n\t"
1138                   "pandn     %%mm7, %%mm4     \n\t"
1139                   "por       %%mm4, %%mm6     \n\t"
1140                   "movq      %%mm6, 16(%%edi) \n\t"
1141
1142                   "movq      24(%%esi), %%mm7 \n\t"
1143                   "pand      %%mm3, %%mm7     \n\t"
1144                   "movq      %%mm3, %%mm5     \n\t"
1145                   "movq      24(%%edi), %%mm4 \n\t"
1146                   "pandn     %%mm4, %%mm5     \n\t"
1147                   "por       %%mm5, %%mm7     \n\t"
1148                   "movq      %%mm7, 24(%%edi) \n\t"
1149
1150                   "addl      $32, %%esi       \n\t" // inc by 32 bytes processed
1151                   "addl      $32, %%edi       \n\t"
1152                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
1153                   "ja        mainloop32       \n\t"
1154
1155                 "mainloop32end:               \n\t"
1156 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
1157                   "movl      %%eax, %%ecx     \n\t"
1158                   "cmpl      $0, %%ecx        \n\t"
1159                   "jz        end32            \n\t"
1160 // preload        "movl      mask, %%edx      \n\t"
1161                   "sall      $24, %%edx       \n\t" // low byte => high byte
1162
1163                 "secondloop32:                \n\t"
1164                   "sall      %%edx            \n\t" // move high bit to CF
1165                   "jnc       skip32           \n\t" // if CF = 0
1166                   "movl      (%%esi), %%eax   \n\t"
1167                   "movl      %%eax, (%%edi)   \n\t"
1168
1169                 "skip32:                      \n\t"
1170                   "addl      $4, %%esi        \n\t"
1171                   "addl      $4, %%edi        \n\t"
1172                   "decl      %%ecx            \n\t"
1173                   "jnz       secondloop32     \n\t"
1174
1175                 "end32:                       \n\t"
1176                   "EMMS                       \n\t" // DONE
1177
1178                   : "=a" (dummy_value_a),           // output regs (dummy)
1179                     "=d" (dummy_value_d),
1180                     "=c" (dummy_value_c),
1181                     "=S" (dummy_value_S),
1182                     "=D" (dummy_value_D)
1183
1184                   : "3" (srcptr),      // esi       // input regs
1185                     "4" (dstptr),      // edi
1186                     "0" (diff),        // eax
1187 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
1188                     "2" (len),         // ecx
1189                     "1" (mask)         // edx
1190
1191 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1192                   : "%mm0", "%mm1", "%mm2", "%mm3"  // clobber list
1193                   , "%mm4", "%mm5", "%mm6", "%mm7"
1194 #endif
1195                );
1196             }
1197             else /* mmx _not supported - Use modified C routine */
1198 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1199             {
1200                register png_uint_32 i;
1201                png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass];
1202                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1203                register int stride = BPP4 * png_pass_inc[png_ptr->pass];
1204                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1205                register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass];
1206                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1207                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1208                int diff = (int) (png_ptr->width & 7); /* amount lost */
1209                register png_uint_32 final_val = BPP4 * len;   /* GRR bugfix */
1210
1211                srcptr = png_ptr->row_buf + 1 + initial_val;
1212                dstptr = row + initial_val;
1213
1214                for (i = initial_val; i < final_val; i += stride)
1215                {
1216                   png_memcpy(dstptr, srcptr, rep_bytes);
1217                   srcptr += stride;
1218                   dstptr += stride;
1219                }
1220                if (diff)  /* number of leftover pixels:  3 for pngtest */
1221                {
1222                   final_val+=diff*BPP4;
1223                   for (; i < final_val; i += stride)
1224                   {
1225                      if (rep_bytes > (int)(final_val-i))
1226                         rep_bytes = (int)(final_val-i);
1227                      png_memcpy(dstptr, srcptr, rep_bytes);
1228                      srcptr += stride;
1229                      dstptr += stride;
1230                   }
1231                }
1232             } /* end of else (_mmx_supported) */
1233
1234             break;
1235          }       /* end 32 bpp */
1236
1237          case 48:       /* png_ptr->row_info.pixel_depth */
1238          {
1239             png_bytep srcptr;
1240             png_bytep dstptr;
1241
1242 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
1243 #if !defined(PNG_1_0_X)
1244             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1245                 /* && _mmx_supported */ )
1246 #else
1247             if (_mmx_supported)
1248 #endif
1249             {
1250                png_uint_32 len;
1251                int diff;
1252                int dummy_value_a;   // fix 'forbidden register spilled' error
1253                int dummy_value_d;
1254                int dummy_value_c;
1255                int dummy_value_S;
1256                int dummy_value_D;
1257                _unmask = ~mask;            // global variable for -fPIC version
1258                srcptr = png_ptr->row_buf + 1;
1259                dstptr = row;
1260                len  = png_ptr->width &~7;  // reduce to multiple of 8
1261                diff = (int) (png_ptr->width & 7); // amount lost //
1262
1263                __asm__ __volatile__ (
1264                   "movd      _unmask, %%mm7   \n\t" // load bit pattern
1265                   "psubb     %%mm6, %%mm6     \n\t" // zero mm6
1266                   "punpcklbw %%mm7, %%mm7     \n\t"
1267                   "punpcklwd %%mm7, %%mm7     \n\t"
1268                   "punpckldq %%mm7, %%mm7     \n\t" // fill reg with 8 masks
1269
1270                   "movq      _mask48_0, %%mm0 \n\t"
1271                   "movq      _mask48_1, %%mm1 \n\t"
1272                   "movq      _mask48_2, %%mm2 \n\t"
1273                   "movq      _mask48_3, %%mm3 \n\t"
1274                   "movq      _mask48_4, %%mm4 \n\t"
1275                   "movq      _mask48_5, %%mm5 \n\t"
1276
1277                   "pand      %%mm7, %%mm0     \n\t"
1278                   "pand      %%mm7, %%mm1     \n\t"
1279                   "pand      %%mm7, %%mm2     \n\t"
1280                   "pand      %%mm7, %%mm3     \n\t"
1281                   "pand      %%mm7, %%mm4     \n\t"
1282                   "pand      %%mm7, %%mm5     \n\t"
1283
1284                   "pcmpeqb   %%mm6, %%mm0     \n\t"
1285                   "pcmpeqb   %%mm6, %%mm1     \n\t"
1286                   "pcmpeqb   %%mm6, %%mm2     \n\t"
1287                   "pcmpeqb   %%mm6, %%mm3     \n\t"
1288                   "pcmpeqb   %%mm6, %%mm4     \n\t"
1289                   "pcmpeqb   %%mm6, %%mm5     \n\t"
1290
1291 // preload        "movl      len, %%ecx       \n\t" // load length of line
1292 // preload        "movl      srcptr, %%esi    \n\t" // load source
1293 // preload        "movl      dstptr, %%edi    \n\t" // load dest
1294
1295                   "cmpl      $0, %%ecx        \n\t"
1296                   "jz        mainloop48end    \n\t"
1297
1298                 "mainloop48:                  \n\t"
1299                   "movq      (%%esi), %%mm7   \n\t"
1300                   "pand      %%mm0, %%mm7     \n\t"
1301                   "movq      %%mm0, %%mm6     \n\t"
1302                   "pandn     (%%edi), %%mm6   \n\t"
1303                   "por       %%mm6, %%mm7     \n\t"
1304                   "movq      %%mm7, (%%edi)   \n\t"
1305
1306                   "movq      8(%%esi), %%mm6  \n\t"
1307                   "pand      %%mm1, %%mm6     \n\t"
1308                   "movq      %%mm1, %%mm7     \n\t"
1309                   "pandn     8(%%edi), %%mm7  \n\t"
1310                   "por       %%mm7, %%mm6     \n\t"
1311                   "movq      %%mm6, 8(%%edi)  \n\t"
1312
1313                   "movq      16(%%esi), %%mm6 \n\t"
1314                   "pand      %%mm2, %%mm6     \n\t"
1315                   "movq      %%mm2, %%mm7     \n\t"
1316                   "pandn     16(%%edi), %%mm7 \n\t"
1317                   "por       %%mm7, %%mm6     \n\t"
1318                   "movq      %%mm6, 16(%%edi) \n\t"
1319
1320                   "movq      24(%%esi), %%mm7 \n\t"
1321                   "pand      %%mm3, %%mm7     \n\t"
1322                   "movq      %%mm3, %%mm6     \n\t"
1323                   "pandn     24(%%edi), %%mm6 \n\t"
1324                   "por       %%mm6, %%mm7     \n\t"
1325                   "movq      %%mm7, 24(%%edi) \n\t"
1326
1327                   "movq      32(%%esi), %%mm6 \n\t"
1328                   "pand      %%mm4, %%mm6     \n\t"
1329                   "movq      %%mm4, %%mm7     \n\t"
1330                   "pandn     32(%%edi), %%mm7 \n\t"
1331                   "por       %%mm7, %%mm6     \n\t"
1332                   "movq      %%mm6, 32(%%edi) \n\t"
1333
1334                   "movq      40(%%esi), %%mm7 \n\t"
1335                   "pand      %%mm5, %%mm7     \n\t"
1336                   "movq      %%mm5, %%mm6     \n\t"
1337                   "pandn     40(%%edi), %%mm6 \n\t"
1338                   "por       %%mm6, %%mm7     \n\t"
1339                   "movq      %%mm7, 40(%%edi) \n\t"
1340
1341                   "addl      $48, %%esi       \n\t" // inc by 48 bytes processed
1342                   "addl      $48, %%edi       \n\t"
1343                   "subl      $8, %%ecx        \n\t" // dec by 8 pixels processed
1344
1345                   "ja        mainloop48       \n\t"
1346
1347                 "mainloop48end:               \n\t"
1348 // preload        "movl      diff, %%ecx      \n\t" // (diff is in eax)
1349                   "movl      %%eax, %%ecx     \n\t"
1350                   "cmpl      $0, %%ecx        \n\t"
1351                   "jz        end48            \n\t"
1352 // preload        "movl      mask, %%edx      \n\t"
1353                   "sall      $24, %%edx       \n\t" // make low byte, high byte
1354
1355                 "secondloop48:                \n\t"
1356                   "sall      %%edx            \n\t" // move high bit to CF
1357                   "jnc       skip48           \n\t" // if CF = 0
1358                   "movl      (%%esi), %%eax   \n\t"
1359                   "movl      %%eax, (%%edi)   \n\t"
1360
1361                 "skip48:                      \n\t"
1362                   "addl      $4, %%esi        \n\t"
1363                   "addl      $4, %%edi        \n\t"
1364                   "decl      %%ecx            \n\t"
1365                   "jnz       secondloop48     \n\t"
1366
1367                 "end48:                       \n\t"
1368                   "EMMS                       \n\t" // DONE
1369
1370                   : "=a" (dummy_value_a),           // output regs (dummy)
1371                     "=d" (dummy_value_d),
1372                     "=c" (dummy_value_c),
1373                     "=S" (dummy_value_S),
1374                     "=D" (dummy_value_D)
1375
1376                   : "3" (srcptr),      // esi       // input regs
1377                     "4" (dstptr),      // edi
1378                     "0" (diff),        // eax
1379 // was (unmask)     "b"    RESERVED    // ebx       // Global Offset Table idx
1380                     "2" (len),         // ecx
1381                     "1" (mask)         // edx
1382
1383 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
1384                   : "%mm0", "%mm1", "%mm2", "%mm3"  // clobber list
1385                   , "%mm4", "%mm5", "%mm6", "%mm7"
1386 #endif
1387                );
1388             }
1389             else /* mmx _not supported - Use modified C routine */
1390 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
1391             {
1392                register png_uint_32 i;
1393                png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass];
1394                  /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1395                register int stride = BPP6 * png_pass_inc[png_ptr->pass];
1396                  /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1397                register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass];
1398                  /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1399                png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1400                int diff = (int) (png_ptr->width & 7); /* amount lost */
1401                register png_uint_32 final_val = BPP6 * len;   /* GRR bugfix */
1402
1403                srcptr = png_ptr->row_buf + 1 + initial_val;
1404                dstptr = row + initial_val;
1405
1406                for (i = initial_val; i < final_val; i += stride)
1407                {
1408                   png_memcpy(dstptr, srcptr, rep_bytes);
1409                   srcptr += stride;
1410                   dstptr += stride;
1411                }
1412                if (diff)  /* number of leftover pixels:  3 for pngtest */
1413                {
1414                   final_val+=diff*BPP6;
1415                   for (; i < final_val; i += stride)
1416                   {
1417                      if (rep_bytes > (int)(final_val-i))
1418                         rep_bytes = (int)(final_val-i);
1419                      png_memcpy(dstptr, srcptr, rep_bytes);
1420                      srcptr += stride;
1421                      dstptr += stride;
1422                   }
1423                }
1424             } /* end of else (_mmx_supported) */
1425
1426             break;
1427          }       /* end 48 bpp */
1428
1429          case 64:       /* png_ptr->row_info.pixel_depth */
1430          {
1431             png_bytep srcptr;
1432             png_bytep dstptr;
1433             register png_uint_32 i;
1434             png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass];
1435               /* png.c:  png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1436             register int stride = BPP8 * png_pass_inc[png_ptr->pass];
1437               /* png.c:  png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1438             register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass];
1439               /* png.c:  png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1440             png_uint_32 len = png_ptr->width &~7;  /* reduce to mult. of 8 */
1441             int diff = (int) (png_ptr->width & 7); /* amount lost */
1442             register png_uint_32 final_val = BPP8 * len;   /* GRR bugfix */
1443
1444             srcptr = png_ptr->row_buf + 1 + initial_val;
1445             dstptr = row + initial_val;
1446
1447             for (i = initial_val; i < final_val; i += stride)
1448             {
1449                png_memcpy(dstptr, srcptr, rep_bytes);
1450                srcptr += stride;
1451                dstptr += stride;
1452             }
1453             if (diff)  /* number of leftover pixels:  3 for pngtest */
1454             {
1455                final_val+=diff*BPP8;
1456                for (; i < final_val; i += stride)
1457                {
1458                   if (rep_bytes > (int)(final_val-i))
1459                      rep_bytes = (int)(final_val-i);
1460                   png_memcpy(dstptr, srcptr, rep_bytes);
1461                   srcptr += stride;
1462                   dstptr += stride;
1463                }
1464             }
1465
1466             break;
1467          }       /* end 64 bpp */
1468
1469          default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */
1470          {
1471             /* this should never happen */
1472             png_warning(png_ptr, "Invalid row_info.pixel_depth in pnggccrd");
1473             break;
1474          }
1475       } /* end switch (png_ptr->row_info.pixel_depth) */
1476
1477    } /* end if (non-trivial mask) */
1478
1479 } /* end png_combine_row() */
1480
1481 #endif /* PNG_HAVE_ASSEMBLER_COMBINE_ROW */
1482
1483
1484
1485
1486 /*===========================================================================*/
1487 /*                                                                           */
1488 /*                 P N G _ D O _ R E A D _ I N T E R L A C E                 */
1489 /*                                                                           */
1490 /*===========================================================================*/
1491
1492 #if defined(PNG_READ_INTERLACING_SUPPORTED)
1493 #if defined(PNG_HAVE_ASSEMBLER_READ_INTERLACE)
1494
1495 /* png_do_read_interlace() is called after any 16-bit to 8-bit conversion
1496  * has taken place.  [GRR: what other steps come before and/or after?]
1497  */
1498
1499 void /* PRIVATE */
1500 png_do_read_interlace(png_structp png_ptr)
1501 {
1502    png_row_infop row_info = &(png_ptr->row_info);
1503    png_bytep row = png_ptr->row_buf + 1;
1504    int pass = png_ptr->pass;
1505 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1506    png_uint_32 transformations = png_ptr->transformations;
1507 #endif
1508
1509    png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n");
1510
1511 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
1512    if (_mmx_supported == 2) {
1513 #if !defined(PNG_1_0_X)
1514        /* this should have happened in png_init_mmx_flags() already */
1515        png_warning(png_ptr, "asm_flags may not have been initialized");
1516 #endif
1517        png_mmx_support();
1518    }
1519 #endif
1520
1521    if (row != NULL && row_info != NULL)
1522    {
1523       png_uint_32 final_width;
1524
1525       final_width = row_info->width * png_pass_inc[pass];
1526
1527       switch (row_info->pixel_depth)
1528       {
1529          case 1:
1530          {
1531             png_bytep sp, dp;
1532             int sshift, dshift;
1533             int s_start, s_end, s_inc;
1534             png_byte v;
1535             png_uint_32 i;
1536             int j;
1537
1538             sp = row + (png_size_t)((row_info->width - 1) >> 3);
1539             dp = row + (png_size_t)((final_width - 1) >> 3);
1540 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1541             if (transformations & PNG_PACKSWAP)
1542             {
1543                sshift = (int)((row_info->width + 7) & 7);
1544                dshift = (int)((final_width + 7) & 7);
1545                s_start = 7;
1546                s_end = 0;
1547                s_inc = -1;
1548             }
1549             else
1550 #endif
1551             {
1552                sshift = 7 - (int)((row_info->width + 7) & 7);
1553                dshift = 7 - (int)((final_width + 7) & 7);
1554                s_start = 0;
1555                s_end = 7;
1556                s_inc = 1;
1557             }
1558
1559             for (i = row_info->width; i; i--)
1560             {
1561                v = (png_byte)((*sp >> sshift) & 0x1);
1562                for (j = 0; j < png_pass_inc[pass]; j++)
1563                {
1564                   *dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff);
1565                   *dp |= (png_byte)(v << dshift);
1566                   if (dshift == s_end)
1567                   {
1568                      dshift = s_start;
1569                      dp--;
1570                   }
1571                   else
1572                      dshift += s_inc;
1573                }
1574                if (sshift == s_end)
1575                {
1576                   sshift = s_start;
1577                   sp--;
1578                }
1579                else
1580                   sshift += s_inc;
1581             }
1582             break;
1583          }
1584
1585          case 2:
1586          {
1587             png_bytep sp, dp;
1588             int sshift, dshift;
1589             int s_start, s_end, s_inc;
1590             png_uint_32 i;
1591
1592             sp = row + (png_size_t)((row_info->width - 1) >> 2);
1593             dp = row + (png_size_t)((final_width - 1) >> 2);
1594 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1595             if (transformations & PNG_PACKSWAP)
1596             {
1597                sshift = (png_size_t)(((row_info->width + 3) & 3) << 1);
1598                dshift = (png_size_t)(((final_width + 3) & 3) << 1);
1599                s_start = 6;
1600                s_end = 0;
1601                s_inc = -2;
1602             }
1603             else
1604 #endif
1605             {
1606                sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1);
1607                dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1);
1608                s_start = 0;
1609                s_end = 6;
1610                s_inc = 2;
1611             }
1612
1613             for (i = row_info->width; i; i--)
1614             {
1615                png_byte v;
1616                int j;
1617
1618                v = (png_byte)((*sp >> sshift) & 0x3);
1619                for (j = 0; j < png_pass_inc[pass]; j++)
1620                {
1621                   *dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff);
1622                   *dp |= (png_byte)(v << dshift);
1623                   if (dshift == s_end)
1624                   {
1625                      dshift = s_start;
1626                      dp--;
1627                   }
1628                   else
1629                      dshift += s_inc;
1630                }
1631                if (sshift == s_end)
1632                {
1633                   sshift = s_start;
1634                   sp--;
1635                }
1636                else
1637                   sshift += s_inc;
1638             }
1639             break;
1640          }
1641
1642          case 4:
1643          {
1644             png_bytep sp, dp;
1645             int sshift, dshift;
1646             int s_start, s_end, s_inc;
1647             png_uint_32 i;
1648
1649             sp = row + (png_size_t)((row_info->width - 1) >> 1);
1650             dp = row + (png_size_t)((final_width - 1) >> 1);
1651 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1652             if (transformations & PNG_PACKSWAP)
1653             {
1654                sshift = (png_size_t)(((row_info->width + 1) & 1) << 2);
1655                dshift = (png_size_t)(((final_width + 1) & 1) << 2);
1656                s_start = 4;
1657                s_end = 0;
1658                s_inc = -4;
1659             }
1660             else
1661 #endif
1662             {
1663                sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2);
1664                dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2);
1665                s_start = 0;
1666                s_end = 4;
1667                s_inc = 4;
1668             }
1669
1670             for (i = row_info->width; i; i--)
1671             {
1672                png_byte v;
1673                int j;
1674
1675                v = (png_byte)((*sp >> sshift) & 0xf);
1676                for (j = 0; j < png_pass_inc[pass]; j++)
1677                {
1678                   *dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff);
1679                   *dp |= (png_byte)(v << dshift);
1680                   if (dshift == s_end)
1681                   {
1682                      dshift = s_start;
1683                      dp--;
1684                   }
1685                   else
1686                      dshift += s_inc;
1687                }
1688                if (sshift == s_end)
1689                {
1690                   sshift = s_start;
1691                   sp--;
1692                }
1693                else
1694                   sshift += s_inc;
1695             }
1696             break;
1697          }
1698
1699        /*====================================================================*/
1700
1701          default: /* 8-bit or larger (this is where the routine is modified) */
1702          {
1703 #if 0
1704 //          static unsigned long long _const4 = 0x0000000000FFFFFFLL;  no good
1705 //          static unsigned long long const4 = 0x0000000000FFFFFFLL;   no good
1706 //          unsigned long long _const4 = 0x0000000000FFFFFFLL;         no good
1707 //          unsigned long long const4 = 0x0000000000FFFFFFLL;          no good
1708 #endif
1709             png_bytep sptr, dp;
1710             png_uint_32 i;
1711             png_size_t pixel_bytes;
1712             int width = (int)row_info->width;
1713
1714             pixel_bytes = (row_info->pixel_depth >> 3);
1715
1716             /* point sptr at the last pixel in the pre-expanded row: */
1717             sptr = row + (width - 1) * pixel_bytes;
1718
1719             /* point dp at the last pixel position in the expanded row: */
1720             dp = row + (final_width - 1) * pixel_bytes;
1721
1722             /* New code by Nirav Chhatrapati - Intel Corporation */
1723
1724 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
1725 #if !defined(PNG_1_0_X)
1726             if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE)
1727                 /* && _mmx_supported */ )
1728 #else
1729             if (_mmx_supported)
1730 #endif
1731             {
1732                //--------------------------------------------------------------
1733                if (pixel_bytes == 3)
1734                {
1735                   if (((pass == 0) || (pass == 1)) && width)
1736                   {
1737                      int dummy_value_c;   // fix 'forbidden register spilled'
1738                      int dummy_value_S;
1739                      int dummy_value_D;
1740
1741                      __asm__ __volatile__ (
1742                         "subl $21, %%edi         \n\t"
1743                                      // (png_pass_inc[pass] - 1)*pixel_bytes
1744
1745                      ".loop3_pass0:              \n\t"
1746                         "movd (%%esi), %%mm0     \n\t" // x x x x x 2 1 0
1747                         "pand _const4, %%mm0     \n\t" // z z z z z 2 1 0
1748                         "movq %%mm0, %%mm1       \n\t" // z z z z z 2 1 0
1749                         "psllq $16, %%mm0        \n\t" // z z z 2 1 0 z z
1750                         "movq %%mm0, %%mm2       \n\t" // z z z 2 1 0 z z
1751                         "psllq $24, %%mm0        \n\t" // 2 1 0 z z z z z
1752                         "psrlq $8, %%mm1         \n\t" // z z z z z z 2 1
1753                         "por %%mm2, %%mm0        \n\t" // 2 1 0 2 1 0 z z
1754                         "por %%mm1, %%mm0        \n\t" // 2 1 0 2 1 0 2 1
1755                         "movq %%mm0, %%mm3       \n\t" // 2 1 0 2 1 0 2 1
1756                         "psllq $16, %%mm0        \n\t" // 0 2 1 0 2 1 z z
1757                         "movq %%mm3, %%mm4       \n\t" // 2 1 0 2 1 0 2 1
1758                         "punpckhdq %%mm0, %%mm3  \n\t" // 0 2 1 0 2 1 0 2
1759                         "movq %%mm4, 16(%%edi)   \n\t"
1760                         "psrlq $32, %%mm0        \n\t" // z z z z 0 2 1 0
1761                         "movq %%mm3, 8(%%edi)    \n\t"
1762                         "punpckldq %%mm4, %%mm0  \n\t" // 1 0 2 1 0 2 1 0
1763                         "subl $3, %%esi          \n\t"
1764                         "movq %%mm0, (%%edi)     \n\t"
1765                         "subl $24, %%edi         \n\t"
1766                         "decl %%ecx              \n\t"
1767                         "jnz .loop3_pass0        \n\t"
1768                         "EMMS                    \n\t" // DONE
1769
1770                         : "=c" (dummy_value_c),        // output regs (dummy)
1771                           "=S" (dummy_value_S),
1772                           "=D" (dummy_value_D)
1773
1774                         : "1" (sptr),      // esi      // input regs
1775                           "2" (dp),        // edi
1776                           "0" (width),     // ecx
1777                           "rim" (_const4)  // %1(?)  (0x0000000000FFFFFFLL)
1778
1779 #if 0  /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
1780                         : "%mm0", "%mm1", "%mm2"       // clobber list
1781                         , "%mm3", "%mm4"
1782 #endif
1783                      );
1784                   }
1785                   else if (((pass == 2) || (pass == 3)) && width)
1786                   {
1787                      int dummy_value_c;   // fix 'forbidden register spilled'
1788                      int dummy_value_S;
1789                      int dummy_value_D;
1790
1791                      __asm__ __volatile__ (
1792                         "subl $9, %%edi          \n\t"
1793                                      // (png_pass_inc[pass] - 1)*pixel_bytes
1794
1795                      ".loop3_pass2:              \n\t"
1796                         "movd (%%esi), %%mm0     \n\t" // x x x x x 2 1 0
1797                         "pand _const4, %%mm0     \n\t" // z z z z z 2 1 0
1798                         "movq %%mm0, %%mm1       \n\t" // z z z z z 2 1 0
1799                         "psllq $16, %%mm0        \n\t" // z z z 2 1 0 z z
1800                         "movq %%mm0, %%mm2       \n\t" // z z z 2 1 0 z z
1801                         "psllq $24, %%mm0        \n\t" // 2 1 0 z z z z z
1802                         "psrlq $8, %%mm1         \n\t" // z z z z z z 2 1
1803                         "por %%mm2, %%mm0        \n\t" // 2 1 0 2 1 0 z z
1804                         "por %%mm1, %%mm0        \n\t" // 2 1 0 2 1 0 2 1
1805                         "movq %%mm0, 4(%%edi)    \n\t"
1806                         "psrlq $16, %%mm0        \n\t" // z z 2 1 0 2 1 0
1807                         "subl $3, %%esi          \n\t"
1808                         "movd %%mm0, (%%edi)     \n\t"
1809                         "subl $12, %%edi         \n\t"
1810                         "decl %%ecx              \n\t"
1811                         "jnz .loop3_pass2        \n\t"
1812                         "EMMS                    \n\t" // DONE
1813
1814                         : "=c" (dummy_value_c),        // output regs (dummy)
1815                           "=S" (dummy_value_S),
1816                           "=D" (dummy_value_D)
1817
1818                         : "1" (sptr),      // esi      // input regs
1819                           "2" (dp),        // edi
1820                           "0" (width),     // ecx
1821                           "rim" (_const4)  // (0x0000000000FFFFFFLL)
1822
1823 #if 0  /* %mm0, ..., %mm2 not supported by gcc 2.7.2.3 or egcs 1.1 */
1824                         : "%mm0", "%mm1", "%mm2"       // clobber list
1825 #endif
1826                      );
1827                   }
1828                   else if (width) /* && ((pass == 4) || (pass == 5)) */
1829                   {
1830                      int width_mmx = ((width >> 1) << 1) - 8;   // GRR:  huh?
1831                      if (width_mmx < 0)
1832                          width_mmx = 0;
1833                      width -= width_mmx;        // 8 or 9 pix, 24 or 27 bytes
1834                      if (width_mmx)
1835                      {
1836                         // png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1};
1837                         // sptr points at last pixel in pre-expanded row
1838                         // dp points at last pixel position in expanded row
1839                         int dummy_value_c;  // fix 'forbidden register spilled'
1840                         int dummy_value_S;
1841                         int dummy_value_D;
1842
1843                         __asm__ __volatile__ (
1844                            "subl $3, %%esi          \n\t"
1845                            "subl $9, %%edi          \n\t"
1846                                         // (png_pass_inc[pass] + 1)*pixel_bytes
1847
1848                         ".loop3_pass4:              \n\t"
1849                            "movq (%%esi), %%mm0     \n\t" // x x 5 4 3 2 1 0
1850                            "movq %%mm0, %%mm1       \n\t" // x x 5 4 3 2 1 0
1851                            "movq %%mm0, %%mm2       \n\t" // x x 5 4 3 2 1 0
1852                            "psllq $24, %%mm0        \n\t" // 4 3 2 1 0 z z z
1853                            "pand _const4, %%mm1     \n\t" // z z z z z 2 1 0
1854                            "psrlq $24, %%mm2        \n\t" // z z z x x 5 4 3
1855                            "por %%mm1, %%mm0        \n\t" // 4 3 2 1 0 2 1 0
1856                            "movq %%mm2, %%mm3       \n\t" // z z z x x 5 4 3
1857                            "psllq $8, %%mm2         \n\t" // z z x x 5 4 3 z
1858                            "movq %%mm0, (%%edi)     \n\t"
1859                            "psrlq $16, %%mm3        \n\t" // z z z z z x x 5
1860                            "pand _const6, %%mm3     \n\t" // z z z z z z z 5
1861                            "por %%mm3, %%mm2        \n\t" // z z x x 5 4 3 5
1862                            "subl $6, %%esi          \n\t"
1863                            "movd %%mm2, 8(%%edi)    \n\t"
1864                            "subl $12, %%edi         \n\t"
1865                            "subl $2, %%ecx          \n\t"
1866                            "jnz .loop3_pass4        \n\t"
1867                            "EMMS                    \n\t" // DONE
1868
1869                            : "=c" (dummy_value_c),        // output regs (dummy)
1870                              "=S" (dummy_value_S),
1871                              "=D" (dummy_value_D)
1872
1873                            : "1" (sptr),      // esi      // input regs
1874                              "2" (dp),        // edi
1875                              "0" (width_mmx), // ecx
1876                              "rim" (_const4), // 0x0000000000FFFFFFLL
1877                              "rim" (_const6)  // 0x00000000000000FFLL
1878
1879 #if 0  /* %mm0, ..., %mm3 not supported by gcc 2.7.2.3 or egcs 1.1 */
1880                            : "%mm0", "%mm1"               // clobber list
1881                            , "%mm2", "%mm3"
1882 #endif
1883                         );
1884                      }
1885
1886                      sptr -= width_mmx*3;
1887                      dp -= width_mmx*6;
1888                      for (i = width; i; i--)
1889                      {
1890                         png_byte v[8];
1891                         int j;
1892
1893                         png_memcpy(v, sptr, 3);
1894                         for (j = 0; j < png_pass_inc[pass]; j++)
1895                         {
1896                            png_memcpy(dp, v, 3);
1897                            dp -= 3;
1898                         }
1899                         sptr -= 3;
1900                      }
1901                   }
1902                } /* end of pixel_bytes == 3 */
1903
1904                //--------------------------------------------------------------
1905                else if (pixel_bytes == 1)
1906                {
1907                   if (((pass == 0) || (pass == 1)) && width)
1908                   {
1909                      int width_mmx = ((width >> 2) << 2);
1910                      width -= width_mmx;        // 0-3 pixels => 0-3 bytes
1911                      if (width_mmx)
1912                      {
1913                         int dummy_value_c;  // fix 'forbidden register spilled'
1914                         int dummy_value_S;
1915                         int dummy_value_D;
1916
1917                         __asm__ __volatile__ (
1918                            "subl $3, %%esi          \n\t"
1919                            "subl $31, %%edi         \n\t"
1920
1921                         ".loop1_pass0:              \n\t"
1922                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
1923                            "movq %%mm0, %%mm1       \n\t" // x x x x 3 2 1 0
1924                            "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
1925                            "movq %%mm0, %%mm2       \n\t" // 3 3 2 2 1 1 0 0
1926                            "punpcklwd %%mm0, %%mm0  \n\t" // 1 1 1 1 0 0 0 0
1927                            "movq %%mm0, %%mm3       \n\t" // 1 1 1 1 0 0 0 0
1928                            "punpckldq %%mm0, %%mm0  \n\t" // 0 0 0 0 0 0 0 0
1929                            "punpckhdq %%mm3, %%mm3  \n\t" // 1 1 1 1 1 1 1 1
1930                            "movq %%mm0, (%%edi)     \n\t"
1931                            "punpckhwd %%mm2, %%mm2  \n\t" // 3 3 3 3 2 2 2 2
1932                            "movq %%mm3, 8(%%edi)    \n\t"
1933                            "movq %%mm2, %%mm4       \n\t" // 3 3 3 3 2 2 2 2
1934                            "punpckldq %%mm2, %%mm2  \n\t" // 2 2 2 2 2 2 2 2
1935                            "punpckhdq %%mm4, %%mm4  \n\t" // 3 3 3 3 3 3 3 3
1936                            "movq %%mm2, 16(%%edi)   \n\t"
1937                            "subl $4, %%esi          \n\t"
1938                            "movq %%mm4, 24(%%edi)   \n\t"
1939                            "subl $32, %%edi         \n\t"
1940                            "subl $4, %%ecx          \n\t"
1941                            "jnz .loop1_pass0        \n\t"
1942                            "EMMS                    \n\t" // DONE
1943
1944                            : "=c" (dummy_value_c),        // output regs (dummy)
1945                              "=S" (dummy_value_S),
1946                              "=D" (dummy_value_D)
1947
1948                            : "1" (sptr),      // esi      // input regs
1949                              "2" (dp),        // edi
1950                              "0" (width_mmx)  // ecx
1951
1952 #if 0  /* %mm0, ..., %mm4 not supported by gcc 2.7.2.3 or egcs 1.1 */
1953                            : "%mm0", "%mm1", "%mm2"       // clobber list
1954                            , "%mm3", "%mm4"
1955 #endif
1956                         );
1957                      }
1958
1959                      sptr -= width_mmx;
1960                      dp -= width_mmx*8;
1961                      for (i = width; i; i--)
1962                      {
1963                         int j;
1964
1965                        /* I simplified this part in version 1.0.4e
1966                         * here and in several other instances where
1967                         * pixel_bytes == 1  -- GR-P
1968                         *
1969                         * Original code:
1970                         *
1971                         * png_byte v[8];
1972                         * png_memcpy(v, sptr, pixel_bytes);
1973                         * for (j = 0; j < png_pass_inc[pass]; j++)
1974                         * {
1975                         *    png_memcpy(dp, v, pixel_bytes);
1976                         *    dp -= pixel_bytes;
1977                         * }
1978                         * sptr -= pixel_bytes;
1979                         *
1980                         * Replacement code is in the next three lines:
1981                         */
1982
1983                         for (j = 0; j < png_pass_inc[pass]; j++)
1984                         {
1985                            *dp-- = *sptr;
1986                         }
1987                         --sptr;
1988                      }
1989                   }
1990                   else if (((pass == 2) || (pass == 3)) && width)
1991                   {
1992                      int width_mmx = ((width >> 2) << 2);
1993                      width -= width_mmx;        // 0-3 pixels => 0-3 bytes
1994                      if (width_mmx)
1995                      {
1996                         int dummy_value_c;  // fix 'forbidden register spilled'
1997                         int dummy_value_S;
1998                         int dummy_value_D;
1999
2000                         __asm__ __volatile__ (
2001                            "subl $3, %%esi          \n\t"
2002                            "subl $15, %%edi         \n\t"
2003
2004                         ".loop1_pass2:              \n\t"
2005                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2006                            "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
2007                            "movq %%mm0, %%mm1       \n\t" // 3 3 2 2 1 1 0 0
2008                            "punpcklwd %%mm0, %%mm0  \n\t" // 1 1 1 1 0 0 0 0
2009                            "punpckhwd %%mm1, %%mm1  \n\t" // 3 3 3 3 2 2 2 2
2010                            "movq %%mm0, (%%edi)     \n\t"
2011                            "subl $4, %%esi          \n\t"
2012                            "movq %%mm1, 8(%%edi)    \n\t"
2013                            "subl $16, %%edi         \n\t"
2014                            "subl $4, %%ecx          \n\t"
2015                            "jnz .loop1_pass2        \n\t"
2016                            "EMMS                    \n\t" // DONE
2017
2018                            : "=c" (dummy_value_c),        // output regs (dummy)
2019                              "=S" (dummy_value_S),
2020                              "=D" (dummy_value_D)
2021
2022                            : "1" (sptr),      // esi      // input regs
2023                              "2" (dp),        // edi
2024                              "0" (width_mmx)  // ecx
2025
2026 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2027                            : "%mm0", "%mm1"               // clobber list
2028 #endif
2029                         );
2030                      }
2031
2032                      sptr -= width_mmx;
2033                      dp -= width_mmx*4;
2034                      for (i = width; i; i--)
2035                      {
2036                         int j;
2037
2038                         for (j = 0; j < png_pass_inc[pass]; j++)
2039                         {
2040                            *dp-- = *sptr;
2041                         }
2042                         --sptr;
2043                      }
2044                   }
2045                   else if (width)  /* && ((pass == 4) || (pass == 5)) */
2046                   {
2047                      int width_mmx = ((width >> 3) << 3);
2048                      width -= width_mmx;        // 0-3 pixels => 0-3 bytes
2049                      if (width_mmx)
2050                      {
2051                         int dummy_value_c;  // fix 'forbidden register spilled'
2052                         int dummy_value_S;
2053                         int dummy_value_D;
2054
2055                         __asm__ __volatile__ (
2056                            "subl $7, %%esi          \n\t"
2057                            "subl $15, %%edi         \n\t"
2058
2059                         ".loop1_pass4:              \n\t"
2060                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2061                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2062                            "punpcklbw %%mm0, %%mm0  \n\t" // 3 3 2 2 1 1 0 0
2063                            "punpckhbw %%mm1, %%mm1  \n\t" // 7 7 6 6 5 5 4 4
2064                            "movq %%mm1, 8(%%edi)    \n\t"
2065                            "subl $8, %%esi          \n\t"
2066                            "movq %%mm0, (%%edi)     \n\t"
2067                            "subl $16, %%edi         \n\t"
2068                            "subl $8, %%ecx          \n\t"
2069                            "jnz .loop1_pass4        \n\t"
2070                            "EMMS                    \n\t" // DONE
2071
2072                            : "=c" (dummy_value_c),        // output regs (none)
2073                              "=S" (dummy_value_S),
2074                              "=D" (dummy_value_D)
2075
2076                            : "1" (sptr),      // esi      // input regs
2077                              "2" (dp),        // edi
2078                              "0" (width_mmx)  // ecx
2079
2080 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2081                            : "%mm0", "%mm1"               // clobber list
2082 #endif
2083                         );
2084                      }
2085
2086                      sptr -= width_mmx;
2087                      dp -= width_mmx*2;
2088                      for (i = width; i; i--)
2089                      {
2090                         int j;
2091
2092                         for (j = 0; j < png_pass_inc[pass]; j++)
2093                         {
2094                            *dp-- = *sptr;
2095                         }
2096                         --sptr;
2097                      }
2098                   }
2099                } /* end of pixel_bytes == 1 */
2100
2101                //--------------------------------------------------------------
2102                else if (pixel_bytes == 2)
2103                {
2104                   if (((pass == 0) || (pass == 1)) && width)
2105                   {
2106                      int width_mmx = ((width >> 1) << 1);
2107                      width -= width_mmx;        // 0,1 pixels => 0,2 bytes
2108                      if (width_mmx)
2109                      {
2110                         int dummy_value_c;  // fix 'forbidden register spilled'
2111                         int dummy_value_S;
2112                         int dummy_value_D;
2113
2114                         __asm__ __volatile__ (
2115                            "subl $2, %%esi          \n\t"
2116                            "subl $30, %%edi         \n\t"
2117
2118                         ".loop2_pass0:              \n\t"
2119                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2120                            "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
2121                            "movq %%mm0, %%mm1       \n\t" // 3 2 3 2 1 0 1 0
2122                            "punpckldq %%mm0, %%mm0  \n\t" // 1 0 1 0 1 0 1 0
2123                            "punpckhdq %%mm1, %%mm1  \n\t" // 3 2 3 2 3 2 3 2
2124                            "movq %%mm0, (%%edi)     \n\t"
2125                            "movq %%mm0, 8(%%edi)    \n\t"
2126                            "movq %%mm1, 16(%%edi)   \n\t"
2127                            "subl $4, %%esi          \n\t"
2128                            "movq %%mm1, 24(%%edi)   \n\t"
2129                            "subl $32, %%edi         \n\t"
2130                            "subl $2, %%ecx          \n\t"
2131                            "jnz .loop2_pass0        \n\t"
2132                            "EMMS                    \n\t" // DONE
2133
2134                            : "=c" (dummy_value_c),        // output regs (dummy)
2135                              "=S" (dummy_value_S),
2136                              "=D" (dummy_value_D)
2137
2138                            : "1" (sptr),      // esi      // input regs
2139                              "2" (dp),        // edi
2140                              "0" (width_mmx)  // ecx
2141
2142 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2143                            : "%mm0", "%mm1"               // clobber list
2144 #endif
2145                         );
2146                      }
2147
2148                      sptr -= (width_mmx*2 - 2); // sign fixed
2149                      dp -= (width_mmx*16 - 2);  // sign fixed
2150                      for (i = width; i; i--)
2151                      {
2152                         png_byte v[8];
2153                         int j;
2154                         sptr -= 2;
2155                         png_memcpy(v, sptr, 2);
2156                         for (j = 0; j < png_pass_inc[pass]; j++)
2157                         {
2158                            dp -= 2;
2159                            png_memcpy(dp, v, 2);
2160                         }
2161                      }
2162                   }
2163                   else if (((pass == 2) || (pass == 3)) && width)
2164                   {
2165                      int width_mmx = ((width >> 1) << 1) ;
2166                      width -= width_mmx;        // 0,1 pixels => 0,2 bytes
2167                      if (width_mmx)
2168                      {
2169                         int dummy_value_c;  // fix 'forbidden register spilled'
2170                         int dummy_value_S;
2171                         int dummy_value_D;
2172
2173                         __asm__ __volatile__ (
2174                            "subl $2, %%esi          \n\t"
2175                            "subl $14, %%edi         \n\t"
2176
2177                         ".loop2_pass2:              \n\t"
2178                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2179                            "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
2180                            "movq %%mm0, %%mm1       \n\t" // 3 2 3 2 1 0 1 0
2181                            "punpckldq %%mm0, %%mm0  \n\t" // 1 0 1 0 1 0 1 0
2182                            "punpckhdq %%mm1, %%mm1  \n\t" // 3 2 3 2 3 2 3 2
2183                            "movq %%mm0, (%%edi)     \n\t"
2184                            "subl $4, %%esi          \n\t"
2185                            "movq %%mm1, 8(%%edi)    \n\t"
2186                            "subl $16, %%edi         \n\t"
2187                            "subl $2, %%ecx          \n\t"
2188                            "jnz .loop2_pass2        \n\t"
2189                            "EMMS                    \n\t" // DONE
2190
2191                            : "=c" (dummy_value_c),        // output regs (dummy)
2192                              "=S" (dummy_value_S),
2193                              "=D" (dummy_value_D)
2194
2195                            : "1" (sptr),      // esi      // input regs
2196                              "2" (dp),        // edi
2197                              "0" (width_mmx)  // ecx
2198
2199 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2200                            : "%mm0", "%mm1"               // clobber list
2201 #endif
2202                         );
2203                      }
2204
2205                      sptr -= (width_mmx*2 - 2); // sign fixed
2206                      dp -= (width_mmx*8 - 2);   // sign fixed
2207                      for (i = width; i; i--)
2208                      {
2209                         png_byte v[8];
2210                         int j;
2211                         sptr -= 2;
2212                         png_memcpy(v, sptr, 2);
2213                         for (j = 0; j < png_pass_inc[pass]; j++)
2214                         {
2215                            dp -= 2;
2216                            png_memcpy(dp, v, 2);
2217                         }
2218                      }
2219                   }
2220                   else if (width)  // pass == 4 or 5
2221                   {
2222                      int width_mmx = ((width >> 1) << 1) ;
2223                      width -= width_mmx;        // 0,1 pixels => 0,2 bytes
2224                      if (width_mmx)
2225                      {
2226                         int dummy_value_c;  // fix 'forbidden register spilled'
2227                         int dummy_value_S;
2228                         int dummy_value_D;
2229
2230                         __asm__ __volatile__ (
2231                            "subl $2, %%esi          \n\t"
2232                            "subl $6, %%edi          \n\t"
2233
2234                         ".loop2_pass4:              \n\t"
2235                            "movd (%%esi), %%mm0     \n\t" // x x x x 3 2 1 0
2236                            "punpcklwd %%mm0, %%mm0  \n\t" // 3 2 3 2 1 0 1 0
2237                            "subl $4, %%esi          \n\t"
2238                            "movq %%mm0, (%%edi)     \n\t"
2239                            "subl $8, %%edi          \n\t"
2240                            "subl $2, %%ecx          \n\t"
2241                            "jnz .loop2_pass4        \n\t"
2242                            "EMMS                    \n\t" // DONE
2243
2244                            : "=c" (dummy_value_c),        // output regs (dummy)
2245                              "=S" (dummy_value_S),
2246                              "=D" (dummy_value_D)
2247
2248                            : "1" (sptr),      // esi      // input regs
2249                              "2" (dp),        // edi
2250                              "0" (width_mmx)  // ecx
2251
2252 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2253                            : "%mm0"                       // clobber list
2254 #endif
2255                         );
2256                      }
2257
2258                      sptr -= (width_mmx*2 - 2); // sign fixed
2259                      dp -= (width_mmx*4 - 2);   // sign fixed
2260                      for (i = width; i; i--)
2261                      {
2262                         png_byte v[8];
2263                         int j;
2264                         sptr -= 2;
2265                         png_memcpy(v, sptr, 2);
2266                         for (j = 0; j < png_pass_inc[pass]; j++)
2267                         {
2268                            dp -= 2;
2269                            png_memcpy(dp, v, 2);
2270                         }
2271                      }
2272                   }
2273                } /* end of pixel_bytes == 2 */
2274
2275                //--------------------------------------------------------------
2276                else if (pixel_bytes == 4)
2277                {
2278                   if (((pass == 0) || (pass == 1)) && width)
2279                   {
2280                      int width_mmx = ((width >> 1) << 1);
2281                      width -= width_mmx;        // 0,1 pixels => 0,4 bytes
2282                      if (width_mmx)
2283                      {
2284                         int dummy_value_c;  // fix 'forbidden register spilled'
2285                         int dummy_value_S;
2286                         int dummy_value_D;
2287
2288                         __asm__ __volatile__ (
2289                            "subl $4, %%esi          \n\t"
2290                            "subl $60, %%edi         \n\t"
2291
2292                         ".loop4_pass0:              \n\t"
2293                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2294                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2295                            "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
2296                            "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
2297                            "movq %%mm0, (%%edi)     \n\t"
2298                            "movq %%mm0, 8(%%edi)    \n\t"
2299                            "movq %%mm0, 16(%%edi)   \n\t"
2300                            "movq %%mm0, 24(%%edi)   \n\t"
2301                            "movq %%mm1, 32(%%edi)   \n\t"
2302                            "movq %%mm1, 40(%%edi)   \n\t"
2303                            "movq %%mm1, 48(%%edi)   \n\t"
2304                            "subl $8, %%esi          \n\t"
2305                            "movq %%mm1, 56(%%edi)   \n\t"
2306                            "subl $64, %%edi         \n\t"
2307                            "subl $2, %%ecx          \n\t"
2308                            "jnz .loop4_pass0        \n\t"
2309                            "EMMS                    \n\t" // DONE
2310
2311                            : "=c" (dummy_value_c),        // output regs (dummy)
2312                              "=S" (dummy_value_S),
2313                              "=D" (dummy_value_D)
2314
2315                            : "1" (sptr),      // esi      // input regs
2316                              "2" (dp),        // edi
2317                              "0" (width_mmx)  // ecx
2318
2319 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2320                            : "%mm0", "%mm1"               // clobber list
2321 #endif
2322                         );
2323                      }
2324
2325                      sptr -= (width_mmx*4 - 4); // sign fixed
2326                      dp -= (width_mmx*32 - 4);  // sign fixed
2327                      for (i = width; i; i--)
2328                      {
2329                         png_byte v[8];
2330                         int j;
2331                         sptr -= 4;
2332                         png_memcpy(v, sptr, 4);
2333                         for (j = 0; j < png_pass_inc[pass]; j++)
2334                         {
2335                            dp -= 4;
2336                            png_memcpy(dp, v, 4);
2337                         }
2338                      }
2339                   }
2340                   else if (((pass == 2) || (pass == 3)) && width)
2341                   {
2342                      int width_mmx = ((width >> 1) << 1);
2343                      width -= width_mmx;        // 0,1 pixels => 0,4 bytes
2344                      if (width_mmx)
2345                      {
2346                         int dummy_value_c;  // fix 'forbidden register spilled'
2347                         int dummy_value_S;
2348                         int dummy_value_D;
2349
2350                         __asm__ __volatile__ (
2351                            "subl $4, %%esi          \n\t"
2352                            "subl $28, %%edi         \n\t"
2353
2354                         ".loop4_pass2:              \n\t"
2355                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2356                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2357                            "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
2358                            "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
2359                            "movq %%mm0, (%%edi)     \n\t"
2360                            "movq %%mm0, 8(%%edi)    \n\t"
2361                            "movq %%mm1, 16(%%edi)   \n\t"
2362                            "movq %%mm1, 24(%%edi)   \n\t"
2363                            "subl $8, %%esi          \n\t"
2364                            "subl $32, %%edi         \n\t"
2365                            "subl $2, %%ecx          \n\t"
2366                            "jnz .loop4_pass2        \n\t"
2367                            "EMMS                    \n\t" // DONE
2368
2369                            : "=c" (dummy_value_c),        // output regs (dummy)
2370                              "=S" (dummy_value_S),
2371                              "=D" (dummy_value_D)
2372
2373                            : "1" (sptr),      // esi      // input regs
2374                              "2" (dp),        // edi
2375                              "0" (width_mmx)  // ecx
2376
2377 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2378                            : "%mm0", "%mm1"               // clobber list
2379 #endif
2380                         );
2381                      }
2382
2383                      sptr -= (width_mmx*4 - 4); // sign fixed
2384                      dp -= (width_mmx*16 - 4);  // sign fixed
2385                      for (i = width; i; i--)
2386                      {
2387                         png_byte v[8];
2388                         int j;
2389                         sptr -= 4;
2390                         png_memcpy(v, sptr, 4);
2391                         for (j = 0; j < png_pass_inc[pass]; j++)
2392                         {
2393                            dp -= 4;
2394                            png_memcpy(dp, v, 4);
2395                         }
2396                      }
2397                   }
2398                   else if (width)  // pass == 4 or 5
2399                   {
2400                      int width_mmx = ((width >> 1) << 1) ;
2401                      width -= width_mmx;        // 0,1 pixels => 0,4 bytes
2402                      if (width_mmx)
2403                      {
2404                         int dummy_value_c;  // fix 'forbidden register spilled'
2405                         int dummy_value_S;
2406                         int dummy_value_D;
2407
2408                         __asm__ __volatile__ (
2409                            "subl $4, %%esi          \n\t"
2410                            "subl $12, %%edi         \n\t"
2411
2412                         ".loop4_pass4:              \n\t"
2413                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2414                            "movq %%mm0, %%mm1       \n\t" // 7 6 5 4 3 2 1 0
2415                            "punpckldq %%mm0, %%mm0  \n\t" // 3 2 1 0 3 2 1 0
2416                            "punpckhdq %%mm1, %%mm1  \n\t" // 7 6 5 4 7 6 5 4
2417                            "movq %%mm0, (%%edi)     \n\t"
2418                            "subl $8, %%esi          \n\t"
2419                            "movq %%mm1, 8(%%edi)    \n\t"
2420                            "subl $16, %%edi         \n\t"
2421                            "subl $2, %%ecx          \n\t"
2422                            "jnz .loop4_pass4        \n\t"
2423                            "EMMS                    \n\t" // DONE
2424
2425                            : "=c" (dummy_value_c),        // output regs (dummy)
2426                              "=S" (dummy_value_S),
2427                              "=D" (dummy_value_D)
2428
2429                            : "1" (sptr),      // esi      // input regs
2430                              "2" (dp),        // edi
2431                              "0" (width_mmx)  // ecx
2432
2433 #if 0  /* %mm0, %mm1 not supported by gcc 2.7.2.3 or egcs 1.1 */
2434                            : "%mm0", "%mm1"               // clobber list
2435 #endif
2436                         );
2437                      }
2438
2439                      sptr -= (width_mmx*4 - 4); // sign fixed
2440                      dp -= (width_mmx*8 - 4);   // sign fixed
2441                      for (i = width; i; i--)
2442                      {
2443                         png_byte v[8];
2444                         int j;
2445                         sptr -= 4;
2446                         png_memcpy(v, sptr, 4);
2447                         for (j = 0; j < png_pass_inc[pass]; j++)
2448                         {
2449                            dp -= 4;
2450                            png_memcpy(dp, v, 4);
2451                         }
2452                      }
2453                   }
2454                } /* end of pixel_bytes == 4 */
2455
2456                //--------------------------------------------------------------
2457                else if (pixel_bytes == 8)
2458                {
2459 // GRR TEST:  should work, but needs testing (special 64-bit version of rpng2?)
2460                   // GRR NOTE:  no need to combine passes here!
2461                   if (((pass == 0) || (pass == 1)) && width)
2462                   {
2463                      int dummy_value_c;  // fix 'forbidden register spilled'
2464                      int dummy_value_S;
2465                      int dummy_value_D;
2466
2467                      // source is 8-byte RRGGBBAA
2468                      // dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ...
2469                      __asm__ __volatile__ (
2470                         "subl $56, %%edi         \n\t" // start of last block
2471
2472                      ".loop8_pass0:              \n\t"
2473                         "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2474                         "movq %%mm0, (%%edi)     \n\t"
2475                         "movq %%mm0, 8(%%edi)    \n\t"
2476                         "movq %%mm0, 16(%%edi)   \n\t"
2477                         "movq %%mm0, 24(%%edi)   \n\t"
2478                         "movq %%mm0, 32(%%edi)   \n\t"
2479                         "movq %%mm0, 40(%%edi)   \n\t"
2480                         "movq %%mm0, 48(%%edi)   \n\t"
2481                         "subl $8, %%esi          \n\t"
2482                         "movq %%mm0, 56(%%edi)   \n\t"
2483                         "subl $64, %%edi         \n\t"
2484                         "decl %%ecx              \n\t"
2485                         "jnz .loop8_pass0        \n\t"
2486                         "EMMS                    \n\t" // DONE
2487
2488                         : "=c" (dummy_value_c),        // output regs (dummy)
2489                           "=S" (dummy_value_S),
2490                           "=D" (dummy_value_D)
2491
2492                         : "1" (sptr),      // esi      // input regs
2493                           "2" (dp),        // edi
2494                           "0" (width)      // ecx
2495
2496 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2497                         : "%mm0"                       // clobber list
2498 #endif
2499                      );
2500                   }
2501                   else if (((pass == 2) || (pass == 3)) && width)
2502                   {
2503                      // source is 8-byte RRGGBBAA
2504                      // dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA
2505                      // (recall that expansion is _in place_:  sptr and dp
2506                      //  both point at locations within same row buffer)
2507                      {
2508                         int dummy_value_c;  // fix 'forbidden register spilled'
2509                         int dummy_value_S;
2510                         int dummy_value_D;
2511
2512                         __asm__ __volatile__ (
2513                            "subl $24, %%edi         \n\t" // start of last block
2514
2515                         ".loop8_pass2:              \n\t"
2516                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2517                            "movq %%mm0, (%%edi)     \n\t"
2518                            "movq %%mm0, 8(%%edi)    \n\t"
2519                            "movq %%mm0, 16(%%edi)   \n\t"
2520                            "subl $8, %%esi          \n\t"
2521                            "movq %%mm0, 24(%%edi)   \n\t"
2522                            "subl $32, %%edi         \n\t"
2523                            "decl %%ecx              \n\t"
2524                            "jnz .loop8_pass2        \n\t"
2525                            "EMMS                    \n\t" // DONE
2526
2527                            : "=c" (dummy_value_c),        // output regs (dummy)
2528                              "=S" (dummy_value_S),
2529                              "=D" (dummy_value_D)
2530
2531                            : "1" (sptr),      // esi      // input regs
2532                              "2" (dp),        // edi
2533                              "0" (width)      // ecx
2534
2535 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2536                            : "%mm0"                       // clobber list
2537 #endif
2538                         );
2539                      }
2540                   }
2541                   else if (width)  // pass == 4 or 5
2542                   {
2543                      // source is 8-byte RRGGBBAA
2544                      // dest is 16-byte RRGGBBAA RRGGBBAA
2545                      {
2546                         int dummy_value_c;  // fix 'forbidden register spilled'
2547                         int dummy_value_S;
2548                         int dummy_value_D;
2549
2550                         __asm__ __volatile__ (
2551                            "subl $8, %%edi          \n\t" // start of last block
2552
2553                         ".loop8_pass4:              \n\t"
2554                            "movq (%%esi), %%mm0     \n\t" // 7 6 5 4 3 2 1 0
2555                            "movq %%mm0, (%%edi)     \n\t"
2556                            "subl $8, %%esi          \n\t"
2557                            "movq %%mm0, 8(%%edi)    \n\t"
2558                            "subl $16, %%edi         \n\t"
2559                            "decl %%ecx              \n\t"
2560                            "jnz .loop8_pass4        \n\t"
2561                            "EMMS                    \n\t" // DONE
2562
2563                            : "=c" (dummy_value_c),        // output regs (dummy)
2564                              "=S" (dummy_value_S),
2565                              "=D" (dummy_value_D)
2566
2567                            : "1" (sptr),      // esi      // input regs
2568                              "2" (dp),        // edi
2569                              "0" (width)      // ecx
2570
2571 #if 0  /* %mm0 not supported by gcc 2.7.2.3 or egcs 1.1 */
2572                            : "%mm0"                       // clobber list
2573 #endif
2574                         );
2575                      }
2576                   }
2577
2578                } /* end of pixel_bytes == 8 */
2579
2580                //--------------------------------------------------------------
2581                else if (pixel_bytes == 6)
2582                {
2583                   for (i = width; i; i--)
2584                   {
2585                      png_byte v[8];
2586                      int j;
2587                      png_memcpy(v, sptr, 6);
2588                      for (j = 0; j < png_pass_inc[pass]; j++)
2589                      {
2590                         png_memcpy(dp, v, 6);
2591                         dp -= 6;
2592                      }
2593                      sptr -= 6;
2594                   }
2595                } /* end of pixel_bytes == 6 */
2596
2597                //--------------------------------------------------------------
2598                else
2599                {
2600                   for (i = width; i; i--)
2601                   {
2602                      png_byte v[8];
2603                      int j;
2604                      png_memcpy(v, sptr, pixel_bytes);
2605                      for (j = 0; j < png_pass_inc[pass]; j++)
2606                      {
2607                         png_memcpy(dp, v, pixel_bytes);
2608                         dp -= pixel_bytes;
2609                      }
2610                      sptr-= pixel_bytes;
2611                   }
2612                }
2613             } // end of _mmx_supported ========================================
2614
2615             else /* MMX not supported:  use modified C code - takes advantage
2616                   *   of inlining of png_memcpy for a constant */
2617                  /* GRR 19991007:  does it?  or should pixel_bytes in each
2618                   *   block be replaced with immediate value (e.g., 1)? */
2619                  /* GRR 19991017:  replaced with constants in each case */
2620 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
2621             {
2622                if (pixel_bytes == 1)
2623                {
2624                   for (i = width; i; i--)
2625                   {
2626                      int j;
2627                      for (j = 0; j < png_pass_inc[pass]; j++)
2628                      {
2629                         *dp-- = *sptr;
2630                      }
2631                      --sptr;
2632                   }
2633                }
2634                else if (pixel_bytes == 3)
2635                {
2636                   for (i = width; i; i--)
2637                   {
2638                      png_byte v[8];
2639                      int j;
2640                      png_memcpy(v, sptr, 3);
2641                      for (j = 0; j < png_pass_inc[pass]; j++)
2642                      {
2643                         png_memcpy(dp, v, 3);
2644                         dp -= 3;
2645                      }
2646                      sptr -= 3;
2647                   }
2648                }
2649                else if (pixel_bytes == 2)
2650                {
2651                   for (i = width; i; i--)
2652                   {
2653                      png_byte v[8];
2654                      int j;
2655                      png_memcpy(v, sptr, 2);
2656                      for (j = 0; j < png_pass_inc[pass]; j++)
2657                      {
2658                         png_memcpy(dp, v, 2);
2659                         dp -= 2;
2660                      }
2661                      sptr -= 2;
2662                   }
2663                }
2664                else if (pixel_bytes == 4)
2665                {
2666                   for (i = width; i; i--)
2667                   {
2668                      png_byte v[8];
2669                      int j;
2670                      png_memcpy(v, sptr, 4);
2671                      for (j = 0; j < png_pass_inc[pass]; j++)
2672                      {
2673 #ifdef PNG_DEBUG
2674                         if (dp < row || dp+3 > row+png_ptr->row_buf_size)
2675                         {
2676                            printf("dp out of bounds: row=%d, dp=%d, rp=%d\n",
2677                              row, dp, row+png_ptr->row_buf_size);
2678                            printf("row_buf=%d\n",png_ptr->row_buf_size);
2679                         }
2680 #endif
2681                         png_memcpy(dp, v, 4);
2682                         dp -= 4;
2683                      }
2684                      sptr -= 4;
2685                   }
2686                }
2687                else if (pixel_bytes == 6)
2688                {
2689                   for (i = width; i; i--)
2690                   {
2691                      png_byte v[8];
2692                      int j;
2693                      png_memcpy(v, sptr, 6);
2694                      for (j = 0; j < png_pass_inc[pass]; j++)
2695                      {
2696                         png_memcpy(dp, v, 6);
2697                         dp -= 6;
2698                      }
2699                      sptr -= 6;
2700                   }
2701                }
2702                else if (pixel_bytes == 8)
2703                {
2704                   for (i = width; i; i--)
2705                   {
2706                      png_byte v[8];
2707                      int j;
2708                      png_memcpy(v, sptr, 8);
2709                      for (j = 0; j < png_pass_inc[pass]; j++)
2710                      {
2711                         png_memcpy(dp, v, 8);
2712                         dp -= 8;
2713                      }
2714                      sptr -= 8;
2715                   }
2716                }
2717                else     /* GRR:  should never be reached */
2718                {
2719                   for (i = width; i; i--)
2720                   {
2721                      png_byte v[8];
2722                      int j;
2723                      png_memcpy(v, sptr, pixel_bytes);
2724                      for (j = 0; j < png_pass_inc[pass]; j++)
2725                      {
2726                         png_memcpy(dp, v, pixel_bytes);
2727                         dp -= pixel_bytes;
2728                      }
2729                      sptr -= pixel_bytes;
2730                   }
2731                }
2732
2733             } /* end if (MMX not supported) */
2734             break;
2735          }
2736       } /* end switch (row_info->pixel_depth) */
2737
2738       row_info->width = final_width;
2739
2740       row_info->rowbytes = PNG_ROWBYTES(row_info->pixel_depth,final_width);
2741    }
2742
2743 } /* end png_do_read_interlace() */
2744
2745 #endif /* PNG_HAVE_ASSEMBLER_READ_INTERLACE */
2746 #endif /* PNG_READ_INTERLACING_SUPPORTED */
2747
2748
2749
2750 #if defined(PNG_HAVE_ASSEMBLER_READ_FILTER_ROW)
2751 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
2752
2753 // These variables are utilized in the functions below.  They are declared
2754 // globally here to ensure alignment on 8-byte boundaries.
2755
2756 union uAll {
2757    long long use;
2758    double  align;
2759 } _LBCarryMask = {0x0101010101010101LL},
2760   _HBClearMask = {0x7f7f7f7f7f7f7f7fLL},
2761   _ActiveMask, _ActiveMask2, _ActiveMaskEnd, _ShiftBpp, _ShiftRem;
2762
2763 #ifdef PNG_THREAD_UNSAFE_OK
2764 //===========================================================================//
2765 //                                                                           //
2766 //           P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G           //
2767 //                                                                           //
2768 //===========================================================================//
2769
2770 // Optimized code for PNG Average filter decoder
2771
2772 static void /* PRIVATE */
2773 png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row,
2774                             png_bytep prev_row)
2775 {
2776    int bpp;
2777    int dummy_value_c;   // fix 'forbidden register 2 (cx) was spilled' error
2778    int dummy_value_S;
2779    int dummy_value_D;
2780
2781    bpp = (row_info->pixel_depth + 7) >> 3;  // get # bytes per pixel
2782    _FullLength  = row_info->rowbytes;       // # of bytes to filter
2783
2784    __asm__ __volatile__ (
2785       // initialize address pointers and offset
2786 #ifdef __PIC__
2787       "pushl %%ebx                 \n\t" // save index to Global Offset Table
2788 #endif
2789 //pre "movl row, %%edi             \n\t" // edi:  Avg(x)
2790       "xorl %%ebx, %%ebx           \n\t" // ebx:  x
2791       "movl %%edi, %%edx           \n\t"
2792 //pre "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
2793 //pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
2794       "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
2795
2796       "xorl %%eax,%%eax            \n\t"
2797
2798       // Compute the Raw value for the first bpp bytes
2799       //    Raw(x) = Avg(x) + (Prior(x)/2)
2800    "avg_rlp:                       \n\t"
2801       "movb (%%esi,%%ebx,),%%al    \n\t" // load al with Prior(x)
2802       "incl %%ebx                  \n\t"
2803       "shrb %%al                   \n\t" // divide by 2
2804       "addb -1(%%edi,%%ebx,),%%al  \n\t" // add Avg(x); -1 to offset inc ebx
2805 //pre "cmpl bpp, %%ebx             \n\t" // (bpp is preloaded into ecx)
2806       "cmpl %%ecx, %%ebx           \n\t"
2807       "movb %%al,-1(%%edi,%%ebx,)  \n\t" // write Raw(x); -1 to offset inc ebx
2808       "jb avg_rlp                  \n\t" // mov does not affect flags
2809
2810       // get # of bytes to alignment
2811       "movl %%edi, _dif            \n\t" // take start of row
2812       "addl %%ebx, _dif            \n\t" // add bpp
2813       "addl $0xf, _dif             \n\t" // add 7+8 to incr past alignment bdry
2814       "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
2815       "subl %%edi, _dif            \n\t" // subtract from start => value ebx at
2816       "jz avg_go                   \n\t" //  alignment
2817
2818       // fix alignment
2819       // Compute the Raw value for the bytes up to the alignment boundary
2820       //    Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
2821       "xorl %%ecx, %%ecx           \n\t"
2822
2823    "avg_lp1:                       \n\t"
2824       "xorl %%eax, %%eax           \n\t"
2825       "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
2826       "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
2827       "addw %%cx, %%ax             \n\t"
2828       "incl %%ebx                  \n\t"
2829       "shrw %%ax                   \n\t" // divide by 2
2830       "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
2831       "cmpl _dif, %%ebx            \n\t" // check if at alignment boundary
2832       "movb %%al, -1(%%edi,%%ebx,) \n\t" // write Raw(x); -1 to offset inc ebx
2833       "jb avg_lp1                  \n\t" // repeat until at alignment boundary
2834
2835    "avg_go:                        \n\t"
2836       "movl _FullLength, %%eax     \n\t"
2837       "movl %%eax, %%ecx           \n\t"
2838       "subl %%ebx, %%eax           \n\t" // subtract alignment fix
2839       "andl $0x00000007, %%eax     \n\t" // calc bytes over mult of 8
2840       "subl %%eax, %%ecx           \n\t" // drop over bytes from original length
2841       "movl %%ecx, _MMXLength      \n\t"
2842 #ifdef __PIC__
2843       "popl %%ebx                  \n\t" // restore index to Global Offset Table
2844 #endif
2845
2846       : "=c" (dummy_value_c),            // output regs (dummy)
2847         "=S" (dummy_value_S),
2848         "=D" (dummy_value_D)
2849
2850       : "0" (bpp),       // ecx          // input regs
2851         "1" (prev_row),  // esi
2852         "2" (row)        // edi
2853
2854       : "%eax", "%edx"                   // clobber list
2855 #ifndef __PIC__
2856       , "%ebx"
2857 #endif
2858       // GRR: INCLUDE "memory" as clobbered? (_dif, _MMXLength)
2859       // (seems to work fine without...)
2860    );
2861
2862    // now do the math for the rest of the row
2863    switch (bpp)
2864    {
2865       case 3:
2866       {
2867          _ActiveMask.use  = 0x0000000000ffffffLL;
2868          _ShiftBpp.use = 24;    // == 3 * 8
2869          _ShiftRem.use = 40;    // == 64 - 24
2870
2871          __asm__ __volatile__ (
2872             // re-init address pointers and offset
2873             "movq _ActiveMask, %%mm7      \n\t"
2874             "movl _dif, %%ecx             \n\t" // ecx:  x = offset to
2875             "movq _LBCarryMask, %%mm5     \n\t" //  alignment boundary
2876 // preload  "movl row, %%edi              \n\t" // edi:  Avg(x)
2877             "movq _HBClearMask, %%mm4     \n\t"
2878 // preload  "movl prev_row, %%esi         \n\t" // esi:  Prior(x)
2879
2880             // prime the pump:  load the first Raw(x-bpp) data set
2881             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
2882                                                 // (correct pos. in loop below)
2883          "avg_3lp:                        \n\t"
2884             "movq (%%edi,%%ecx,), %%mm0   \n\t" // load mm0 with Avg(x)
2885             "movq %%mm5, %%mm3            \n\t"
2886             "psrlq _ShiftRem, %%mm2       \n\t" // correct position Raw(x-bpp)
2887                                                 // data
2888             "movq (%%esi,%%ecx,), %%mm1   \n\t" // load mm1 with Prior(x)
2889             "movq %%mm7, %%mm6            \n\t"
2890             "pand %%mm1, %%mm3            \n\t" // get lsb for each prev_row byte
2891             "psrlq $1, %%mm1              \n\t" // divide prev_row bytes by 2
2892             "pand  %%mm4, %%mm1           \n\t" // clear invalid bit 7 of each
2893                                                 // byte
2894             "paddb %%mm1, %%mm0           \n\t" // add (Prev_row/2) to Avg for
2895                                                 // each byte
2896             // add 1st active group (Raw(x-bpp)/2) to average with LBCarry
2897             "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
2898                                                 // LBCarrys
2899             "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
2900                                                 // where both
2901                                // lsb's were == 1 (only valid for active group)
2902             "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
2903             "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
2904                                                 // byte
2905             "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2906                                                 // for each byte
2907             "pand %%mm6, %%mm2            \n\t" // leave only Active Group 1
2908                                                 // bytes to add to Avg
2909             "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
2910                                                 // Avg for each Active
2911                                //  byte
2912             // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
2913             "psllq _ShiftBpp, %%mm6       \n\t" // shift the mm6 mask to cover
2914                                                 // bytes 3-5
2915             "movq %%mm0, %%mm2            \n\t" // mov updated Raws to mm2
2916             "psllq _ShiftBpp, %%mm2       \n\t" // shift data to pos. correctly
2917             "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
2918                                                 // LBCarrys
2919             "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
2920                                                 // where both
2921                                // lsb's were == 1 (only valid for active group)
2922             "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
2923             "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
2924                                                 // byte
2925             "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2926                                                 // for each byte
2927             "pand %%mm6, %%mm2            \n\t" // leave only Active Group 2
2928                                                 // bytes to add to Avg
2929             "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
2930                                                 // Avg for each Active
2931                                //  byte
2932
2933             // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
2934             "psllq _ShiftBpp, %%mm6       \n\t" // shift mm6 mask to cover last
2935                                                 // two
2936                                  // bytes
2937             "movq %%mm0, %%mm2            \n\t" // mov updated Raws to mm2
2938             "psllq _ShiftBpp, %%mm2       \n\t" // shift data to pos. correctly
2939                               // Data only needs to be shifted once here to
2940                               // get the correct x-bpp offset.
2941             "movq %%mm3, %%mm1            \n\t" // now use mm1 for getting
2942                                                 // LBCarrys
2943             "pand %%mm2, %%mm1            \n\t" // get LBCarrys for each byte
2944                                                 // where both
2945                               // lsb's were == 1 (only valid for active group)
2946             "psrlq $1, %%mm2              \n\t" // divide raw bytes by 2
2947             "pand  %%mm4, %%mm2           \n\t" // clear invalid bit 7 of each
2948                                                 // byte
2949             "paddb %%mm1, %%mm2           \n\t" // add LBCarrys to (Raw(x-bpp)/2)
2950                                                 // for each byte
2951             "pand %%mm6, %%mm2            \n\t" // leave only Active Group 2
2952                                                 // bytes to add to Avg
2953             "addl $8, %%ecx               \n\t"
2954             "paddb %%mm2, %%mm0           \n\t" // add (Raw/2) + LBCarrys to
2955                                                 // Avg for each Active
2956                                                 // byte
2957             // now ready to write back to memory
2958             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
2959             // move updated Raw(x) to use as Raw(x-bpp) for next loop
2960             "cmpl _MMXLength, %%ecx       \n\t"
2961             "movq %%mm0, %%mm2            \n\t" // mov updated Raw(x) to mm2
2962             "jb avg_3lp                   \n\t"
2963
2964             : "=S" (dummy_value_S),             // output regs (dummy)
2965               "=D" (dummy_value_D)
2966
2967             : "0" (prev_row),  // esi           // input regs
2968               "1" (row)        // edi
2969
2970             : "%ecx"                            // clobber list
2971 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
2972             , "%mm0", "%mm1", "%mm2", "%mm3"
2973             , "%mm4", "%mm5", "%mm6", "%mm7"
2974 #endif
2975          );
2976       }
2977       break;  // end 3 bpp
2978
2979       case 6:
2980       case 4:
2981       //case 7:   // who wrote this?  PNG doesn't support 5 or 7 bytes/pixel
2982       //case 5:   // GRR BOGUS
2983       {
2984          _ActiveMask.use  = 0xffffffffffffffffLL; // use shift below to clear
2985                                                   // appropriate inactive bytes
2986          _ShiftBpp.use = bpp << 3;
2987          _ShiftRem.use = 64 - _ShiftBpp.use;
2988
2989          __asm__ __volatile__ (
2990             "movq _HBClearMask, %%mm4    \n\t"
2991
2992             // re-init address pointers and offset
2993             "movl _dif, %%ecx            \n\t" // ecx:  x = offset to
2994                                                // alignment boundary
2995
2996             // load _ActiveMask and clear all bytes except for 1st active group
2997             "movq _ActiveMask, %%mm7     \n\t"
2998 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
2999             "psrlq _ShiftRem, %%mm7      \n\t"
3000 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3001             "movq %%mm7, %%mm6           \n\t"
3002             "movq _LBCarryMask, %%mm5    \n\t"
3003             "psllq _ShiftBpp, %%mm6      \n\t" // create mask for 2nd active
3004                                                // group
3005
3006             // prime the pump:  load the first Raw(x-bpp) data set
3007             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3008                                           // (we correct pos. in loop below)
3009          "avg_4lp:                       \n\t"
3010             "movq (%%edi,%%ecx,), %%mm0  \n\t"
3011             "psrlq _ShiftRem, %%mm2      \n\t" // shift data to pos. correctly
3012             "movq (%%esi,%%ecx,), %%mm1  \n\t"
3013             // add (Prev_row/2) to average
3014             "movq %%mm5, %%mm3           \n\t"
3015             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3016             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3017             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
3018                                                // byte
3019             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
3020                                                // each byte
3021             // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3022             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3023                                                // LBCarrys
3024             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3025                                                // where both
3026                               // lsb's were == 1 (only valid for active group)
3027             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3028             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3029                                                // byte
3030             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3031                                                // for each byte
3032             "pand %%mm7, %%mm2           \n\t" // leave only Active Group 1
3033                                                // bytes to add to Avg
3034             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to Avg
3035                                                // for each Active
3036                               // byte
3037             // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3038             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3039             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3040             "addl $8, %%ecx              \n\t"
3041             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3042                                                // LBCarrys
3043             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3044                                                // where both
3045                               // lsb's were == 1 (only valid for active group)
3046             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3047             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3048                                                // byte
3049             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3050                                                // for each byte
3051             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3052                                                // bytes to add to Avg
3053             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3054                                                // Avg for each Active
3055                               // byte
3056             "cmpl _MMXLength, %%ecx      \n\t"
3057             // now ready to write back to memory
3058             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3059             // prep Raw(x-bpp) for next loop
3060             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3061             "jb avg_4lp                  \n\t"
3062
3063             : "=S" (dummy_value_S),            // output regs (dummy)
3064               "=D" (dummy_value_D)
3065
3066             : "0" (prev_row),  // esi          // input regs
3067               "1" (row)        // edi
3068
3069             : "%ecx"                           // clobber list
3070 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3071             , "%mm0", "%mm1", "%mm2", "%mm3"
3072             , "%mm4", "%mm5", "%mm6", "%mm7"
3073 #endif
3074          );
3075       }
3076       break;  // end 4,6 bpp
3077
3078       case 2:
3079       {
3080          _ActiveMask.use  = 0x000000000000ffffLL;
3081          _ShiftBpp.use = 16;   // == 2 * 8
3082          _ShiftRem.use = 48;   // == 64 - 16
3083
3084          __asm__ __volatile__ (
3085             // load _ActiveMask
3086             "movq _ActiveMask, %%mm7     \n\t"
3087             // re-init address pointers and offset
3088             "movl _dif, %%ecx            \n\t" // ecx:  x = offset to alignment
3089                                                // boundary
3090             "movq _LBCarryMask, %%mm5    \n\t"
3091 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
3092             "movq _HBClearMask, %%mm4    \n\t"
3093 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3094
3095             // prime the pump:  load the first Raw(x-bpp) data set
3096             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3097                               // (we correct pos. in loop below)
3098          "avg_2lp:                       \n\t"
3099             "movq (%%edi,%%ecx,), %%mm0  \n\t"
3100             "psrlq _ShiftRem, %%mm2      \n\t" // shift data to pos. correctly
3101             "movq (%%esi,%%ecx,), %%mm1  \n\t" //  (GRR BUGFIX:  was psllq)
3102             // add (Prev_row/2) to average
3103             "movq %%mm5, %%mm3           \n\t"
3104             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3105             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3106             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
3107                                                // byte
3108             "movq %%mm7, %%mm6           \n\t"
3109             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
3110                                                // each byte
3111
3112             // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3113             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3114                                                // LBCarrys
3115             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3116                                                // where both
3117                                                // lsb's were == 1 (only valid
3118                                                // for active group)
3119             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3120             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3121                                                // byte
3122             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3123                                                // for each byte
3124             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 1
3125                                                // bytes to add to Avg
3126             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to Avg
3127                                                // for each Active byte
3128
3129             // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3130             "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
3131                                                // bytes 2 & 3
3132             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3133             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3134             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3135                                                // LBCarrys
3136             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3137                                                // where both
3138                                                // lsb's were == 1 (only valid
3139                                                // for active group)
3140             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3141             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3142                                                // byte
3143             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3144                                                // for each byte
3145             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3146                                                // bytes to add to Avg
3147             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3148                                                // Avg for each Active byte
3149
3150             // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
3151             "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
3152                                                // bytes 4 & 5
3153             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3154             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3155             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3156                                                // LBCarrys
3157             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3158                                                // where both lsb's were == 1
3159                                                // (only valid for active group)
3160             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3161             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3162                                                // byte
3163             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3164                                                // for each byte
3165             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3166                                                // bytes to add to Avg
3167             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3168                                                // Avg for each Active byte
3169
3170             // add 4th active group (Raw(x-bpp)/2) to average with _LBCarry
3171             "psllq _ShiftBpp, %%mm6      \n\t" // shift the mm6 mask to cover
3172                                                // bytes 6 & 7
3173             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3174             "psllq _ShiftBpp, %%mm2      \n\t" // shift data to pos. correctly
3175             "addl $8, %%ecx              \n\t"
3176             "movq %%mm3, %%mm1           \n\t" // now use mm1 for getting
3177                                                // LBCarrys
3178             "pand %%mm2, %%mm1           \n\t" // get LBCarrys for each byte
3179                                                // where both
3180                                                // lsb's were == 1 (only valid
3181                                                // for active group)
3182             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3183             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3184                                                // byte
3185             "paddb %%mm1, %%mm2          \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3186                                                // for each byte
3187             "pand %%mm6, %%mm2           \n\t" // leave only Active Group 2
3188                                                // bytes to add to Avg
3189             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) + LBCarrys to
3190                                                // Avg for each Active byte
3191
3192             "cmpl _MMXLength, %%ecx      \n\t"
3193             // now ready to write back to memory
3194             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3195             // prep Raw(x-bpp) for next loop
3196             "movq %%mm0, %%mm2           \n\t" // mov updated Raws to mm2
3197             "jb avg_2lp                  \n\t"
3198
3199             : "=S" (dummy_value_S),            // output regs (dummy)
3200               "=D" (dummy_value_D)
3201
3202             : "0" (prev_row),  // esi          // input regs
3203               "1" (row)        // edi
3204
3205             : "%ecx"                           // clobber list
3206 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3207             , "%mm0", "%mm1", "%mm2", "%mm3"
3208             , "%mm4", "%mm5", "%mm6", "%mm7"
3209 #endif
3210          );
3211       }
3212       break;  // end 2 bpp
3213
3214       case 1:
3215       {
3216          __asm__ __volatile__ (
3217             // re-init address pointers and offset
3218 #ifdef __PIC__
3219             "pushl %%ebx                 \n\t" // save Global Offset Table index
3220 #endif
3221             "movl _dif, %%ebx            \n\t" // ebx:  x = offset to alignment
3222                                                // boundary
3223 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
3224             "cmpl _FullLength, %%ebx     \n\t" // test if offset at end of array
3225             "jnb avg_1end                \n\t"
3226             // do Paeth decode for remaining bytes
3227 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3228             "movl %%edi, %%edx           \n\t"
3229 // preload  "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
3230             "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
3231             "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx
3232                                                //  in loop below
3233          "avg_1lp:                       \n\t"
3234             // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3235             "xorl %%eax, %%eax           \n\t"
3236             "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
3237             "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
3238             "addw %%cx, %%ax             \n\t"
3239             "incl %%ebx                  \n\t"
3240             "shrw %%ax                   \n\t" // divide by 2
3241             "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset
3242                                                // inc ebx
3243             "cmpl _FullLength, %%ebx     \n\t" // check if at end of array
3244             "movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x);
3245                          // mov does not affect flags; -1 to offset inc ebx
3246             "jb avg_1lp                  \n\t"
3247
3248          "avg_1end:                      \n\t"
3249 #ifdef __PIC__
3250             "popl %%ebx                  \n\t" // Global Offset Table index
3251 #endif
3252
3253             : "=c" (dummy_value_c),            // output regs (dummy)
3254               "=S" (dummy_value_S),
3255               "=D" (dummy_value_D)
3256
3257             : "0" (bpp),       // ecx          // input regs
3258               "1" (prev_row),  // esi
3259               "2" (row)        // edi
3260
3261             : "%eax", "%edx"                   // clobber list
3262 #ifndef __PIC__
3263             , "%ebx"
3264 #endif
3265          );
3266       }
3267       return;  // end 1 bpp
3268
3269       case 8:
3270       {
3271          __asm__ __volatile__ (
3272             // re-init address pointers and offset
3273             "movl _dif, %%ecx            \n\t" // ecx:  x == offset to alignment
3274             "movq _LBCarryMask, %%mm5    \n\t" //            boundary
3275 // preload  "movl row, %%edi             \n\t" // edi:  Avg(x)
3276             "movq _HBClearMask, %%mm4    \n\t"
3277 // preload  "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3278
3279             // prime the pump:  load the first Raw(x-bpp) data set
3280             "movq -8(%%edi,%%ecx,), %%mm2 \n\t" // load previous aligned 8 bytes
3281                                       // (NO NEED to correct pos. in loop below)
3282
3283          "avg_8lp:                       \n\t"
3284             "movq (%%edi,%%ecx,), %%mm0  \n\t"
3285             "movq %%mm5, %%mm3           \n\t"
3286             "movq (%%esi,%%ecx,), %%mm1  \n\t"
3287             "addl $8, %%ecx              \n\t"
3288             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3289             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3290             "pand %%mm2, %%mm3           \n\t" // get LBCarrys for each byte
3291                                                //  where both lsb's were == 1
3292             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3293             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7, each byte
3294             "paddb %%mm3, %%mm0          \n\t" // add LBCarrys to Avg, each byte
3295             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7, each byte
3296             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg, each
3297             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) to Avg for each
3298             "cmpl _MMXLength, %%ecx      \n\t"
3299             "movq %%mm0, -8(%%edi,%%ecx,) \n\t"
3300             "movq %%mm0, %%mm2           \n\t" // reuse as Raw(x-bpp)
3301             "jb avg_8lp                  \n\t"
3302
3303             : "=S" (dummy_value_S),            // output regs (dummy)
3304               "=D" (dummy_value_D)
3305
3306             : "0" (prev_row),  // esi          // input regs
3307               "1" (row)        // edi
3308
3309             : "%ecx"                           // clobber list
3310 #if 0  /* %mm0, ..., %mm5 not supported by gcc 2.7.2.3 or egcs 1.1 */
3311             , "%mm0", "%mm1", "%mm2"
3312             , "%mm3", "%mm4", "%mm5"
3313 #endif
3314          );
3315       }
3316       break;  // end 8 bpp
3317
3318       default:                  // bpp greater than 8 (!= 1,2,3,4,[5],6,[7],8)
3319       {
3320
3321 #ifdef PNG_DEBUG
3322          // GRR:  PRINT ERROR HERE:  SHOULD NEVER BE REACHED
3323         png_debug(1,
3324         "Internal logic error in pnggccrd (png_read_filter_row_mmx_avg())\n");
3325 #endif
3326
3327 #if 0
3328         __asm__ __volatile__ (
3329             "movq _LBCarryMask, %%mm5    \n\t"
3330             // re-init address pointers and offset
3331             "movl _dif, %%ebx            \n\t" // ebx:  x = offset to
3332                                                // alignment boundary
3333             "movl row, %%edi             \n\t" // edi:  Avg(x)
3334             "movq _HBClearMask, %%mm4    \n\t"
3335             "movl %%edi, %%edx           \n\t"
3336             "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3337             "subl bpp, %%edx             \n\t" // edx:  Raw(x-bpp)
3338          "avg_Alp:                       \n\t"
3339             "movq (%%edi,%%ebx,), %%mm0  \n\t"
3340             "movq %%mm5, %%mm3           \n\t"
3341             "movq (%%esi,%%ebx,), %%mm1  \n\t"
3342             "pand %%mm1, %%mm3           \n\t" // get lsb for each prev_row byte
3343             "movq (%%edx,%%ebx,), %%mm2  \n\t"
3344             "psrlq $1, %%mm1             \n\t" // divide prev_row bytes by 2
3345             "pand %%mm2, %%mm3           \n\t" // get LBCarrys for each byte
3346                                                // where both lsb's were == 1
3347             "psrlq $1, %%mm2             \n\t" // divide raw bytes by 2
3348             "pand  %%mm4, %%mm1          \n\t" // clear invalid bit 7 of each
3349                                                // byte
3350             "paddb %%mm3, %%mm0          \n\t" // add LBCarrys to Avg for each
3351                                                // byte
3352             "pand  %%mm4, %%mm2          \n\t" // clear invalid bit 7 of each
3353                                                // byte
3354             "paddb %%mm1, %%mm0          \n\t" // add (Prev_row/2) to Avg for
3355                                                // each byte
3356             "addl $8, %%ebx              \n\t"
3357             "paddb %%mm2, %%mm0          \n\t" // add (Raw/2) to Avg for each
3358                                                // byte
3359             "cmpl _MMXLength, %%ebx      \n\t"
3360             "movq %%mm0, -8(%%edi,%%ebx,) \n\t"
3361             "jb avg_Alp                  \n\t"
3362
3363             : // FIXASM: output regs/vars go here, e.g.:  "=m" (memory_var)
3364
3365             : // FIXASM: input regs, e.g.:  "c" (count), "S" (src), "D" (dest)
3366
3367             : "%ebx", "%edx", "%edi", "%esi" // CHECKASM: clobber list
3368          );
3369 #endif /* 0 - NEVER REACHED */
3370       }
3371       break;
3372
3373    } // end switch (bpp)
3374
3375    __asm__ __volatile__ (
3376       // MMX acceleration complete; now do clean-up
3377       // check if any remaining bytes left to decode
3378 #ifdef __PIC__
3379       "pushl %%ebx                 \n\t" // save index to Global Offset Table
3380 #endif
3381       "movl _MMXLength, %%ebx      \n\t" // ebx:  x == offset bytes after MMX
3382 //pre "movl row, %%edi             \n\t" // edi:  Avg(x)
3383       "cmpl _FullLength, %%ebx     \n\t" // test if offset at end of array
3384       "jnb avg_end                 \n\t"
3385
3386       // do Avg decode for remaining bytes
3387 //pre "movl prev_row, %%esi        \n\t" // esi:  Prior(x)
3388       "movl %%edi, %%edx           \n\t"
3389 //pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
3390       "subl %%ecx, %%edx           \n\t" // edx:  Raw(x-bpp)
3391       "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx below
3392
3393    "avg_lp2:                       \n\t"
3394       // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3395       "xorl %%eax, %%eax           \n\t"
3396       "movb (%%esi,%%ebx,), %%cl   \n\t" // load cl with Prior(x)
3397       "movb (%%edx,%%ebx,), %%al   \n\t" // load al with Raw(x-bpp)
3398       "addw %%cx, %%ax             \n\t"
3399       "incl %%ebx                  \n\t"
3400       "shrw %%ax                   \n\t" // divide by 2
3401       "addb -1(%%edi,%%ebx,), %%al \n\t" // add Avg(x); -1 to offset inc ebx
3402       "cmpl _FullLength, %%ebx     \n\t" // check if at end of array
3403       "movb %%al, -1(%%edi,%%ebx,) \n\t" // write back Raw(x) [mov does not
3404       "jb avg_lp2                  \n\t" //  affect flags; -1 to offset inc ebx]
3405
3406    "avg_end:                       \n\t"
3407       "EMMS                        \n\t" // end MMX; prep for poss. FP instrs.
3408 #ifdef __PIC__
3409       "popl %%ebx                  \n\t" // restore index to Global Offset Table
3410 #endif
3411
3412       : "=c" (dummy_value_c),            // output regs (dummy)
3413         "=S" (dummy_value_S),
3414         "=D" (dummy_value_D)
3415
3416       : "0" (bpp),       // ecx          // input regs
3417         "1" (prev_row),  // esi
3418         "2" (row)        // edi
3419
3420       : "%eax", "%edx"                   // clobber list
3421 #ifndef __PIC__
3422       , "%ebx"
3423 #endif
3424    );
3425
3426 } /* end png_read_filter_row_mmx_avg() */
3427 #endif
3428
3429
3430
3431 #ifdef PNG_THREAD_UNSAFE_OK
3432 //===========================================================================//
3433 //                                                                           //
3434 //         P N G _ R E A D _ F I L T E R _ R O W _ M M X _ P A E T H         //
3435 //                                                                           //
3436 //===========================================================================//
3437
3438 // Optimized code for PNG Paeth filter decoder
3439
3440 static void /* PRIVATE */
3441 png_read_filter_row_mmx_paeth(png_row_infop row_info, png_bytep row,
3442                               png_bytep prev_row)
3443 {
3444    int bpp;
3445    int dummy_value_c;   // fix 'forbidden register 2 (cx) was spilled' error
3446    int dummy_value_S;
3447    int dummy_value_D;
3448
3449    bpp = (row_info->pixel_depth + 7) >> 3; // Get # bytes per pixel
3450    _FullLength  = row_info->rowbytes; // # of bytes to filter
3451
3452    __asm__ __volatile__ (
3453 #ifdef __PIC__
3454       "pushl %%ebx                 \n\t" // save index to Global Offset Table
3455 #endif
3456       "xorl %%ebx, %%ebx           \n\t" // ebx:  x offset
3457 //pre "movl row, %%edi             \n\t"
3458       "xorl %%edx, %%edx           \n\t" // edx:  x-bpp offset
3459 //pre "movl prev_row, %%esi        \n\t"
3460       "xorl %%eax, %%eax           \n\t"
3461
3462       // Compute the Raw value for the first bpp bytes
3463       // Note: the formula works out to be always
3464       //   Paeth(x) = Raw(x) + Prior(x)      where x < bpp
3465    "paeth_rlp:                     \n\t"
3466       "movb (%%edi,%%ebx,), %%al   \n\t"
3467       "addb (%%esi,%%ebx,), %%al   \n\t"
3468       "incl %%ebx                  \n\t"
3469 //pre "cmpl bpp, %%ebx             \n\t" (bpp is preloaded into ecx)
3470       "cmpl %%ecx, %%ebx           \n\t"
3471       "movb %%al, -1(%%edi,%%ebx,) \n\t"
3472       "jb paeth_rlp                \n\t"
3473       // get # of bytes to alignment
3474       "movl %%edi, _dif            \n\t" // take start of row
3475       "addl %%ebx, _dif            \n\t" // add bpp
3476       "xorl %%ecx, %%ecx           \n\t"
3477       "addl $0xf, _dif             \n\t" // add 7 + 8 to incr past alignment
3478                                          // boundary
3479       "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
3480       "subl %%edi, _dif            \n\t" // subtract from start ==> value ebx
3481                                          // at alignment
3482       "jz paeth_go                 \n\t"
3483       // fix alignment
3484
3485    "paeth_lp1:                     \n\t"
3486       "xorl %%eax, %%eax           \n\t"
3487       // pav = p - a = (a + b - c) - a = b - c
3488       "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
3489       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
3490       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
3491       "movl %%eax, _patemp         \n\t" // Save pav for later use
3492       "xorl %%eax, %%eax           \n\t"
3493       // pbv = p - b = (a + b - c) - b = a - c
3494       "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
3495       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
3496       "movl %%eax, %%ecx           \n\t"
3497       // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3498       "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
3499       // pc = abs(pcv)
3500       "testl $0x80000000, %%eax    \n\t"
3501       "jz paeth_pca                \n\t"
3502       "negl %%eax                  \n\t" // reverse sign of neg values
3503
3504    "paeth_pca:                     \n\t"
3505       "movl %%eax, _pctemp         \n\t" // save pc for later use
3506       // pb = abs(pbv)
3507       "testl $0x80000000, %%ecx    \n\t"
3508       "jz paeth_pba                \n\t"
3509       "negl %%ecx                  \n\t" // reverse sign of neg values
3510
3511    "paeth_pba:                     \n\t"
3512       "movl %%ecx, _pbtemp         \n\t" // save pb for later use
3513       // pa = abs(pav)
3514       "movl _patemp, %%eax         \n\t"
3515       "testl $0x80000000, %%eax    \n\t"
3516       "jz paeth_paa                \n\t"
3517       "negl %%eax                  \n\t" // reverse sign of neg values
3518
3519    "paeth_paa:                     \n\t"
3520       "movl %%eax, _patemp         \n\t" // save pa for later use
3521       // test if pa <= pb
3522       "cmpl %%ecx, %%eax           \n\t"
3523       "jna paeth_abb               \n\t"
3524       // pa > pb; now test if pb <= pc
3525       "cmpl _pctemp, %%ecx         \n\t"
3526       "jna paeth_bbc               \n\t"
3527       // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
3528       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
3529       "jmp paeth_paeth             \n\t"
3530
3531    "paeth_bbc:                     \n\t"
3532       // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
3533       "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
3534       "jmp paeth_paeth             \n\t"
3535
3536    "paeth_abb:                     \n\t"
3537       // pa <= pb; now test if pa <= pc
3538       "cmpl _pctemp, %%eax         \n\t"
3539       "jna paeth_abc               \n\t"
3540       // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
3541       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
3542       "jmp paeth_paeth             \n\t"
3543
3544    "paeth_abc:                     \n\t"
3545       // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
3546       "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
3547
3548    "paeth_paeth:                   \n\t"
3549       "incl %%ebx                  \n\t"
3550       "incl %%edx                  \n\t"
3551       // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
3552       "addb %%cl, -1(%%edi,%%ebx,) \n\t"
3553       "cmpl _dif, %%ebx            \n\t"
3554       "jb paeth_lp1                \n\t"
3555
3556    "paeth_go:                      \n\t"
3557       "movl _FullLength, %%ecx     \n\t"
3558       "movl %%ecx, %%eax           \n\t"
3559       "subl %%ebx, %%eax           \n\t" // subtract alignment fix
3560       "andl $0x00000007, %%eax     \n\t" // calc bytes over mult of 8
3561       "subl %%eax, %%ecx           \n\t" // drop over bytes from original length
3562       "movl %%ecx, _MMXLength      \n\t"
3563 #ifdef __PIC__
3564       "popl %%ebx                  \n\t" // restore index to Global Offset Table
3565 #endif
3566
3567       : "=c" (dummy_value_c),            // output regs (dummy)
3568         "=S" (dummy_value_S),
3569         "=D" (dummy_value_D)
3570
3571       : "0" (bpp),       // ecx          // input regs
3572         "1" (prev_row),  // esi
3573         "2" (row)        // edi
3574
3575       : "%eax", "%edx"                   // clobber list
3576 #ifndef __PIC__
3577       , "%ebx"
3578 #endif
3579    );
3580
3581    // now do the math for the rest of the row
3582    switch (bpp)
3583    {
3584       case 3:
3585       {
3586          _ActiveMask.use = 0x0000000000ffffffLL;
3587          _ActiveMaskEnd.use = 0xffff000000000000LL;
3588          _ShiftBpp.use = 24;    // == bpp(3) * 8
3589          _ShiftRem.use = 40;    // == 64 - 24
3590
3591          __asm__ __volatile__ (
3592             "movl _dif, %%ecx            \n\t"
3593 // preload  "movl row, %%edi             \n\t"
3594 // preload  "movl prev_row, %%esi        \n\t"
3595             "pxor %%mm0, %%mm0           \n\t"
3596             // prime the pump:  load the first Raw(x-bpp) data set
3597             "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3598          "paeth_3lp:                     \n\t"
3599             "psrlq _ShiftRem, %%mm1      \n\t" // shift last 3 bytes to 1st
3600                                                // 3 bytes
3601             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3602             "punpcklbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3603             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // prep c=Prior(x-bpp) bytes
3604             "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3605             "psrlq _ShiftRem, %%mm3      \n\t" // shift last 3 bytes to 1st
3606                                                // 3 bytes
3607             // pav = p - a = (a + b - c) - a = b - c
3608             "movq %%mm2, %%mm4           \n\t"
3609             "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3610             // pbv = p - b = (a + b - c) - b = a - c
3611             "movq %%mm1, %%mm5           \n\t"
3612             "psubw %%mm3, %%mm4          \n\t"
3613             "pxor %%mm7, %%mm7           \n\t"
3614             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3615             "movq %%mm4, %%mm6           \n\t"
3616             "psubw %%mm3, %%mm5          \n\t"
3617
3618             // pa = abs(p-a) = abs(pav)
3619             // pb = abs(p-b) = abs(pbv)
3620             // pc = abs(p-c) = abs(pcv)
3621             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3622             "paddw %%mm5, %%mm6          \n\t"
3623             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3624             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3625             "psubw %%mm0, %%mm4          \n\t"
3626             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3627             "psubw %%mm0, %%mm4          \n\t"
3628             "psubw %%mm7, %%mm5          \n\t"
3629             "pxor %%mm0, %%mm0           \n\t"
3630             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3631             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3632             "psubw %%mm7, %%mm5          \n\t"
3633             "psubw %%mm0, %%mm6          \n\t"
3634             //  test pa <= pb
3635             "movq %%mm4, %%mm7           \n\t"
3636             "psubw %%mm0, %%mm6          \n\t"
3637             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3638             "movq %%mm7, %%mm0           \n\t"
3639             // use mm7 mask to merge pa & pb
3640             "pand %%mm7, %%mm5           \n\t"
3641             // use mm0 mask copy to merge a & b
3642             "pand %%mm0, %%mm2           \n\t"
3643             "pandn %%mm4, %%mm7          \n\t"
3644             "pandn %%mm1, %%mm0          \n\t"
3645             "paddw %%mm5, %%mm7          \n\t"
3646             "paddw %%mm2, %%mm0          \n\t"
3647             //  test  ((pa <= pb)? pa:pb) <= pc
3648             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3649             "pxor %%mm1, %%mm1           \n\t"
3650             "pand %%mm7, %%mm3           \n\t"
3651             "pandn %%mm0, %%mm7          \n\t"
3652             "paddw %%mm3, %%mm7          \n\t"
3653             "pxor %%mm0, %%mm0           \n\t"
3654             "packuswb %%mm1, %%mm7       \n\t"
3655             "movq (%%esi,%%ecx,), %%mm3  \n\t" // load c=Prior(x-bpp)
3656             "pand _ActiveMask, %%mm7     \n\t"
3657             "movq %%mm3, %%mm2           \n\t" // load b=Prior(x) step 1
3658             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
3659             "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3660             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
3661             "movq %%mm7, %%mm1           \n\t" // now mm1 will be used as
3662                                                // Raw(x-bpp)
3663             // now do Paeth for 2nd set of bytes (3-5)
3664             "psrlq _ShiftBpp, %%mm2      \n\t" // load b=Prior(x) step 2
3665             "punpcklbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3666             "pxor %%mm7, %%mm7           \n\t"
3667             "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3668             // pbv = p - b = (a + b - c) - b = a - c
3669             "movq %%mm1, %%mm5           \n\t"
3670             // pav = p - a = (a + b - c) - a = b - c
3671             "movq %%mm2, %%mm4           \n\t"
3672             "psubw %%mm3, %%mm5          \n\t"
3673             "psubw %%mm3, %%mm4          \n\t"
3674             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) =
3675             //       pav + pbv = pbv + pav
3676             "movq %%mm5, %%mm6           \n\t"
3677             "paddw %%mm4, %%mm6          \n\t"
3678
3679             // pa = abs(p-a) = abs(pav)
3680             // pb = abs(p-b) = abs(pbv)
3681             // pc = abs(p-c) = abs(pcv)
3682             "pcmpgtw %%mm5, %%mm0        \n\t" // create mask pbv bytes < 0
3683             "pcmpgtw %%mm4, %%mm7        \n\t" // create mask pav bytes < 0
3684             "pand %%mm5, %%mm0           \n\t" // only pbv bytes < 0 in mm0
3685             "pand %%mm4, %%mm7           \n\t" // only pav bytes < 0 in mm7
3686             "psubw %%mm0, %%mm5          \n\t"
3687             "psubw %%mm7, %%mm4          \n\t"
3688             "psubw %%mm0, %%mm5          \n\t"
3689             "psubw %%mm7, %%mm4          \n\t"
3690             "pxor %%mm0, %%mm0           \n\t"
3691             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3692             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3693             "psubw %%mm0, %%mm6          \n\t"
3694             //  test pa <= pb
3695             "movq %%mm4, %%mm7           \n\t"
3696             "psubw %%mm0, %%mm6          \n\t"
3697             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3698             "movq %%mm7, %%mm0           \n\t"
3699             // use mm7 mask to merge pa & pb
3700             "pand %%mm7, %%mm5           \n\t"
3701             // use mm0 mask copy to merge a & b
3702             "pand %%mm0, %%mm2           \n\t"
3703             "pandn %%mm4, %%mm7          \n\t"
3704             "pandn %%mm1, %%mm0          \n\t"
3705             "paddw %%mm5, %%mm7          \n\t"
3706             "paddw %%mm2, %%mm0          \n\t"
3707             //  test  ((pa <= pb)? pa:pb) <= pc
3708             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3709             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3710             "pand %%mm7, %%mm3           \n\t"
3711             "pandn %%mm0, %%mm7          \n\t"
3712             "pxor %%mm1, %%mm1           \n\t"
3713             "paddw %%mm3, %%mm7          \n\t"
3714             "pxor %%mm0, %%mm0           \n\t"
3715             "packuswb %%mm1, %%mm7       \n\t"
3716             "movq %%mm2, %%mm3           \n\t" // load c=Prior(x-bpp) step 1
3717             "pand _ActiveMask, %%mm7     \n\t"
3718             "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3719             "psllq _ShiftBpp, %%mm7      \n\t" // shift bytes to 2nd group of
3720                                                // 3 bytes
3721              // pav = p - a = (a + b - c) - a = b - c
3722             "movq %%mm2, %%mm4           \n\t"
3723             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
3724             "psllq _ShiftBpp, %%mm3      \n\t" // load c=Prior(x-bpp) step 2
3725             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
3726             "movq %%mm7, %%mm1           \n\t"
3727             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3728             "psllq _ShiftBpp, %%mm1      \n\t" // shift bytes
3729                                     // now mm1 will be used as Raw(x-bpp)
3730             // now do Paeth for 3rd, and final, set of bytes (6-7)
3731             "pxor %%mm7, %%mm7           \n\t"
3732             "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3733             "psubw %%mm3, %%mm4          \n\t"
3734             // pbv = p - b = (a + b - c) - b = a - c
3735             "movq %%mm1, %%mm5           \n\t"
3736             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3737             "movq %%mm4, %%mm6           \n\t"
3738             "psubw %%mm3, %%mm5          \n\t"
3739             "pxor %%mm0, %%mm0           \n\t"
3740             "paddw %%mm5, %%mm6          \n\t"
3741
3742             // pa = abs(p-a) = abs(pav)
3743             // pb = abs(p-b) = abs(pbv)
3744             // pc = abs(p-c) = abs(pcv)
3745             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3746             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3747             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3748             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3749             "psubw %%mm0, %%mm4          \n\t"
3750             "psubw %%mm7, %%mm5          \n\t"
3751             "psubw %%mm0, %%mm4          \n\t"
3752             "psubw %%mm7, %%mm5          \n\t"
3753             "pxor %%mm0, %%mm0           \n\t"
3754             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3755             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3756             "psubw %%mm0, %%mm6          \n\t"
3757             //  test pa <= pb
3758             "movq %%mm4, %%mm7           \n\t"
3759             "psubw %%mm0, %%mm6          \n\t"
3760             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3761             "movq %%mm7, %%mm0           \n\t"
3762             // use mm0 mask copy to merge a & b
3763             "pand %%mm0, %%mm2           \n\t"
3764             // use mm7 mask to merge pa & pb
3765             "pand %%mm7, %%mm5           \n\t"
3766             "pandn %%mm1, %%mm0          \n\t"
3767             "pandn %%mm4, %%mm7          \n\t"
3768             "paddw %%mm2, %%mm0          \n\t"
3769             "paddw %%mm5, %%mm7          \n\t"
3770             //  test  ((pa <= pb)? pa:pb) <= pc
3771             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3772             "pand %%mm7, %%mm3           \n\t"
3773             "pandn %%mm0, %%mm7          \n\t"
3774             "paddw %%mm3, %%mm7          \n\t"
3775             "pxor %%mm1, %%mm1           \n\t"
3776             "packuswb %%mm7, %%mm1       \n\t"
3777             // step ecx to next set of 8 bytes and repeat loop til done
3778             "addl $8, %%ecx              \n\t"
3779             "pand _ActiveMaskEnd, %%mm1  \n\t"
3780             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with
3781                                                  // Raw(x)
3782
3783             "cmpl _MMXLength, %%ecx      \n\t"
3784             "pxor %%mm0, %%mm0           \n\t" // pxor does not affect flags
3785             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
3786                                  // mm1 will be used as Raw(x-bpp) next loop
3787                            // mm3 ready to be used as Prior(x-bpp) next loop
3788             "jb paeth_3lp                \n\t"
3789
3790             : "=S" (dummy_value_S),             // output regs (dummy)
3791               "=D" (dummy_value_D)
3792
3793             : "0" (prev_row),  // esi           // input regs
3794               "1" (row)        // edi
3795
3796             : "%ecx"                            // clobber list
3797 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3798             , "%mm0", "%mm1", "%mm2", "%mm3"
3799             , "%mm4", "%mm5", "%mm6", "%mm7"
3800 #endif
3801          );
3802       }
3803       break;  // end 3 bpp
3804
3805       case 6:
3806       //case 7:   // GRR BOGUS
3807       //case 5:   // GRR BOGUS
3808       {
3809          _ActiveMask.use  = 0x00000000ffffffffLL;
3810          _ActiveMask2.use = 0xffffffff00000000LL;
3811          _ShiftBpp.use = bpp << 3;    // == bpp * 8
3812          _ShiftRem.use = 64 - _ShiftBpp.use;
3813
3814          __asm__ __volatile__ (
3815             "movl _dif, %%ecx            \n\t"
3816 // preload  "movl row, %%edi             \n\t"
3817 // preload  "movl prev_row, %%esi        \n\t"
3818             // prime the pump:  load the first Raw(x-bpp) data set
3819             "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3820             "pxor %%mm0, %%mm0           \n\t"
3821
3822          "paeth_6lp:                     \n\t"
3823             // must shift to position Raw(x-bpp) data
3824             "psrlq _ShiftRem, %%mm1      \n\t"
3825             // do first set of 4 bytes
3826             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
3827             "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
3828             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3829             "punpcklbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
3830             // must shift to position Prior(x-bpp) data
3831             "psrlq _ShiftRem, %%mm3      \n\t"
3832             // pav = p - a = (a + b - c) - a = b - c
3833             "movq %%mm2, %%mm4           \n\t"
3834             "punpcklbw %%mm0, %%mm3      \n\t" // unpack Low bytes of c
3835             // pbv = p - b = (a + b - c) - b = a - c
3836             "movq %%mm1, %%mm5           \n\t"
3837             "psubw %%mm3, %%mm4          \n\t"
3838             "pxor %%mm7, %%mm7           \n\t"
3839             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3840             "movq %%mm4, %%mm6           \n\t"
3841             "psubw %%mm3, %%mm5          \n\t"
3842             // pa = abs(p-a) = abs(pav)
3843             // pb = abs(p-b) = abs(pbv)
3844             // pc = abs(p-c) = abs(pcv)
3845             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3846             "paddw %%mm5, %%mm6          \n\t"
3847             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3848             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3849             "psubw %%mm0, %%mm4          \n\t"
3850             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3851             "psubw %%mm0, %%mm4          \n\t"
3852             "psubw %%mm7, %%mm5          \n\t"
3853             "pxor %%mm0, %%mm0           \n\t"
3854             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3855             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3856             "psubw %%mm7, %%mm5          \n\t"
3857             "psubw %%mm0, %%mm6          \n\t"
3858             //  test pa <= pb
3859             "movq %%mm4, %%mm7           \n\t"
3860             "psubw %%mm0, %%mm6          \n\t"
3861             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3862             "movq %%mm7, %%mm0           \n\t"
3863             // use mm7 mask to merge pa & pb
3864             "pand %%mm7, %%mm5           \n\t"
3865             // use mm0 mask copy to merge a & b
3866             "pand %%mm0, %%mm2           \n\t"
3867             "pandn %%mm4, %%mm7          \n\t"
3868             "pandn %%mm1, %%mm0          \n\t"
3869             "paddw %%mm5, %%mm7          \n\t"
3870             "paddw %%mm2, %%mm0          \n\t"
3871             //  test  ((pa <= pb)? pa:pb) <= pc
3872             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3873             "pxor %%mm1, %%mm1           \n\t"
3874             "pand %%mm7, %%mm3           \n\t"
3875             "pandn %%mm0, %%mm7          \n\t"
3876             "paddw %%mm3, %%mm7          \n\t"
3877             "pxor %%mm0, %%mm0           \n\t"
3878             "packuswb %%mm1, %%mm7       \n\t"
3879             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // load c=Prior(x-bpp)
3880             "pand _ActiveMask, %%mm7     \n\t"
3881             "psrlq _ShiftRem, %%mm3      \n\t"
3882             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x) step 1
3883             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor and Raw(x)
3884             "movq %%mm2, %%mm6           \n\t"
3885             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
3886             "movq -8(%%edi,%%ecx,), %%mm1 \n\t"
3887             "psllq _ShiftBpp, %%mm6      \n\t"
3888             "movq %%mm7, %%mm5           \n\t"
3889             "psrlq _ShiftRem, %%mm1      \n\t"
3890             "por %%mm6, %%mm3            \n\t"
3891             "psllq _ShiftBpp, %%mm5      \n\t"
3892             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3893             "por %%mm5, %%mm1            \n\t"
3894             // do second set of 4 bytes
3895             "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3896             "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
3897             // pav = p - a = (a + b - c) - a = b - c
3898             "movq %%mm2, %%mm4           \n\t"
3899             // pbv = p - b = (a + b - c) - b = a - c
3900             "movq %%mm1, %%mm5           \n\t"
3901             "psubw %%mm3, %%mm4          \n\t"
3902             "pxor %%mm7, %%mm7           \n\t"
3903             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3904             "movq %%mm4, %%mm6           \n\t"
3905             "psubw %%mm3, %%mm5          \n\t"
3906             // pa = abs(p-a) = abs(pav)
3907             // pb = abs(p-b) = abs(pbv)
3908             // pc = abs(p-c) = abs(pcv)
3909             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3910             "paddw %%mm5, %%mm6          \n\t"
3911             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
3912             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
3913             "psubw %%mm0, %%mm4          \n\t"
3914             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
3915             "psubw %%mm0, %%mm4          \n\t"
3916             "psubw %%mm7, %%mm5          \n\t"
3917             "pxor %%mm0, %%mm0           \n\t"
3918             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
3919             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
3920             "psubw %%mm7, %%mm5          \n\t"
3921             "psubw %%mm0, %%mm6          \n\t"
3922             //  test pa <= pb
3923             "movq %%mm4, %%mm7           \n\t"
3924             "psubw %%mm0, %%mm6          \n\t"
3925             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
3926             "movq %%mm7, %%mm0           \n\t"
3927             // use mm7 mask to merge pa & pb
3928             "pand %%mm7, %%mm5           \n\t"
3929             // use mm0 mask copy to merge a & b
3930             "pand %%mm0, %%mm2           \n\t"
3931             "pandn %%mm4, %%mm7          \n\t"
3932             "pandn %%mm1, %%mm0          \n\t"
3933             "paddw %%mm5, %%mm7          \n\t"
3934             "paddw %%mm2, %%mm0          \n\t"
3935             //  test  ((pa <= pb)? pa:pb) <= pc
3936             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
3937             "pxor %%mm1, %%mm1           \n\t"
3938             "pand %%mm7, %%mm3           \n\t"
3939             "pandn %%mm0, %%mm7          \n\t"
3940             "pxor %%mm1, %%mm1           \n\t"
3941             "paddw %%mm3, %%mm7          \n\t"
3942             "pxor %%mm0, %%mm0           \n\t"
3943             // step ecx to next set of 8 bytes and repeat loop til done
3944             "addl $8, %%ecx              \n\t"
3945             "packuswb %%mm7, %%mm1       \n\t"
3946             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
3947             "cmpl _MMXLength, %%ecx      \n\t"
3948             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
3949                                 // mm1 will be used as Raw(x-bpp) next loop
3950             "jb paeth_6lp                \n\t"
3951
3952             : "=S" (dummy_value_S),             // output regs (dummy)
3953               "=D" (dummy_value_D)
3954
3955             : "0" (prev_row),  // esi           // input regs
3956               "1" (row)        // edi
3957
3958             : "%ecx"                            // clobber list
3959 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
3960             , "%mm0", "%mm1", "%mm2", "%mm3"
3961             , "%mm4", "%mm5", "%mm6", "%mm7"
3962 #endif
3963          );
3964       }
3965       break;  // end 6 bpp
3966
3967       case 4:
3968       {
3969          _ActiveMask.use  = 0x00000000ffffffffLL;
3970
3971          __asm__ __volatile__ (
3972             "movl _dif, %%ecx            \n\t"
3973 // preload  "movl row, %%edi             \n\t"
3974 // preload  "movl prev_row, %%esi        \n\t"
3975             "pxor %%mm0, %%mm0           \n\t"
3976             // prime the pump:  load the first Raw(x-bpp) data set
3977             "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
3978                                      //  a=Raw(x-bpp) bytes
3979          "paeth_4lp:                     \n\t"
3980             // do first set of 4 bytes
3981             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
3982             "punpckhbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
3983             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
3984             "punpcklbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
3985             // pav = p - a = (a + b - c) - a = b - c
3986             "movq %%mm2, %%mm4           \n\t"
3987             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
3988             // pbv = p - b = (a + b - c) - b = a - c
3989             "movq %%mm1, %%mm5           \n\t"
3990             "psubw %%mm3, %%mm4          \n\t"
3991             "pxor %%mm7, %%mm7           \n\t"
3992             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
3993             "movq %%mm4, %%mm6           \n\t"
3994             "psubw %%mm3, %%mm5          \n\t"
3995             // pa = abs(p-a) = abs(pav)
3996             // pb = abs(p-b) = abs(pbv)
3997             // pc = abs(p-c) = abs(pcv)
3998             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
3999             "paddw %%mm5, %%mm6          \n\t"
4000             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4001             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4002             "psubw %%mm0, %%mm4          \n\t"
4003             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4004             "psubw %%mm0, %%mm4          \n\t"
4005             "psubw %%mm7, %%mm5          \n\t"
4006             "pxor %%mm0, %%mm0           \n\t"
4007             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4008             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4009             "psubw %%mm7, %%mm5          \n\t"
4010             "psubw %%mm0, %%mm6          \n\t"
4011             //  test pa <= pb
4012             "movq %%mm4, %%mm7           \n\t"
4013             "psubw %%mm0, %%mm6          \n\t"
4014             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4015             "movq %%mm7, %%mm0           \n\t"
4016             // use mm7 mask to merge pa & pb
4017             "pand %%mm7, %%mm5           \n\t"
4018             // use mm0 mask copy to merge a & b
4019             "pand %%mm0, %%mm2           \n\t"
4020             "pandn %%mm4, %%mm7          \n\t"
4021             "pandn %%mm1, %%mm0          \n\t"
4022             "paddw %%mm5, %%mm7          \n\t"
4023             "paddw %%mm2, %%mm0          \n\t"
4024             //  test  ((pa <= pb)? pa:pb) <= pc
4025             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4026             "pxor %%mm1, %%mm1           \n\t"
4027             "pand %%mm7, %%mm3           \n\t"
4028             "pandn %%mm0, %%mm7          \n\t"
4029             "paddw %%mm3, %%mm7          \n\t"
4030             "pxor %%mm0, %%mm0           \n\t"
4031             "packuswb %%mm1, %%mm7       \n\t"
4032             "movq (%%esi,%%ecx,), %%mm3  \n\t" // load c=Prior(x-bpp)
4033             "pand _ActiveMask, %%mm7     \n\t"
4034             "movq %%mm3, %%mm2           \n\t" // load b=Prior(x) step 1
4035             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
4036             "punpcklbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
4037             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
4038             "movq %%mm7, %%mm1           \n\t" // now mm1 will be used as Raw(x-bpp)
4039             // do second set of 4 bytes
4040             "punpckhbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
4041             "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
4042             // pav = p - a = (a + b - c) - a = b - c
4043             "movq %%mm2, %%mm4           \n\t"
4044             // pbv = p - b = (a + b - c) - b = a - c
4045             "movq %%mm1, %%mm5           \n\t"
4046             "psubw %%mm3, %%mm4          \n\t"
4047             "pxor %%mm7, %%mm7           \n\t"
4048             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4049             "movq %%mm4, %%mm6           \n\t"
4050             "psubw %%mm3, %%mm5          \n\t"
4051             // pa = abs(p-a) = abs(pav)
4052             // pb = abs(p-b) = abs(pbv)
4053             // pc = abs(p-c) = abs(pcv)
4054             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
4055             "paddw %%mm5, %%mm6          \n\t"
4056             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4057             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4058             "psubw %%mm0, %%mm4          \n\t"
4059             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4060             "psubw %%mm0, %%mm4          \n\t"
4061             "psubw %%mm7, %%mm5          \n\t"
4062             "pxor %%mm0, %%mm0           \n\t"
4063             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4064             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4065             "psubw %%mm7, %%mm5          \n\t"
4066             "psubw %%mm0, %%mm6          \n\t"
4067             //  test pa <= pb
4068             "movq %%mm4, %%mm7           \n\t"
4069             "psubw %%mm0, %%mm6          \n\t"
4070             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4071             "movq %%mm7, %%mm0           \n\t"
4072             // use mm7 mask to merge pa & pb
4073             "pand %%mm7, %%mm5           \n\t"
4074             // use mm0 mask copy to merge a & b
4075             "pand %%mm0, %%mm2           \n\t"
4076             "pandn %%mm4, %%mm7          \n\t"
4077             "pandn %%mm1, %%mm0          \n\t"
4078             "paddw %%mm5, %%mm7          \n\t"
4079             "paddw %%mm2, %%mm0          \n\t"
4080             //  test  ((pa <= pb)? pa:pb) <= pc
4081             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4082             "pxor %%mm1, %%mm1           \n\t"
4083             "pand %%mm7, %%mm3           \n\t"
4084             "pandn %%mm0, %%mm7          \n\t"
4085             "pxor %%mm1, %%mm1           \n\t"
4086             "paddw %%mm3, %%mm7          \n\t"
4087             "pxor %%mm0, %%mm0           \n\t"
4088             // step ecx to next set of 8 bytes and repeat loop til done
4089             "addl $8, %%ecx              \n\t"
4090             "packuswb %%mm7, %%mm1       \n\t"
4091             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add predictor with Raw(x)
4092             "cmpl _MMXLength, %%ecx      \n\t"
4093             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
4094                                 // mm1 will be used as Raw(x-bpp) next loop
4095             "jb paeth_4lp                \n\t"
4096
4097             : "=S" (dummy_value_S),             // output regs (dummy)
4098               "=D" (dummy_value_D)
4099
4100             : "0" (prev_row),  // esi           // input regs
4101               "1" (row)        // edi
4102
4103             : "%ecx"                            // clobber list
4104 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
4105             , "%mm0", "%mm1", "%mm2", "%mm3"
4106             , "%mm4", "%mm5", "%mm6", "%mm7"
4107 #endif
4108          );
4109       }
4110       break;  // end 4 bpp
4111
4112       case 8:                          // bpp == 8
4113       {
4114          _ActiveMask.use  = 0x00000000ffffffffLL;
4115
4116          __asm__ __volatile__ (
4117             "movl _dif, %%ecx            \n\t"
4118 // preload  "movl row, %%edi             \n\t"
4119 // preload  "movl prev_row, %%esi        \n\t"
4120             "pxor %%mm0, %%mm0           \n\t"
4121             // prime the pump:  load the first Raw(x-bpp) data set
4122             "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // only time should need to read
4123                                        //  a=Raw(x-bpp) bytes
4124          "paeth_8lp:                     \n\t"
4125             // do first set of 4 bytes
4126             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4127             "punpcklbw %%mm0, %%mm1      \n\t" // unpack Low bytes of a
4128             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
4129             "punpcklbw %%mm0, %%mm2      \n\t" // unpack Low bytes of b
4130             // pav = p - a = (a + b - c) - a = b - c
4131             "movq %%mm2, %%mm4           \n\t"
4132             "punpcklbw %%mm0, %%mm3      \n\t" // unpack Low bytes of c
4133             // pbv = p - b = (a + b - c) - b = a - c
4134             "movq %%mm1, %%mm5           \n\t"
4135             "psubw %%mm3, %%mm4          \n\t"
4136             "pxor %%mm7, %%mm7           \n\t"
4137             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4138             "movq %%mm4, %%mm6           \n\t"
4139             "psubw %%mm3, %%mm5          \n\t"
4140             // pa = abs(p-a) = abs(pav)
4141             // pb = abs(p-b) = abs(pbv)
4142             // pc = abs(p-c) = abs(pcv)
4143             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
4144             "paddw %%mm5, %%mm6          \n\t"
4145             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4146             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4147             "psubw %%mm0, %%mm4          \n\t"
4148             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4149             "psubw %%mm0, %%mm4          \n\t"
4150             "psubw %%mm7, %%mm5          \n\t"
4151             "pxor %%mm0, %%mm0           \n\t"
4152             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4153             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4154             "psubw %%mm7, %%mm5          \n\t"
4155             "psubw %%mm0, %%mm6          \n\t"
4156             //  test pa <= pb
4157             "movq %%mm4, %%mm7           \n\t"
4158             "psubw %%mm0, %%mm6          \n\t"
4159             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4160             "movq %%mm7, %%mm0           \n\t"
4161             // use mm7 mask to merge pa & pb
4162             "pand %%mm7, %%mm5           \n\t"
4163             // use mm0 mask copy to merge a & b
4164             "pand %%mm0, %%mm2           \n\t"
4165             "pandn %%mm4, %%mm7          \n\t"
4166             "pandn %%mm1, %%mm0          \n\t"
4167             "paddw %%mm5, %%mm7          \n\t"
4168             "paddw %%mm2, %%mm0          \n\t"
4169             //  test  ((pa <= pb)? pa:pb) <= pc
4170             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4171             "pxor %%mm1, %%mm1           \n\t"
4172             "pand %%mm7, %%mm3           \n\t"
4173             "pandn %%mm0, %%mm7          \n\t"
4174             "paddw %%mm3, %%mm7          \n\t"
4175             "pxor %%mm0, %%mm0           \n\t"
4176             "packuswb %%mm1, %%mm7       \n\t"
4177             "movq -8(%%esi,%%ecx,), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4178             "pand _ActiveMask, %%mm7     \n\t"
4179             "movq (%%esi,%%ecx,), %%mm2  \n\t" // load b=Prior(x)
4180             "paddb (%%edi,%%ecx,), %%mm7 \n\t" // add Paeth predictor with Raw(x)
4181             "punpckhbw %%mm0, %%mm3      \n\t" // unpack High bytes of c
4182             "movq %%mm7, (%%edi,%%ecx,)  \n\t" // write back updated value
4183             "movq -8(%%edi,%%ecx,), %%mm1 \n\t" // read a=Raw(x-bpp) bytes
4184
4185             // do second set of 4 bytes
4186             "punpckhbw %%mm0, %%mm2      \n\t" // unpack High bytes of b
4187             "punpckhbw %%mm0, %%mm1      \n\t" // unpack High bytes of a
4188             // pav = p - a = (a + b - c) - a = b - c
4189             "movq %%mm2, %%mm4           \n\t"
4190             // pbv = p - b = (a + b - c) - b = a - c
4191             "movq %%mm1, %%mm5           \n\t"
4192             "psubw %%mm3, %%mm4          \n\t"
4193             "pxor %%mm7, %%mm7           \n\t"
4194             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4195             "movq %%mm4, %%mm6           \n\t"
4196             "psubw %%mm3, %%mm5          \n\t"
4197             // pa = abs(p-a) = abs(pav)
4198             // pb = abs(p-b) = abs(pbv)
4199             // pc = abs(p-c) = abs(pcv)
4200             "pcmpgtw %%mm4, %%mm0        \n\t" // create mask pav bytes < 0
4201             "paddw %%mm5, %%mm6          \n\t"
4202             "pand %%mm4, %%mm0           \n\t" // only pav bytes < 0 in mm7
4203             "pcmpgtw %%mm5, %%mm7        \n\t" // create mask pbv bytes < 0
4204             "psubw %%mm0, %%mm4          \n\t"
4205             "pand %%mm5, %%mm7           \n\t" // only pbv bytes < 0 in mm0
4206             "psubw %%mm0, %%mm4          \n\t"
4207             "psubw %%mm7, %%mm5          \n\t"
4208             "pxor %%mm0, %%mm0           \n\t"
4209             "pcmpgtw %%mm6, %%mm0        \n\t" // create mask pcv bytes < 0
4210             "pand %%mm6, %%mm0           \n\t" // only pav bytes < 0 in mm7
4211             "psubw %%mm7, %%mm5          \n\t"
4212             "psubw %%mm0, %%mm6          \n\t"
4213             //  test pa <= pb
4214             "movq %%mm4, %%mm7           \n\t"
4215             "psubw %%mm0, %%mm6          \n\t"
4216             "pcmpgtw %%mm5, %%mm7        \n\t" // pa > pb?
4217             "movq %%mm7, %%mm0           \n\t"
4218             // use mm7 mask to merge pa & pb
4219             "pand %%mm7, %%mm5           \n\t"
4220             // use mm0 mask copy to merge a & b
4221             "pand %%mm0, %%mm2           \n\t"
4222             "pandn %%mm4, %%mm7          \n\t"
4223             "pandn %%mm1, %%mm0          \n\t"
4224             "paddw %%mm5, %%mm7          \n\t"
4225             "paddw %%mm2, %%mm0          \n\t"
4226             //  test  ((pa <= pb)? pa:pb) <= pc
4227             "pcmpgtw %%mm6, %%mm7        \n\t" // pab > pc?
4228             "pxor %%mm1, %%mm1           \n\t"
4229             "pand %%mm7, %%mm3           \n\t"
4230             "pandn %%mm0, %%mm7          \n\t"
4231             "pxor %%mm1, %%mm1           \n\t"
4232             "paddw %%mm3, %%mm7          \n\t"
4233             "pxor %%mm0, %%mm0           \n\t"
4234             // step ecx to next set of 8 bytes and repeat loop til done
4235             "addl $8, %%ecx              \n\t"
4236             "packuswb %%mm7, %%mm1       \n\t"
4237             "paddb -8(%%edi,%%ecx,), %%mm1 \n\t" // add Paeth predictor with Raw(x)
4238             "cmpl _MMXLength, %%ecx      \n\t"
4239             "movq %%mm1, -8(%%edi,%%ecx,) \n\t" // write back updated value
4240                             // mm1 will be used as Raw(x-bpp) next loop
4241             "jb paeth_8lp                \n\t"
4242
4243             : "=S" (dummy_value_S),             // output regs (dummy)
4244               "=D" (dummy_value_D)
4245
4246             : "0" (prev_row),  // esi           // input regs
4247               "1" (row)        // edi
4248
4249             : "%ecx"                            // clobber list
4250 #if 0  /* %mm0, ..., %mm7 not supported by gcc 2.7.2.3 or egcs 1.1 */
4251             , "%mm0", "%mm1", "%mm2", "%mm3"
4252             , "%mm4", "%mm5", "%mm6", "%mm7"
4253 #endif
4254          );
4255       }
4256       break;  // end 8 bpp
4257
4258       case 1:                // bpp = 1
4259       case 2:                // bpp = 2
4260       default:               // bpp > 8
4261       {
4262          __asm__ __volatile__ (
4263 #ifdef __PIC__
4264             "pushl %%ebx                 \n\t" // save Global Offset Table index
4265 #endif
4266             "movl _dif, %%ebx            \n\t"
4267             "cmpl _FullLength, %%ebx     \n\t"
4268             "jnb paeth_dend              \n\t"
4269
4270 // preload  "movl row, %%edi             \n\t"
4271 // preload  "movl prev_row, %%esi        \n\t"
4272             // do Paeth decode for remaining bytes
4273             "movl %%ebx, %%edx           \n\t"
4274 // preload  "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
4275             "subl %%ecx, %%edx           \n\t" // edx = ebx - bpp
4276             "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx
4277
4278          "paeth_dlp:                     \n\t"
4279             "xorl %%eax, %%eax           \n\t"
4280             // pav = p - a = (a + b - c) - a = b - c
4281             "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
4282             "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4283             "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4284             "movl %%eax, _patemp         \n\t" // Save pav for later use
4285             "xorl %%eax, %%eax           \n\t"
4286             // pbv = p - b = (a + b - c) - b = a - c
4287             "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
4288             "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4289             "movl %%eax, %%ecx           \n\t"
4290             // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4291             "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
4292             // pc = abs(pcv)
4293             "testl $0x80000000, %%eax    \n\t"
4294             "jz paeth_dpca               \n\t"
4295             "negl %%eax                  \n\t" // reverse sign of neg values
4296
4297          "paeth_dpca:                    \n\t"
4298             "movl %%eax, _pctemp         \n\t" // save pc for later use
4299             // pb = abs(pbv)
4300             "testl $0x80000000, %%ecx    \n\t"
4301             "jz paeth_dpba               \n\t"
4302             "negl %%ecx                  \n\t" // reverse sign of neg values
4303
4304          "paeth_dpba:                    \n\t"
4305             "movl %%ecx, _pbtemp         \n\t" // save pb for later use
4306             // pa = abs(pav)
4307             "movl _patemp, %%eax         \n\t"
4308             "testl $0x80000000, %%eax    \n\t"
4309             "jz paeth_dpaa               \n\t"
4310             "negl %%eax                  \n\t" // reverse sign of neg values
4311
4312          "paeth_dpaa:                    \n\t"
4313             "movl %%eax, _patemp         \n\t" // save pa for later use
4314             // test if pa <= pb
4315             "cmpl %%ecx, %%eax           \n\t"
4316             "jna paeth_dabb              \n\t"
4317             // pa > pb; now test if pb <= pc
4318             "cmpl _pctemp, %%ecx         \n\t"
4319             "jna paeth_dbbc              \n\t"
4320             // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4321             "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4322             "jmp paeth_dpaeth            \n\t"
4323
4324          "paeth_dbbc:                    \n\t"
4325             // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4326             "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
4327             "jmp paeth_dpaeth            \n\t"
4328
4329          "paeth_dabb:                    \n\t"
4330             // pa <= pb; now test if pa <= pc
4331             "cmpl _pctemp, %%eax         \n\t"
4332             "jna paeth_dabc              \n\t"
4333             // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4334             "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4335             "jmp paeth_dpaeth            \n\t"
4336
4337          "paeth_dabc:                    \n\t"
4338             // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4339             "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
4340
4341          "paeth_dpaeth:                  \n\t"
4342             "incl %%ebx                  \n\t"
4343             "incl %%edx                  \n\t"
4344             // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4345             "addb %%cl, -1(%%edi,%%ebx,) \n\t"
4346             "cmpl _FullLength, %%ebx     \n\t"
4347             "jb paeth_dlp                \n\t"
4348
4349          "paeth_dend:                    \n\t"
4350 #ifdef __PIC__
4351             "popl %%ebx                  \n\t" // index to Global Offset Table
4352 #endif
4353
4354             : "=c" (dummy_value_c),            // output regs (dummy)
4355               "=S" (dummy_value_S),
4356               "=D" (dummy_value_D)
4357
4358             : "0" (bpp),       // ecx          // input regs
4359               "1" (prev_row),  // esi
4360               "2" (row)        // edi
4361
4362             : "%eax", "%edx"                   // clobber list
4363 #ifndef __PIC__
4364             , "%ebx"
4365 #endif
4366          );
4367       }
4368       return;                   // No need to go further with this one
4369
4370    } // end switch (bpp)
4371
4372    __asm__ __volatile__ (
4373       // MMX acceleration complete; now do clean-up
4374       // check if any remaining bytes left to decode
4375 #ifdef __PIC__
4376       "pushl %%ebx                 \n\t" // save index to Global Offset Table
4377 #endif
4378       "movl _MMXLength, %%ebx      \n\t"
4379       "cmpl _FullLength, %%ebx     \n\t"
4380       "jnb paeth_end               \n\t"
4381 //pre "movl row, %%edi             \n\t"
4382 //pre "movl prev_row, %%esi        \n\t"
4383       // do Paeth decode for remaining bytes
4384       "movl %%ebx, %%edx           \n\t"
4385 //pre "subl bpp, %%edx             \n\t" // (bpp is preloaded into ecx)
4386       "subl %%ecx, %%edx           \n\t" // edx = ebx - bpp
4387       "xorl %%ecx, %%ecx           \n\t" // zero ecx before using cl & cx below
4388
4389    "paeth_lp2:                     \n\t"
4390       "xorl %%eax, %%eax           \n\t"
4391       // pav = p - a = (a + b - c) - a = b - c
4392       "movb (%%esi,%%ebx,), %%al   \n\t" // load Prior(x) into al
4393       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4394       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4395       "movl %%eax, _patemp         \n\t" // Save pav for later use
4396       "xorl %%eax, %%eax           \n\t"
4397       // pbv = p - b = (a + b - c) - b = a - c
4398       "movb (%%edi,%%edx,), %%al   \n\t" // load Raw(x-bpp) into al
4399       "subl %%ecx, %%eax           \n\t" // subtract Prior(x-bpp)
4400       "movl %%eax, %%ecx           \n\t"
4401       // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4402       "addl _patemp, %%eax         \n\t" // pcv = pav + pbv
4403       // pc = abs(pcv)
4404       "testl $0x80000000, %%eax    \n\t"
4405       "jz paeth_pca2               \n\t"
4406       "negl %%eax                  \n\t" // reverse sign of neg values
4407
4408    "paeth_pca2:                    \n\t"
4409       "movl %%eax, _pctemp         \n\t" // save pc for later use
4410       // pb = abs(pbv)
4411       "testl $0x80000000, %%ecx    \n\t"
4412       "jz paeth_pba2               \n\t"
4413       "negl %%ecx                  \n\t" // reverse sign of neg values
4414
4415    "paeth_pba2:                    \n\t"
4416       "movl %%ecx, _pbtemp         \n\t" // save pb for later use
4417       // pa = abs(pav)
4418       "movl _patemp, %%eax         \n\t"
4419       "testl $0x80000000, %%eax    \n\t"
4420       "jz paeth_paa2               \n\t"
4421       "negl %%eax                  \n\t" // reverse sign of neg values
4422
4423    "paeth_paa2:                    \n\t"
4424       "movl %%eax, _patemp         \n\t" // save pa for later use
4425       // test if pa <= pb
4426       "cmpl %%ecx, %%eax           \n\t"
4427       "jna paeth_abb2              \n\t"
4428       // pa > pb; now test if pb <= pc
4429       "cmpl _pctemp, %%ecx         \n\t"
4430       "jna paeth_bbc2              \n\t"
4431       // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4432       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4433       "jmp paeth_paeth2            \n\t"
4434
4435    "paeth_bbc2:                    \n\t"
4436       // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4437       "movb (%%esi,%%ebx,), %%cl   \n\t" // load Prior(x) into cl
4438       "jmp paeth_paeth2            \n\t"
4439
4440    "paeth_abb2:                    \n\t"
4441       // pa <= pb; now test if pa <= pc
4442       "cmpl _pctemp, %%eax         \n\t"
4443       "jna paeth_abc2              \n\t"
4444       // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4445       "movb (%%esi,%%edx,), %%cl   \n\t" // load Prior(x-bpp) into cl
4446       "jmp paeth_paeth2            \n\t"
4447
4448    "paeth_abc2:                    \n\t"
4449       // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4450       "movb (%%edi,%%edx,), %%cl   \n\t" // load Raw(x-bpp) into cl
4451
4452    "paeth_paeth2:                  \n\t"
4453       "incl %%ebx                  \n\t"
4454       "incl %%edx                  \n\t"
4455       // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4456       "addb %%cl, -1(%%edi,%%ebx,) \n\t"
4457       "cmpl _FullLength, %%ebx     \n\t"
4458       "jb paeth_lp2                \n\t"
4459
4460    "paeth_end:                     \n\t"
4461       "EMMS                        \n\t" // end MMX; prep for poss. FP instrs.
4462 #ifdef __PIC__
4463       "popl %%ebx                  \n\t" // restore index to Global Offset Table
4464 #endif
4465
4466       : "=c" (dummy_value_c),            // output regs (dummy)
4467         "=S" (dummy_value_S),
4468         "=D" (dummy_value_D)
4469
4470       : "0" (bpp),       // ecx          // input regs
4471         "1" (prev_row),  // esi
4472         "2" (row)        // edi
4473
4474       : "%eax", "%edx"                   // clobber list (no input regs!)
4475 #ifndef __PIC__
4476       , "%ebx"
4477 #endif
4478    );
4479
4480 } /* end png_read_filter_row_mmx_paeth() */
4481 #endif
4482
4483
4484
4485
4486 #ifdef PNG_THREAD_UNSAFE_OK
4487 //===========================================================================//
4488 //                                                                           //
4489 //           P N G _ R E A D _ F I L T E R _ R O W _ M M X _ S U B           //
4490 //                                                                           //
4491 //===========================================================================//
4492
4493 // Optimized code for PNG Sub filter decoder
4494
4495 static void /* PRIVATE */
4496 png_read_filter_row_mmx_sub(png_row_infop row_info, png_bytep row)
4497 {
4498    int bpp;
4499    int dummy_value_a;
4500    int dummy_value_D;
4501
4502    bpp = (row_info->pixel_depth + 7) >> 3;   // calc number of bytes per pixel
4503    _FullLength = row_info->rowbytes - bpp;   // number of bytes to filter
4504
4505    __asm__ __volatile__ (
4506 //pre "movl row, %%edi             \n\t"
4507       "movl %%edi, %%esi           \n\t" // lp = row
4508 //pre "movl bpp, %%eax             \n\t"
4509       "addl %%eax, %%edi           \n\t" // rp = row + bpp
4510 //irr "xorl %%eax, %%eax           \n\t"
4511       // get # of bytes to alignment
4512       "movl %%edi, _dif            \n\t" // take start of row
4513       "addl $0xf, _dif             \n\t" // add 7 + 8 to incr past
4514                                          //  alignment boundary
4515       "xorl %%ecx, %%ecx           \n\t"
4516       "andl $0xfffffff8, _dif      \n\t" // mask to alignment boundary
4517       "subl %%edi, _dif            \n\t" // subtract from start ==> value
4518       "jz sub_go                   \n\t" //  ecx at alignment
4519
4520    "sub_lp1:                       \n\t" // fix alignment
4521       "movb (%%esi,%%ecx,), %%al   \n\t"
4522       "addb %%al, (%%edi,%%ecx,)   \n\t"
4523       "incl %%ecx                  \n\t"
4524       "cmpl _dif, %%ecx            \n\t"
4525       "jb sub_lp1                  \n\t"
4526
4527    "sub_go:                        \n\t"
4528       "movl _FullLength, %%eax     \n\t"
4529       "movl %%eax, %%edx           \n\t"
4530       "subl %%ecx, %%edx           \n\t" // subtract alignment fix
4531       "andl $0x00000007, %%edx     \n\t" // calc bytes over mult of 8
4532       "subl %%edx, %%eax           \n\t" // drop over bytes from length
4533       "movl %%eax, _MMXLength      \n\t"
4534
4535       : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4536         "=D" (dummy_value_D)    // 1
4537
4538       : "0" (bpp),              // eax    // input regs
4539         "1" (row)               // edi
4540
4541       : "%esi", "%ecx", "%edx"            // clobber list
4542
4543 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4544       , "%mm0", "%mm1", "%mm2", "%mm3"
4545       , "%mm4", "%mm5", "%mm6", "%mm7"
4546 #endif
4547    );
4548
4549    // now do the math for the rest of the row
4550    switch (bpp)
4551    {
4552       case 3:
4553       {
4554          _ActiveMask.use  = 0x0000ffffff000000LL;
4555          _ShiftBpp.use = 24;       // == 3 * 8
4556          _ShiftRem.use  = 40;      // == 64 - 24
4557
4558          __asm__ __volatile__ (
4559 // preload  "movl row, %%edi              \n\t"
4560             "movq _ActiveMask, %%mm7       \n\t" // load _ActiveMask for 2nd
4561                                                 //  active byte group
4562             "movl %%edi, %%esi            \n\t" // lp = row
4563 // preload  "movl bpp, %%eax              \n\t"
4564             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4565             "movq %%mm7, %%mm6            \n\t"
4566             "movl _dif, %%edx             \n\t"
4567             "psllq _ShiftBpp, %%mm6       \n\t" // move mask in mm6 to cover
4568                                                 //  3rd active byte group
4569             // prime the pump:  load the first Raw(x-bpp) data set
4570             "movq -8(%%edi,%%edx,), %%mm1 \n\t"
4571
4572          "sub_3lp:                        \n\t" // shift data for adding first
4573             "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
4574                                                 //  shift clears inactive bytes)
4575             // add 1st active group
4576             "movq (%%edi,%%edx,), %%mm0   \n\t"
4577             "paddb %%mm1, %%mm0           \n\t"
4578
4579             // add 2nd active group
4580             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4581             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4582             "pand %%mm7, %%mm1            \n\t" // mask to use 2nd active group
4583             "paddb %%mm1, %%mm0           \n\t"
4584
4585             // add 3rd active group
4586             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4587             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4588             "pand %%mm6, %%mm1            \n\t" // mask to use 3rd active group
4589             "addl $8, %%edx               \n\t"
4590             "paddb %%mm1, %%mm0           \n\t"
4591
4592             "cmpl _MMXLength, %%edx       \n\t"
4593             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
4594             "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
4595             "jb sub_3lp                   \n\t"
4596
4597             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4598               "=D" (dummy_value_D)    // 1
4599
4600             : "0" (bpp),              // eax    // input regs
4601               "1" (row)               // edi
4602
4603             : "%edx", "%esi"                    // clobber list
4604 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4605             , "%mm0", "%mm1", "%mm6", "%mm7"
4606 #endif
4607          );
4608       }
4609       break;
4610
4611       case 1:
4612       {
4613          __asm__ __volatile__ (
4614             "movl _dif, %%edx            \n\t"
4615 // preload  "movl row, %%edi             \n\t"
4616             "cmpl _FullLength, %%edx     \n\t"
4617             "jnb sub_1end                \n\t"
4618             "movl %%edi, %%esi           \n\t" // lp = row
4619             "xorl %%eax, %%eax           \n\t"
4620 // preload  "movl bpp, %%eax             \n\t"
4621             "addl %%eax, %%edi           \n\t" // rp = row + bpp
4622
4623          "sub_1lp:                       \n\t"
4624             "movb (%%esi,%%edx,), %%al   \n\t"
4625             "addb %%al, (%%edi,%%edx,)   \n\t"
4626             "incl %%edx                  \n\t"
4627             "cmpl _FullLength, %%edx     \n\t"
4628             "jb sub_1lp                  \n\t"
4629
4630          "sub_1end:                      \n\t"
4631
4632             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4633               "=D" (dummy_value_D)    // 1
4634
4635             : "0" (bpp),              // eax    // input regs
4636               "1" (row)               // edi
4637
4638             : "%edx", "%esi"                    // clobber list
4639          );
4640       }
4641       return;
4642
4643       case 6:
4644       case 4:
4645       //case 7:   // GRR BOGUS
4646       //case 5:   // GRR BOGUS
4647       {
4648          _ShiftBpp.use = bpp << 3;
4649          _ShiftRem.use = 64 - _ShiftBpp.use;
4650
4651          __asm__ __volatile__ (
4652 // preload  "movl row, %%edi              \n\t"
4653             "movl _dif, %%edx             \n\t"
4654             "movl %%edi, %%esi            \n\t" // lp = row
4655 // preload  "movl bpp, %%eax              \n\t"
4656             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4657
4658             // prime the pump:  load the first Raw(x-bpp) data set
4659             "movq -8(%%edi,%%edx,), %%mm1 \n\t"
4660
4661          "sub_4lp:                        \n\t" // shift data for adding first
4662             "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
4663                                                 //  shift clears inactive bytes)
4664             "movq (%%edi,%%edx,), %%mm0   \n\t"
4665             "paddb %%mm1, %%mm0           \n\t"
4666
4667             // add 2nd active group
4668             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4669             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4670             "addl $8, %%edx               \n\t"
4671             "paddb %%mm1, %%mm0           \n\t"
4672
4673             "cmpl _MMXLength, %%edx       \n\t"
4674             "movq %%mm0, -8(%%edi,%%edx,) \n\t"
4675             "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
4676             "jb sub_4lp                   \n\t"
4677
4678             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4679               "=D" (dummy_value_D)    // 1
4680
4681             : "0" (bpp),              // eax    // input regs
4682               "1" (row)               // edi
4683
4684             : "%edx", "%esi"                    // clobber list
4685 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4686             , "%mm0", "%mm1"
4687 #endif
4688          );
4689       }
4690       break;
4691
4692       case 2:
4693       {
4694          _ActiveMask.use = 0x00000000ffff0000LL;
4695          _ShiftBpp.use = 16;       // == 2 * 8
4696          _ShiftRem.use = 48;       // == 64 - 16
4697
4698          __asm__ __volatile__ (
4699             "movq _ActiveMask, %%mm7      \n\t" // load _ActiveMask for 2nd
4700                                                 //  active byte group
4701             "movl _dif, %%edx             \n\t"
4702             "movq %%mm7, %%mm6            \n\t"
4703 // preload  "movl row, %%edi              \n\t"
4704             "psllq _ShiftBpp, %%mm6       \n\t" // move mask in mm6 to cover
4705                                                 //  3rd active byte group
4706             "movl %%edi, %%esi            \n\t" // lp = row
4707             "movq %%mm6, %%mm5            \n\t"
4708 // preload  "movl bpp, %%eax              \n\t"
4709             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4710             "psllq _ShiftBpp, %%mm5       \n\t" // move mask in mm5 to cover
4711                                                 //  4th active byte group
4712             // prime the pump:  load the first Raw(x-bpp) data set
4713             "movq -8(%%edi,%%edx,), %%mm1 \n\t"
4714
4715          "sub_2lp:                        \n\t" // shift data for adding first
4716             "psrlq _ShiftRem, %%mm1       \n\t" //  bpp bytes (no need for mask;
4717                                                 //  shift clears inactive bytes)
4718             // add 1st active group
4719             "movq (%%edi,%%edx,), %%mm0   \n\t"
4720             "paddb %%mm1, %%mm0           \n\t"
4721
4722             // add 2nd active group
4723             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4724             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4725             "pand %%mm7, %%mm1            \n\t" // mask to use 2nd active group
4726             "paddb %%mm1, %%mm0           \n\t"
4727
4728             // add 3rd active group
4729             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4730             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4731             "pand %%mm6, %%mm1            \n\t" // mask to use 3rd active group
4732             "paddb %%mm1, %%mm0           \n\t"
4733
4734             // add 4th active group
4735             "movq %%mm0, %%mm1            \n\t" // mov updated Raws to mm1
4736             "psllq _ShiftBpp, %%mm1       \n\t" // shift data to pos. correctly
4737             "pand %%mm5, %%mm1            \n\t" // mask to use 4th active group
4738             "addl $8, %%edx               \n\t"
4739             "paddb %%mm1, %%mm0           \n\t"
4740             "cmpl _MMXLength, %%edx       \n\t"
4741             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // write updated Raws to array
4742             "movq %%mm0, %%mm1            \n\t" // prep 1st add at top of loop
4743             "jb sub_2lp                   \n\t"
4744
4745             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4746               "=D" (dummy_value_D)    // 1
4747
4748             : "0" (bpp),              // eax    // input regs
4749               "1" (row)               // edi
4750
4751             : "%edx", "%esi"                    // clobber list
4752 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4753             , "%mm0", "%mm1", "%mm5", "%mm6", "%mm7"
4754 #endif
4755          );
4756       }
4757       break;
4758
4759       case 8:
4760       {
4761          __asm__ __volatile__ (
4762 // preload  "movl row, %%edi              \n\t"
4763             "movl _dif, %%edx             \n\t"
4764             "movl %%edi, %%esi            \n\t" // lp = row
4765 // preload  "movl bpp, %%eax              \n\t"
4766             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4767             "movl _MMXLength, %%ecx       \n\t"
4768
4769             // prime the pump:  load the first Raw(x-bpp) data set
4770             "movq -8(%%edi,%%edx,), %%mm7 \n\t"
4771             "andl $0x0000003f, %%ecx      \n\t" // calc bytes over mult of 64
4772
4773          "sub_8lp:                        \n\t"
4774             "movq (%%edi,%%edx,), %%mm0   \n\t" // load Sub(x) for 1st 8 bytes
4775             "paddb %%mm7, %%mm0           \n\t"
4776             "movq 8(%%edi,%%edx,), %%mm1  \n\t" // load Sub(x) for 2nd 8 bytes
4777             "movq %%mm0, (%%edi,%%edx,)   \n\t" // write Raw(x) for 1st 8 bytes
4778
4779             // Now mm0 will be used as Raw(x-bpp) for the 2nd group of 8 bytes.
4780             // This will be repeated for each group of 8 bytes with the 8th
4781             // group being used as the Raw(x-bpp) for the 1st group of the
4782             // next loop.
4783
4784             "paddb %%mm0, %%mm1           \n\t"
4785             "movq 16(%%edi,%%edx,), %%mm2 \n\t" // load Sub(x) for 3rd 8 bytes
4786             "movq %%mm1, 8(%%edi,%%edx,)  \n\t" // write Raw(x) for 2nd 8 bytes
4787             "paddb %%mm1, %%mm2           \n\t"
4788             "movq 24(%%edi,%%edx,), %%mm3 \n\t" // load Sub(x) for 4th 8 bytes
4789             "movq %%mm2, 16(%%edi,%%edx,) \n\t" // write Raw(x) for 3rd 8 bytes
4790             "paddb %%mm2, %%mm3           \n\t"
4791             "movq 32(%%edi,%%edx,), %%mm4 \n\t" // load Sub(x) for 5th 8 bytes
4792             "movq %%mm3, 24(%%edi,%%edx,) \n\t" // write Raw(x) for 4th 8 bytes
4793             "paddb %%mm3, %%mm4           \n\t"
4794             "movq 40(%%edi,%%edx,), %%mm5 \n\t" // load Sub(x) for 6th 8 bytes
4795             "movq %%mm4, 32(%%edi,%%edx,) \n\t" // write Raw(x) for 5th 8 bytes
4796             "paddb %%mm4, %%mm5           \n\t"
4797             "movq 48(%%edi,%%edx,), %%mm6 \n\t" // load Sub(x) for 7th 8 bytes
4798             "movq %%mm5, 40(%%edi,%%edx,) \n\t" // write Raw(x) for 6th 8 bytes
4799             "paddb %%mm5, %%mm6           \n\t"
4800             "movq 56(%%edi,%%edx,), %%mm7 \n\t" // load Sub(x) for 8th 8 bytes
4801             "movq %%mm6, 48(%%edi,%%edx,) \n\t" // write Raw(x) for 7th 8 bytes
4802             "addl $64, %%edx              \n\t"
4803             "paddb %%mm6, %%mm7           \n\t"
4804             "cmpl %%ecx, %%edx            \n\t"
4805             "movq %%mm7, -8(%%edi,%%edx,) \n\t" // write Raw(x) for 8th 8 bytes
4806             "jb sub_8lp                   \n\t"
4807
4808             "cmpl _MMXLength, %%edx       \n\t"
4809             "jnb sub_8lt8                 \n\t"
4810
4811          "sub_8lpA:                       \n\t"
4812             "movq (%%edi,%%edx,), %%mm0   \n\t"
4813             "addl $8, %%edx               \n\t"
4814             "paddb %%mm7, %%mm0           \n\t"
4815             "cmpl _MMXLength, %%edx       \n\t"
4816             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // -8 to offset early addl edx
4817             "movq %%mm0, %%mm7            \n\t" // move calculated Raw(x) data
4818                                                 //  to mm1 to be new Raw(x-bpp)
4819                                                 //  for next loop
4820             "jb sub_8lpA                  \n\t"
4821
4822          "sub_8lt8:                       \n\t"
4823
4824             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4825               "=D" (dummy_value_D)    // 1
4826
4827             : "0" (bpp),              // eax    // input regs
4828               "1" (row)               // edi
4829
4830             : "%ecx", "%edx", "%esi"            // clobber list
4831 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4832             , "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7"
4833 #endif
4834          );
4835       }
4836       break;
4837
4838       default:                // bpp greater than 8 bytes   GRR BOGUS
4839       {
4840          __asm__ __volatile__ (
4841             "movl _dif, %%edx             \n\t"
4842 // preload  "movl row, %%edi              \n\t"
4843             "movl %%edi, %%esi            \n\t" // lp = row
4844 // preload  "movl bpp, %%eax              \n\t"
4845             "addl %%eax, %%edi            \n\t" // rp = row + bpp
4846
4847          "sub_Alp:                        \n\t"
4848             "movq (%%edi,%%edx,), %%mm0   \n\t"
4849             "movq (%%esi,%%edx,), %%mm1   \n\t"
4850             "addl $8, %%edx               \n\t"
4851             "paddb %%mm1, %%mm0           \n\t"
4852             "cmpl _MMXLength, %%edx       \n\t"
4853             "movq %%mm0, -8(%%edi,%%edx,) \n\t" // mov does not affect flags;
4854                                                 //  -8 to offset addl edx
4855             "jb sub_Alp                   \n\t"
4856
4857             : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4858               "=D" (dummy_value_D)    // 1
4859
4860             : "0" (bpp),              // eax    // input regs
4861               "1" (row)               // edi
4862
4863             : "%edx", "%esi"                    // clobber list
4864 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
4865             , "%mm0", "%mm1"
4866 #endif
4867          );
4868       }
4869       break;
4870
4871    } // end switch (bpp)
4872
4873    __asm__ __volatile__ (
4874       "movl _MMXLength, %%edx       \n\t"
4875 //pre "movl row, %%edi              \n\t"
4876       "cmpl _FullLength, %%edx      \n\t"
4877       "jnb sub_end                  \n\t"
4878
4879       "movl %%edi, %%esi            \n\t" // lp = row
4880 //pre "movl bpp, %%eax              \n\t"
4881       "addl %%eax, %%edi            \n\t" // rp = row + bpp
4882       "xorl %%eax, %%eax            \n\t"
4883
4884    "sub_lp2:                        \n\t"
4885       "movb (%%esi,%%edx,), %%al    \n\t"
4886       "addb %%al, (%%edi,%%edx,)    \n\t"
4887       "incl %%edx                   \n\t"
4888       "cmpl _FullLength, %%edx      \n\t"
4889       "jb sub_lp2                   \n\t"
4890
4891    "sub_end:                        \n\t"
4892       "EMMS                         \n\t" // end MMX instructions
4893
4894       : "=a" (dummy_value_a),   // 0      // output regs (dummy)
4895         "=D" (dummy_value_D)    // 1
4896
4897       : "0" (bpp),              // eax    // input regs
4898         "1" (row)               // edi
4899
4900       : "%edx", "%esi"                    // clobber list
4901    );
4902
4903 } // end of png_read_filter_row_mmx_sub()
4904 #endif
4905
4906
4907
4908
4909 //===========================================================================//
4910 //                                                                           //
4911 //            P N G _ R E A D _ F I L T E R _ R O W _ M M X _ U P            //
4912 //                                                                           //
4913 //===========================================================================//
4914
4915 // Optimized code for PNG Up filter decoder
4916
4917 static void /* PRIVATE */
4918 png_read_filter_row_mmx_up(png_row_infop row_info, png_bytep row,
4919                            png_bytep prev_row)
4920 {
4921    png_uint_32 len;
4922    int dummy_value_d;   // fix 'forbidden register 3 (dx) was spilled' error
4923    int dummy_value_S;
4924    int dummy_value_D;
4925
4926    len = row_info->rowbytes;              // number of bytes to filter
4927
4928    __asm__ __volatile__ (
4929 //pre "movl row, %%edi              \n\t"
4930       // get # of bytes to alignment
4931 #ifdef __PIC__
4932       "pushl %%ebx                  \n\t"
4933 #endif
4934       "movl %%edi, %%ecx            \n\t"
4935       "xorl %%ebx, %%ebx            \n\t"
4936       "addl $0x7, %%ecx             \n\t"
4937       "xorl %%eax, %%eax            \n\t"
4938       "andl $0xfffffff8, %%ecx      \n\t"
4939 //pre "movl prev_row, %%esi         \n\t"
4940       "subl %%edi, %%ecx            \n\t"
4941       "jz up_go                     \n\t"
4942
4943    "up_lp1:                         \n\t" // fix alignment
4944       "movb (%%edi,%%ebx,), %%al    \n\t"
4945       "addb (%%esi,%%ebx,), %%al    \n\t"
4946       "incl %%ebx                   \n\t"
4947       "cmpl %%ecx, %%ebx            \n\t"
4948       "movb %%al, -1(%%edi,%%ebx,)  \n\t" // mov does not affect flags; -1 to
4949       "jb up_lp1                    \n\t" //  offset incl ebx
4950
4951    "up_go:                          \n\t"
4952 //pre "movl len, %%edx              \n\t"
4953       "movl %%edx, %%ecx            \n\t"
4954       "subl %%ebx, %%edx            \n\t" // subtract alignment fix
4955       "andl $0x0000003f, %%edx      \n\t" // calc bytes over mult of 64
4956       "subl %%edx, %%ecx            \n\t" // drop over bytes from length
4957
4958       // unrolled loop - use all MMX registers and interleave to reduce
4959       // number of branch instructions (loops) and reduce partial stalls
4960    "up_loop:                        \n\t"
4961       "movq (%%esi,%%ebx,), %%mm1   \n\t"
4962       "movq (%%edi,%%ebx,), %%mm0   \n\t"
4963       "movq 8(%%esi,%%ebx,), %%mm3  \n\t"
4964       "paddb %%mm1, %%mm0           \n\t"
4965       "movq 8(%%edi,%%ebx,), %%mm2  \n\t"
4966       "movq %%mm0, (%%edi,%%ebx,)   \n\t"
4967       "paddb %%mm3, %%mm2           \n\t"
4968       "movq 16(%%esi,%%ebx,), %%mm5 \n\t"
4969       "movq %%mm2, 8(%%edi,%%ebx,)  \n\t"
4970       "movq 16(%%edi,%%ebx,), %%mm4 \n\t"
4971       "movq 24(%%esi,%%ebx,), %%mm7 \n\t"
4972       "paddb %%mm5, %%mm4           \n\t"
4973       "movq 24(%%edi,%%ebx,), %%mm6 \n\t"
4974       "movq %%mm4, 16(%%edi,%%ebx,) \n\t"
4975       "paddb %%mm7, %%mm6           \n\t"
4976       "movq 32(%%esi,%%ebx,), %%mm1 \n\t"
4977       "movq %%mm6, 24(%%edi,%%ebx,) \n\t"
4978       "movq 32(%%edi,%%ebx,), %%mm0 \n\t"
4979       "movq 40(%%esi,%%ebx,), %%mm3 \n\t"
4980       "paddb %%mm1, %%mm0           \n\t"
4981       "movq 40(%%edi,%%ebx,), %%mm2 \n\t"
4982       "movq %%mm0, 32(%%edi,%%ebx,) \n\t"
4983       "paddb %%mm3, %%mm2           \n\t"
4984       "movq 48(%%esi,%%ebx,), %%mm5 \n\t"
4985       "movq %%mm2, 40(%%edi,%%ebx,) \n\t"
4986       "movq 48(%%edi,%%ebx,), %%mm4 \n\t"
4987       "movq 56(%%esi,%%ebx,), %%mm7 \n\t"
4988       "paddb %%mm5, %%mm4           \n\t"
4989       "movq 56(%%edi,%%ebx,), %%mm6 \n\t"
4990       "movq %%mm4, 48(%%edi,%%ebx,) \n\t"
4991       "addl $64, %%ebx              \n\t"
4992       "paddb %%mm7, %%mm6           \n\t"
4993       "cmpl %%ecx, %%ebx            \n\t"
4994       "movq %%mm6, -8(%%edi,%%ebx,) \n\t" // (+56)movq does not affect flags;
4995       "jb up_loop                   \n\t" //  -8 to offset addl ebx
4996
4997       "cmpl $0, %%edx               \n\t" // test for bytes over mult of 64
4998       "jz up_end                    \n\t"
4999
5000       "cmpl $8, %%edx               \n\t" // test for less than 8 bytes
5001       "jb up_lt8                    \n\t" //  [added by lcreeve at netins.net]
5002
5003       "addl %%edx, %%ecx            \n\t"
5004       "andl $0x00000007, %%edx      \n\t" // calc bytes over mult of 8
5005       "subl %%edx, %%ecx            \n\t" // drop over bytes from length
5006       "jz up_lt8                    \n\t"
5007
5008    "up_lpA:                         \n\t" // use MMX regs to update 8 bytes sim.
5009       "movq (%%esi,%%ebx,), %%mm1   \n\t"
5010       "movq (%%edi,%%ebx,), %%mm0   \n\t"
5011       "addl $8, %%ebx               \n\t"
5012       "paddb %%mm1, %%mm0           \n\t"
5013       "cmpl %%ecx, %%ebx            \n\t"
5014       "movq %%mm0, -8(%%edi,%%ebx,) \n\t" // movq does not affect flags; -8 to
5015       "jb up_lpA                    \n\t" //  offset add ebx
5016       "cmpl $0, %%edx               \n\t" // test for bytes over mult of 8
5017       "jz up_end                    \n\t"
5018
5019    "up_lt8:                         \n\t"
5020       "xorl %%eax, %%eax            \n\t"
5021       "addl %%edx, %%ecx            \n\t" // move over byte count into counter
5022
5023    "up_lp2:                         \n\t" // use x86 regs for remaining bytes
5024       "movb (%%edi,%%ebx,), %%al    \n\t"
5025       "addb (%%esi,%%ebx,), %%al    \n\t"
5026       "incl %%ebx                   \n\t"
5027       "cmpl %%ecx, %%ebx            \n\t"
5028       "movb %%al, -1(%%edi,%%ebx,)  \n\t" // mov does not affect flags; -1 to
5029       "jb up_lp2                    \n\t" //  offset inc ebx
5030
5031    "up_end:                         \n\t"
5032       "EMMS                         \n\t" // conversion of filtered row complete
5033 #ifdef __PIC__
5034       "popl %%ebx                   \n\t"
5035 #endif
5036
5037       : "=d" (dummy_value_d),   // 0      // output regs (dummy)
5038         "=S" (dummy_value_S),   // 1
5039         "=D" (dummy_value_D)    // 2
5040
5041       : "0" (len),              // edx    // input regs
5042         "1" (prev_row),         // esi
5043         "2" (row)               // edi
5044
5045       : "%eax", "%ecx"            // clobber list (no input regs!)
5046 #ifndef __PIC__
5047       , "%ebx"
5048 #endif
5049
5050 #if 0  /* MMX regs (%mm0, etc.) not supported by gcc 2.7.2.3 or egcs 1.1 */
5051       , "%mm0", "%mm1", "%mm2", "%mm3"
5052       , "%mm4", "%mm5", "%mm6", "%mm7"
5053 #endif
5054    );
5055
5056 } // end of png_read_filter_row_mmx_up()
5057
5058 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5059
5060
5061
5062
5063 /*===========================================================================*/
5064 /*                                                                           */
5065 /*                   P N G _ R E A D _ F I L T E R _ R O W                   */
5066 /*                                                                           */
5067 /*===========================================================================*/
5068
5069
5070 /* Optimized png_read_filter_row routines */
5071
5072 void /* PRIVATE */
5073 png_read_filter_row(png_structp png_ptr, png_row_infop row_info, png_bytep
5074    row, png_bytep prev_row, int filter)
5075 {
5076 #ifdef PNG_DEBUG
5077    char filnm[10];
5078 #endif
5079
5080 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
5081 /* GRR:  these are superseded by png_ptr->asm_flags: */
5082 #define UseMMX_sub    1   // GRR:  converted 20000730
5083 #define UseMMX_up     1   // GRR:  converted 20000729
5084 #define UseMMX_avg    1   // GRR:  converted 20000828 (+ 16-bit bugfix 20000916)
5085 #define UseMMX_paeth  1   // GRR:  converted 20000828
5086
5087    if (_mmx_supported == 2) {
5088        /* this should have happened in png_init_mmx_flags() already */
5089 #if !defined(PNG_1_0_X)
5090        png_warning(png_ptr, "asm_flags may not have been initialized");
5091 #endif
5092        png_mmx_support();
5093    }
5094 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5095
5096 #ifdef PNG_DEBUG
5097    png_debug(1, "in png_read_filter_row (pnggccrd.c)\n");
5098    switch (filter)
5099    {
5100       case 0: sprintf(filnm, "none");
5101          break;
5102       case 1: sprintf(filnm, "sub-%s",
5103 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5104 #if !defined(PNG_1_0_X)
5105         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB)? "MMX" :
5106 #endif
5107 #endif
5108 "x86");
5109          break;
5110       case 2: sprintf(filnm, "up-%s",
5111 #ifdef PNG_ASSEMBLER_CODE_SUPPORTED
5112 #if !defined(PNG_1_0_X)
5113         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP)? "MMX" :
5114 #endif
5115 #endif
5116  "x86");
5117          break;
5118       case 3: sprintf(filnm, "avg-%s",
5119 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5120 #if !defined(PNG_1_0_X)
5121         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG)? "MMX" :
5122 #endif
5123 #endif
5124  "x86");
5125          break;
5126       case 4: sprintf(filnm, "Paeth-%s",
5127 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5128 #if !defined(PNG_1_0_X)
5129         (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH)? "MMX":
5130 #endif
5131 #endif
5132 "x86");
5133          break;
5134       default: sprintf(filnm, "unknw");
5135          break;
5136    }
5137    png_debug2(0, "row_number=%5ld, %5s, ", png_ptr->row_number, filnm);
5138    png_debug1(0, "row=0x%08lx, ", (unsigned long)row);
5139    png_debug2(0, "pixdepth=%2d, bytes=%d, ", (int)row_info->pixel_depth,
5140       (int)((row_info->pixel_depth + 7) >> 3));
5141    png_debug1(0,"rowbytes=%8ld\n", row_info->rowbytes);
5142 #endif /* PNG_DEBUG */
5143
5144    switch (filter)
5145    {
5146       case PNG_FILTER_VALUE_NONE:
5147          break;
5148
5149       case PNG_FILTER_VALUE_SUB:
5150 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5151 #if !defined(PNG_1_0_X)
5152          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) &&
5153              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5154              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5155 #else
5156          if (_mmx_supported)
5157 #endif
5158          {
5159             png_read_filter_row_mmx_sub(row_info, row);
5160          }
5161          else
5162 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5163          {
5164             png_uint_32 i;
5165             png_uint_32 istop = row_info->rowbytes;
5166             png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5167             png_bytep rp = row + bpp;
5168             png_bytep lp = row;
5169
5170             for (i = bpp; i < istop; i++)
5171             {
5172                *rp = (png_byte)(((int)(*rp) + (int)(*lp++)) & 0xff);
5173                rp++;
5174             }
5175          }  /* end !UseMMX_sub */
5176          break;
5177
5178       case PNG_FILTER_VALUE_UP:
5179 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED)
5180 #if !defined(PNG_1_0_X)
5181          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) &&
5182              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5183              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5184 #else
5185          if (_mmx_supported)
5186 #endif
5187          {
5188             png_read_filter_row_mmx_up(row_info, row, prev_row);
5189          }
5190           else
5191 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5192          {
5193             png_uint_32 i;
5194             png_uint_32 istop = row_info->rowbytes;
5195             png_bytep rp = row;
5196             png_bytep pp = prev_row;
5197
5198             for (i = 0; i < istop; ++i)
5199             {
5200                *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5201                rp++;
5202             }
5203          }  /* end !UseMMX_up */
5204          break;
5205
5206       case PNG_FILTER_VALUE_AVG:
5207 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5208 #if !defined(PNG_1_0_X)
5209          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) &&
5210              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5211              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5212 #else
5213          if (_mmx_supported)
5214 #endif
5215          {
5216             png_read_filter_row_mmx_avg(row_info, row, prev_row);
5217          }
5218          else
5219 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5220          {
5221             png_uint_32 i;
5222             png_bytep rp = row;
5223             png_bytep pp = prev_row;
5224             png_bytep lp = row;
5225             png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5226             png_uint_32 istop = row_info->rowbytes - bpp;
5227
5228             for (i = 0; i < bpp; i++)
5229             {
5230                *rp = (png_byte)(((int)(*rp) +
5231                   ((int)(*pp++) >> 1)) & 0xff);
5232                rp++;
5233             }
5234
5235             for (i = 0; i < istop; i++)
5236             {
5237                *rp = (png_byte)(((int)(*rp) +
5238                   ((int)(*pp++ + *lp++) >> 1)) & 0xff);
5239                rp++;
5240             }
5241          }  /* end !UseMMX_avg */
5242          break;
5243
5244       case PNG_FILTER_VALUE_PAETH:
5245 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_THREAD_UNSAFE_OK)
5246 #if !defined(PNG_1_0_X)
5247          if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) &&
5248              (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5249              (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5250 #else
5251          if (_mmx_supported)
5252 #endif
5253          {
5254             png_read_filter_row_mmx_paeth(row_info, row, prev_row);
5255          }
5256          else
5257 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED */
5258          {
5259             png_uint_32 i;
5260             png_bytep rp = row;
5261             png_bytep pp = prev_row;
5262             png_bytep lp = row;
5263             png_bytep cp = prev_row;
5264             png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5265             png_uint_32 istop = row_info->rowbytes - bpp;
5266
5267             for (i = 0; i < bpp; i++)
5268             {
5269                *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5270                rp++;
5271             }
5272
5273             for (i = 0; i < istop; i++)   /* use leftover rp,pp */
5274             {
5275                int a, b, c, pa, pb, pc, p;
5276
5277                a = *lp++;
5278                b = *pp++;
5279                c = *cp++;
5280
5281                p = b - c;
5282                pc = a - c;
5283
5284 #ifdef PNG_USE_ABS
5285                pa = abs(p);
5286                pb = abs(pc);
5287                pc = abs(p + pc);
5288 #else
5289                pa = p < 0 ? -p : p;
5290                pb = pc < 0 ? -pc : pc;
5291                pc = (p + pc) < 0 ? -(p + pc) : p + pc;
5292 #endif
5293
5294                /*
5295                   if (pa <= pb && pa <= pc)
5296                      p = a;
5297                   else if (pb <= pc)
5298                      p = b;
5299                   else
5300                      p = c;
5301                 */
5302
5303                p = (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c;
5304
5305                *rp = (png_byte)(((int)(*rp) + p) & 0xff);
5306                rp++;
5307             }
5308          }  /* end !UseMMX_paeth */
5309          break;
5310
5311       default:
5312          png_warning(png_ptr, "Ignoring bad row-filter type");
5313          *row=0;
5314          break;
5315    }
5316 }
5317
5318 #endif /* PNG_HAVE_ASSEMBLER_READ_FILTER_ROW */
5319
5320
5321 /*===========================================================================*/
5322 /*                                                                           */
5323 /*                      P N G _ M M X _ S U P P O R T                        */
5324 /*                                                                           */
5325 /*===========================================================================*/
5326
5327 /* GRR NOTES:  (1) the following code assumes 386 or better (pushfl/popfl)
5328  *             (2) all instructions compile with gcc 2.7.2.3 and later
5329  *             (3) the function is moved down here to prevent gcc from
5330  *                  inlining it in multiple places and then barfing be-
5331  *                  cause the ".NOT_SUPPORTED" label is multiply defined
5332  *             [is there a way to signal that a *single* function should
5333  *              not be inlined?  is there a way to modify the label for
5334  *              each inlined instance, e.g., by appending _1, _2, etc.?
5335  *              maybe if don't use leading "." in label name? (nope...sigh)]
5336  */
5337
5338 int PNGAPI
5339 png_mmx_support(void)
5340 {
5341 #if defined(PNG_MMX_CODE_SUPPORTED)
5342     __asm__ __volatile__ (
5343         "pushl %%ebx          \n\t"  // ebx gets clobbered by CPUID instruction
5344         "pushl %%ecx          \n\t"  // so does ecx...
5345         "pushl %%edx          \n\t"  // ...and edx (but ecx & edx safe on Linux)
5346 //      ".byte  0x66          \n\t"  // convert 16-bit pushf to 32-bit pushfd
5347 //      "pushf                \n\t"  // 16-bit pushf
5348         "pushfl               \n\t"  // save Eflag to stack
5349         "popl %%eax           \n\t"  // get Eflag from stack into eax
5350         "movl %%eax, %%ecx    \n\t"  // make another copy of Eflag in ecx
5351         "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21)
5352         "pushl %%eax          \n\t"  // save modified Eflag back to stack
5353 //      ".byte  0x66          \n\t"  // convert 16-bit popf to 32-bit popfd
5354 //      "popf                 \n\t"  // 16-bit popf
5355         "popfl                \n\t"  // restore modified value to Eflag reg
5356         "pushfl               \n\t"  // save Eflag to stack
5357         "popl %%eax           \n\t"  // get Eflag from stack
5358         "pushl %%ecx          \n\t"  // save original Eflag to stack
5359         "popfl                \n\t"  // restore original Eflag
5360         "xorl %%ecx, %%eax    \n\t"  // compare new Eflag with original Eflag
5361         "jz 0f                \n\t"  // if same, CPUID instr. is not supported
5362
5363         "xorl %%eax, %%eax    \n\t"  // set eax to zero
5364 //      ".byte  0x0f, 0xa2    \n\t"  // CPUID instruction (two-byte opcode)
5365         "cpuid                \n\t"  // get the CPU identification info
5366         "cmpl $1, %%eax       \n\t"  // make sure eax return non-zero value
5367         "jl 0f                \n\t"  // if eax is zero, MMX is not supported
5368
5369         "xorl %%eax, %%eax    \n\t"  // set eax to zero and...
5370         "incl %%eax           \n\t"  // ...increment eax to 1.  This pair is
5371                                      // faster than the instruction "mov eax, 1"
5372         "cpuid                \n\t"  // get the CPU identification info again
5373         "andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23)
5374         "cmpl $0, %%edx       \n\t"  // 0 = MMX not supported
5375         "jz 0f                \n\t"  // non-zero = yes, MMX IS supported
5376
5377         "movl $1, %%eax       \n\t"  // set return value to 1
5378         "jmp  1f              \n\t"  // DONE:  have MMX support
5379
5380     "0:                       \n\t"  // .NOT_SUPPORTED: target label for jump instructions
5381         "movl $0, %%eax       \n\t"  // set return value to 0
5382     "1:                       \n\t"  // .RETURN: target label for jump instructions
5383         "movl %%eax, _mmx_supported \n\t" // save in global static variable, too
5384         "popl %%edx           \n\t"  // restore edx
5385         "popl %%ecx           \n\t"  // restore ecx
5386         "popl %%ebx           \n\t"  // restore ebx
5387
5388 //      "ret                  \n\t"  // DONE:  no MMX support
5389                                      // (fall through to standard C "ret")
5390
5391         :                            // output list (none)
5392
5393         :                            // any variables used on input (none)
5394
5395         : "%eax"                     // clobber list
5396 //      , "%ebx", "%ecx", "%edx"     // GRR:  we handle these manually
5397 //      , "memory"   // if write to a variable gcc thought was in a reg
5398 //      , "cc"       // "condition codes" (flag bits)
5399     );
5400 #else
5401     _mmx_supported = 0;
5402 #endif /* PNG_MMX_CODE_SUPPORTED */
5403
5404     return _mmx_supported;
5405 }
5406
5407
5408 #endif /* PNG_USE_PNGGCCRD */