sim/arm/iwmmxt.c

/*  iwmmxt.c -- Intel(r) Wireless MMX(tm) technology co-processor interface.
    Copyright (C) 2002-2024 Free Software Foundation, Inc.
    Contributed by matthew green (mrg@redhat.com).

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

/* This must come before any other includes.  */
#include "defs.h"

#include <stdlib.h>
#include <string.h>

#include "armdefs.h"
#include "armos.h"
#include "armemu.h"
#include "ansidecl.h"
#include "iwmmxt.h"

/* #define DEBUG 1 */

/* Intel(r) Wireless MMX(tm) technology co-processor.
   It uses co-processor numbers (0 and 1).  There are 16 vector registers wRx
   and 16 control registers wCx.  Co-processors 0 and 1 are used in MCR/MRC
   to access wRx and wCx respectively.  */

static ARMdword wR[16];
static ARMword  wC[16] = { 0x69051010 };

#define SUBSTR(w,t,m,n) ((t)(w <<  ((sizeof (t) * 8 - 1) - (n))) \
                               >> (((sizeof (t) * 8 - 1) - (n)) + (m)))
#define wCBITS(w,x,y)   SUBSTR (wC[w], ARMword, x, y)
#define wRBITS(w,x,y)   SUBSTR (wR[w], ARMdword, x, y)
#define wCID   0
#define wCon   1
#define wCSSF  2
#define wCASF  3
#define wCGR0  8
#define wCGR1  9
#define wCGR2 10
#define wCGR3 11

/* Bits in the wCon register.  */
#define WCON_CUP	(1 << 0)
#define WCON_MUP	(1 << 1)

/* Set the SIMD wCASF flags for 8, 16, 32 or 64-bit operations.  */
#define SIMD8_SET(x,  v, n, b)	(x) |= ((v != 0) << ((((b) + 1) * 4) + (n)))
#define SIMD16_SET(x, v, n, h)	(x) |= ((v != 0) << ((((h) + 1) * 8) + (n)))
#define SIMD32_SET(x, v, n, w)	(x) |= ((v != 0) << ((((w) + 1) * 16) + (n)))
#define SIMD64_SET(x, v, n)	(x) |= ((v != 0) << (32 + (n)))

/* Flags to pass as "n" above.  */
#define SIMD_NBIT	-1
#define SIMD_ZBIT	-2
#define SIMD_CBIT	-3
#define SIMD_VBIT	-4

/* Various status bit macros.  */
#define NBIT8(x)	((x) & 0x80)
#define NBIT16(x)	((x) & 0x8000)
#define NBIT32(x)	((x) & 0x80000000)
#define NBIT64(x)	((x) & 0x8000000000000000ULL)
#define ZBIT8(x)	(((x) & 0xff) == 0)
#define ZBIT16(x)	(((x) & 0xffff) == 0)
#define ZBIT32(x)	(((x) & 0xffffffff) == 0)
#define ZBIT64(x)	(x == 0)

/* Access byte/half/word "n" of register "x".  */
#define wRBYTE(x,n)	wRBITS ((x), (n) * 8, (n) * 8 + 7)
#define wRHALF(x,n)	wRBITS ((x), (n) * 16, (n) * 16 + 15)
#define wRWORD(x,n)	wRBITS ((x), (n) * 32, (n) * 32 + 31)

/* Macro to handle how the G bit selects wCGR registers.  */
#define DECODE_G_BIT(state, instr, shift)	\
{						\
  unsigned int reg;				\
						\
  reg = BITS (0, 3);				\
						\
  if (BIT (8))	/* G */				\
    {						\
      if (reg < wCGR0 || reg > wCGR3)		\
	{					\
	  ARMul_UndefInstr (state, instr);	\
	  return ARMul_DONE;			\
	}					\
      shift = wC [reg];				\
    }						\
  else						\
    shift = wR [reg];				\
						\
  shift &= 0xff;				\
}

/* Index calculations for the satrv[] array.  */
#define BITIDX8(x)	(x)
#define BITIDX16(x)	(((x) + 1) * 2 - 1)
#define BITIDX32(x)	(((x) + 1) * 4 - 1)

/* Sign extension macros.  */
#define EXTEND8(a)	((a) & 0x80 ? ((a) | 0xffffff00) : (a))
#define EXTEND16(a)	((a) & 0x8000 ? ((a) | 0xffff0000) : (a))
#define EXTEND32(a)	((a) & 0x80000000ULL ? ((a) | 0xffffffff00000000ULL) : (a))

/* Set the wCSSF from 8 values.  */
#define SET_wCSSF(a,b,c,d,e,f,g,h) \
  wC[wCSSF] = (((h) != 0) << 7) | (((g) != 0) << 6) \
            | (((f) != 0) << 5) | (((e) != 0) << 4) \
            | (((d) != 0) << 3) | (((c) != 0) << 2) \
            | (((b) != 0) << 1) | (((a) != 0) << 0);

/* Set the wCSSR from an array with 8 values.  */
#define SET_wCSSFvec(v) \
  SET_wCSSF((v)[0],(v)[1],(v)[2],(v)[3],(v)[4],(v)[5],(v)[6],(v)[7])

/* Size qualifiers for vector operations.  */
#define Bqual 			0
#define Hqual 			1
#define Wqual 			2
#define Dqual 			3

/* Saturation qualifiers for vector operations.  */
#define NoSaturation 		0
#define UnsignedSaturation	1
#define SignedSaturation	3


/* Prototypes.  */
static ARMword         Add32  (ARMword,  ARMword,  int *, int *, ARMword);
static ARMdword        AddS32 (ARMdword, ARMdword, int *, int *);
static ARMdword        AddU32 (ARMdword, ARMdword, int *, int *);
static ARMword         AddS16 (ARMword,  ARMword,  int *, int *);
static ARMword         AddU16 (ARMword,  ARMword,  int *, int *);
static ARMword         AddS8  (ARMword,  ARMword,  int *, int *);
static ARMword         AddU8  (ARMword,  ARMword,  int *, int *);
static ARMword         Sub32  (ARMword,  ARMword,  int *, int *, ARMword);
static ARMdword        SubS32 (ARMdword, ARMdword, int *, int *);
static ARMdword        SubU32 (ARMdword, ARMdword, int *, int *);
static ARMword         SubS16 (ARMword,  ARMword,  int *, int *);
static ARMword         SubS8  (ARMword,  ARMword,  int *, int *);
static ARMword         SubU16 (ARMword,  ARMword,  int *, int *);
static ARMword         SubU8  (ARMword,  ARMword,  int *, int *);
static unsigned char   IwmmxtSaturateU8  (signed short, int *);
static signed char     IwmmxtSaturateS8  (signed short, int *);
static unsigned short  IwmmxtSaturateU16 (signed int, int *);
static signed short    IwmmxtSaturateS16 (signed int, int *);
static unsigned long   IwmmxtSaturateU32 (signed long long, int *);
static signed long     IwmmxtSaturateS32 (signed long long, int *);
static ARMword         Compute_Iwmmxt_Address   (ARMul_State *, ARMword, int *);
static ARMdword        Iwmmxt_Load_Double_Word  (ARMul_State *, ARMword);
static ARMword         Iwmmxt_Load_Word         (ARMul_State *, ARMword);
static ARMword         Iwmmxt_Load_Half_Word    (ARMul_State *, ARMword);
static ARMword         Iwmmxt_Load_Byte         (ARMul_State *, ARMword);
static void            Iwmmxt_Store_Double_Word (ARMul_State *, ARMword, ARMdword);
static void            Iwmmxt_Store_Word        (ARMul_State *, ARMword, ARMword);
static void            Iwmmxt_Store_Half_Word   (ARMul_State *, ARMword, ARMword);
static void            Iwmmxt_Store_Byte        (ARMul_State *, ARMword, ARMword);
static int             Process_Instruction      (ARMul_State *, ARMword);

static int TANDC    (ARMul_State *, ARMword);
static int TBCST    (ARMul_State *, ARMword);
static int TEXTRC   (ARMul_State *, ARMword);
static int TEXTRM   (ARMul_State *, ARMword);
static int TINSR    (ARMul_State *, ARMword);
static int TMCR     (ARMul_State *, ARMword);
static int TMCRR    (ARMul_State *, ARMword);
static int TMIA     (ARMul_State *, ARMword);
static int TMIAPH   (ARMul_State *, ARMword);
static int TMIAxy   (ARMul_State *, ARMword);
static int TMOVMSK  (ARMul_State *, ARMword);
static int TMRC     (ARMul_State *, ARMword);
static int TMRRC    (ARMul_State *, ARMword);
static int TORC     (ARMul_State *, ARMword);
static int WACC     (ARMul_State *, ARMword);
static int WADD     (ARMul_State *, ARMword);
static int WALIGNI  (ARMword);
static int WALIGNR  (ARMul_State *, ARMword);
static int WAND     (ARMword);
static int WANDN    (ARMword);
static int WAVG2    (ARMword);
static int WCMPEQ   (ARMul_State *, ARMword);
static int WCMPGT   (ARMul_State *, ARMword);
static int WLDR     (ARMul_State *, ARMword);
static int WMAC     (ARMword);
static int WMADD    (ARMword);
static int WMAX     (ARMul_State *, ARMword);
static int WMIN     (ARMul_State *, ARMword);
static int WMUL     (ARMword);
static int WOR      (ARMword);
static int WPACK    (ARMul_State *, ARMword);
static int WROR     (ARMul_State *, ARMword);
static int WSAD     (ARMword);
static int WSHUFH   (ARMword);
static int WSLL     (ARMul_State *, ARMword);
static int WSRA     (ARMul_State *, ARMword);
static int WSRL     (ARMul_State *, ARMword);
static int WSTR     (ARMul_State *, ARMword);
static int WSUB     (ARMul_State *, ARMword);
static int WUNPCKEH (ARMul_State *, ARMword);
static int WUNPCKEL (ARMul_State *, ARMword);
static int WUNPCKIH (ARMul_State *, ARMword);
static int WUNPCKIL (ARMul_State *, ARMword);
static int WXOR     (ARMword);

/* This function does the work of adding two 32bit values
   together, and calculating if a carry has occurred.  */

static ARMword
Add32 (ARMword a1,
       ARMword a2,
       int * carry_ptr,
       int * overflow_ptr,
       ARMword sign_mask)
{
  ARMword result = (a1 + a2);
  unsigned int uresult = (unsigned int) result;
  unsigned int ua1 = (unsigned int) a1;

  /* If (result == a1) and (a2 == 0),
     or (result > a2) then we have no carry.  */
  * carry_ptr = ((uresult == ua1) ? (a2 != 0) : (uresult < ua1));

  /* Overflow occurs when both arguments are the
     same sign, but the result is a different sign.  */
  * overflow_ptr = (   ( (result & sign_mask) && !(a1 & sign_mask) && !(a2 & sign_mask))
		    || (!(result & sign_mask) &&  (a1 & sign_mask) &&  (a2 & sign_mask)));

  return result;
}

static ARMdword
AddS32 (ARMdword a1, ARMdword a2, int * carry_ptr, int * overflow_ptr)
{
  ARMdword     result;
  unsigned int uresult;
  unsigned int ua1;

  a1 = EXTEND32 (a1);
  a2 = EXTEND32 (a2);

  result  = a1 + a2;
  uresult = (unsigned int) result;
  ua1     = (unsigned int) a1;

  * carry_ptr = ((uresult == a1) ? (a2 != 0) : (uresult < ua1));

  * overflow_ptr = (   ( (result & 0x80000000ULL) && !(a1 & 0x80000000ULL) && !(a2 & 0x80000000ULL))
		    || (!(result & 0x80000000ULL) &&  (a1 & 0x80000000ULL) &&  (a2 & 0x80000000ULL)));

  return result;
}

static ARMdword
AddU32 (ARMdword a1, ARMdword a2, int * carry_ptr, int * overflow_ptr)
{
  ARMdword     result;
  unsigned int uresult;
  unsigned int ua1;

  a1 &= 0xffffffff;
  a2 &= 0xffffffff;

  result  = a1 + a2;
  uresult = (unsigned int) result;
  ua1     = (unsigned int) a1;

  * carry_ptr = ((uresult == a1) ? (a2 != 0) : (uresult < ua1));

  * overflow_ptr = (   ( (result & 0x80000000ULL) && !(a1 & 0x80000000ULL) && !(a2 & 0x80000000ULL))
		    || (!(result & 0x80000000ULL) &&  (a1 & 0x80000000ULL) &&  (a2 & 0x80000000ULL)));

  return result;
}

static ARMword
AddS16 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 = EXTEND16 (a1);
  a2 = EXTEND16 (a2);

  return Add32 (a1, a2, carry_ptr, overflow_ptr, 0x8000);
}

static ARMword
AddU16 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 &= 0xffff;
  a2 &= 0xffff;

  return Add32 (a1, a2, carry_ptr, overflow_ptr, 0x8000);
}

static ARMword
AddS8 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 = EXTEND8 (a1);
  a2 = EXTEND8 (a2);

  return Add32 (a1, a2, carry_ptr, overflow_ptr, 0x80);
}

static ARMword
AddU8 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 &= 0xff;
  a2 &= 0xff;

  return Add32 (a1, a2, carry_ptr, overflow_ptr, 0x80);
}

static ARMword
Sub32 (ARMword a1,
       ARMword a2,
       int * borrow_ptr,
       int * overflow_ptr,
       ARMword sign_mask)
{
  ARMword result = (a1 - a2);
  unsigned int ua1 = (unsigned int) a1;
  unsigned int ua2 = (unsigned int) a2;

  /* A borrow occurs if a2 is (unsigned) larger than a1.
     However the carry flag is *cleared* if a borrow occurs.  */
  * borrow_ptr = ! (ua2 > ua1);

  /* Overflow occurs when a negative number is subtracted from a
     positive number and the result is negative or a positive
     number is subtracted from a negative number and the result is
     positive.  */
  * overflow_ptr = ( (! (a1 & sign_mask) &&   (a2 & sign_mask) &&   (result & sign_mask))
		    || ((a1 & sign_mask) && ! (a2 & sign_mask) && ! (result & sign_mask)));

  return result;
}

static ARMdword
SubS32 (ARMdword a1, ARMdword a2, int * borrow_ptr, int * overflow_ptr)
{
  ARMdword     result;
  unsigned int ua1;
  unsigned int ua2;

  a1 = EXTEND32 (a1);
  a2 = EXTEND32 (a2);

  result = a1 - a2;
  ua1    = (unsigned int) a1;
  ua2    = (unsigned int) a2;

  * borrow_ptr = ! (ua2 > ua1);

  * overflow_ptr = ( (! (a1 & 0x80000000ULL) &&   (a2 & 0x80000000ULL) &&   (result & 0x80000000ULL))
		    || ((a1 & 0x80000000ULL) && ! (a2 & 0x80000000ULL) && ! (result & 0x80000000ULL)));

  return result;
}

static ARMword
SubS16 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 = EXTEND16 (a1);
  a2 = EXTEND16 (a2);

  return Sub32 (a1, a2, carry_ptr, overflow_ptr, 0x8000);
}

static ARMword
SubS8 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 = EXTEND8 (a1);
  a2 = EXTEND8 (a2);

  return Sub32 (a1, a2, carry_ptr, overflow_ptr, 0x80);
}

static ARMword
SubU16 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 &= 0xffff;
  a2 &= 0xffff;

  return Sub32 (a1, a2, carry_ptr, overflow_ptr, 0x8000);
}

static ARMword
SubU8 (ARMword a1, ARMword a2, int * carry_ptr, int * overflow_ptr)
{
  a1 &= 0xff;
  a2 &= 0xff;

  return Sub32 (a1, a2, carry_ptr, overflow_ptr, 0x80);
}

static ARMdword
SubU32 (ARMdword a1, ARMdword a2, int * borrow_ptr, int * overflow_ptr)
{
  ARMdword     result;
  unsigned int ua1;
  unsigned int ua2;

  a1 &= 0xffffffff;
  a2 &= 0xffffffff;

  result = a1 - a2;
  ua1    = (unsigned int) a1;
  ua2    = (unsigned int) a2;

  * borrow_ptr = ! (ua2 > ua1);

  * overflow_ptr = ( (! (a1 & 0x80000000ULL) &&   (a2 & 0x80000000ULL) &&   (result & 0x80000000ULL))
		    || ((a1 & 0x80000000ULL) && ! (a2 & 0x80000000ULL) && ! (result & 0x80000000ULL)));

  return result;
}

/* For the saturation.  */

static unsigned char
IwmmxtSaturateU8 (signed short val, int * sat)
{
  unsigned char rv;

  if (val < 0)
    {
      rv = 0;
      *sat = 1;
    }
  else if (val > 0xff)
    {
      rv = 0xff;
      *sat = 1;
    }
  else
    {
      rv = val & 0xff;
      *sat = 0;
    }
  return rv;
}

static signed char
IwmmxtSaturateS8 (signed short val, int * sat)
{
  signed char rv;

  if (val < -0x80)
    {
      rv = -0x80;
      *sat = 1;
    }
  else if (val > 0x7f)
    {
      rv = 0x7f;
      *sat = 1;
    }
  else
    {
      rv = val & 0xff;
      *sat = 0;
    }
  return rv;
}

static unsigned short
IwmmxtSaturateU16 (signed int val, int * sat)
{
  unsigned short rv;

  if (val < 0)
    {
      rv = 0;
      *sat = 1;
    }
  else if (val > 0xffff)
    {
      rv = 0xffff;
      *sat = 1;
    }
  else
    {
      rv = val & 0xffff;
      *sat = 0;
    }
  return rv;
}

static signed short
IwmmxtSaturateS16 (signed int val, int * sat)
{
  signed short rv;

  if (val < -0x8000)
    {
      rv = - 0x8000;
      *sat = 1;
    }
  else if (val > 0x7fff)
    {
      rv = 0x7fff;
      *sat = 1;
    }
  else
    {
      rv = val & 0xffff;
      *sat = 0;
    }
  return rv;
}

static unsigned long
IwmmxtSaturateU32 (signed long long val, int * sat)
{
  unsigned long rv;

  if (val < 0)
    {
      rv = 0;
      *sat = 1;
    }
  else if (val > 0xffffffff)
    {
      rv = 0xffffffff;
      *sat = 1;
    }
  else
    {
      rv = val & 0xffffffff;
      *sat = 0;
    }
  return rv;
}

static signed long
IwmmxtSaturateS32 (signed long long val, int * sat)
{
  signed long rv;

  if (val < -0x80000000LL)
    {
      rv = -0x80000000;
      *sat = 1;
    }
  else if (val > 0x7fffffff)
    {
      rv = 0x7fffffff;
      *sat = 1;
    }
  else
    {
      rv = val & 0xffffffff;
      *sat = 0;
    }
  return rv;
}

/* Intel(r) Wireless MMX(tm) technology Acessor functions.  */

unsigned
IwmmxtLDC (ARMul_State * state ATTRIBUTE_UNUSED,
	   unsigned      type  ATTRIBUTE_UNUSED,
	   ARMword       instr,
	   ARMword       data)
{
  return ARMul_CANT;
}

unsigned
IwmmxtSTC (ARMul_State * state ATTRIBUTE_UNUSED,
	   unsigned      type  ATTRIBUTE_UNUSED,
	   ARMword       instr,
	   ARMword *     data)
{
  return ARMul_CANT;
}

unsigned
IwmmxtMRC (ARMul_State * state ATTRIBUTE_UNUSED,
	   unsigned      type  ATTRIBUTE_UNUSED,
	   ARMword       instr,
	   ARMword *     value)
{
  return ARMul_CANT;
}

unsigned
IwmmxtMCR (ARMul_State * state ATTRIBUTE_UNUSED,
	   unsigned      type  ATTRIBUTE_UNUSED,
	   ARMword       instr,
	   ARMword       value)
{
  return ARMul_CANT;
}

unsigned
IwmmxtCDP (ARMul_State * state, unsigned type, ARMword instr)
{
  return ARMul_CANT;
}

/* Intel(r) Wireless MMX(tm) technology instruction implementations.  */

static int
TANDC (ARMul_State * state, ARMword instr)
{
  ARMword cpsr;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tandc\n");
#endif

  /* The Rd field must be r15.  */
  if (BITS (12, 15) != 15)
    return ARMul_CANT;

  /* The CRn field must be r3.  */
  if (BITS (16, 19) != 3)
    return ARMul_CANT;

  /* The CRm field must be r0.  */
  if (BITS (0, 3) != 0)
    return ARMul_CANT;

  cpsr = ARMul_GetCPSR (state) & 0x0fffffff;

  switch (BITS (22, 23))
    {
    case Bqual:
      cpsr |= (  (wCBITS (wCASF, 28, 31) & wCBITS (wCASF, 24, 27)
		& wCBITS (wCASF, 20, 23) & wCBITS (wCASF, 16, 19)
		& wCBITS (wCASF, 12, 15) & wCBITS (wCASF,  8, 11)
		& wCBITS (wCASF,  4,  7) & wCBITS (wCASF,  0,  3)) << 28);
      break;

    case Hqual:
      cpsr |= (  (wCBITS (wCASF, 28, 31) & wCBITS (wCASF, 20, 23)
		& wCBITS (wCASF, 12, 15) & wCBITS (wCASF,  4, 7)) << 28);
      break;

    case Wqual:
      cpsr |= ((wCBITS (wCASF, 28, 31) & wCBITS (wCASF, 12, 15)) << 28);
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  ARMul_SetCPSR (state, cpsr);

  return ARMul_DONE;
}

static int
TBCST (ARMul_State * state, ARMword instr)
{
  ARMdword Rn;
  int wRd;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tbcst\n");
#endif

  Rn  = state->Reg [BITS (12, 15)];
  if (BITS (12, 15) == 15)
    Rn &= 0xfffffffc;

  wRd = BITS (16, 19);

  switch (BITS (6, 7))
    {
    case Bqual:
      Rn &= 0xff;
      wR [wRd] = (Rn << 56) | (Rn << 48) | (Rn << 40) | (Rn << 32)
	       | (Rn << 24) | (Rn << 16) | (Rn << 8) | Rn;
      break;

    case Hqual:
      Rn &= 0xffff;
      wR [wRd] = (Rn << 48) | (Rn << 32) | (Rn << 16) | Rn;
      break;

    case Wqual:
      Rn &= 0xffffffff;
      wR [wRd] = (Rn << 32) | Rn;
      break;

    default:
      ARMul_UndefInstr (state, instr);
      break;
    }

  wC [wCon] |= WCON_MUP;
  return ARMul_DONE;
}

static int
TEXTRC (ARMul_State * state, ARMword instr)
{
  ARMword cpsr;
  ARMword selector;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "textrc\n");
#endif

  /* The Rd field must be r15.  */
  if (BITS (12, 15) != 15)
    return ARMul_CANT;

  /* The CRn field must be r3.  */
  if (BITS (16, 19) != 3)
    return ARMul_CANT;

  /* The CRm field must be 0xxx.  */
  if (BIT (3) != 0)
    return ARMul_CANT;

  selector = BITS (0, 2);
  cpsr = ARMul_GetCPSR (state) & 0x0fffffff;

  switch (BITS (22, 23))
    {
    case Bqual: selector *= 4; break;
    case Hqual: selector = ((selector & 3) * 8) + 4; break;
    case Wqual: selector = ((selector & 1) * 16) + 12; break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  cpsr |= wCBITS (wCASF, selector, selector + 3) << 28;
  ARMul_SetCPSR (state, cpsr);

  return ARMul_DONE;
}

static int
TEXTRM (ARMul_State * state, ARMword instr)
{
  ARMword Rd;
  int     offset;
  int     wRn;
  int     sign;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "textrm\n");
#endif

  wRn    = BITS (16, 19);
  sign   = BIT (3);
  offset = BITS (0, 2);

  switch (BITS (22, 23))
    {
    case Bqual:
      offset *= 8;
      Rd = wRBITS (wRn, offset, offset + 7);
      if (sign)
	Rd = EXTEND8 (Rd);
      break;

    case Hqual:
      offset = (offset & 3) * 16;
      Rd = wRBITS (wRn, offset, offset + 15);
      if (sign)
	Rd = EXTEND16 (Rd);
      break;

    case Wqual:
      offset = (offset & 1) * 32;
      Rd = wRBITS (wRn, offset, offset + 31);
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  if (BITS (12, 15) == 15)
    ARMul_UndefInstr (state, instr);
  else
    state->Reg [BITS (12, 15)] = Rd;

  return ARMul_DONE;
}

static int
TINSR (ARMul_State * state, ARMword instr)
{
  ARMdword data;
  ARMword  offset;
  int      wRd;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tinsr\n");
#endif

  wRd = BITS (16, 19);
  data = state->Reg [BITS (12, 15)];
  offset = BITS (0, 2);

  switch (BITS (6, 7))
    {
    case Bqual:
      data &= 0xff;
      switch (offset)
	{
	case 0: wR [wRd] = data | (wRBITS (wRd, 8, 63) << 8); break;
	case 1: wR [wRd] = wRBITS (wRd, 0,  7) | (data <<  8) | (wRBITS (wRd, 16, 63) << 16); break;
	case 2: wR [wRd] = wRBITS (wRd, 0, 15) | (data << 16) | (wRBITS (wRd, 24, 63) << 24); break;
	case 3: wR [wRd] = wRBITS (wRd, 0, 23) | (data << 24) | (wRBITS (wRd, 32, 63) << 32); break;
	case 4: wR [wRd] = wRBITS (wRd, 0, 31) | (data << 32) | (wRBITS (wRd, 40, 63) << 40); break;
	case 5: wR [wRd] = wRBITS (wRd, 0, 39) | (data << 40) | (wRBITS (wRd, 48, 63) << 48); break;
	case 6: wR [wRd] = wRBITS (wRd, 0, 47) | (data << 48) | (wRBITS (wRd, 56, 63) << 56); break;
	case 7: wR [wRd] = wRBITS (wRd, 0, 55) | (data << 56); break;
	}
      break;

    case Hqual:
      data &= 0xffff;

      switch (offset & 3)
	{
	case 0: wR [wRd] = data | (wRBITS (wRd, 16, 63) << 16); break;
	case 1: wR [wRd] = wRBITS (wRd, 0, 15) | (data << 16) | (wRBITS (wRd, 32, 63) << 32); break;
	case 2: wR [wRd] = wRBITS (wRd, 0, 31) | (data << 32) | (wRBITS (wRd, 48, 63) << 48); break;
	case 3: wR [wRd] = wRBITS (wRd, 0, 47) | (data << 48); break;
	}
      break;

    case Wqual:
      if (offset & 1)
	wR [wRd] = wRBITS (wRd, 0, 31) | (data << 32);
      else
	wR [wRd] = (wRBITS (wRd, 32, 63) << 32) | data;
      break;

    default:
      ARMul_UndefInstr (state, instr);
      break;
    }

  wC [wCon] |= WCON_MUP;
  return ARMul_DONE;
}

static int
TMCR (ARMul_State * state, ARMword instr)
{
  ARMword val;
  int     wCreg;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmcr\n");
#endif

  if (BITS (0, 3) != 0)
    return ARMul_CANT;

  val = state->Reg [BITS (12, 15)];
  if (BITS (12, 15) == 15)
    val &= 0xfffffffc;

  wCreg = BITS (16, 19);

  switch (wCreg)
    {
    case wCID:
      /* The wCID register is read only.  */
      break;

    case wCon:
      /* Writing to the MUP or CUP bits clears them.  */
      wC [wCon] &= ~ (val & 0x3);
      break;

    case wCSSF:
      /* Only the bottom 8 bits can be written to.
          The higher bits write as zero.  */
      wC [wCSSF] = (val & 0xff);
      wC [wCon] |= WCON_CUP;
      break;

    default:
      wC [wCreg] = val;
      wC [wCon] |= WCON_CUP;
      break;
    }

  return ARMul_DONE;
}

static int
TMCRR (ARMul_State * state, ARMword instr)
{
  ARMdword RdHi = state->Reg [BITS (16, 19)];
  ARMword  RdLo = state->Reg [BITS (12, 15)];

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmcrr\n");
#endif

  if ((BITS (16, 19) == 15) || (BITS (12, 15) == 15))
    return ARMul_CANT;

  wR [BITS (0, 3)] = (RdHi << 32) | RdLo;

  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
TMIA (ARMul_State * state, ARMword instr)
{
  signed long long a, b;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmia\n");
#endif

  if ((BITS (0, 3) == 15) || (BITS (12, 15) == 15))
    {
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  a = state->Reg [BITS (0, 3)];
  b = state->Reg [BITS (12, 15)];

  a = EXTEND32 (a);
  b = EXTEND32 (b);

  wR [BITS (5, 8)] += a * b;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
TMIAPH (ARMul_State * state, ARMword instr)
{
  signed long a, b, result;
  signed long long r;
  ARMword Rm = state->Reg [BITS (0, 3)];
  ARMword Rs = state->Reg [BITS (12, 15)];

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmiaph\n");
#endif

  if (BITS (0, 3) == 15 || BITS (12, 15) == 15)
    {
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  a = SUBSTR (Rs, ARMword, 16, 31);
  b = SUBSTR (Rm, ARMword, 16, 31);

  a = EXTEND16 (a);
  b = EXTEND16 (b);

  result = a * b;

  r = result;
  r = EXTEND32 (r);

  wR [BITS (5, 8)] += r;

  a = SUBSTR (Rs, ARMword,  0, 15);
  b = SUBSTR (Rm, ARMword,  0, 15);

  a = EXTEND16 (a);
  b = EXTEND16 (b);

  result = a * b;

  r = result;
  r = EXTEND32 (r);

  wR [BITS (5, 8)] += r;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
TMIAxy (ARMul_State * state, ARMword instr)
{
  ARMword Rm;
  ARMword Rs;
  long long temp;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmiaxy\n");
#endif

  if (BITS (0, 3) == 15 || BITS (12, 15) == 15)
    {
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  Rm = state->Reg [BITS (0, 3)];
  if (BIT (17))
    Rm >>= 16;
  else
    Rm &= 0xffff;

  Rs = state->Reg [BITS (12, 15)];
  if (BIT (16))
    Rs >>= 16;
  else
    Rs &= 0xffff;

  if (Rm & (1 << 15))
    Rm -= 1 << 16;

  if (Rs & (1 << 15))
    Rs -= 1 << 16;

  Rm *= Rs;
  temp = Rm;

  if (temp & (1 << 31))
    temp -= 1ULL << 32;

  wR [BITS (5, 8)] += temp;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
TMOVMSK (ARMul_State * state, ARMword instr)
{
  ARMdword result;
  int      wRn;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmovmsk\n");
#endif

  /* The CRm field must be r0.  */
  if (BITS (0, 3) != 0)
    return ARMul_CANT;

  wRn = BITS (16, 19);

  switch (BITS (22, 23))
    {
    case Bqual:
      result = (  (wRBITS (wRn, 63, 63) << 7)
		| (wRBITS (wRn, 55, 55) << 6)
		| (wRBITS (wRn, 47, 47) << 5)
		| (wRBITS (wRn, 39, 39) << 4)
		| (wRBITS (wRn, 31, 31) << 3)
		| (wRBITS (wRn, 23, 23) << 2)
		| (wRBITS (wRn, 15, 15) << 1)
		| (wRBITS (wRn,  7,  7) << 0));
      break;

    case Hqual:
      result = (  (wRBITS (wRn, 63, 63) << 3)
		| (wRBITS (wRn, 47, 47) << 2)
		| (wRBITS (wRn, 31, 31) << 1)
		| (wRBITS (wRn, 15, 15) << 0));
      break;

    case Wqual:
      result = (wRBITS (wRn, 63, 63) << 1) | wRBITS (wRn, 31, 31);
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  state->Reg [BITS (12, 15)] = result;

  return ARMul_DONE;
}

static int
TMRC (ARMul_State * state, ARMword instr)
{
  int reg = BITS (12, 15);

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmrc\n");
#endif

  if (BITS (0, 3) != 0)
    return ARMul_CANT;

  if (reg == 15)
    ARMul_UndefInstr (state, instr);
  else
    state->Reg [reg] = wC [BITS (16, 19)];

  return ARMul_DONE;
}

static int
TMRRC (ARMul_State * state, ARMword instr)
{
  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "tmrrc\n");
#endif

  if ((BITS (16, 19) == 15) || (BITS (12, 15) == 15) || (BITS (4, 11) != 0))
    ARMul_UndefInstr (state, instr);
  else
    {
      state->Reg [BITS (16, 19)] = wRBITS (BITS (0, 3), 32, 63);
      state->Reg [BITS (12, 15)] = wRBITS (BITS (0, 3),  0, 31);
    }

  return ARMul_DONE;
}

static int
TORC (ARMul_State * state, ARMword instr)
{
  ARMword cpsr = ARMul_GetCPSR (state);

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "torc\n");
#endif

  /* The Rd field must be r15.  */
  if (BITS (12, 15) != 15)
    return ARMul_CANT;

  /* The CRn field must be r3.  */
  if (BITS (16, 19) != 3)
    return ARMul_CANT;

  /* The CRm field must be r0.  */
  if (BITS (0, 3) != 0)
    return ARMul_CANT;

  cpsr &= 0x0fffffff;

  switch (BITS (22, 23))
    {
    case Bqual:
      cpsr |= (  (wCBITS (wCASF, 28, 31) | wCBITS (wCASF, 24, 27)
		| wCBITS (wCASF, 20, 23) | wCBITS (wCASF, 16, 19)
		| wCBITS (wCASF, 12, 15) | wCBITS (wCASF,  8, 11)
		| wCBITS (wCASF,  4,  7) | wCBITS (wCASF,  0,  3)) << 28);
      break;

    case Hqual:
      cpsr |= (  (wCBITS (wCASF, 28, 31) | wCBITS (wCASF, 20, 23)
		| wCBITS (wCASF, 12, 15) | wCBITS (wCASF,  4,  7)) << 28);
      break;

    case Wqual:
      cpsr |= ((wCBITS (wCASF, 28, 31) | wCBITS (wCASF, 12, 15)) << 28);
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  ARMul_SetCPSR (state, cpsr);

  return ARMul_DONE;
}

static int
WACC (ARMul_State * state, ARMword instr)
{
  int wRn;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wacc\n");
#endif

  wRn = BITS (16, 19);

  switch (BITS (22, 23))
    {
    case Bqual:
      wR [BITS (12, 15)] =
	  wRBITS (wRn, 56, 63) + wRBITS (wRn, 48, 55)
	+ wRBITS (wRn, 40, 47) + wRBITS (wRn, 32, 39)
	+ wRBITS (wRn, 24, 31) + wRBITS (wRn, 16, 23)
	+ wRBITS (wRn,  8, 15) + wRBITS (wRn,  0,  7);
      break;

    case Hqual:
      wR [BITS (12, 15)] =
	  wRBITS (wRn, 48, 63) + wRBITS (wRn, 32, 47)
	+ wRBITS (wRn, 16, 31) + wRBITS (wRn,  0, 15);
      break;

    case Wqual:
      wR [BITS (12, 15)] = wRBITS (wRn, 32, 63) + wRBITS (wRn, 0, 31);
      break;

    default:
      ARMul_UndefInstr (state, instr);
      break;
    }

  wC [wCon] |= WCON_MUP;
  return ARMul_DONE;
}

static int
WADD (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMdword x;
  ARMdword s;
  ARMword  psr = 0;
  int      i;
  int      carry;
  int      overflow;
  int      satrv[8];

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wadd\n");
#endif

  /* Add two numbers using the specified function,
     leaving setting the carry bit as required.  */
#define ADDx(x, y, m, f) \
   (*f) (wRBITS (BITS (16, 19), (x), (y)) & (m), \
         wRBITS (BITS ( 0,  3), (x), (y)) & (m), \
        & carry, & overflow)

  switch (BITS (22, 23))
    {
    case Bqual:
      for (i = 0; i < 8; i++)
        {
	  switch (BITS (20, 21))
	    {
	    case NoSaturation:
	      s = ADDx ((i * 8), (i * 8) + 7, 0xff, AddS8);
	      satrv [BITIDX8 (i)] = 0;
	      r |= (s & 0xff) << (i * 8);
	      SIMD8_SET (psr, NBIT8 (s), SIMD_NBIT, i);
	      SIMD8_SET (psr, ZBIT8 (s), SIMD_ZBIT, i);
	      SIMD8_SET (psr, carry,     SIMD_CBIT, i);
	      SIMD8_SET (psr, overflow,  SIMD_VBIT, i);
	      break;

	    case UnsignedSaturation:
	      s = ADDx ((i * 8), (i * 8) + 7, 0xff, AddU8);
	      x = IwmmxtSaturateU8 (s, satrv + BITIDX8 (i));
	      r |= (x & 0xff) << (i * 8);
	      SIMD8_SET (psr, NBIT8 (x), SIMD_NBIT, i);
	      SIMD8_SET (psr, ZBIT8 (x), SIMD_ZBIT, i);
	      if (! satrv [BITIDX8 (i)])
		{
		  SIMD8_SET (psr, carry,    SIMD_CBIT, i);
		  SIMD8_SET (psr, overflow, SIMD_VBIT, i);
		}
	      break;

	    case SignedSaturation:
	      s = ADDx ((i * 8), (i * 8) + 7, 0xff, AddS8);
	      x = IwmmxtSaturateS8 (s, satrv + BITIDX8 (i));
	      r |= (x & 0xff) << (i * 8);
	      SIMD8_SET (psr, NBIT8 (x), SIMD_NBIT, i);
	      SIMD8_SET (psr, ZBIT8 (x), SIMD_ZBIT, i);
	      if (! satrv [BITIDX8 (i)])
		{
		  SIMD8_SET (psr, carry,    SIMD_CBIT, i);
		  SIMD8_SET (psr, overflow, SIMD_VBIT, i);
		}
	      break;

	    default:
	      ARMul_UndefInstr (state, instr);
	      return ARMul_DONE;
	    }
	}
      break;

    case Hqual:
      satrv[0] = satrv[2] = satrv[4] = satrv[6] = 0;

      for (i = 0; i < 4; i++)
	{
	  switch (BITS (20, 21))
	    {
	    case NoSaturation:
	      s = ADDx ((i * 16), (i * 16) + 15, 0xffff, AddS16);
	      satrv [BITIDX16 (i)] = 0;
	      r |= (s & 0xffff) << (i * 16);
	      SIMD16_SET (psr, NBIT16 (s), SIMD_NBIT, i);
	      SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	      SIMD16_SET (psr, carry,      SIMD_CBIT, i);
	      SIMD16_SET (psr, overflow,   SIMD_VBIT, i);
	      break;

	    case UnsignedSaturation:
	      s = ADDx ((i * 16), (i * 16) + 15, 0xffff, AddU16);
	      x = IwmmxtSaturateU16 (s, satrv + BITIDX16 (i));
	      r |= (x & 0xffff) << (i * 16);
	      SIMD16_SET (psr, NBIT16 (x), SIMD_NBIT, i);
	      SIMD16_SET (psr, ZBIT16 (x), SIMD_ZBIT, i);
	      if (! satrv [BITIDX16 (i)])
		{
		  SIMD16_SET (psr, carry,    SIMD_CBIT, i);
		  SIMD16_SET (psr, overflow, SIMD_VBIT, i);
		}
	      break;

	    case SignedSaturation:
	      s = ADDx ((i * 16), (i * 16) + 15, 0xffff, AddS16);
	      x = IwmmxtSaturateS16 (s, satrv + BITIDX16 (i));
	      r |= (x & 0xffff) << (i * 16);
	      SIMD16_SET (psr, NBIT16 (x), SIMD_NBIT, i);
	      SIMD16_SET (psr, ZBIT16 (x), SIMD_ZBIT, i);
	      if (! satrv [BITIDX16 (i)])
		{
		  SIMD16_SET (psr, carry,    SIMD_CBIT, i);
		  SIMD16_SET (psr, overflow, SIMD_VBIT, i);
		}
	      break;

	    default:
	      ARMul_UndefInstr (state, instr);
	      return ARMul_DONE;
	    }
	}
      break;

    case Wqual:
      satrv[0] = satrv[1] = satrv[2] = satrv[4] = satrv[5] = satrv[6] = 0;

      for (i = 0; i < 2; i++)
	{
	  switch (BITS (20, 21))
	    {
	    case NoSaturation:
	      s = ADDx ((i * 32), (i * 32) + 31, 0xffffffff, AddS32);
	      satrv [BITIDX32 (i)] = 0;
	      r |= (s & 0xffffffff) << (i * 32);
	      SIMD32_SET (psr, NBIT32 (s), SIMD_NBIT, i);
	      SIMD32_SET (psr, ZBIT32 (s), SIMD_ZBIT, i);
	      SIMD32_SET (psr, carry,      SIMD_CBIT, i);
	      SIMD32_SET (psr, overflow,   SIMD_VBIT, i);
	      break;

	    case UnsignedSaturation:
	      s = ADDx ((i * 32), (i * 32) + 31, 0xffffffff, AddU32);
	      x = IwmmxtSaturateU32 (s, satrv + BITIDX32 (i));
	      r |= (x & 0xffffffff) << (i * 32);
	      SIMD32_SET (psr, NBIT32 (x), SIMD_NBIT, i);
	      SIMD32_SET (psr, ZBIT32 (x), SIMD_ZBIT, i);
	      if (! satrv [BITIDX32 (i)])
		{
		  SIMD32_SET (psr, carry,    SIMD_CBIT, i);
		  SIMD32_SET (psr, overflow, SIMD_VBIT, i);
		}
	      break;

	    case SignedSaturation:
	      s = ADDx ((i * 32), (i * 32) + 31, 0xffffffff, AddS32);
	      x = IwmmxtSaturateS32 (s, satrv + BITIDX32 (i));
	      r |= (x & 0xffffffff) << (i * 32);
	      SIMD32_SET (psr, NBIT32 (x), SIMD_NBIT, i);
	      SIMD32_SET (psr, ZBIT32 (x), SIMD_ZBIT, i);
	      if (! satrv [BITIDX32 (i)])
		{
		  SIMD32_SET (psr, carry,    SIMD_CBIT, i);
		  SIMD32_SET (psr, overflow, SIMD_VBIT, i);
		}
	      break;

	    default:
	      ARMul_UndefInstr (state, instr);
	      return ARMul_DONE;
	    }
	}
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wC [wCASF] = psr;
  wR [BITS (12, 15)] = r;
  wC [wCon] |= (WCON_MUP | WCON_CUP);

  SET_wCSSFvec (satrv);

#undef ADDx

  return ARMul_DONE;
}

static int
WALIGNI (ARMword instr)
{
  int shift = BITS (20, 22) * 8;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "waligni\n");
#endif

  if (shift)
    wR [BITS (12, 15)] =
      wRBITS (BITS (16, 19), shift, 63)
      | (wRBITS (BITS (0, 3), 0, shift) << ((64 - shift)));
  else
    wR [BITS (12, 15)] = wR [BITS (16, 19)];

  wC [wCon] |= WCON_MUP;
  return ARMul_DONE;
}

static int
WALIGNR (ARMul_State * state, ARMword instr)
{
  int shift = (wC [BITS (20, 21) + 8] & 0x7) * 8;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "walignr\n");
#endif

  if (shift)
    wR [BITS (12, 15)] =
      wRBITS (BITS (16, 19), shift, 63)
      | (wRBITS (BITS (0, 3), 0, shift) << ((64 - shift)));
  else
    wR [BITS (12, 15)] = wR [BITS (16, 19)];

  wC [wCon] |= WCON_MUP;
  return ARMul_DONE;
}

static int
WAND (ARMword instr)
{
  ARMdword result;
  ARMword  psr = 0;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wand\n");
#endif

  result = wR [BITS (16, 19)] & wR [BITS (0, 3)];
  wR [BITS (12, 15)] = result;

  SIMD64_SET (psr, (result == 0), SIMD_ZBIT);
  SIMD64_SET (psr, (result & (1ULL << 63)), SIMD_NBIT);

  wC [wCASF] = psr;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WANDN (ARMword instr)
{
  ARMdword result;
  ARMword  psr = 0;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wandn\n");
#endif

  result = wR [BITS (16, 19)] & ~ wR [BITS (0, 3)];
  wR [BITS (12, 15)] = result;

  SIMD64_SET (psr, (result == 0), SIMD_ZBIT);
  SIMD64_SET (psr, (result & (1ULL << 63)), SIMD_NBIT);

  wC [wCASF] = psr;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WAVG2 (ARMword instr)
{
  ARMdword r = 0;
  ARMword  psr = 0;
  ARMdword s;
  int      i;
  int      round = BIT (20) ? 1 : 0;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wavg2\n");
#endif

#define AVG2x(x, y, m) (((wRBITS (BITS (16, 19), (x), (y)) & (m)) \
		       + (wRBITS (BITS ( 0,  3), (x), (y)) & (m)) \
		       + round) / 2)

  if (BIT (22))
    {
      for (i = 0; i < 4; i++)
	{
	  s = AVG2x ((i * 16), (i * 16) + 15, 0xffff) & 0xffff;
	  SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	  r |= s << (i * 16);
	}
    }
  else
    {
      for (i = 0; i < 8; i++)
	{
	  s = AVG2x ((i * 8), (i * 8) + 7, 0xff) & 0xff;
	  SIMD8_SET (psr, ZBIT8 (s), SIMD_ZBIT, i);
	  r |= s << (i * 8);
	}
    }

  wR [BITS (12, 15)] = r;
  wC [wCASF] = psr;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WCMPEQ (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMword  psr = 0;
  ARMdword s;
  int      i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wcmpeq\n");
#endif

  switch (BITS (22, 23))
    {
    case Bqual:
      for (i = 0; i < 8; i++)
	{
	  s = wRBYTE (BITS (16, 19), i) == wRBYTE (BITS (0, 3), i) ? 0xff : 0;
	  r |= s << (i * 8);
	  SIMD8_SET (psr, NBIT8 (s), SIMD_NBIT, i);
	  SIMD8_SET (psr, ZBIT8 (s), SIMD_ZBIT, i);
	}
      break;

    case Hqual:
      for (i = 0; i < 4; i++)
	{
	  s = wRHALF (BITS (16, 19), i) == wRHALF (BITS (0, 3), i) ? 0xffff : 0;
	  r |= s << (i * 16);
	  SIMD16_SET (psr, NBIT16 (s), SIMD_NBIT, i);
	  SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	}
      break;

    case Wqual:
      for (i = 0; i < 2; i++)
	{
	  s = wRWORD (BITS (16, 19), i) == wRWORD (BITS (0, 3), i) ? 0xffffffff : 0;
	  r |= s << (i * 32);
	  SIMD32_SET (psr, NBIT32 (s), SIMD_NBIT, i);
	  SIMD32_SET (psr, ZBIT32 (s), SIMD_ZBIT, i);
	}
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wC [wCASF] = psr;
  wR [BITS (12, 15)] = r;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WCMPGT (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMword  psr = 0;
  ARMdword s;
  int      i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wcmpgt\n");
#endif

  switch (BITS (22, 23))
    {
    case Bqual:
      if (BIT (21))
	{
	  /* Use a signed comparison.  */
	  for (i = 0; i < 8; i++)
	    {
	      signed char a, b;

	      a = wRBYTE (BITS (16, 19), i);
	      b = wRBYTE (BITS (0, 3), i);

	      s = (a > b) ? 0xff : 0;
	      r |= s << (i * 8);
	      SIMD8_SET (psr, NBIT8 (s), SIMD_NBIT, i);
	      SIMD8_SET (psr, ZBIT8 (s), SIMD_ZBIT, i);
	    }
	}
      else
	{
	  for (i = 0; i < 8; i++)
	    {
	      s = (wRBYTE (BITS (16, 19), i) > wRBYTE (BITS (0, 3), i))
		? 0xff : 0;
	      r |= s << (i * 8);
	      SIMD8_SET (psr, NBIT8 (s), SIMD_NBIT, i);
	      SIMD8_SET (psr, ZBIT8 (s), SIMD_ZBIT, i);
	    }
	}
      break;

    case Hqual:
      if (BIT (21))
	{
	  for (i = 0; i < 4; i++)
	    {
	      signed int a, b;

	      a = wRHALF (BITS (16, 19), i);
	      a = EXTEND16 (a);

	      b = wRHALF (BITS (0, 3), i);
	      b = EXTEND16 (b);

	      s = (a > b) ? 0xffff : 0;		
	      r |= s << (i * 16);
	      SIMD16_SET (psr, NBIT16 (s), SIMD_NBIT, i);
	      SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	    }
	}
      else
	{
	  for (i = 0; i < 4; i++)
	    {
	      s = (wRHALF (BITS (16, 19), i) > wRHALF (BITS (0, 3), i))
		? 0xffff : 0;
	      r |= s << (i * 16);
	      SIMD16_SET (psr, NBIT16 (s), SIMD_NBIT, i);
	      SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	    }
	}
      break;

    case Wqual:
      if (BIT (21))
	{
	  for (i = 0; i < 2; i++)
	    {
	      signed long a, b;

	      a = EXTEND32 (wRWORD (BITS (16, 19), i));
	      b = EXTEND32 (wRWORD (BITS (0, 3), i));

	      s = (a > b) ? 0xffffffff : 0;
	      r |= s << (i * 32);

	      SIMD32_SET (psr, NBIT32 (s), SIMD_NBIT, i);
	      SIMD32_SET (psr, ZBIT32 (s), SIMD_ZBIT, i);
	    }
	}
      else
	{
	  for (i = 0; i < 2; i++)
	    {
	      s = (wRWORD (BITS (16, 19), i) > wRWORD (BITS (0, 3), i))
		? 0xffffffff : 0;
	      r |= s << (i * 32);
	      SIMD32_SET (psr, NBIT32 (s), SIMD_NBIT, i);
	      SIMD32_SET (psr, ZBIT32 (s), SIMD_ZBIT, i);
	    }
	}
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wC [wCASF] = psr;
  wR [BITS (12, 15)] = r;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static ARMword
Compute_Iwmmxt_Address (ARMul_State * state, ARMword instr, int * pFailed)
{
  ARMword  Rn;
  ARMword  addr;
  ARMword  offset;
  ARMword  multiplier;

  * pFailed  = 0;
  Rn         = BITS (16, 19);
  addr       = state->Reg [Rn];
  offset     = BITS (0, 7);
  multiplier = BIT (8) ? 4 : 1;

  if (BIT (24)) /* P */
    {
      /* Pre Indexed Addressing.  */
      if (BIT (23))
	addr += offset * multiplier;
      else
	addr -= offset * multiplier;

      /* Immediate Pre-Indexed.  */
      if (BIT (21)) /* W */
	{
	  if (Rn == 15)
	    {
	      /* Writeback into R15 is UNPREDICTABLE.  */
#ifdef DEBUG
	      fprintf (stderr, "iWMMXt: writeback into r15\n");
#endif
	      * pFailed = 1;
	    }
	  else
	    state->Reg [Rn] = addr;
	}
    }
  else
    {
      /* Post Indexed Addressing.  */
      if (BIT (21)) /* W */
	{
	  /* Handle the write back of the final address.  */
	  if (Rn == 15)
	    {
	      /* Writeback into R15 is UNPREDICTABLE.  */
#ifdef DEBUG
	      fprintf (stderr, "iWMMXt: writeback into r15\n");
#endif
	      * pFailed = 1;
	    }
	  else
	    {
	      ARMword  increment;

	      if (BIT (23))
		increment = offset * multiplier;
	      else
		increment = - (offset * multiplier);

	      state->Reg [Rn] = addr + increment;
	    }
	}
      else
	{
	  /* P == 0, W == 0, U == 0 is UNPREDICTABLE.  */
	  if (BIT (23) == 0)
	    {
#ifdef DEBUG
	      fprintf (stderr, "iWMMXt: undefined addressing mode\n");
#endif
	      * pFailed = 1;
	    }
	}
    }

  return addr;
}

static ARMdword
Iwmmxt_Load_Double_Word (ARMul_State * state, ARMword address)
{
  ARMdword value;

  /* The address must be aligned on a 8 byte boundary.  */
  if (address & 0x7)
    {
      fprintf (stderr, "iWMMXt: At addr 0x%x: Unaligned double word load from 0x%x\n",
	       (state->Reg[15] - 8) & ~0x3, address);
#ifdef DEBUG
#endif
      /* No need to check for alignment traps.  An unaligned
	 double word load with alignment trapping disabled is
	 UNPREDICTABLE.  */
      ARMul_Abort (state, ARMul_DataAbortV);
    }

  /* Load the words.  */
  if (! state->bigendSig)
    {
      value = ARMul_LoadWordN (state, address + 4);
      value <<= 32;
      value |= ARMul_LoadWordN (state, address);
    }
  else
    {
      value = ARMul_LoadWordN (state, address);
      value <<= 32;
      value |= ARMul_LoadWordN (state, address + 4);
    }

  /* Check for data aborts.  */
  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
  else
    ARMul_Icycles (state, 2, 0L);

  return value;
}

static ARMword
Iwmmxt_Load_Word (ARMul_State * state, ARMword address)
{
  ARMword value;

  /* Check for a misaligned address.  */
  if (address & 3)
    {
      if ((read_cp15_reg (1, 0, 0) & ARMul_CP15_R1_ALIGN))
	ARMul_Abort (state, ARMul_DataAbortV);
      else
	address &= ~ 3;
    }

  value = ARMul_LoadWordN (state, address);

  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
  else
    ARMul_Icycles (state, 1, 0L);

  return value;
}

static ARMword
Iwmmxt_Load_Half_Word (ARMul_State * state, ARMword address)
{
  ARMword value;

  /* Check for a misaligned address.  */
  if (address & 1)
    {
      if ((read_cp15_reg (1, 0, 0) & ARMul_CP15_R1_ALIGN))
	ARMul_Abort (state, ARMul_DataAbortV);
      else
	address &= ~ 1;
    }

  value = ARMul_LoadHalfWord (state, address);

  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
  else
    ARMul_Icycles (state, 1, 0L);

  return value;
}

static ARMword
Iwmmxt_Load_Byte (ARMul_State * state, ARMword address)
{
  ARMword value;

  value = ARMul_LoadByte (state, address);

  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
  else
    ARMul_Icycles (state, 1, 0L);

  return value;
}

static void
Iwmmxt_Store_Double_Word (ARMul_State * state, ARMword address, ARMdword value)
{
  /* The address must be aligned on a 8 byte boundary.  */
  if (address & 0x7)
    {
      fprintf (stderr, "iWMMXt: At addr 0x%x: Unaligned double word store to 0x%x\n",
	       (state->Reg[15] - 8) & ~0x3, address);
#ifdef DEBUG
#endif
      /* No need to check for alignment traps.  An unaligned
	 double word store with alignment trapping disabled is
	 UNPREDICTABLE.  */
      ARMul_Abort (state, ARMul_DataAbortV);
    }

  /* Store the words.  */
  if (! state->bigendSig)
    {
      ARMul_StoreWordN (state, address, value);
      ARMul_StoreWordN (state, address + 4, value >> 32);
    }
  else
    {
      ARMul_StoreWordN (state, address + 4, value);
      ARMul_StoreWordN (state, address, value >> 32);
    }

  /* Check for data aborts.  */
  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
  else
    ARMul_Icycles (state, 2, 0L);
}

static void
Iwmmxt_Store_Word (ARMul_State * state, ARMword address, ARMword value)
{
  /* Check for a misaligned address.  */
  if (address & 3)
    {
      if ((read_cp15_reg (1, 0, 0) & ARMul_CP15_R1_ALIGN))
	ARMul_Abort (state, ARMul_DataAbortV);
      else
	address &= ~ 3;
    }

  ARMul_StoreWordN (state, address, value);

  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
}

static void
Iwmmxt_Store_Half_Word (ARMul_State * state, ARMword address, ARMword value)
{
  /* Check for a misaligned address.  */
  if (address & 1)
    {
      if ((read_cp15_reg (1, 0, 0) & ARMul_CP15_R1_ALIGN))
	ARMul_Abort (state, ARMul_DataAbortV);
      else
	address &= ~ 1;
    }

  ARMul_StoreHalfWord (state, address, value);

  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
}

static void
Iwmmxt_Store_Byte (ARMul_State * state, ARMword address, ARMword value)
{
  ARMul_StoreByte (state, address, value);

  if (state->Aborted)
    ARMul_Abort (state, ARMul_DataAbortV);
}

static int
WLDR (ARMul_State * state, ARMword instr)
{
  ARMword address;
  int failed;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wldr\n");
#endif

  address = Compute_Iwmmxt_Address (state, instr, & failed);
  if (failed)
    return ARMul_CANT;

  if (BITS (28, 31) == 0xf)
    {
      /* WLDRW wCx */
      wC [BITS (12, 15)] = Iwmmxt_Load_Word (state, address);
    }
  else if (BIT (8) == 0)
    {
      if (BIT (22) == 0)
	/* WLDRB */
	wR [BITS (12, 15)] = Iwmmxt_Load_Byte (state, address);
      else
	/* WLDRH */
	wR [BITS (12, 15)] = Iwmmxt_Load_Half_Word (state, address);
    }
  else
    {
      if (BIT (22) == 0)
	/* WLDRW wRd */
	wR [BITS (12, 15)] = Iwmmxt_Load_Word (state, address);
      else
	/* WLDRD */
	wR [BITS (12, 15)] = Iwmmxt_Load_Double_Word (state, address);
    }

  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WMAC (ARMword instr)
{
  int      i;
  ARMdword t = 0;
  ARMword  a, b;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wmac\n");
#endif

  for (i = 0; i < 4; i++)
    {
      if (BIT (21))
        {
	  /* Signed.  */
	  signed long s;

	  a = wRHALF (BITS (16, 19), i);
	  a = EXTEND16 (a);

	  b = wRHALF (BITS (0, 3), i);
	  b = EXTEND16 (b);

	  s = (signed long) a * (signed long) b;

	  t = t + (ARMdword) s;
        }
      else
        {
	  /* Unsigned.  */
	  a = wRHALF (BITS (16, 19), i);
	  b = wRHALF (BITS ( 0,  3), i);

	  t += a * b;
        }
    }

  if (BIT (21))
    t = EXTEND32 (t);
  else
    t &= 0xffffffff;

  if (BIT (20))
    wR [BITS (12, 15)] = t;
  else
    wR[BITS (12, 15)] += t;

  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WMADD (ARMword instr)
{
  ARMdword r = 0;
  int i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wmadd\n");
#endif

  for (i = 0; i < 2; i++)
    {
      ARMdword s1, s2;

      if (BIT (21))	/* Signed.  */
        {
	  signed long a, b;

	  a = wRHALF (BITS (16, 19), i * 2);
	  a = EXTEND16 (a);

	  b = wRHALF (BITS (0, 3), i * 2);
	  b = EXTEND16 (b);

	  s1 = (ARMdword) (a * b);

	  a = wRHALF (BITS (16, 19), i * 2 + 1);
	  a = EXTEND16 (a);

	  b = wRHALF (BITS (0, 3), i * 2 + 1);
	  b = EXTEND16 (b);

	  s2 = (ARMdword) (a * b);
        }
      else			/* Unsigned.  */
        {
	  unsigned long a, b;

	  a = wRHALF (BITS (16, 19), i * 2);
	  b = wRHALF (BITS ( 0,  3), i * 2);

	  s1 = (ARMdword) (a * b);

	  a = wRHALF (BITS (16, 19), i * 2 + 1);
	  b = wRHALF (BITS ( 0,  3), i * 2 + 1);

	  s2 = (ARMdword) a * b;
        }

      r |= (ARMdword) ((s1 + s2) & 0xffffffff) << (i ? 32 : 0);
    }

  wR [BITS (12, 15)] = r;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WMAX (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMdword s;
  int      i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wmax\n");
#endif

  switch (BITS (22, 23))
    {
    case Bqual:
      for (i = 0; i < 8; i++)
	if (BIT (21))	/* Signed.  */
	  {
	    int a, b;

	    a = wRBYTE (BITS (16, 19), i);
	    a = EXTEND8 (a);

	    b = wRBYTE (BITS (0, 3), i);
	    b = EXTEND8 (b);

	    if (a > b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xff) << (i * 8);
	  }
	else	 	/* Unsigned.  */
	  {
	    unsigned int a, b;

	    a = wRBYTE (BITS (16, 19), i);
	    b = wRBYTE (BITS (0, 3), i);

	    if (a > b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xff) << (i * 8);
          }
      break;

    case Hqual:
      for (i = 0; i < 4; i++)
	if (BIT (21))	/* Signed.  */
	  {
	    int a, b;

	    a = wRHALF (BITS (16, 19), i);
	    a = EXTEND16 (a);

	    b = wRHALF (BITS (0, 3), i);
	    b = EXTEND16 (b);

	    if (a > b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffff) << (i * 16);
	  }
	else	 	/* Unsigned.  */
	  {
	    unsigned int a, b;

	    a = wRHALF (BITS (16, 19), i);
	    b = wRHALF (BITS (0, 3), i);

	    if (a > b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffff) << (i * 16);
          }
      break;

    case Wqual:
      for (i = 0; i < 2; i++)
	if (BIT (21))	/* Signed.  */
	  {
	    int a, b;

	    a = wRWORD (BITS (16, 19), i);
	    b = wRWORD (BITS (0, 3), i);

	    if (a > b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffffffff) << (i * 32);
	  }
	else
	  {
	    unsigned int a, b;

	    a = wRWORD (BITS (16, 19), i);
	    b = wRWORD (BITS (0, 3), i);

	    if (a > b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffffffff) << (i * 32);
          }
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wR [BITS (12, 15)] = r;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WMIN (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMdword s;
  int      i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wmin\n");
#endif

  switch (BITS (22, 23))
    {
    case Bqual:
      for (i = 0; i < 8; i++)
	if (BIT (21))	/* Signed.  */
	  {
	    int a, b;

	    a = wRBYTE (BITS (16, 19), i);
	    a = EXTEND8 (a);

	    b = wRBYTE (BITS (0, 3), i);
	    b = EXTEND8 (b);

	    if (a < b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xff) << (i * 8);
	  }
	else	 	/* Unsigned.  */
	  {
	    unsigned int a, b;

	    a = wRBYTE (BITS (16, 19), i);
	    b = wRBYTE (BITS (0, 3), i);

	    if (a < b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xff) << (i * 8);
          }
      break;

    case Hqual:
      for (i = 0; i < 4; i++)
	if (BIT (21))	/* Signed.  */
	  {
	    int a, b;

	    a = wRHALF (BITS (16, 19), i);
	    a = EXTEND16 (a);

	    b = wRHALF (BITS (0, 3), i);
	    b = EXTEND16 (b);

	    if (a < b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffff) << (i * 16);
	  }
	else
	  {
	    /* Unsigned.  */
	    unsigned int a, b;

	    a = wRHALF (BITS (16, 19), i);
	    b = wRHALF (BITS ( 0,  3), i);

	    if (a < b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffff) << (i * 16);
          }
      break;

    case Wqual:
      for (i = 0; i < 2; i++)
	if (BIT (21))	/* Signed.  */
	  {
	    int a, b;

	    a = wRWORD (BITS (16, 19), i);
	    b = wRWORD (BITS ( 0,  3), i);

	    if (a < b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffffffff) << (i * 32);
	  }
	else
	  {
	    unsigned int a, b;

	    a = wRWORD (BITS (16, 19), i);
	    b = wRWORD (BITS (0, 3), i);

	    if (a < b)
	      s = a;
	    else
	      s = b;

	    r |= (s & 0xffffffff) << (i * 32);
          }
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wR [BITS (12, 15)] = r;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WMUL (ARMword instr)
{
  ARMdword r = 0;
  ARMdword s;
  int      i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wmul\n");
#endif

  for (i = 0; i < 4; i++)
    if (BIT (21))	/* Signed.  */
      {
	long a, b;

	a = wRHALF (BITS (16, 19), i);
	a = EXTEND16 (a);

	b = wRHALF (BITS (0, 3), i);
	b = EXTEND16 (b);

	s = a * b;

	if (BIT (20))
	  r |= ((s >> 16) & 0xffff) << (i * 16);
	else
	  r |= (s & 0xffff) << (i * 16);
      }
    else		/* Unsigned.  */
      {
	unsigned long a, b;

	a = wRHALF (BITS (16, 19), i);
	b = wRHALF (BITS (0, 3), i);

	s = a * b;

	if (BIT (20))
	  r |= ((s >> 16) & 0xffff) << (i * 16);
	else
	  r |= (s & 0xffff) << (i * 16);
      }

  wR [BITS (12, 15)] = r;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WOR (ARMword instr)
{
  ARMword psr = 0;
  ARMdword result;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wor\n");
#endif

  result = wR [BITS (16, 19)] | wR [BITS (0, 3)];
  wR [BITS (12, 15)] = result;

  SIMD64_SET (psr, (result == 0), SIMD_ZBIT);
  SIMD64_SET (psr, (result & (1ULL << 63)), SIMD_NBIT);

  wC [wCASF] = psr;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WPACK (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMword  psr = 0;
  ARMdword x;
  ARMdword s;
  int      i;
  int      satrv[8];

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wpack\n");
#endif

  switch (BITS (22, 23))
    {
    case Hqual:
      for (i = 0; i < 8; i++)
	{
	  x = wRHALF (i < 4 ? BITS (16, 19) : BITS (0, 3), i & 3);

	  switch (BITS (20, 21))
	    {
	    case UnsignedSaturation:
	      s = IwmmxtSaturateU8 (x, satrv + BITIDX8 (i));
	      break;

	    case SignedSaturation:
	      s = IwmmxtSaturateS8 (x, satrv + BITIDX8 (i));
	      break;

	    default:
	      ARMul_UndefInstr (state, instr);
	      return ARMul_DONE;
	    }

	  r |= (s & 0xff) << (i * 8);
	  SIMD8_SET (psr, NBIT8 (s), SIMD_NBIT, i);
	  SIMD8_SET (psr, ZBIT8 (s), SIMD_ZBIT, i);
	}
      break;

    case Wqual:
      satrv[0] = satrv[2] = satrv[4] = satrv[6] = 0;

      for (i = 0; i < 4; i++)
	{
	  x = wRWORD (i < 2 ? BITS (16, 19) : BITS (0, 3), i & 1);

	  switch (BITS (20, 21))
	    {
	    case UnsignedSaturation:
	      s = IwmmxtSaturateU16 (x, satrv + BITIDX16 (i));
	      break;

	    case SignedSaturation:
	      s = IwmmxtSaturateS16 (x, satrv + BITIDX16 (i));
	      break;

	    default:
	      ARMul_UndefInstr (state, instr);
	      return ARMul_DONE;
	    }

	  r |= (s & 0xffff) << (i * 16);
	  SIMD16_SET (psr, NBIT16 (s), SIMD_NBIT, i);
	  SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	}
      break;

    case Dqual:
      satrv[0] = satrv[1] = satrv[2] = satrv[4] = satrv[5] = satrv[6] = 0;

      for (i = 0; i < 2; i++)
	{
	  x = wR [i ? BITS (0, 3) : BITS (16, 19)];

	  switch (BITS (20, 21))
	    {
	    case UnsignedSaturation:
	      s = IwmmxtSaturateU32 (x, satrv + BITIDX32 (i));
	      break;

	    case SignedSaturation:
	      s = IwmmxtSaturateS32 (x, satrv + BITIDX32 (i));
	      break;

	    default:
	      ARMul_UndefInstr (state, instr);
	      return ARMul_DONE;
	    }

	  r |= (s & 0xffffffff) << (i * 32);
	  SIMD32_SET (psr, NBIT32 (s), SIMD_NBIT, i);
	  SIMD32_SET (psr, ZBIT32 (s), SIMD_ZBIT, i);
	}
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wC [wCASF] = psr;
  wR [BITS (12, 15)] = r;
  SET_wCSSFvec (satrv);
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WROR (ARMul_State * state, ARMword instr)
{
  ARMdword r = 0;
  ARMdword s;
  ARMword  psr = 0;
  int      i;
  int      shift;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wror\n");
#endif

  DECODE_G_BIT (state, instr, shift);

  switch (BITS (22, 23))
    {
    case Hqual:
      shift &= 0xf;
      for (i = 0; i < 4; i++)
	{
	  s = ((wRHALF (BITS (16, 19), i) & 0xffff) << (16 - shift))
	    | ((wRHALF (BITS (16, 19), i) & 0xffff) >> shift);
	  r |= (s & 0xffff) << (i * 16);
	  SIMD16_SET (psr, NBIT16 (s), SIMD_NBIT, i);
	  SIMD16_SET (psr, ZBIT16 (s), SIMD_ZBIT, i);
	}
      break;

    case Wqual:
      shift &= 0x1f;
      for (i = 0; i < 2; i++)
	{
	  s = ((wRWORD (BITS (16, 19), i) & 0xffffffff) << (32 - shift))
	    | ((wRWORD (BITS (16, 19), i) & 0xffffffff) >> shift);
	  r |= (s & 0xffffffff) << (i * 32);
	  SIMD32_SET (psr, NBIT32 (s), SIMD_NBIT, i);
	  SIMD32_SET (psr, ZBIT32 (s), SIMD_ZBIT, i);
	}
      break;

    case Dqual:
      shift &= 0x3f;
      r = (wR [BITS (16, 19)] >> shift)
	| (wR [BITS (16, 19)] << (64 - shift));

      SIMD64_SET (psr, NBIT64 (r), SIMD_NBIT);
      SIMD64_SET (psr, ZBIT64 (r), SIMD_ZBIT);
      break;

    default:
      ARMul_UndefInstr (state, instr);
      return ARMul_DONE;
    }

  wC [wCASF] = psr;
  wR [BITS (12, 15)] = r;
  wC [wCon] |= (WCON_CUP | WCON_MUP);

  return ARMul_DONE;
}

static int
WSAD (ARMword instr)
{
  ARMdword r;
  int      s;
  int      i;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)
    return ARMul_CANT;

#ifdef DEBUG
  fprintf (stderr, "wsad\n");
#endif

  /* Z bit.  */
  r = BIT (20) ? 0 : (wR [BITS (12, 15)] & 0xffffffff);

  if (BIT (22))
    /* Half.  */
    for (i = 0; i < 4; i++)
      {
	s = (wRHALF (BITS (16, 19), i) - wRHALF (BITS (0, 3), i));
	r += abs (s);
      }
  else
    /* Byte.  */
    for (i = 0; i < 8; i++)
      {
	s = (wRBYTE (BITS (16, 19), i) - wRBYTE (BITS (0, 3), i));
	r += abs (s);
      }

  wR [BITS (12, 15)] = r;
  wC [wCon] |= WCON_MUP;

  return ARMul_DONE;
}

static int
WSHUFH (ARMword instr)
{
  ARMdword r = 0;
  ARMword  psr = 0;
  ARMdword s;
  int      i;
  int      imm8;

  if ((read_cp15_reg (15, 0, 1) & 3) != 3)