x86: refine when to trigger optimizations

Checking just the base opcode without also checking this isn't a VEX
encoding, and without there being other insn properties avoiding a match
once respective VEX/XOP/EXEX-encoded insns would appear, is at least
dangerous. Add respective checks. At the same time there's no real need
to check the extension opcode to be None for the 0xA8 form - there's
nothing it can be confused with, and non-VEX-and-alike forms also can't
appear.

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 26a0075e0792225758fb5d872b18f0a313adb137..660b9a205e95a3cb9a4a35266c415b906dda1b1c 100644 (file)
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,9 @@
+2020-01-09  Jan Beulich  <jbeulich@suse.com>
+
+       * config/tc-i386.c (optimize_encoding): Add
+       is_any_vex_encoding() invocations. Drop respective
+       i.tm.extension_opcode == None checks.
+
 2020-01-09  Jan Beulich  <jbeulich@suse.com>
 
        * config/tc-i386.c (md_assemble): Check RegRex is clear during

diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 645fd38a12c1de6d86dec292a9d87aadc1c2bc79..2c087e42e0de4486ce1b89d6c2d1b1d36ded1a9e 100644 (file)
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -3990,13 +3990,13 @@ optimize_encoding (void)
   unsigned int j;
 
   if (optimize_for_space
+      && !is_any_vex_encoding (&i.tm)
       && i.reg_operands == 1
       && i.imm_operands == 1
       && !i.types[1].bitfield.byte
       && i.op[0].imms->X_op == O_constant
       && fits_in_imm7 (i.op[0].imms->X_add_number)
-      && ((i.tm.base_opcode == 0xa8
-          && i.tm.extension_opcode == None)
+      && (i.tm.base_opcode == 0xa8
          || (i.tm.base_opcode == 0xf6
              && i.tm.extension_opcode == 0x0)))
     {
@@ -4022,6 +4022,7 @@ optimize_encoding (void)
        }
     }
   else if (flag_code == CODE_64BIT
+          && !is_any_vex_encoding (&i.tm)
           && ((i.types[1].bitfield.qword
                && i.reg_operands == 1
                && i.imm_operands == 1
@@ -4030,9 +4031,8 @@ optimize_encoding (void)
                     && i.tm.extension_opcode == None
                     && fits_in_unsigned_long (i.op[0].imms->X_add_number))
                    || (fits_in_imm31 (i.op[0].imms->X_add_number)
-                       && (((i.tm.base_opcode == 0x24
-                             || i.tm.base_opcode == 0xa8)
-                            && i.tm.extension_opcode == None)
+                       && ((i.tm.base_opcode == 0x24
+                            || i.tm.base_opcode == 0xa8)
                            || (i.tm.base_opcode == 0x80
                                && i.tm.extension_opcode == 0x4)
                            || ((i.tm.base_opcode == 0xf6
@@ -4044,13 +4044,11 @@ optimize_encoding (void)
               || (i.types[0].bitfield.qword
                   && ((i.reg_operands == 2
                        && i.op[0].regs == i.op[1].regs
-                       && ((i.tm.base_opcode == 0x30
-                            || i.tm.base_opcode == 0x28)
-                           && i.tm.extension_opcode == None))
+                       && (i.tm.base_opcode == 0x30
+                           || i.tm.base_opcode == 0x28))
                       || (i.reg_operands == 1
                           && i.operands == 1
-                          && i.tm.base_opcode == 0x30
-                          && i.tm.extension_opcode == None)))))
+                          && i.tm.base_opcode == 0x30)))))
     {
       /* Optimize: -O:
           andq $imm31, %r64   -> andl $imm31, %r32
@@ -4091,6 +4089,7 @@ optimize_encoding (void)
     }
   else if (optimize > 1
           && !optimize_for_space
+          && !is_any_vex_encoding (&i.tm)
           && i.reg_operands == 2
           && i.op[0].regs == i.op[1].regs
           && ((i.tm.base_opcode & ~(Opcode_D | 1)) == 0x8