From: Jan Beulich <jbeulich@suse.com>
Date: Thu, 9 Jan 2020 10:40:04 +0000 (+0100)
Subject: x86: refine when to trigger optimizations
X-Git-Url: http://git.efficios.com/?p=deliverable%2Fbinutils-gdb.git;a=commitdiff_plain;h=72aea32839b5f4a2c8fa67e60451169b841941c0

x86: refine when to trigger optimizations

Checking just the base opcode without also checking this isn't a VEX
encoding, and without there being other insn properties avoiding a match
once respective VEX/XOP/EXEX-encoded insns would appear, is at least
dangerous. Add respective checks. At the same time there's no real need
to check the extension opcode to be None for the 0xA8 form - there's
nothing it can be confused with, and non-VEX-and-alike forms also can't
appear.
---

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 26a0075e07..660b9a205e 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,9 @@
+2020-01-09  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (optimize_encoding): Add
+	is_any_vex_encoding() invocations. Drop respective
+	i.tm.extension_opcode == None checks.
+
 2020-01-09  Jan Beulich  <jbeulich@suse.com>
 
 	* config/tc-i386.c (md_assemble): Check RegRex is clear during
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 645fd38a12..2c087e42e0 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -3990,13 +3990,13 @@ optimize_encoding (void)
   unsigned int j;
 
   if (optimize_for_space
+      && !is_any_vex_encoding (&i.tm)
       && i.reg_operands == 1
       && i.imm_operands == 1
       && !i.types[1].bitfield.byte
       && i.op[0].imms->X_op == O_constant
       && fits_in_imm7 (i.op[0].imms->X_add_number)
-      && ((i.tm.base_opcode == 0xa8
-	   && i.tm.extension_opcode == None)
+      && (i.tm.base_opcode == 0xa8
 	  || (i.tm.base_opcode == 0xf6
 	      && i.tm.extension_opcode == 0x0)))
     {
@@ -4022,6 +4022,7 @@ optimize_encoding (void)
 	}
     }
   else if (flag_code == CODE_64BIT
+	   && !is_any_vex_encoding (&i.tm)
 	   && ((i.types[1].bitfield.qword
 		&& i.reg_operands == 1
 		&& i.imm_operands == 1
@@ -4030,9 +4031,8 @@ optimize_encoding (void)
 		     && i.tm.extension_opcode == None
 		     && fits_in_unsigned_long (i.op[0].imms->X_add_number))
 		    || (fits_in_imm31 (i.op[0].imms->X_add_number)
-			&& (((i.tm.base_opcode == 0x24
-			      || i.tm.base_opcode == 0xa8)
-			     && i.tm.extension_opcode == None)
+			&& ((i.tm.base_opcode == 0x24
+			     || i.tm.base_opcode == 0xa8)
 			    || (i.tm.base_opcode == 0x80
 				&& i.tm.extension_opcode == 0x4)
 			    || ((i.tm.base_opcode == 0xf6
@@ -4044,13 +4044,11 @@ optimize_encoding (void)
 	       || (i.types[0].bitfield.qword
 		   && ((i.reg_operands == 2
 			&& i.op[0].regs == i.op[1].regs
-			&& ((i.tm.base_opcode == 0x30
-			     || i.tm.base_opcode == 0x28)
-			    && i.tm.extension_opcode == None))
+			&& (i.tm.base_opcode == 0x30
+			    || i.tm.base_opcode == 0x28))
 		       || (i.reg_operands == 1
 			   && i.operands == 1
-			   && i.tm.base_opcode == 0x30
-			   && i.tm.extension_opcode == None)))))
+			   && i.tm.base_opcode == 0x30)))))
     {
       /* Optimize: -O:
 	   andq $imm31, %r64   -> andl $imm31, %r32
@@ -4091,6 +4089,7 @@ optimize_encoding (void)
     }
   else if (optimize > 1
 	   && !optimize_for_space
+	   && !is_any_vex_encoding (&i.tm)
 	   && i.reg_operands == 2
 	   && i.op[0].regs == i.op[1].regs
 	   && ((i.tm.base_opcode & ~(Opcode_D | 1)) == 0x8