[deliverable/binutils-gdb.git] / gdb / nat / aarch64-sve-linux-ptrace.c

/* Common target dependent for AArch64 systems.

   Copyright (C) 2018 Free Software Foundation, Inc.

   This file is part of GDB.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

#include <sys/utsname.h>
#include <sys/uio.h>
#include "common-defs.h"
#include "elf/external.h"
#include "elf/common.h"
#include "aarch64-sve-linux-ptrace.h"
#include "arch/aarch64.h"
#include "common-regcache.h"
#include "common/byte-vector.h"

static bool vq_change_warned = false;

/* See nat/aarch64-sve-linux-ptrace.h.  */

uint64_t
aarch64_sve_get_vq (int tid)
{
  struct iovec iovec;
  struct user_sve_header header;

  iovec.iov_len = sizeof (header);
  iovec.iov_base = &header;

  /* Ptrace gives the vector length in bytes.  Convert it to VQ, the number of
     128bit chunks in a Z register.  We use VQ because 128bits is the minimum
     a Z register can increase in size.  */

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* SVE is not supported.  */
      return 0;
    }

  uint64_t vq = sve_vq_from_vl (header.vl);

  if (!sve_vl_valid (header.vl))
    {
      warning (_("Invalid SVE state from kernel; SVE disabled."));
      return 0;
    }

  return vq;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

std::unique_ptr<gdb_byte[]>
aarch64_sve_get_sveregs (int tid)
{
  struct iovec iovec;
  uint64_t vq = aarch64_sve_get_vq (tid);

  if (vq == 0)
    perror_with_name (_("Unable to fetch SVE register header"));

  /* A ptrace call with NT_ARM_SVE will return a header followed by either a
     dump of all the SVE and FP registers, or an fpsimd structure (identical to
     the one returned by NT_FPREGSET) if the kernel has not yet executed any
     SVE code.  Make sure we allocate enough space for a full SVE dump.  */

  iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
  std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
  iovec.iov_base = buf.get ();

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    perror_with_name (_("Unable to fetch SVE registers"));

  return buf;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
				  const void *buf)
{
  char *base = (char *) buf;
  struct user_sve_header *header = (struct user_sve_header *) buf;
  uint64_t vq, vg_reg_buf = 0;

  vq = sve_vq_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);

  if (vg_reg_buf == 0)
    {
      /* VG has not been set.  */
      vg_reg_buf = sve_vg_from_vl (header->vl);
      reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
    }
  else if (vg_reg_buf != sve_vg_from_vl (header->vl) && !vq_change_warned)
    {
      /* Vector length on the running process has changed.  GDB currently does
	 not support this and will result in GDB showing incorrect partially
	 incorrect data for the vector registers.  Warn once and continue.  We
	 do not expect many programs to exhibit this behaviour.  To fix this
	 we need to spot the change earlier and generate a new target
	 descriptor.  */
      warning (_("SVE Vector length has changed (%ld to %d). "
		 "Vector registers may show incorrect data."),
	       vg_reg_buf, sve_vg_from_vl (header->vl));
      vq_change_warned = true;
    }

  if (HAS_SVE_STATE (*header))
    {
      /* The register dump contains a set of SVE registers.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
			     base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
			     base + SVE_PT_SVE_PREG_OFFSET (vq, i));

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
			   base + SVE_PT_SVE_FFR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
			   base + SVE_PT_SVE_FPSR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
			   base + SVE_PT_SVE_FPCR_OFFSET (vq));
    }
  else
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  These registers still exist in the hardware, but
	 the kernel has not yet initialised them, and so they will be null.  */

      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      /* Copy across the V registers from fpsimd structure to the Z registers,
	 ensuring the non overlapping state is set to null.  */

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
	  reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	}

      reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

      /* Clear the SVE only registers.  */

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
    }
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
				    void *buf)
{
  struct user_sve_header *header = (struct user_sve_header *) buf;
  char *base = (char *) buf;
  uint64_t vq, vg_reg_buf = 0;

  vq = sve_vq_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);

  if (vg_reg_buf != 0 && vg_reg_buf != sve_vg_from_vl (header->vl))
    {
      /* Vector length on the running process has changed.  GDB currently does
	 not support this and will result in GDB writing invalid data back to
	 the vector registers.  Error and exit.  We do not expect many programs
	 to exhibit this behaviour.  To fix this we need to spot the change
	 earlier and generate a new target descriptor.  */
      error (_("SVE Vector length has changed (%ld to %d). "
	       "Cannot write back registers."),
	     vg_reg_buf, sve_vg_from_vl (header->vl));
    }

  if (!HAS_SVE_STATE (*header))
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  Where possible we want to write the reg_buf data
	 back to the kernel using the fpsimd structure.  However, if we cannot
	 then we'll need to reformat the fpsimd into a full SVE structure,
	 resulting in the initialization of SVE state written back to the
	 kernel, which is why we try to avoid it.  */

      bool has_sve_state = false;
      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      /* Check in the reg_buf if any of the Z registers are set after the
	 first 128 bits, or if any of the other SVE registers are set.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
						 zero_reg, sizeof (__int128_t));
	  if (has_sve_state)
	    break;
	}

      if (!has_sve_state)
	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	  {
	    has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
						   zero_reg, 0);
	    if (has_sve_state)
	      break;
	  }

      if (!has_sve_state)
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
						 zero_reg, 0);

      /* If no SVE state exists, then use the existing fpsimd structure to
	 write out state and return.  */
      if (!has_sve_state)
	{
	  /* The collects of the Z registers will overflow the size of a vreg.
	     There is enough space in the structure to allow for this, but we
	     cannot overflow into the next register as we might not be
	     collecting every register.  */

	  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	    {
	      if (REG_VALID
		  == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
		{
		  reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
		  memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
		}
	    }

	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

	  return;
	}

      /* Otherwise, reformat the fpsimd structure into a full SVE set, by
	 expanding the V registers (working backwards so we don't splat
	 registers before they are copied) and using null for everything else.
	 Note that enough space for a full SVE dump was originally allocated
	 for base.  */

      header->flags |= SVE_PT_REGS_SVE;
      header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);

      memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
	      sizeof (uint32_t));
      memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
	      sizeof (uint32_t));

      for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
	{
	  memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
		  sizeof (__int128_t));
	}
    }

  /* Replace the kernel values with those from reg_buf.  */

  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
			    base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

  for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
			    base + SVE_PT_SVE_PREG_OFFSET (vq, i));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
			  base + SVE_PT_SVE_FFR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
			  base + SVE_PT_SVE_FPSR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
			  base + SVE_PT_SVE_FPCR_OFFSET (vq));

}
Commit	Line	Data
122394f1 AH	1	/* Common target dependent for AArch64 systems.
	2
	3	Copyright (C) 2018 Free Software Foundation, Inc.
	4
	5	This file is part of GDB.
	6
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of the GNU General Public License as published by
	9	the Free Software Foundation; either version 3 of the License, or
	10	(at your option) any later version.
	11
	12	This program is distributed in the hope that it will be useful,
	13	but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	GNU General Public License for more details.
	16
	17	You should have received a copy of the GNU General Public License
	18	along with this program. If not, see <http://www.gnu.org/licenses/>. */
	19
	20	#include <sys/utsname.h>
	21	#include <sys/uio.h>
	22	#include "common-defs.h"
	23	#include "elf/external.h"
	24	#include "elf/common.h"
	25	#include "aarch64-sve-linux-ptrace.h"
	26	#include "arch/aarch64.h"
e9902bfc AH	27	#include "common-regcache.h"
	28	#include "common/byte-vector.h"
	29
	30	static bool vq_change_warned = false;
122394f1 AH	31
	32	/* See nat/aarch64-sve-linux-ptrace.h. */
	33
39bfb937	34	uint64_t
122394f1 AH	35	aarch64_sve_get_vq (int tid)
	36	{
	37	struct iovec iovec;
	38	struct user_sve_header header;
	39
	40	iovec.iov_len = sizeof (header);
	41	iovec.iov_base = &header;
	42
	43	/* Ptrace gives the vector length in bytes. Convert it to VQ, the number of
	44	128bit chunks in a Z register. We use VQ because 128bits is the minimum
	45	a Z register can increase in size. */
	46
	47	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	48	{
	49	/* SVE is not supported. */
	50	return 0;
	51	}
	52
e9902bfc	53	uint64_t vq = sve_vq_from_vl (header.vl);
122394f1 AH	54
	55	if (!sve_vl_valid (header.vl))
	56	{
	57	warning (_("Invalid SVE state from kernel; SVE disabled."));
	58	return 0;
	59	}
	60
	61	return vq;
	62	}
e9902bfc AH	63
	64	/* See nat/aarch64-sve-linux-ptrace.h. */
	65
	66	std::unique_ptr<gdb_byte[]>
	67	aarch64_sve_get_sveregs (int tid)
	68	{
	69	struct iovec iovec;
e9902bfc AH	70	uint64_t vq = aarch64_sve_get_vq (tid);
	71
	72	if (vq == 0)
	73	perror_with_name (_("Unable to fetch SVE register header"));
	74
	75	/* A ptrace call with NT_ARM_SVE will return a header followed by either a
	76	dump of all the SVE and FP registers, or an fpsimd structure (identical to
	77	the one returned by NT_FPREGSET) if the kernel has not yet executed any
	78	SVE code. Make sure we allocate enough space for a full SVE dump. */
	79
	80	iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
	81	std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
	82	iovec.iov_base = buf.get ();
	83
	84	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	85	perror_with_name (_("Unable to fetch SVE registers"));
	86
	87	return buf;
	88	}
	89
	90	/* See nat/aarch64-sve-linux-ptrace.h. */
	91
	92	void
	93	aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
	94	const void *buf)
	95	{
	96	char base = (char ) buf;
	97	struct user_sve_header header = (struct user_sve_header ) buf;
	98	uint64_t vq, vg_reg_buf = 0;
	99
	100	vq = sve_vq_from_vl (header->vl);
	101
	102	/* Sanity check the data in the header. */
	103	if (!sve_vl_valid (header->vl)
	104	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
	105	error (_("Invalid SVE header from kernel."));
	106
	107	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
	108	reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
	109
	110	if (vg_reg_buf == 0)
	111	{
	112	/* VG has not been set. */
	113	vg_reg_buf = sve_vg_from_vl (header->vl);
	114	reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
	115	}
	116	else if (vg_reg_buf != sve_vg_from_vl (header->vl) && !vq_change_warned)
	117	{
	118	/* Vector length on the running process has changed. GDB currently does
	119	not support this and will result in GDB showing incorrect partially
	120	incorrect data for the vector registers. Warn once and continue. We
	121	do not expect many programs to exhibit this behaviour. To fix this
	122	we need to spot the change earlier and generate a new target
	123	descriptor. */
	124	warning (_("SVE Vector length has changed (%ld to %d). "
	125	"Vector registers may show incorrect data."),
	126	vg_reg_buf, sve_vg_from_vl (header->vl));
	127	vq_change_warned = true;
	128	}
	129
	130	if (HAS_SVE_STATE (*header))
	131	{
	132	/* The register dump contains a set of SVE registers. */
	133
134	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
135	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
136	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
137
138	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
139	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
140	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
141
142	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
143	base + SVE_PT_SVE_FFR_OFFSET (vq));
144	reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
145	base + SVE_PT_SVE_FPSR_OFFSET (vq));
146	reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
147	base + SVE_PT_SVE_FPCR_OFFSET (vq));
148	}
149	else
150	{
151	/* There is no SVE state yet - the register dump contains a fpsimd
152	structure instead. These registers still exist in the hardware, but
153	the kernel has not yet initialised them, and so they will be null. */
154
155	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
156	struct user_fpsimd_state *fpsimd
157	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
158
159	/* Copy across the V registers from fpsimd structure to the Z registers,
160	ensuring the non overlapping state is set to null. */
161
162	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
163
164	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
165	{
166	memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
167	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
168	}
169
170	reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
171	reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
172
173	/* Clear the SVE only registers. */
174
175	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
176	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);
177
178	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
179	}
180	}
181
182	/* See nat/aarch64-sve-linux-ptrace.h. */
183
184	void
185	aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
186	void *buf)
187	{
188	struct user_sve_header header = (struct user_sve_header ) buf;
189	char base = (char ) buf;
190	uint64_t vq, vg_reg_buf = 0;
191
192	vq = sve_vq_from_vl (header->vl);
193
194	/* Sanity check the data in the header. */
195	if (!sve_vl_valid (header->vl)
196	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
197	error (_("Invalid SVE header from kernel."));
198
199	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM))
200	reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &vg_reg_buf);
201
202	if (vg_reg_buf != 0 && vg_reg_buf != sve_vg_from_vl (header->vl))
203	{
204	/* Vector length on the running process has changed. GDB currently does
205	not support this and will result in GDB writing invalid data back to
206	the vector registers. Error and exit. We do not expect many programs
207	to exhibit this behaviour. To fix this we need to spot the change
208	earlier and generate a new target descriptor. */
209	error (_("SVE Vector length has changed (%ld to %d). "
210	"Cannot write back registers."),
211	vg_reg_buf, sve_vg_from_vl (header->vl));
212	}
213
214	if (!HAS_SVE_STATE (*header))
215	{
216	/* There is no SVE state yet - the register dump contains a fpsimd
217	structure instead. Where possible we want to write the reg_buf data
218	back to the kernel using the fpsimd structure. However, if we cannot
219	then we'll need to reformat the fpsimd into a full SVE structure,
220	resulting in the initialization of SVE state written back to the
221	kernel, which is why we try to avoid it. */
222
223	bool has_sve_state = false;
224	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
225	struct user_fpsimd_state *fpsimd
226	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
227
228	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
229
230	/* Check in the reg_buf if any of the Z registers are set after the
231	first 128 bits, or if any of the other SVE registers are set. */
232
233	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
234	{
235	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
236	zero_reg, sizeof (__int128_t));
237	if (has_sve_state)
238	break;
239	}
240
241	if (!has_sve_state)
242	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
243	{
244	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
245	zero_reg, 0);
246	if (has_sve_state)
247	break;
248	}
249
250	if (!has_sve_state)
251	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
252	zero_reg, 0);
253
254	/* If no SVE state exists, then use the existing fpsimd structure to
255	write out state and return. */
256	if (!has_sve_state)
257	{
258	/* The collects of the Z registers will overflow the size of a vreg.
259	There is enough space in the structure to allow for this, but we
260	cannot overflow into the next register as we might not be
261	collecting every register. */
262
263	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
264	{
265	if (REG_VALID
266	== reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
267	{
268	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
269	memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
270	}
271	}
272
273	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
274	reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
275	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
276	reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
277
278	return;
279	}
280
281	/* Otherwise, reformat the fpsimd structure into a full SVE set, by
282	expanding the V registers (working backwards so we don't splat
283	registers before they are copied) and using null for everything else.
284	Note that enough space for a full SVE dump was originally allocated
285	for base. */
286
287	header->flags \|= SVE_PT_REGS_SVE;
288	header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
289
290	memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
291	sizeof (uint32_t));
292	memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
293	sizeof (uint32_t));
294
295	for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
296	{
297	memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
298	sizeof (__int128_t));
299	}
300	}
301
302	/* Replace the kernel values with those from reg_buf. */
303
304	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
305	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
306	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
307	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
308
309	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
310	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
311	reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
312	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
313
314	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
315	reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
316	base + SVE_PT_SVE_FFR_OFFSET (vq));
317	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
318	reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
319	base + SVE_PT_SVE_FPSR_OFFSET (vq));
320	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
321	reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
322	base + SVE_PT_SVE_FPCR_OFFSET (vq));
323
324	}