[deliverable/binutils-gdb.git] / gdb / nat / aarch64-sve-linux-ptrace.c

/* Common target dependent for AArch64 systems.

   Copyright (C) 2018-2019 Free Software Foundation, Inc.

   This file is part of GDB.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

#include <sys/utsname.h>
#include <sys/uio.h>
#include "gdbsupport/common-defs.h"
#include "elf/external.h"
#include "elf/common.h"
#include "aarch64-sve-linux-ptrace.h"
#include "arch/aarch64.h"
#include "gdbsupport/common-regcache.h"
#include "gdbsupport/byte-vector.h"

/* See nat/aarch64-sve-linux-ptrace.h.  */

uint64_t
aarch64_sve_get_vq (int tid)
{
  struct iovec iovec;
  struct user_sve_header header;

  iovec.iov_len = sizeof (header);
  iovec.iov_base = &header;

  /* Ptrace gives the vector length in bytes.  Convert it to VQ, the number of
     128bit chunks in a Z register.  We use VQ because 128bits is the minimum
     a Z register can increase in size.  */

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* SVE is not supported.  */
      return 0;
    }

  uint64_t vq = sve_vq_from_vl (header.vl);

  if (!sve_vl_valid (header.vl))
    {
      warning (_("Invalid SVE state from kernel; SVE disabled."));
      return 0;
    }

  return vq;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

bool
aarch64_sve_set_vq (int tid, uint64_t vq)
{
  struct iovec iovec;
  struct user_sve_header header;

  iovec.iov_len = sizeof (header);
  iovec.iov_base = &header;

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* SVE is not supported.  */
      return false;
    }

  header.vl = sve_vl_from_vq (vq);

  if (ptrace (PTRACE_SETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* Vector length change failed.  */
      return false;
    }

  return true;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

bool
aarch64_sve_set_vq (int tid, struct reg_buffer_common *reg_buf)
{
  if (reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM) != REG_VALID)
    return false;

  uint64_t reg_vg = 0;
  reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &reg_vg);

  return aarch64_sve_set_vq (tid, sve_vq_from_vg (reg_vg));
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

std::unique_ptr<gdb_byte[]>
aarch64_sve_get_sveregs (int tid)
{
  struct iovec iovec;
  uint64_t vq = aarch64_sve_get_vq (tid);

  if (vq == 0)
    perror_with_name (_("Unable to fetch SVE register header"));

  /* A ptrace call with NT_ARM_SVE will return a header followed by either a
     dump of all the SVE and FP registers, or an fpsimd structure (identical to
     the one returned by NT_FPREGSET) if the kernel has not yet executed any
     SVE code.  Make sure we allocate enough space for a full SVE dump.  */

  iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
  std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
  iovec.iov_base = buf.get ();

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    perror_with_name (_("Unable to fetch SVE registers"));

  return buf;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
				  const void *buf)
{
  char *base = (char *) buf;
  struct user_sve_header *header = (struct user_sve_header *) buf;

  uint64_t vq = sve_vq_from_vl (header->vl);
  uint64_t vg = sve_vg_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  /* Update VG.  Note, the registers in the regcache will already be of the
     correct length.  */
  reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg);

  if (HAS_SVE_STATE (*header))
    {
      /* The register dump contains a set of SVE registers.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
			     base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
			     base + SVE_PT_SVE_PREG_OFFSET (vq, i));

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
			   base + SVE_PT_SVE_FFR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
			   base + SVE_PT_SVE_FPSR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
			   base + SVE_PT_SVE_FPCR_OFFSET (vq));
    }
  else
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  These registers still exist in the hardware, but
	 the kernel has not yet initialised them, and so they will be null.  */

      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      /* Copy across the V registers from fpsimd structure to the Z registers,
	 ensuring the non overlapping state is set to null.  */

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
	  reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	}

      reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

      /* Clear the SVE only registers.  */

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
    }
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
				    void *buf)
{
  struct user_sve_header *header = (struct user_sve_header *) buf;
  char *base = (char *) buf;
  uint64_t vq = sve_vq_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  if (!HAS_SVE_STATE (*header))
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  Where possible we want to write the reg_buf data
	 back to the kernel using the fpsimd structure.  However, if we cannot
	 then we'll need to reformat the fpsimd into a full SVE structure,
	 resulting in the initialization of SVE state written back to the
	 kernel, which is why we try to avoid it.  */

      bool has_sve_state = false;
      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      /* Check in the reg_buf if any of the Z registers are set after the
	 first 128 bits, or if any of the other SVE registers are set.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
						 zero_reg, sizeof (__int128_t));
	  if (has_sve_state)
	    break;
	}

      if (!has_sve_state)
	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	  {
	    has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
						   zero_reg, 0);
	    if (has_sve_state)
	      break;
	  }

      if (!has_sve_state)
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
						 zero_reg, 0);

      /* If no SVE state exists, then use the existing fpsimd structure to
	 write out state and return.  */
      if (!has_sve_state)
	{
	  /* The collects of the Z registers will overflow the size of a vreg.
	     There is enough space in the structure to allow for this, but we
	     cannot overflow into the next register as we might not be
	     collecting every register.  */

	  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	    {
	      if (REG_VALID
		  == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
		{
		  reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
		  memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
		}
	    }

	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

	  return;
	}

      /* Otherwise, reformat the fpsimd structure into a full SVE set, by
	 expanding the V registers (working backwards so we don't splat
	 registers before they are copied) and using null for everything else.
	 Note that enough space for a full SVE dump was originally allocated
	 for base.  */

      header->flags |= SVE_PT_REGS_SVE;
      header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);

      memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
	      sizeof (uint32_t));
      memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
	      sizeof (uint32_t));

      for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
	{
	  memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
		  sizeof (__int128_t));
	}
    }

  /* Replace the kernel values with those from reg_buf.  */

  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
			    base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

  for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
			    base + SVE_PT_SVE_PREG_OFFSET (vq, i));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
			  base + SVE_PT_SVE_FFR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
			  base + SVE_PT_SVE_FPSR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
			  base + SVE_PT_SVE_FPCR_OFFSET (vq));

}
Commit	Line	Data
122394f1 AH	1	/* Common target dependent for AArch64 systems.
122394f1 AH	2
42a4f53d	3	Copyright (C) 2018-2019 Free Software Foundation, Inc.
122394f1 AH	4
	5	This file is part of GDB.
	6
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of the GNU General Public License as published by
	9	the Free Software Foundation; either version 3 of the License, or
	10	(at your option) any later version.
	11
	12	This program is distributed in the hope that it will be useful,
	13	but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	GNU General Public License for more details.
	16
	17	You should have received a copy of the GNU General Public License
	18	along with this program. If not, see <http://www.gnu.org/licenses/>. */
	19
	20	#include <sys/utsname.h>
	21	#include <sys/uio.h>
268a13a5	22	#include "gdbsupport/common-defs.h"
122394f1 AH	23	#include "elf/external.h"
	24	#include "elf/common.h"
	25	#include "aarch64-sve-linux-ptrace.h"
	26	#include "arch/aarch64.h"
268a13a5 TT	27	#include "gdbsupport/common-regcache.h"
268a13a5 TT	28	#include "gdbsupport/byte-vector.h"
e9902bfc	29
122394f1 AH	30	/* See nat/aarch64-sve-linux-ptrace.h. */
122394f1 AH	31
39bfb937	32	uint64_t
122394f1 AH	33	aarch64_sve_get_vq (int tid)
	34	{
	35	struct iovec iovec;
	36	struct user_sve_header header;
	37
	38	iovec.iov_len = sizeof (header);
	39	iovec.iov_base = &header;
	40
	41	/* Ptrace gives the vector length in bytes. Convert it to VQ, the number of
	42	128bit chunks in a Z register. We use VQ because 128bits is the minimum
	43	a Z register can increase in size. */
	44
	45	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	46	{
	47	/* SVE is not supported. */
	48	return 0;
	49	}
	50
e9902bfc	51	uint64_t vq = sve_vq_from_vl (header.vl);
122394f1 AH	52
	53	if (!sve_vl_valid (header.vl))
	54	{
	55	warning (_("Invalid SVE state from kernel; SVE disabled."));
	56	return 0;
	57	}
	58
	59	return vq;
	60	}
e9902bfc AH	61
	62	/* See nat/aarch64-sve-linux-ptrace.h. */
	63
48574d91 AH	64	bool
	65	aarch64_sve_set_vq (int tid, uint64_t vq)
	66	{
	67	struct iovec iovec;
	68	struct user_sve_header header;
	69
	70	iovec.iov_len = sizeof (header);
	71	iovec.iov_base = &header;
	72
	73	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	74	{
	75	/* SVE is not supported. */
	76	return false;
	77	}
	78
	79	header.vl = sve_vl_from_vq (vq);
	80
	81	if (ptrace (PTRACE_SETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	82	{
	83	/* Vector length change failed. */
	84	return false;
	85	}
	86
	87	return true;
	88	}
	89
	90	/* See nat/aarch64-sve-linux-ptrace.h. */
	91
	92	bool
	93	aarch64_sve_set_vq (int tid, struct reg_buffer_common *reg_buf)
	94	{
	95	if (reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM) != REG_VALID)
	96	return false;
	97
	98	uint64_t reg_vg = 0;
	99	reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &reg_vg);
	100
	101	return aarch64_sve_set_vq (tid, sve_vq_from_vg (reg_vg));
	102	}
	103
	104	/* See nat/aarch64-sve-linux-ptrace.h. */
	105
e9902bfc AH	106	std::unique_ptr<gdb_byte[]>
	107	aarch64_sve_get_sveregs (int tid)
	108	{
	109	struct iovec iovec;
e9902bfc AH	110	uint64_t vq = aarch64_sve_get_vq (tid);
	111
	112	if (vq == 0)
	113	perror_with_name (_("Unable to fetch SVE register header"));
	114
	115	/* A ptrace call with NT_ARM_SVE will return a header followed by either a
	116	dump of all the SVE and FP registers, or an fpsimd structure (identical to
	117	the one returned by NT_FPREGSET) if the kernel has not yet executed any
	118	SVE code. Make sure we allocate enough space for a full SVE dump. */
	119
	120	iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
	121	std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
	122	iovec.iov_base = buf.get ();
	123
	124	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	125	perror_with_name (_("Unable to fetch SVE registers"));
	126
	127	return buf;
	128	}
	129
	130	/* See nat/aarch64-sve-linux-ptrace.h. */
	131
	132	void
	133	aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
	134	const void *buf)
	135	{
	136	char base = (char ) buf;
	137	struct user_sve_header header = (struct user_sve_header ) buf;
e9902bfc	138
48574d91 AH	139	uint64_t vq = sve_vq_from_vl (header->vl);
48574d91 AH	140	uint64_t vg = sve_vg_from_vl (header->vl);
e9902bfc AH	141
	142	/* Sanity check the data in the header. */
	143	if (!sve_vl_valid (header->vl)
	144	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
	145	error (_("Invalid SVE header from kernel."));
	146
48574d91 AH	147	/* Update VG. Note, the registers in the regcache will already be of the
	148	correct length. */
	149	reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg);
e9902bfc AH	150
	151	if (HAS_SVE_STATE (*header))
	152	{
	153	/* The register dump contains a set of SVE registers. */
	154
	155	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	156	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
	157	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
	158
	159	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	160	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
	161	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
	162
	163	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
	164	base + SVE_PT_SVE_FFR_OFFSET (vq));
	165	reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
	166	base + SVE_PT_SVE_FPSR_OFFSET (vq));
	167	reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
	168	base + SVE_PT_SVE_FPCR_OFFSET (vq));
	169	}
	170	else
	171	{
	172	/* There is no SVE state yet - the register dump contains a fpsimd
	173	structure instead. These registers still exist in the hardware, but
	174	the kernel has not yet initialised them, and so they will be null. */
	175
	176	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
	177	struct user_fpsimd_state *fpsimd
	178	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
	179
	180	/* Copy across the V registers from fpsimd structure to the Z registers,
	181	ensuring the non overlapping state is set to null. */
	182
	183	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
	184
	185	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	186	{
	187	memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
	188	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	189	}
	190
	191	reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	192	reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
	193
	194	/* Clear the SVE only registers. */
	195
	196	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	197	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);
	198
	199	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
	200	}
	201	}
	202
	203	/* See nat/aarch64-sve-linux-ptrace.h. */
	204
	205	void
	206	aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
	207	void *buf)
	208	{
	209	struct user_sve_header header = (struct user_sve_header ) buf;
	210	char base = (char ) buf;
48574d91	211	uint64_t vq = sve_vq_from_vl (header->vl);
e9902bfc AH	212
	213	/* Sanity check the data in the header. */
	214	if (!sve_vl_valid (header->vl)
	215	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
	216	error (_("Invalid SVE header from kernel."));
	217
e9902bfc AH	218	if (!HAS_SVE_STATE (*header))
	219	{
	220	/* There is no SVE state yet - the register dump contains a fpsimd
	221	structure instead. Where possible we want to write the reg_buf data
	222	back to the kernel using the fpsimd structure. However, if we cannot
	223	then we'll need to reformat the fpsimd into a full SVE structure,
	224	resulting in the initialization of SVE state written back to the
	225	kernel, which is why we try to avoid it. */
	226
	227	bool has_sve_state = false;
	228	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
	229	struct user_fpsimd_state *fpsimd
	230	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
	231
	232	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
	233
	234	/* Check in the reg_buf if any of the Z registers are set after the
	235	first 128 bits, or if any of the other SVE registers are set. */
	236
	237	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	238	{
	239	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
	240	zero_reg, sizeof (__int128_t));
	241	if (has_sve_state)
	242	break;
	243	}
	244
	245	if (!has_sve_state)
	246	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	247	{
	248	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
	249	zero_reg, 0);
	250	if (has_sve_state)
	251	break;
	252	}
	253
	254	if (!has_sve_state)
	255	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
	256	zero_reg, 0);
	257
	258	/* If no SVE state exists, then use the existing fpsimd structure to
	259	write out state and return. */
	260	if (!has_sve_state)
	261	{
	262	/* The collects of the Z registers will overflow the size of a vreg.
	263	There is enough space in the structure to allow for this, but we
	264	cannot overflow into the next register as we might not be
	265	collecting every register. */
	266
	267	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	268	{
	269	if (REG_VALID
	270	== reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
	271	{
	272	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	273	memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
	274	}
	275	}
	276
	277	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
	278	reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	279	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
	280	reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
	281
282	return;
283	}
284
285	/* Otherwise, reformat the fpsimd structure into a full SVE set, by
286	expanding the V registers (working backwards so we don't splat
287	registers before they are copied) and using null for everything else.
288	Note that enough space for a full SVE dump was originally allocated
289	for base. */
290
291	header->flags \|= SVE_PT_REGS_SVE;
292	header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
293
294	memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
295	sizeof (uint32_t));
296	memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
297	sizeof (uint32_t));
298
299	for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
300	{
301	memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
302	sizeof (__int128_t));
303	}
304	}
305
306	/* Replace the kernel values with those from reg_buf. */
307
308	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
309	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
310	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
311	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
312
313	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
314	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
315	reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
316	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
317
318	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
319	reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
320	base + SVE_PT_SVE_FFR_OFFSET (vq));
321	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
322	reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
323	base + SVE_PT_SVE_FPSR_OFFSET (vq));
324	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
325	reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
326	base + SVE_PT_SVE_FPCR_OFFSET (vq));
327
328	}