[deliverable/binutils-gdb.git] / gdb / nat / aarch64-sve-linux-ptrace.c

/* Common target dependent for AArch64 systems.

   Copyright (C) 2018-2020 Free Software Foundation, Inc.

   This file is part of GDB.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

#include <sys/utsname.h>
#include <sys/uio.h>
#include "gdbsupport/common-defs.h"
#include "elf/external.h"
#include "elf/common.h"
#include "aarch64-sve-linux-ptrace.h"
#include "arch/aarch64.h"
#include "gdbsupport/common-regcache.h"
#include "gdbsupport/byte-vector.h"

/* See nat/aarch64-sve-linux-ptrace.h.  */

uint64_t
aarch64_sve_get_vq (int tid)
{
  struct iovec iovec;
  struct user_sve_header header;

  iovec.iov_len = sizeof (header);
  iovec.iov_base = &header;

  /* Ptrace gives the vector length in bytes.  Convert it to VQ, the number of
     128bit chunks in a Z register.  We use VQ because 128bits is the minimum
     a Z register can increase in size.  */

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* SVE is not supported.  */
      return 0;
    }

  uint64_t vq = sve_vq_from_vl (header.vl);

  if (!sve_vl_valid (header.vl))
    {
      warning (_("Invalid SVE state from kernel; SVE disabled."));
      return 0;
    }

  return vq;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

bool
aarch64_sve_set_vq (int tid, uint64_t vq)
{
  struct iovec iovec;
  struct user_sve_header header;

  iovec.iov_len = sizeof (header);
  iovec.iov_base = &header;

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* SVE is not supported.  */
      return false;
    }

  header.vl = sve_vl_from_vq (vq);

  if (ptrace (PTRACE_SETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    {
      /* Vector length change failed.  */
      return false;
    }

  return true;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

bool
aarch64_sve_set_vq (int tid, struct reg_buffer_common *reg_buf)
{
  uint64_t reg_vg = 0;

  /* The VG register may not be valid if we've not collected any value yet.
     This can happen, for example,  if we're restoring the regcache after an
     inferior function call, and the VG register comes after the Z
     registers.  */
  if (reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM) != REG_VALID)
  {
    /* If vg is not available yet, fetch it from ptrace.  The VG value from
       ptrace is likely the correct one.  */
    uint64_t vq = aarch64_sve_get_vq (tid);

    /* If something went wrong, just bail out.  */
    if (vq == 0)
      return false;

    reg_vg = sve_vg_from_vq (vq);
  }
  else
    reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &reg_vg);

  return aarch64_sve_set_vq (tid, sve_vq_from_vg (reg_vg));
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

std::unique_ptr<gdb_byte[]>
aarch64_sve_get_sveregs (int tid)
{
  struct iovec iovec;
  uint64_t vq = aarch64_sve_get_vq (tid);

  if (vq == 0)
    perror_with_name (_("Unable to fetch SVE register header"));

  /* A ptrace call with NT_ARM_SVE will return a header followed by either a
     dump of all the SVE and FP registers, or an fpsimd structure (identical to
     the one returned by NT_FPREGSET) if the kernel has not yet executed any
     SVE code.  Make sure we allocate enough space for a full SVE dump.  */

  iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
  std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
  iovec.iov_base = buf.get ();

  if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
    perror_with_name (_("Unable to fetch SVE registers"));

  return buf;
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
				  const void *buf)
{
  char *base = (char *) buf;
  struct user_sve_header *header = (struct user_sve_header *) buf;

  uint64_t vq = sve_vq_from_vl (header->vl);
  uint64_t vg = sve_vg_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  /* Update VG.  Note, the registers in the regcache will already be of the
     correct length.  */
  reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg);

  if (HAS_SVE_STATE (*header))
    {
      /* The register dump contains a set of SVE registers.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
			     base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
			     base + SVE_PT_SVE_PREG_OFFSET (vq, i));

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
			   base + SVE_PT_SVE_FFR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
			   base + SVE_PT_SVE_FPSR_OFFSET (vq));
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
			   base + SVE_PT_SVE_FPCR_OFFSET (vq));
    }
  else
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  These registers still exist in the hardware, but
	 the kernel has not yet initialised them, and so they will be null.  */

      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      /* Copy across the V registers from fpsimd structure to the Z registers,
	 ensuring the non overlapping state is set to null.  */

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
	  reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	}

      reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
      reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

      /* Clear the SVE only registers.  */

      for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);

      reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
    }
}

/* See nat/aarch64-sve-linux-ptrace.h.  */

void
aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
				    void *buf)
{
  struct user_sve_header *header = (struct user_sve_header *) buf;
  char *base = (char *) buf;
  uint64_t vq = sve_vq_from_vl (header->vl);

  /* Sanity check the data in the header.  */
  if (!sve_vl_valid (header->vl)
      || SVE_PT_SIZE (vq, header->flags) != header->size)
    error (_("Invalid SVE header from kernel."));

  if (!HAS_SVE_STATE (*header))
    {
      /* There is no SVE state yet - the register dump contains a fpsimd
	 structure instead.  Where possible we want to write the reg_buf data
	 back to the kernel using the fpsimd structure.  However, if we cannot
	 then we'll need to reformat the fpsimd into a full SVE structure,
	 resulting in the initialization of SVE state written back to the
	 kernel, which is why we try to avoid it.  */

      bool has_sve_state = false;
      char *zero_reg = (char *) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
      struct user_fpsimd_state *fpsimd
	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);

      memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));

      /* Check in the reg_buf if any of the Z registers are set after the
	 first 128 bits, or if any of the other SVE registers are set.  */

      for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	{
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
						 zero_reg, sizeof (__int128_t));
	  if (has_sve_state)
	    break;
	}

      if (!has_sve_state)
	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	  {
	    has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
						   zero_reg, 0);
	    if (has_sve_state)
	      break;
	  }

      if (!has_sve_state)
	  has_sve_state |= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
						 zero_reg, 0);

      /* If no SVE state exists, then use the existing fpsimd structure to
	 write out state and return.  */
      if (!has_sve_state)
	{
	  /* The collects of the Z registers will overflow the size of a vreg.
	     There is enough space in the structure to allow for this, but we
	     cannot overflow into the next register as we might not be
	     collecting every register.  */

	  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	    {
	      if (REG_VALID
		  == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
		{
		  reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
		  memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
		}
	    }

	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
	    reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);

	  return;
	}

      /* Otherwise, reformat the fpsimd structure into a full SVE set, by
	 expanding the V registers (working backwards so we don't splat
	 registers before they are copied) and using null for everything else.
	 Note that enough space for a full SVE dump was originally allocated
	 for base.  */

      header->flags |= SVE_PT_REGS_SVE;
      header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);

      memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
	      sizeof (uint32_t));
      memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
	      sizeof (uint32_t));

      for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
	{
	  memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
		  sizeof (__int128_t));
	}
    }

  /* Replace the kernel values with those from reg_buf.  */

  for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
			    base + SVE_PT_SVE_ZREG_OFFSET (vq, i));

  for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
    if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
      reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
			    base + SVE_PT_SVE_PREG_OFFSET (vq, i));

  if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
    reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
			  base + SVE_PT_SVE_FFR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
			  base + SVE_PT_SVE_FPSR_OFFSET (vq));
  if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
    reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
			  base + SVE_PT_SVE_FPCR_OFFSET (vq));

}
Commit	Line	Data
122394f1 AH	1	/* Common target dependent for AArch64 systems.
122394f1 AH	2
b811d2c2	3	Copyright (C) 2018-2020 Free Software Foundation, Inc.
122394f1 AH	4
	5	This file is part of GDB.
	6
	7	This program is free software; you can redistribute it and/or modify
	8	it under the terms of the GNU General Public License as published by
	9	the Free Software Foundation; either version 3 of the License, or
	10	(at your option) any later version.
	11
	12	This program is distributed in the hope that it will be useful,
	13	but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	GNU General Public License for more details.
	16
	17	You should have received a copy of the GNU General Public License
	18	along with this program. If not, see <http://www.gnu.org/licenses/>. */
	19
	20	#include <sys/utsname.h>
	21	#include <sys/uio.h>
268a13a5	22	#include "gdbsupport/common-defs.h"
122394f1 AH	23	#include "elf/external.h"
	24	#include "elf/common.h"
	25	#include "aarch64-sve-linux-ptrace.h"
	26	#include "arch/aarch64.h"
268a13a5 TT	27	#include "gdbsupport/common-regcache.h"
268a13a5 TT	28	#include "gdbsupport/byte-vector.h"
e9902bfc	29
122394f1 AH	30	/* See nat/aarch64-sve-linux-ptrace.h. */
122394f1 AH	31
39bfb937	32	uint64_t
122394f1 AH	33	aarch64_sve_get_vq (int tid)
	34	{
	35	struct iovec iovec;
	36	struct user_sve_header header;
	37
	38	iovec.iov_len = sizeof (header);
	39	iovec.iov_base = &header;
	40
	41	/* Ptrace gives the vector length in bytes. Convert it to VQ, the number of
	42	128bit chunks in a Z register. We use VQ because 128bits is the minimum
	43	a Z register can increase in size. */
	44
	45	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	46	{
	47	/* SVE is not supported. */
	48	return 0;
	49	}
	50
e9902bfc	51	uint64_t vq = sve_vq_from_vl (header.vl);
122394f1 AH	52
	53	if (!sve_vl_valid (header.vl))
	54	{
	55	warning (_("Invalid SVE state from kernel; SVE disabled."));
	56	return 0;
	57	}
	58
	59	return vq;
	60	}
e9902bfc AH	61
	62	/* See nat/aarch64-sve-linux-ptrace.h. */
	63
48574d91 AH	64	bool
	65	aarch64_sve_set_vq (int tid, uint64_t vq)
	66	{
	67	struct iovec iovec;
	68	struct user_sve_header header;
	69
	70	iovec.iov_len = sizeof (header);
	71	iovec.iov_base = &header;
	72
	73	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	74	{
	75	/* SVE is not supported. */
	76	return false;
	77	}
	78
	79	header.vl = sve_vl_from_vq (vq);
	80
	81	if (ptrace (PTRACE_SETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	82	{
	83	/* Vector length change failed. */
	84	return false;
	85	}
	86
	87	return true;
	88	}
	89
	90	/* See nat/aarch64-sve-linux-ptrace.h. */
	91
	92	bool
	93	aarch64_sve_set_vq (int tid, struct reg_buffer_common *reg_buf)
	94	{
2d07da27 LM	95	uint64_t reg_vg = 0;
	96
	97	/* The VG register may not be valid if we've not collected any value yet.
	98	This can happen, for example, if we're restoring the regcache after an
	99	inferior function call, and the VG register comes after the Z
	100	registers. */
48574d91	101	if (reg_buf->get_register_status (AARCH64_SVE_VG_REGNUM) != REG_VALID)
2d07da27 LM	102	{
	103	/* If vg is not available yet, fetch it from ptrace. The VG value from
	104	ptrace is likely the correct one. */
	105	uint64_t vq = aarch64_sve_get_vq (tid);
48574d91	106
2d07da27 LM	107	/* If something went wrong, just bail out. */
	108	if (vq == 0)
	109	return false;
	110
	111	reg_vg = sve_vg_from_vq (vq);
	112	}
	113	else
	114	reg_buf->raw_collect (AARCH64_SVE_VG_REGNUM, &reg_vg);
48574d91 AH	115
	116	return aarch64_sve_set_vq (tid, sve_vq_from_vg (reg_vg));
	117	}
	118
	119	/* See nat/aarch64-sve-linux-ptrace.h. */
	120
e9902bfc AH	121	std::unique_ptr<gdb_byte[]>
	122	aarch64_sve_get_sveregs (int tid)
	123	{
	124	struct iovec iovec;
e9902bfc AH	125	uint64_t vq = aarch64_sve_get_vq (tid);
	126
	127	if (vq == 0)
	128	perror_with_name (_("Unable to fetch SVE register header"));
	129
	130	/* A ptrace call with NT_ARM_SVE will return a header followed by either a
	131	dump of all the SVE and FP registers, or an fpsimd structure (identical to
	132	the one returned by NT_FPREGSET) if the kernel has not yet executed any
	133	SVE code. Make sure we allocate enough space for a full SVE dump. */
	134
	135	iovec.iov_len = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
	136	std::unique_ptr<gdb_byte[]> buf (new gdb_byte[iovec.iov_len]);
	137	iovec.iov_base = buf.get ();
	138
	139	if (ptrace (PTRACE_GETREGSET, tid, NT_ARM_SVE, &iovec) < 0)
	140	perror_with_name (_("Unable to fetch SVE registers"));
	141
	142	return buf;
	143	}
	144
	145	/* See nat/aarch64-sve-linux-ptrace.h. */
	146
	147	void
	148	aarch64_sve_regs_copy_to_reg_buf (struct reg_buffer_common *reg_buf,
	149	const void *buf)
	150	{
	151	char base = (char ) buf;
	152	struct user_sve_header header = (struct user_sve_header ) buf;
e9902bfc	153
48574d91 AH	154	uint64_t vq = sve_vq_from_vl (header->vl);
48574d91 AH	155	uint64_t vg = sve_vg_from_vl (header->vl);
e9902bfc AH	156
	157	/* Sanity check the data in the header. */
	158	if (!sve_vl_valid (header->vl)
	159	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
	160	error (_("Invalid SVE header from kernel."));
	161
48574d91 AH	162	/* Update VG. Note, the registers in the regcache will already be of the
	163	correct length. */
	164	reg_buf->raw_supply (AARCH64_SVE_VG_REGNUM, &vg);
e9902bfc AH	165
	166	if (HAS_SVE_STATE (*header))
	167	{
	168	/* The register dump contains a set of SVE registers. */
	169
	170	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	171	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i,
	172	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
	173
	174	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	175	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i,
	176	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
	177
	178	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM,
	179	base + SVE_PT_SVE_FFR_OFFSET (vq));
	180	reg_buf->raw_supply (AARCH64_FPSR_REGNUM,
	181	base + SVE_PT_SVE_FPSR_OFFSET (vq));
	182	reg_buf->raw_supply (AARCH64_FPCR_REGNUM,
	183	base + SVE_PT_SVE_FPCR_OFFSET (vq));
	184	}
	185	else
	186	{
	187	/* There is no SVE state yet - the register dump contains a fpsimd
	188	structure instead. These registers still exist in the hardware, but
	189	the kernel has not yet initialised them, and so they will be null. */
	190
	191	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
	192	struct user_fpsimd_state *fpsimd
	193	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
	194
	195	/* Copy across the V registers from fpsimd structure to the Z registers,
	196	ensuring the non overlapping state is set to null. */
	197
	198	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
	199
	200	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	201	{
	202	memcpy (zero_reg, &fpsimd->vregs[i], sizeof (__int128_t));
	203	reg_buf->raw_supply (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	204	}
	205
	206	reg_buf->raw_supply (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	207	reg_buf->raw_supply (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
	208
	209	/* Clear the SVE only registers. */
	210
	211	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	212	reg_buf->raw_supply (AARCH64_SVE_P0_REGNUM + i, zero_reg);
	213
	214	reg_buf->raw_supply (AARCH64_SVE_FFR_REGNUM, zero_reg);
	215	}
	216	}
	217
	218	/* See nat/aarch64-sve-linux-ptrace.h. */
	219
	220	void
	221	aarch64_sve_regs_copy_from_reg_buf (const struct reg_buffer_common *reg_buf,
	222	void *buf)
	223	{
	224	struct user_sve_header header = (struct user_sve_header ) buf;
	225	char base = (char ) buf;
48574d91	226	uint64_t vq = sve_vq_from_vl (header->vl);
e9902bfc AH	227
	228	/* Sanity check the data in the header. */
	229	if (!sve_vl_valid (header->vl)
	230	\|\| SVE_PT_SIZE (vq, header->flags) != header->size)
	231	error (_("Invalid SVE header from kernel."));
	232
e9902bfc AH	233	if (!HAS_SVE_STATE (*header))
	234	{
	235	/* There is no SVE state yet - the register dump contains a fpsimd
	236	structure instead. Where possible we want to write the reg_buf data
	237	back to the kernel using the fpsimd structure. However, if we cannot
	238	then we'll need to reformat the fpsimd into a full SVE structure,
	239	resulting in the initialization of SVE state written back to the
	240	kernel, which is why we try to avoid it. */
	241
	242	bool has_sve_state = false;
	243	char zero_reg = (char ) alloca (SVE_PT_SVE_ZREG_SIZE (vq));
	244	struct user_fpsimd_state *fpsimd
	245	= (struct user_fpsimd_state *)(base + SVE_PT_FPSIMD_OFFSET);
	246
	247	memset (zero_reg, 0, SVE_PT_SVE_ZREG_SIZE (vq));
	248
	249	/* Check in the reg_buf if any of the Z registers are set after the
	250	first 128 bits, or if any of the other SVE registers are set. */
	251
	252	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	253	{
	254	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_Z0_REGNUM + i,
	255	zero_reg, sizeof (__int128_t));
	256	if (has_sve_state)
	257	break;
	258	}
	259
	260	if (!has_sve_state)
	261	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
	262	{
	263	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_P0_REGNUM + i,
	264	zero_reg, 0);
	265	if (has_sve_state)
	266	break;
	267	}
	268
	269	if (!has_sve_state)
	270	has_sve_state \|= reg_buf->raw_compare (AARCH64_SVE_FFR_REGNUM,
	271	zero_reg, 0);
	272
	273	/* If no SVE state exists, then use the existing fpsimd structure to
	274	write out state and return. */
	275	if (!has_sve_state)
	276	{
	277	/* The collects of the Z registers will overflow the size of a vreg.
	278	There is enough space in the structure to allow for this, but we
	279	cannot overflow into the next register as we might not be
	280	collecting every register. */
	281
	282	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
	283	{
	284	if (REG_VALID
	285	== reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
	286	{
	287	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i, zero_reg);
	288	memcpy (&fpsimd->vregs[i], zero_reg, sizeof (__int128_t));
	289	}
	290	}
	291
	292	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
	293	reg_buf->raw_collect (AARCH64_FPSR_REGNUM, &fpsimd->fpsr);
	294	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
	295	reg_buf->raw_collect (AARCH64_FPCR_REGNUM, &fpsimd->fpcr);
	296
297	return;
298	}
299
300	/* Otherwise, reformat the fpsimd structure into a full SVE set, by
301	expanding the V registers (working backwards so we don't splat
302	registers before they are copied) and using null for everything else.
303	Note that enough space for a full SVE dump was originally allocated
304	for base. */
305
306	header->flags \|= SVE_PT_REGS_SVE;
307	header->size = SVE_PT_SIZE (vq, SVE_PT_REGS_SVE);
308
309	memcpy (base + SVE_PT_SVE_FPSR_OFFSET (vq), &fpsimd->fpsr,
310	sizeof (uint32_t));
311	memcpy (base + SVE_PT_SVE_FPCR_OFFSET (vq), &fpsimd->fpcr,
312	sizeof (uint32_t));
313
314	for (int i = AARCH64_SVE_Z_REGS_NUM; i >= 0 ; i--)
315	{
316	memcpy (base + SVE_PT_SVE_ZREG_OFFSET (vq, i), &fpsimd->vregs[i],
317	sizeof (__int128_t));
318	}
319	}
320
321	/* Replace the kernel values with those from reg_buf. */
322
323	for (int i = 0; i < AARCH64_SVE_Z_REGS_NUM; i++)
324	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_Z0_REGNUM + i))
325	reg_buf->raw_collect (AARCH64_SVE_Z0_REGNUM + i,
326	base + SVE_PT_SVE_ZREG_OFFSET (vq, i));
327
328	for (int i = 0; i < AARCH64_SVE_P_REGS_NUM; i++)
329	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_P0_REGNUM + i))
330	reg_buf->raw_collect (AARCH64_SVE_P0_REGNUM + i,
331	base + SVE_PT_SVE_PREG_OFFSET (vq, i));
332
333	if (REG_VALID == reg_buf->get_register_status (AARCH64_SVE_FFR_REGNUM))
334	reg_buf->raw_collect (AARCH64_SVE_FFR_REGNUM,
335	base + SVE_PT_SVE_FFR_OFFSET (vq));
336	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPSR_REGNUM))
337	reg_buf->raw_collect (AARCH64_FPSR_REGNUM,
338	base + SVE_PT_SVE_FPSR_OFFSET (vq));
339	if (REG_VALID == reg_buf->get_register_status (AARCH64_FPCR_REGNUM))
340	reg_buf->raw_collect (AARCH64_FPCR_REGNUM,
341	base + SVE_PT_SVE_FPCR_OFFSET (vq));
342
343	}