[deliverable/linux.git] / net / rxrpc / ar-output.c

/* RxRPC packet transmission
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/net.h>
#include <linux/gfp.h>
#include <linux/skbuff.h>
#include <linux/circ_buf.h>
#include <linux/export.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"

/*
 * Time till packet resend (in jiffies).
 */
unsigned int rxrpc_resend_timeout = 4 * HZ;

static int rxrpc_send_data(struct rxrpc_sock *rx,
			   struct rxrpc_call *call,
			   struct msghdr *msg, size_t len);

/*
 * extract control messages from the sendmsg() control buffer
 */
static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
			      unsigned long *user_call_ID,
			      enum rxrpc_command *command,
			      u32 *abort_code,
			      bool server)
{
	struct cmsghdr *cmsg;
	int len;

	*command = RXRPC_CMD_SEND_DATA;

	if (msg->msg_controllen == 0)
		return -EINVAL;

	for_each_cmsghdr(cmsg, msg) {
		if (!CMSG_OK(msg, cmsg))
			return -EINVAL;

		len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
		_debug("CMSG %d, %d, %d",
		       cmsg->cmsg_level, cmsg->cmsg_type, len);

		if (cmsg->cmsg_level != SOL_RXRPC)
			continue;

		switch (cmsg->cmsg_type) {
		case RXRPC_USER_CALL_ID:
			if (msg->msg_flags & MSG_CMSG_COMPAT) {
				if (len != sizeof(u32))
					return -EINVAL;
				*user_call_ID = *(u32 *) CMSG_DATA(cmsg);
			} else {
				if (len != sizeof(unsigned long))
					return -EINVAL;
				*user_call_ID = *(unsigned long *)
					CMSG_DATA(cmsg);
			}
			_debug("User Call ID %lx", *user_call_ID);
			break;

		case RXRPC_ABORT:
			if (*command != RXRPC_CMD_SEND_DATA)
				return -EINVAL;
			*command = RXRPC_CMD_SEND_ABORT;
			if (len != sizeof(*abort_code))
				return -EINVAL;
			*abort_code = *(unsigned int *) CMSG_DATA(cmsg);
			_debug("Abort %x", *abort_code);
			if (*abort_code == 0)
				return -EINVAL;
			break;

		case RXRPC_ACCEPT:
			if (*command != RXRPC_CMD_SEND_DATA)
				return -EINVAL;
			*command = RXRPC_CMD_ACCEPT;
			if (len != 0)
				return -EINVAL;
			if (!server)
				return -EISCONN;
			break;

		default:
			return -EINVAL;
		}
	}

	_leave(" = 0");
	return 0;
}

/*
 * abort a call, sending an ABORT packet to the peer
 */
static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
{
	write_lock_bh(&call->state_lock);

	if (call->state <= RXRPC_CALL_COMPLETE) {
		call->state = RXRPC_CALL_LOCALLY_ABORTED;
		call->abort_code = abort_code;
		set_bit(RXRPC_CALL_EV_ABORT, &call->events);
		del_timer_sync(&call->resend_timer);
		del_timer_sync(&call->ack_timer);
		clear_bit(RXRPC_CALL_EV_RESEND_TIMER, &call->events);
		clear_bit(RXRPC_CALL_EV_ACK, &call->events);
		clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
		rxrpc_queue_call(call);
	}

	write_unlock_bh(&call->state_lock);
}

/*
 * send a message forming part of a client call through an RxRPC socket
 * - caller holds the socket locked
 * - the socket may be either a client socket or a server socket
 */
int rxrpc_client_sendmsg(struct rxrpc_sock *rx, struct rxrpc_transport *trans,
			 struct msghdr *msg, size_t len)
{
	struct rxrpc_conn_bundle *bundle;
	enum rxrpc_command cmd;
	struct rxrpc_call *call;
	unsigned long user_call_ID = 0;
	struct key *key;
	u16 service_id;
	u32 abort_code = 0;
	int ret;

	_enter("");

	ASSERT(trans != NULL);

	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
				 false);
	if (ret < 0)
		return ret;

	bundle = NULL;
	if (trans) {
		service_id = rx->srx.srx_service;
		if (msg->msg_name) {
			DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
					 msg->msg_name);
			service_id = srx->srx_service;
		}
		key = rx->key;
		if (key && !rx->key->payload.data[0])
			key = NULL;
		bundle = rxrpc_get_bundle(rx, trans, key, service_id,
					  GFP_KERNEL);
		if (IS_ERR(bundle))
			return PTR_ERR(bundle);
	}

	call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
				     abort_code == 0, GFP_KERNEL);
	if (trans)
		rxrpc_put_bundle(trans, bundle);
	if (IS_ERR(call)) {
		_leave(" = %ld", PTR_ERR(call));
		return PTR_ERR(call);
	}

	_debug("CALL %d USR %lx ST %d on CONN %p",
	       call->debug_id, call->user_call_ID, call->state, call->conn);

	if (call->state >= RXRPC_CALL_COMPLETE) {
		/* it's too late for this call */
		ret = -ESHUTDOWN;
	} else if (cmd == RXRPC_CMD_SEND_ABORT) {
		rxrpc_send_abort(call, abort_code);
	} else if (cmd != RXRPC_CMD_SEND_DATA) {
		ret = -EINVAL;
	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
		/* request phase complete for this client call */
		ret = -EPROTO;
	} else {
		ret = rxrpc_send_data(rx, call, msg, len);
	}

	rxrpc_put_call(call);
	_leave(" = %d", ret);
	return ret;
}

/**
 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 * @call: The call to send data through
 * @msg: The data to send
 * @len: The amount of data to send
 *
 * Allow a kernel service to send data on a call.  The call must be in an state
 * appropriate to sending data.  No control data should be supplied in @msg,
 * nor should an address be supplied.  MSG_MORE should be flagged if there's
 * more data to come, otherwise this data will end the transmission phase.
 */
int rxrpc_kernel_send_data(struct rxrpc_call *call, struct msghdr *msg,
			   size_t len)
{
	int ret;

	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);

	ASSERTCMP(msg->msg_name, ==, NULL);
	ASSERTCMP(msg->msg_control, ==, NULL);

	lock_sock(&call->socket->sk);

	_debug("CALL %d USR %lx ST %d on CONN %p",
	       call->debug_id, call->user_call_ID, call->state, call->conn);

	if (call->state >= RXRPC_CALL_COMPLETE) {
		ret = -ESHUTDOWN; /* it's too late for this call */
	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
		   call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
		   call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
		ret = -EPROTO; /* request phase complete for this client call */
	} else {
		ret = rxrpc_send_data(call->socket, call, msg, len);
	}

	release_sock(&call->socket->sk);
	_leave(" = %d", ret);
	return ret;
}

EXPORT_SYMBOL(rxrpc_kernel_send_data);

/**
 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 * @call: The call to be aborted
 * @abort_code: The abort code to stick into the ABORT packet
 *
 * Allow a kernel service to abort a call, if it's still in an abortable state.
 */
void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
{
	_enter("{%d},%d", call->debug_id, abort_code);

	lock_sock(&call->socket->sk);

	_debug("CALL %d USR %lx ST %d on CONN %p",
	       call->debug_id, call->user_call_ID, call->state, call->conn);

	if (call->state < RXRPC_CALL_COMPLETE)
		rxrpc_send_abort(call, abort_code);

	release_sock(&call->socket->sk);
	_leave("");
}

EXPORT_SYMBOL(rxrpc_kernel_abort_call);

/*
 * send a message through a server socket
 * - caller holds the socket locked
 */
int rxrpc_server_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
{
	enum rxrpc_command cmd;
	struct rxrpc_call *call;
	unsigned long user_call_ID = 0;
	u32 abort_code = 0;
	int ret;

	_enter("");

	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
				 true);
	if (ret < 0)
		return ret;

	if (cmd == RXRPC_CMD_ACCEPT) {
		call = rxrpc_accept_call(rx, user_call_ID);
		if (IS_ERR(call))
			return PTR_ERR(call);
		rxrpc_put_call(call);
		return 0;
	}

	call = rxrpc_find_server_call(rx, user_call_ID);
	if (!call)
		return -EBADSLT;
	if (call->state >= RXRPC_CALL_COMPLETE) {
		ret = -ESHUTDOWN;
		goto out;
	}

	switch (cmd) {
	case RXRPC_CMD_SEND_DATA:
		if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
		    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
		    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
			/* Tx phase not yet begun for this call */
			ret = -EPROTO;
			break;
		}

		ret = rxrpc_send_data(rx, call, msg, len);
		break;

	case RXRPC_CMD_SEND_ABORT:
		rxrpc_send_abort(call, abort_code);
		break;
	default:
		BUG();
	}

	out:
	rxrpc_put_call(call);
	_leave(" = %d", ret);
	return ret;
}

/*
 * send a packet through the transport endpoint
 */
int rxrpc_send_packet(struct rxrpc_transport *trans, struct sk_buff *skb)
{
	struct kvec iov[1];
	struct msghdr msg;
	int ret, opt;

	_enter(",{%d}", skb->len);

	iov[0].iov_base = skb->head;
	iov[0].iov_len = skb->len;

	msg.msg_name = &trans->peer->srx.transport.sin;
	msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags = 0;

	/* send the packet with the don't fragment bit set if we currently
	 * think it's small enough */
	if (skb->len - sizeof(struct rxrpc_wire_header) < trans->peer->maxdata) {
		down_read(&trans->local->defrag_sem);
		/* send the packet by UDP
		 * - returns -EMSGSIZE if UDP would have to fragment the packet
		 *   to go out of the interface
		 *   - in which case, we'll have processed the ICMP error
		 *     message and update the peer record
		 */
		ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
				     iov[0].iov_len);

		up_read(&trans->local->defrag_sem);
		if (ret == -EMSGSIZE)
			goto send_fragmentable;

		_leave(" = %d [%u]", ret, trans->peer->maxdata);
		return ret;
	}

send_fragmentable:
	/* attempt to send this message with fragmentation enabled */
	_debug("send fragment");

	down_write(&trans->local->defrag_sem);
	opt = IP_PMTUDISC_DONT;
	ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
				(char *) &opt, sizeof(opt));
	if (ret == 0) {
		ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
				     iov[0].iov_len);

		opt = IP_PMTUDISC_DO;
		kernel_setsockopt(trans->local->socket, SOL_IP,
				  IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
	}

	up_write(&trans->local->defrag_sem);
	_leave(" = %d [frag %u]", ret, trans->peer->maxdata);
	return ret;
}

/*
 * wait for space to appear in the transmit/ACK window
 * - caller holds the socket locked
 */
static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
				    struct rxrpc_call *call,
				    long *timeo)
{
	DECLARE_WAITQUEUE(myself, current);
	int ret;

	_enter(",{%d},%ld",
	       CIRC_SPACE(call->acks_head, ACCESS_ONCE(call->acks_tail),
			  call->acks_winsz),
	       *timeo);

	add_wait_queue(&call->tx_waitq, &myself);

	for (;;) {
		set_current_state(TASK_INTERRUPTIBLE);
		ret = 0;
		if (CIRC_SPACE(call->acks_head, ACCESS_ONCE(call->acks_tail),
			       call->acks_winsz) > 0)
			break;
		if (signal_pending(current)) {
			ret = sock_intr_errno(*timeo);
			break;
		}

		release_sock(&rx->sk);
		*timeo = schedule_timeout(*timeo);
		lock_sock(&rx->sk);
	}

	remove_wait_queue(&call->tx_waitq, &myself);
	set_current_state(TASK_RUNNING);
	_leave(" = %d", ret);
	return ret;
}

/*
 * attempt to schedule an instant Tx resend
 */
static inline void rxrpc_instant_resend(struct rxrpc_call *call)
{
	read_lock_bh(&call->state_lock);
	if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
		clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
		if (call->state < RXRPC_CALL_COMPLETE &&
		    !test_and_set_bit(RXRPC_CALL_EV_RESEND_TIMER, &call->events))
			rxrpc_queue_call(call);
	}
	read_unlock_bh(&call->state_lock);
}

/*
 * queue a packet for transmission, set the resend timer and attempt
 * to send the packet immediately
 */
static void rxrpc_queue_packet(struct rxrpc_call *call, struct sk_buff *skb,
			       bool last)
{
	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
	int ret;

	_net("queue skb %p [%d]", skb, call->acks_head);

	ASSERT(call->acks_window != NULL);
	call->acks_window[call->acks_head] = (unsigned long) skb;
	smp_wmb();
	call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);

	if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
		_debug("________awaiting reply/ACK__________");
		write_lock_bh(&call->state_lock);
		switch (call->state) {
		case RXRPC_CALL_CLIENT_SEND_REQUEST:
			call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
			break;
		case RXRPC_CALL_SERVER_ACK_REQUEST:
			call->state = RXRPC_CALL_SERVER_SEND_REPLY;
			if (!last)
				break;
		case RXRPC_CALL_SERVER_SEND_REPLY:
			call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
			break;
		default:
			break;
		}
		write_unlock_bh(&call->state_lock);
	}

	_proto("Tx DATA %%%u { #%u }", sp->hdr.serial, sp->hdr.seq);

	sp->need_resend = false;
	sp->resend_at = jiffies + rxrpc_resend_timeout;
	if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
		_debug("run timer");
		call->resend_timer.expires = sp->resend_at;
		add_timer(&call->resend_timer);
	}

	/* attempt to cancel the rx-ACK timer, deferring reply transmission if
	 * we're ACK'ing the request phase of an incoming call */
	ret = -EAGAIN;
	if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
		/* the packet may be freed by rxrpc_process_call() before this
		 * returns */
		ret = rxrpc_send_packet(call->conn->trans, skb);
		_net("sent skb %p", skb);
	} else {
		_debug("failed to delete ACK timer");
	}

	if (ret < 0) {
		_debug("need instant resend %d", ret);
		sp->need_resend = true;
		rxrpc_instant_resend(call);
	}

	_leave("");
}

/*
 * Convert a host-endian header into a network-endian header.
 */
static void rxrpc_insert_header(struct sk_buff *skb)
{
	struct rxrpc_wire_header whdr;
	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

	whdr.epoch	= htonl(sp->hdr.epoch);
	whdr.cid	= htonl(sp->hdr.cid);
	whdr.callNumber	= htonl(sp->hdr.callNumber);
	whdr.seq	= htonl(sp->hdr.seq);
	whdr.serial	= htonl(sp->hdr.serial);
	whdr.type	= sp->hdr.type;
	whdr.flags	= sp->hdr.flags;
	whdr.userStatus	= sp->hdr.userStatus;
	whdr.securityIndex = sp->hdr.securityIndex;
	whdr._rsvd	= htons(sp->hdr._rsvd);
	whdr.serviceId	= htons(sp->hdr.serviceId);

	memcpy(skb->head, &whdr, sizeof(whdr));
}

/*
 * send data through a socket
 * - must be called in process context
 * - caller holds the socket locked
 */
static int rxrpc_send_data(struct rxrpc_sock *rx,
			   struct rxrpc_call *call,
			   struct msghdr *msg, size_t len)
{
	struct rxrpc_skb_priv *sp;
	struct sk_buff *skb;
	struct sock *sk = &rx->sk;
	long timeo;
	bool more;
	int ret, copied;

	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

	/* this should be in poll */
	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);

	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
		return -EPIPE;

	more = msg->msg_flags & MSG_MORE;

	skb = call->tx_pending;
	call->tx_pending = NULL;

	copied = 0;
	do {
		if (!skb) {
			size_t size, chunk, max, space;

			_debug("alloc");

			if (CIRC_SPACE(call->acks_head,
				       ACCESS_ONCE(call->acks_tail),
				       call->acks_winsz) <= 0) {
				ret = -EAGAIN;
				if (msg->msg_flags & MSG_DONTWAIT)
					goto maybe_error;
				ret = rxrpc_wait_for_tx_window(rx, call,
							       &timeo);
				if (ret < 0)
					goto maybe_error;
			}

			max = call->conn->trans->peer->maxdata;
			max -= call->conn->security_size;
			max &= ~(call->conn->size_align - 1UL);

			chunk = max;
			if (chunk > msg_data_left(msg) && !more)
				chunk = msg_data_left(msg);

			space = chunk + call->conn->size_align;
			space &= ~(call->conn->size_align - 1UL);

			size = space + call->conn->header_size;

			_debug("SIZE: %zu/%zu/%zu", chunk, space, size);

			/* create a buffer that we can retain until it's ACK'd */
			skb = sock_alloc_send_skb(
				sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
			if (!skb)
				goto maybe_error;

			rxrpc_new_skb(skb);

			_debug("ALLOC SEND %p", skb);

			ASSERTCMP(skb->mark, ==, 0);

			_debug("HS: %u", call->conn->header_size);
			skb_reserve(skb, call->conn->header_size);
			skb->len += call->conn->header_size;

			sp = rxrpc_skb(skb);
			sp->remain = chunk;
			if (sp->remain > skb_tailroom(skb))
				sp->remain = skb_tailroom(skb);

			_net("skb: hr %d, tr %d, hl %d, rm %d",
			       skb_headroom(skb),
			       skb_tailroom(skb),
			       skb_headlen(skb),
			       sp->remain);

			skb->ip_summed = CHECKSUM_UNNECESSARY;
		}

		_debug("append");
		sp = rxrpc_skb(skb);

		/* append next segment of data to the current buffer */
		if (msg_data_left(msg) > 0) {
			int copy = skb_tailroom(skb);
			ASSERTCMP(copy, >, 0);
			if (copy > msg_data_left(msg))
				copy = msg_data_left(msg);
			if (copy > sp->remain)
				copy = sp->remain;

			_debug("add");
			ret = skb_add_data(skb, &msg->msg_iter, copy);
			_debug("added");
			if (ret < 0)
				goto efault;
			sp->remain -= copy;
			skb->mark += copy;
			copied += copy;
		}

		/* check for the far side aborting the call or a network error
		 * occurring */
		if (call->state > RXRPC_CALL_COMPLETE)
			goto call_aborted;

		/* add the packet to the send queue if it's now full */
		if (sp->remain <= 0 ||
		    (msg_data_left(msg) == 0 && !more)) {
			struct rxrpc_connection *conn = call->conn;
			uint32_t seq;
			size_t pad;

			/* pad out if we're using security */
			if (conn->security) {
				pad = conn->security_size + skb->mark;
				pad = conn->size_align - pad;
				pad &= conn->size_align - 1;
				_debug("pad %zu", pad);
				if (pad)
					memset(skb_put(skb, pad), 0, pad);
			}

			seq = atomic_inc_return(&call->sequence);

			sp->hdr.epoch	= conn->epoch;
			sp->hdr.cid	= call->cid;
			sp->hdr.callNumber = call->call_id;
			sp->hdr.seq	= seq;
			sp->hdr.serial	= atomic_inc_return(&conn->serial);
			sp->hdr.type	= RXRPC_PACKET_TYPE_DATA;
			sp->hdr.userStatus = 0;
			sp->hdr.securityIndex = conn->security_ix;
			sp->hdr._rsvd	= 0;
			sp->hdr.serviceId = call->service_id;

			sp->hdr.flags = conn->out_clientflag;
			if (msg_data_left(msg) == 0 && !more)
				sp->hdr.flags |= RXRPC_LAST_PACKET;
			else if (CIRC_SPACE(call->acks_head,
					    ACCESS_ONCE(call->acks_tail),
					    call->acks_winsz) > 1)
				sp->hdr.flags |= RXRPC_MORE_PACKETS;
			if (more && seq & 1)
				sp->hdr.flags |= RXRPC_REQUEST_ACK;

			ret = rxrpc_secure_packet(
				call, skb, skb->mark,
				skb->head + sizeof(struct rxrpc_wire_header));
			if (ret < 0)
				goto out;

			rxrpc_insert_header(skb);
			rxrpc_queue_packet(call, skb, !msg_data_left(msg) && !more);
			skb = NULL;
		}
	} while (msg_data_left(msg) > 0);

success:
	ret = copied;
out:
	call->tx_pending = skb;
	_leave(" = %d", ret);
	return ret;

call_aborted:
	rxrpc_free_skb(skb);
	if (call->state == RXRPC_CALL_NETWORK_ERROR)
		ret = call->conn->trans->peer->net_error;
	else
		ret = -ECONNABORTED;
	_leave(" = %d", ret);
	return ret;

maybe_error:
	if (copied)
		goto success;
	goto out;

efault:
	ret = -EFAULT;
	goto out;
}
Commit	Line	Data
17926a79 DH	1	/* RxRPC packet transmission
	2	*
	3	* Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
	4	* Written by David Howells (dhowells@redhat.com)
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/net.h>
5a0e3ad6	13	#include <linux/gfp.h>
17926a79 DH	14	#include <linux/skbuff.h>
17926a79 DH	15	#include <linux/circ_buf.h>
bc3b2d7f	16	#include <linux/export.h>
17926a79 DH	17	#include <net/sock.h>
	18	#include <net/af_rxrpc.h>
	19	#include "ar-internal.h"
	20
5873c083 DH	21	/*
	22	* Time till packet resend (in jiffies).
	23	*/
dad8aff7	24	unsigned int rxrpc_resend_timeout = 4 * HZ;
17926a79	25
1b784140	26	static int rxrpc_send_data(struct rxrpc_sock *rx,
17926a79 DH	27	struct rxrpc_call *call,
	28	struct msghdr *msg, size_t len);
	29
	30	/*
	31	* extract control messages from the sendmsg() control buffer
	32	*/
	33	static int rxrpc_sendmsg_cmsg(struct rxrpc_sock rx, struct msghdr msg,
	34	unsigned long *user_call_ID,
	35	enum rxrpc_command *command,
	36	u32 *abort_code,
	37	bool server)
	38	{
	39	struct cmsghdr *cmsg;
	40	int len;
	41
	42	*command = RXRPC_CMD_SEND_DATA;
	43
	44	if (msg->msg_controllen == 0)
	45	return -EINVAL;
	46
f95b414e	47	for_each_cmsghdr(cmsg, msg) {
17926a79 DH	48	if (!CMSG_OK(msg, cmsg))
	49	return -EINVAL;
	50
	51	len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
	52	_debug("CMSG %d, %d, %d",
	53	cmsg->cmsg_level, cmsg->cmsg_type, len);
	54
	55	if (cmsg->cmsg_level != SOL_RXRPC)
	56	continue;
	57
	58	switch (cmsg->cmsg_type) {
	59	case RXRPC_USER_CALL_ID:
	60	if (msg->msg_flags & MSG_CMSG_COMPAT) {
	61	if (len != sizeof(u32))
	62	return -EINVAL;
	63	user_call_ID = (u32 *) CMSG_DATA(cmsg);
	64	} else {
	65	if (len != sizeof(unsigned long))
	66	return -EINVAL;
	67	user_call_ID = (unsigned long *)
	68	CMSG_DATA(cmsg);
	69	}
	70	_debug("User Call ID %lx", *user_call_ID);
	71	break;
	72
	73	case RXRPC_ABORT:
	74	if (*command != RXRPC_CMD_SEND_DATA)
	75	return -EINVAL;
	76	*command = RXRPC_CMD_SEND_ABORT;
	77	if (len != sizeof(*abort_code))
	78	return -EINVAL;
	79	abort_code = (unsigned int *) CMSG_DATA(cmsg);
	80	_debug("Abort %x", *abort_code);
	81	if (*abort_code == 0)
	82	return -EINVAL;
	83	break;
	84
	85	case RXRPC_ACCEPT:
	86	if (*command != RXRPC_CMD_SEND_DATA)
	87	return -EINVAL;
	88	*command = RXRPC_CMD_ACCEPT;
	89	if (len != 0)
	90	return -EINVAL;
	91	if (!server)
	92	return -EISCONN;
	93	break;
	94
	95	default:
	96	return -EINVAL;
	97	}
	98	}
	99
	100	_leave(" = 0");
	101	return 0;
	102	}
	103
	104	/*
	105	* abort a call, sending an ABORT packet to the peer
	106	*/
	107	static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
	108	{
	109	write_lock_bh(&call->state_lock);
	110
	111	if (call->state <= RXRPC_CALL_COMPLETE) {
112	call->state = RXRPC_CALL_LOCALLY_ABORTED;
113	call->abort_code = abort_code;
4c198ad1	114	set_bit(RXRPC_CALL_EV_ABORT, &call->events);
17926a79 DH	115	del_timer_sync(&call->resend_timer);
17926a79 DH	116	del_timer_sync(&call->ack_timer);
4c198ad1 DH	117	clear_bit(RXRPC_CALL_EV_RESEND_TIMER, &call->events);
4c198ad1 DH	118	clear_bit(RXRPC_CALL_EV_ACK, &call->events);
17926a79	119	clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
651350d1	120	rxrpc_queue_call(call);
17926a79 DH	121	}
	122
	123	write_unlock_bh(&call->state_lock);
	124	}
	125
	126	/*
	127	* send a message forming part of a client call through an RxRPC socket
	128	* - caller holds the socket locked
	129	* - the socket may be either a client socket or a server socket
	130	*/
1b784140 YX	131	int rxrpc_client_sendmsg(struct rxrpc_sock rx, struct rxrpc_transport trans,
1b784140 YX	132	struct msghdr *msg, size_t len)
17926a79 DH	133	{
	134	struct rxrpc_conn_bundle *bundle;
	135	enum rxrpc_command cmd;
	136	struct rxrpc_call *call;
	137	unsigned long user_call_ID = 0;
	138	struct key *key;
0d12f8a4	139	u16 service_id;
17926a79 DH	140	u32 abort_code = 0;
	141	int ret;
	142
	143	_enter("");
	144
	145	ASSERT(trans != NULL);
	146
	147	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
	148	false);
	149	if (ret < 0)
	150	return ret;
	151
	152	bundle = NULL;
	153	if (trans) {
0d12f8a4	154	service_id = rx->srx.srx_service;
17926a79	155	if (msg->msg_name) {
342dfc30 SH	156	DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
342dfc30 SH	157	msg->msg_name);
0d12f8a4	158	service_id = srx->srx_service;
17926a79 DH	159	}
17926a79 DH	160	key = rx->key;
146aa8b1	161	if (key && !rx->key->payload.data[0])
17926a79 DH	162	key = NULL;
	163	bundle = rxrpc_get_bundle(rx, trans, key, service_id,
	164	GFP_KERNEL);
	165	if (IS_ERR(bundle))
	166	return PTR_ERR(bundle);
	167	}
	168
	169	call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
	170	abort_code == 0, GFP_KERNEL);
	171	if (trans)
	172	rxrpc_put_bundle(trans, bundle);
	173	if (IS_ERR(call)) {
	174	_leave(" = %ld", PTR_ERR(call));
	175	return PTR_ERR(call);
	176	}
	177
	178	_debug("CALL %d USR %lx ST %d on CONN %p",
	179	call->debug_id, call->user_call_ID, call->state, call->conn);
	180
	181	if (call->state >= RXRPC_CALL_COMPLETE) {
	182	/* it's too late for this call */
	183	ret = -ESHUTDOWN;
	184	} else if (cmd == RXRPC_CMD_SEND_ABORT) {
	185	rxrpc_send_abort(call, abort_code);
	186	} else if (cmd != RXRPC_CMD_SEND_DATA) {
	187	ret = -EINVAL;
	188	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
	189	/* request phase complete for this client call */
	190	ret = -EPROTO;
	191	} else {
1b784140	192	ret = rxrpc_send_data(rx, call, msg, len);
17926a79 DH	193	}
	194
	195	rxrpc_put_call(call);
	196	_leave(" = %d", ret);
	197	return ret;
	198	}
	199
651350d1 DH	200	/**
	201	* rxrpc_kernel_send_data - Allow a kernel service to send data on a call
	202	* @call: The call to send data through
	203	* @msg: The data to send
	204	* @len: The amount of data to send
	205	*
	206	* Allow a kernel service to send data on a call. The call must be in an state
	207	* appropriate to sending data. No control data should be supplied in @msg,
	208	* nor should an address be supplied. MSG_MORE should be flagged if there's
	209	* more data to come, otherwise this data will end the transmission phase.
	210	*/
	211	int rxrpc_kernel_send_data(struct rxrpc_call call, struct msghdr msg,
	212	size_t len)
	213	{
	214	int ret;
	215
	216	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);
	217
	218	ASSERTCMP(msg->msg_name, ==, NULL);
	219	ASSERTCMP(msg->msg_control, ==, NULL);
	220
	221	lock_sock(&call->socket->sk);
	222
	223	_debug("CALL %d USR %lx ST %d on CONN %p",
	224	call->debug_id, call->user_call_ID, call->state, call->conn);
	225
	226	if (call->state >= RXRPC_CALL_COMPLETE) {
	227	ret = -ESHUTDOWN; /* it's too late for this call */
	228	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
	229	call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
	230	call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
	231	ret = -EPROTO; /* request phase complete for this client call */
	232	} else {
1b784140	233	ret = rxrpc_send_data(call->socket, call, msg, len);
651350d1 DH	234	}
	235
	236	release_sock(&call->socket->sk);
	237	_leave(" = %d", ret);
	238	return ret;
	239	}
	240
	241	EXPORT_SYMBOL(rxrpc_kernel_send_data);
	242
2c53040f	243	/**
651350d1 DH	244	* rxrpc_kernel_abort_call - Allow a kernel service to abort a call
	245	* @call: The call to be aborted
	246	* @abort_code: The abort code to stick into the ABORT packet
	247	*
	248	* Allow a kernel service to abort a call, if it's still in an abortable state.
	249	*/
	250	void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
	251	{
	252	_enter("{%d},%d", call->debug_id, abort_code);
	253
	254	lock_sock(&call->socket->sk);
	255
	256	_debug("CALL %d USR %lx ST %d on CONN %p",
	257	call->debug_id, call->user_call_ID, call->state, call->conn);
	258
	259	if (call->state < RXRPC_CALL_COMPLETE)
	260	rxrpc_send_abort(call, abort_code);
	261
	262	release_sock(&call->socket->sk);
	263	_leave("");
	264	}
	265
	266	EXPORT_SYMBOL(rxrpc_kernel_abort_call);
	267
17926a79 DH	268	/*
	269	* send a message through a server socket
	270	* - caller holds the socket locked
	271	*/
1b784140	272	int rxrpc_server_sendmsg(struct rxrpc_sock rx, struct msghdr msg, size_t len)
17926a79 DH	273	{
	274	enum rxrpc_command cmd;
	275	struct rxrpc_call *call;
	276	unsigned long user_call_ID = 0;
	277	u32 abort_code = 0;
	278	int ret;
	279
	280	_enter("");
	281
	282	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
	283	true);
	284	if (ret < 0)
	285	return ret;
	286
651350d1 DH	287	if (cmd == RXRPC_CMD_ACCEPT) {
	288	call = rxrpc_accept_call(rx, user_call_ID);
	289	if (IS_ERR(call))
	290	return PTR_ERR(call);
	291	rxrpc_put_call(call);
	292	return 0;
	293	}
17926a79 DH	294
	295	call = rxrpc_find_server_call(rx, user_call_ID);
	296	if (!call)
	297	return -EBADSLT;
	298	if (call->state >= RXRPC_CALL_COMPLETE) {
	299	ret = -ESHUTDOWN;
	300	goto out;
	301	}
	302
	303	switch (cmd) {
	304	case RXRPC_CMD_SEND_DATA:
	305	if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
	306	call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
	307	call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
	308	/* Tx phase not yet begun for this call */
	309	ret = -EPROTO;
	310	break;
	311	}
	312
1b784140	313	ret = rxrpc_send_data(rx, call, msg, len);
17926a79 DH	314	break;
	315
	316	case RXRPC_CMD_SEND_ABORT:
	317	rxrpc_send_abort(call, abort_code);
	318	break;
	319	default:
	320	BUG();
	321	}
	322
	323	out:
	324	rxrpc_put_call(call);
	325	_leave(" = %d", ret);
	326	return ret;
	327	}
	328
	329	/*
	330	* send a packet through the transport endpoint
	331	*/
	332	int rxrpc_send_packet(struct rxrpc_transport trans, struct sk_buff skb)
	333	{
	334	struct kvec iov[1];
	335	struct msghdr msg;
	336	int ret, opt;
	337
	338	_enter(",{%d}", skb->len);
	339
	340	iov[0].iov_base = skb->head;
	341	iov[0].iov_len = skb->len;
	342
	343	msg.msg_name = &trans->peer->srx.transport.sin;
	344	msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
	345	msg.msg_control = NULL;
	346	msg.msg_controllen = 0;
	347	msg.msg_flags = 0;
	348
	349	/* send the packet with the don't fragment bit set if we currently
	350	* think it's small enough */
0d12f8a4	351	if (skb->len - sizeof(struct rxrpc_wire_header) < trans->peer->maxdata) {
17926a79 DH	352	down_read(&trans->local->defrag_sem);
	353	/* send the packet by UDP
	354	* - returns -EMSGSIZE if UDP would have to fragment the packet
	355	* to go out of the interface
	356	* - in which case, we'll have processed the ICMP error
	357	* message and update the peer record
	358	*/
	359	ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
	360	iov[0].iov_len);
	361
	362	up_read(&trans->local->defrag_sem);
	363	if (ret == -EMSGSIZE)
	364	goto send_fragmentable;
	365
	366	_leave(" = %d [%u]", ret, trans->peer->maxdata);
	367	return ret;
	368	}
	369
	370	send_fragmentable:
	371	/* attempt to send this message with fragmentation enabled */
	372	_debug("send fragment");
	373
	374	down_write(&trans->local->defrag_sem);
	375	opt = IP_PMTUDISC_DONT;
	376	ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
	377	(char *) &opt, sizeof(opt));
	378	if (ret == 0) {
	379	ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
	380	iov[0].iov_len);
	381
	382	opt = IP_PMTUDISC_DO;
	383	kernel_setsockopt(trans->local->socket, SOL_IP,
	384	IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
	385	}
	386
	387	up_write(&trans->local->defrag_sem);
	388	_leave(" = %d [frag %u]", ret, trans->peer->maxdata);
	389	return ret;
	390	}
	391
	392	/*
	393	* wait for space to appear in the transmit/ACK window
	394	* - caller holds the socket locked
	395	*/
	396	static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
	397	struct rxrpc_call *call,
	398	long *timeo)
	399	{
	400	DECLARE_WAITQUEUE(myself, current);
	401	int ret;
	402
	403	_enter(",{%d},%ld",
ee72b9fd DH	404	CIRC_SPACE(call->acks_head, ACCESS_ONCE(call->acks_tail),
ee72b9fd DH	405	call->acks_winsz),
17926a79 DH	406	*timeo);
	407
	408	add_wait_queue(&call->tx_waitq, &myself);
	409
	410	for (;;) {
	411	set_current_state(TASK_INTERRUPTIBLE);
	412	ret = 0;
ee72b9fd	413	if (CIRC_SPACE(call->acks_head, ACCESS_ONCE(call->acks_tail),
17926a79 DH	414	call->acks_winsz) > 0)
	415	break;
	416	if (signal_pending(current)) {
	417	ret = sock_intr_errno(*timeo);
	418	break;
	419	}
	420
	421	release_sock(&rx->sk);
	422	timeo = schedule_timeout(timeo);
	423	lock_sock(&rx->sk);
	424	}
	425
	426	remove_wait_queue(&call->tx_waitq, &myself);
	427	set_current_state(TASK_RUNNING);
	428	_leave(" = %d", ret);
	429	return ret;
	430	}
	431
	432	/*
	433	* attempt to schedule an instant Tx resend
	434	*/
	435	static inline void rxrpc_instant_resend(struct rxrpc_call *call)
	436	{
	437	read_lock_bh(&call->state_lock);
	438	if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
	439	clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
	440	if (call->state < RXRPC_CALL_COMPLETE &&
4c198ad1	441	!test_and_set_bit(RXRPC_CALL_EV_RESEND_TIMER, &call->events))
651350d1	442	rxrpc_queue_call(call);
17926a79 DH	443	}
	444	read_unlock_bh(&call->state_lock);
	445	}
	446
	447	/*
	448	* queue a packet for transmission, set the resend timer and attempt
	449	* to send the packet immediately
	450	*/
	451	static void rxrpc_queue_packet(struct rxrpc_call call, struct sk_buff skb,
	452	bool last)
	453	{
	454	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
	455	int ret;
	456
	457	_net("queue skb %p [%d]", skb, call->acks_head);
	458
	459	ASSERT(call->acks_window != NULL);
	460	call->acks_window[call->acks_head] = (unsigned long) skb;
	461	smp_wmb();
	462	call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);
	463
	464	if (last \|\| call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
	465	_debug("________awaiting reply/ACK__________");
	466	write_lock_bh(&call->state_lock);
	467	switch (call->state) {
	468	case RXRPC_CALL_CLIENT_SEND_REQUEST:
	469	call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
	470	break;
	471	case RXRPC_CALL_SERVER_ACK_REQUEST:
	472	call->state = RXRPC_CALL_SERVER_SEND_REPLY;
	473	if (!last)
	474	break;
	475	case RXRPC_CALL_SERVER_SEND_REPLY:
	476	call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
	477	break;
	478	default:
	479	break;
	480	}
	481	write_unlock_bh(&call->state_lock);
	482	}
	483
0d12f8a4	484	_proto("Tx DATA %%%u { #%u }", sp->hdr.serial, sp->hdr.seq);
17926a79	485
3db1cd5c	486	sp->need_resend = false;
5873c083	487	sp->resend_at = jiffies + rxrpc_resend_timeout;
17926a79 DH	488	if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
	489	_debug("run timer");
	490	call->resend_timer.expires = sp->resend_at;
	491	add_timer(&call->resend_timer);
	492	}
	493
	494	/* attempt to cancel the rx-ACK timer, deferring reply transmission if
	495	* we're ACK'ing the request phase of an incoming call */
	496	ret = -EAGAIN;
	497	if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
	498	/* the packet may be freed by rxrpc_process_call() before this
	499	* returns */
	500	ret = rxrpc_send_packet(call->conn->trans, skb);
	501	_net("sent skb %p", skb);
	502	} else {
	503	_debug("failed to delete ACK timer");
	504	}
	505
	506	if (ret < 0) {
	507	_debug("need instant resend %d", ret);
3db1cd5c	508	sp->need_resend = true;
17926a79 DH	509	rxrpc_instant_resend(call);
	510	}
	511
	512	_leave("");
	513	}
	514
0d12f8a4 DH	515	/*
	516	* Convert a host-endian header into a network-endian header.
	517	*/
	518	static void rxrpc_insert_header(struct sk_buff *skb)
	519	{
	520	struct rxrpc_wire_header whdr;
	521	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
	522
	523	whdr.epoch = htonl(sp->hdr.epoch);
	524	whdr.cid = htonl(sp->hdr.cid);
	525	whdr.callNumber = htonl(sp->hdr.callNumber);
	526	whdr.seq = htonl(sp->hdr.seq);
	527	whdr.serial = htonl(sp->hdr.serial);
	528	whdr.type = sp->hdr.type;
	529	whdr.flags = sp->hdr.flags;
	530	whdr.userStatus = sp->hdr.userStatus;
	531	whdr.securityIndex = sp->hdr.securityIndex;
	532	whdr._rsvd = htons(sp->hdr._rsvd);
	533	whdr.serviceId = htons(sp->hdr.serviceId);
	534
	535	memcpy(skb->head, &whdr, sizeof(whdr));
	536	}
	537
17926a79 DH	538	/*
	539	* send data through a socket
	540	* - must be called in process context
	541	* - caller holds the socket locked
	542	*/
1b784140	543	static int rxrpc_send_data(struct rxrpc_sock *rx,
17926a79 DH	544	struct rxrpc_call *call,
	545	struct msghdr *msg, size_t len)
	546	{
	547	struct rxrpc_skb_priv *sp;
17926a79	548	struct sk_buff *skb;
17926a79 DH	549	struct sock *sk = &rx->sk;
	550	long timeo;
	551	bool more;
af2b040e	552	int ret, copied;
17926a79	553
17926a79 DH	554	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
	555
	556	/* this should be in poll */
9cd3e072	557	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
17926a79 DH	558
	559	if (sk->sk_err \|\| (sk->sk_shutdown & SEND_SHUTDOWN))
	560	return -EPIPE;
	561
17926a79 DH	562	more = msg->msg_flags & MSG_MORE;
	563
	564	skb = call->tx_pending;
	565	call->tx_pending = NULL;
	566
	567	copied = 0;
3af6878e	568	do {
17926a79 DH	569	if (!skb) {
	570	size_t size, chunk, max, space;
	571
	572	_debug("alloc");
	573
ee72b9fd DH	574	if (CIRC_SPACE(call->acks_head,
ee72b9fd DH	575	ACCESS_ONCE(call->acks_tail),
17926a79 DH	576	call->acks_winsz) <= 0) {
	577	ret = -EAGAIN;
	578	if (msg->msg_flags & MSG_DONTWAIT)
	579	goto maybe_error;
	580	ret = rxrpc_wait_for_tx_window(rx, call,
	581	&timeo);
	582	if (ret < 0)
	583	goto maybe_error;
	584	}
	585
	586	max = call->conn->trans->peer->maxdata;
	587	max -= call->conn->security_size;
	588	max &= ~(call->conn->size_align - 1UL);
	589
	590	chunk = max;
01e97e65 AV	591	if (chunk > msg_data_left(msg) && !more)
01e97e65 AV	592	chunk = msg_data_left(msg);
17926a79 DH	593
	594	space = chunk + call->conn->size_align;
	595	space &= ~(call->conn->size_align - 1UL);
	596
	597	size = space + call->conn->header_size;
	598
	599	_debug("SIZE: %zu/%zu/%zu", chunk, space, size);
	600
	601	/* create a buffer that we can retain until it's ACK'd */
	602	skb = sock_alloc_send_skb(
	603	sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
	604	if (!skb)
	605	goto maybe_error;
	606
	607	rxrpc_new_skb(skb);
	608
	609	_debug("ALLOC SEND %p", skb);
	610
	611	ASSERTCMP(skb->mark, ==, 0);
	612
	613	_debug("HS: %u", call->conn->header_size);
	614	skb_reserve(skb, call->conn->header_size);
	615	skb->len += call->conn->header_size;
	616
	617	sp = rxrpc_skb(skb);
	618	sp->remain = chunk;
	619	if (sp->remain > skb_tailroom(skb))
	620	sp->remain = skb_tailroom(skb);
	621
	622	_net("skb: hr %d, tr %d, hl %d, rm %d",
	623	skb_headroom(skb),
	624	skb_tailroom(skb),
	625	skb_headlen(skb),
	626	sp->remain);
	627
	628	skb->ip_summed = CHECKSUM_UNNECESSARY;
	629	}
	630
	631	_debug("append");
	632	sp = rxrpc_skb(skb);
	633
	634	/* append next segment of data to the current buffer */
01e97e65	635	if (msg_data_left(msg) > 0) {
aab94830 DH	636	int copy = skb_tailroom(skb);
aab94830 DH	637	ASSERTCMP(copy, >, 0);
01e97e65 AV	638	if (copy > msg_data_left(msg))
01e97e65 AV	639	copy = msg_data_left(msg);
aab94830 DH	640	if (copy > sp->remain)
	641	copy = sp->remain;
	642
	643	_debug("add");
	644	ret = skb_add_data(skb, &msg->msg_iter, copy);
	645	_debug("added");
	646	if (ret < 0)
	647	goto efault;
	648	sp->remain -= copy;
	649	skb->mark += copy;
	650	copied += copy;
aab94830	651	}
17926a79 DH	652
	653	/* check for the far side aborting the call or a network error
	654	* occurring */
	655	if (call->state > RXRPC_CALL_COMPLETE)
	656	goto call_aborted;
	657
	658	/* add the packet to the send queue if it's now full */
382d7974	659	if (sp->remain <= 0 \|\|
01e97e65	660	(msg_data_left(msg) == 0 && !more)) {
17926a79	661	struct rxrpc_connection *conn = call->conn;
e8388eb1	662	uint32_t seq;
17926a79 DH	663	size_t pad;
	664
	665	/* pad out if we're using security */
	666	if (conn->security) {
	667	pad = conn->security_size + skb->mark;
	668	pad = conn->size_align - pad;
	669	pad &= conn->size_align - 1;
	670	_debug("pad %zu", pad);
	671	if (pad)
	672	memset(skb_put(skb, pad), 0, pad);
	673	}
	674
e8388eb1 DH	675	seq = atomic_inc_return(&call->sequence);
e8388eb1 DH	676
0d12f8a4 DH	677	sp->hdr.epoch = conn->epoch;
0d12f8a4 DH	678	sp->hdr.cid = call->cid;
17926a79	679	sp->hdr.callNumber = call->call_id;
0d12f8a4 DH	680	sp->hdr.seq = seq;
	681	sp->hdr.serial = atomic_inc_return(&conn->serial);
	682	sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
17926a79 DH	683	sp->hdr.userStatus = 0;
17926a79 DH	684	sp->hdr.securityIndex = conn->security_ix;
0d12f8a4 DH	685	sp->hdr._rsvd = 0;
0d12f8a4 DH	686	sp->hdr.serviceId = call->service_id;
17926a79 DH	687
17926a79 DH	688	sp->hdr.flags = conn->out_clientflag;
01e97e65	689	if (msg_data_left(msg) == 0 && !more)
17926a79	690	sp->hdr.flags \|= RXRPC_LAST_PACKET;
ee72b9fd DH	691	else if (CIRC_SPACE(call->acks_head,
ee72b9fd DH	692	ACCESS_ONCE(call->acks_tail),
17926a79 DH	693	call->acks_winsz) > 1)
17926a79 DH	694	sp->hdr.flags \|= RXRPC_MORE_PACKETS;
e8388eb1 DH	695	if (more && seq & 1)
e8388eb1 DH	696	sp->hdr.flags \|= RXRPC_REQUEST_ACK;
17926a79 DH	697
	698	ret = rxrpc_secure_packet(
	699	call, skb, skb->mark,
0d12f8a4	700	skb->head + sizeof(struct rxrpc_wire_header));
17926a79 DH	701	if (ret < 0)
	702	goto out;
	703
0d12f8a4	704	rxrpc_insert_header(skb);
01e97e65	705	rxrpc_queue_packet(call, skb, !msg_data_left(msg) && !more);
17926a79 DH	706	skb = NULL;
17926a79 DH	707	}
01e97e65	708	} while (msg_data_left(msg) > 0);
17926a79	709
19e6454c DH	710	success:
19e6454c DH	711	ret = copied;
17926a79 DH	712	out:
	713	call->tx_pending = skb;
	714	_leave(" = %d", ret);
	715	return ret;
	716
	717	call_aborted:
	718	rxrpc_free_skb(skb);
	719	if (call->state == RXRPC_CALL_NETWORK_ERROR)
	720	ret = call->conn->trans->peer->net_error;
	721	else
	722	ret = -ECONNABORTED;
	723	_leave(" = %d", ret);
	724	return ret;
	725
	726	maybe_error:
	727	if (copied)
19e6454c	728	goto success;
17926a79 DH	729	goto out;
	730
	731	efault:
	732	ret = -EFAULT;
	733	goto out;
	734	}