OpenCloudOS-Kernel/arch/sparc/kernel/ldc.c

// SPDX-License-Identifier: GPL-2.0
/* ldc.c: Logical Domain Channel link-layer protocol driver.
 *
 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/scatterlist.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/bitmap.h>
#include <linux/iommu-common.h>

#include <asm/hypervisor.h>
#include <asm/iommu.h>
#include <asm/page.h>
#include <asm/ldc.h>
#include <asm/mdesc.h>

#define DRV_MODULE_NAME		"ldc"
#define PFX DRV_MODULE_NAME	": "
#define DRV_MODULE_VERSION	"1.1"
#define DRV_MODULE_RELDATE	"July 22, 2008"

#define COOKIE_PGSZ_CODE	0xf000000000000000ULL
#define COOKIE_PGSZ_CODE_SHIFT	60ULL


static char version[] =
	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

/* Packet header layout for unreliable and reliable mode frames.
 * When in RAW mode, packets are simply straight 64-byte payloads
 * with no headers.
 */
struct ldc_packet {
	u8			type;
#define LDC_CTRL		0x01
#define LDC_DATA		0x02
#define LDC_ERR			0x10

	u8			stype;
#define LDC_INFO		0x01
#define LDC_ACK			0x02
#define LDC_NACK		0x04

	u8			ctrl;
#define LDC_VERS		0x01 /* Link Version		*/
#define LDC_RTS			0x02 /* Request To Send		*/
#define LDC_RTR			0x03 /* Ready To Receive	*/
#define LDC_RDX			0x04 /* Ready for Data eXchange	*/
#define LDC_CTRL_MSK		0x0f

	u8			env;
#define LDC_LEN			0x3f
#define LDC_FRAG_MASK		0xc0
#define LDC_START		0x40
#define LDC_STOP		0x80

	u32			seqid;

	union {
		u8		u_data[LDC_PACKET_SIZE - 8];
		struct {
			u32	pad;
			u32	ackid;
			u8	r_data[LDC_PACKET_SIZE - 8 - 8];
		} r;
	} u;
};

struct ldc_version {
	u16 major;
	u16 minor;
};

/* Ordered from largest major to lowest.  */
static struct ldc_version ver_arr[] = {
	{ .major = 1, .minor = 0 },
};

#define LDC_DEFAULT_MTU			(4 * LDC_PACKET_SIZE)
#define LDC_DEFAULT_NUM_ENTRIES		(PAGE_SIZE / LDC_PACKET_SIZE)

struct ldc_channel;

struct ldc_mode_ops {
	int (*write)(struct ldc_channel *, const void *, unsigned int);
	int (*read)(struct ldc_channel *, void *, unsigned int);
};

static const struct ldc_mode_ops raw_ops;
static const struct ldc_mode_ops nonraw_ops;
static const struct ldc_mode_ops stream_ops;

int ldom_domaining_enabled;

struct ldc_iommu {
	/* Protects ldc_unmap.  */
	spinlock_t			lock;
	struct ldc_mtable_entry		*page_table;
	struct iommu_map_table		iommu_map_table;
};

struct ldc_channel {
	/* Protects all operations that depend upon channel state.  */
	spinlock_t			lock;

	unsigned long			id;

	u8				*mssbuf;
	u32				mssbuf_len;
	u32				mssbuf_off;

	struct ldc_packet		*tx_base;
	unsigned long			tx_head;
	unsigned long			tx_tail;
	unsigned long			tx_num_entries;
	unsigned long			tx_ra;

	unsigned long			tx_acked;

	struct ldc_packet		*rx_base;
	unsigned long			rx_head;
	unsigned long			rx_tail;
	unsigned long			rx_num_entries;
	unsigned long			rx_ra;

	u32				rcv_nxt;
	u32				snd_nxt;

	unsigned long			chan_state;

	struct ldc_channel_config	cfg;
	void				*event_arg;

	const struct ldc_mode_ops	*mops;

	struct ldc_iommu		iommu;

	struct ldc_version		ver;

	u8				hs_state;
#define LDC_HS_CLOSED			0x00
#define LDC_HS_OPEN			0x01
#define LDC_HS_GOTVERS			0x02
#define LDC_HS_SENTRTR			0x03
#define LDC_HS_GOTRTR			0x04
#define LDC_HS_COMPLETE			0x10

	u8				flags;
#define LDC_FLAG_ALLOCED_QUEUES		0x01
#define LDC_FLAG_REGISTERED_QUEUES	0x02
#define LDC_FLAG_REGISTERED_IRQS	0x04
#define LDC_FLAG_RESET			0x10

	u8				mss;
	u8				state;

#define LDC_IRQ_NAME_MAX		32
	char				rx_irq_name[LDC_IRQ_NAME_MAX];
	char				tx_irq_name[LDC_IRQ_NAME_MAX];

	struct hlist_head		mh_list;

	struct hlist_node		list;
};

#define ldcdbg(TYPE, f, a...) \
do {	if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
		printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
} while (0)

#define	LDC_ABORT(lp)	ldc_abort((lp), __func__)

static const char *state_to_str(u8 state)
{
	switch (state) {
	case LDC_STATE_INVALID:
		return "INVALID";
	case LDC_STATE_INIT:
		return "INIT";
	case LDC_STATE_BOUND:
		return "BOUND";
	case LDC_STATE_READY:
		return "READY";
	case LDC_STATE_CONNECTED:
		return "CONNECTED";
	default:
		return "<UNKNOWN>";
	}
}

static unsigned long __advance(unsigned long off, unsigned long num_entries)
{
	off += LDC_PACKET_SIZE;
	if (off == (num_entries * LDC_PACKET_SIZE))
		off = 0;

	return off;
}

static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
{
	return __advance(off, lp->rx_num_entries);
}

static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
{
	return __advance(off, lp->tx_num_entries);
}

static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
						  unsigned long *new_tail)
{
	struct ldc_packet *p;
	unsigned long t;

	t = tx_advance(lp, lp->tx_tail);
	if (t == lp->tx_head)
		return NULL;

	*new_tail = t;

	p = lp->tx_base;
	return p + (lp->tx_tail / LDC_PACKET_SIZE);
}

/* When we are in reliable or stream mode, have to track the next packet
 * we haven't gotten an ACK for in the TX queue using tx_acked.  We have
 * to be careful not to stomp over the queue past that point.  During
 * the handshake, we don't have TX data packets pending in the queue
 * and that's why handshake_get_tx_packet() need not be mindful of
 * lp->tx_acked.
 */
static unsigned long head_for_data(struct ldc_channel *lp)
{
	if (lp->cfg.mode == LDC_MODE_STREAM)
		return lp->tx_acked;
	return lp->tx_head;
}

static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
{
	unsigned long limit, tail, new_tail, diff;
	unsigned int mss;

	limit = head_for_data(lp);
	tail = lp->tx_tail;
	new_tail = tx_advance(lp, tail);
	if (new_tail == limit)
		return 0;

	if (limit > new_tail)
		diff = limit - new_tail;
	else
		diff = (limit +
			((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
	diff /= LDC_PACKET_SIZE;
	mss = lp->mss;

	if (diff * mss < size)
		return 0;

	return 1;
}

static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
					     unsigned long *new_tail)
{
	struct ldc_packet *p;
	unsigned long h, t;

	h = head_for_data(lp);
	t = tx_advance(lp, lp->tx_tail);
	if (t == h)
		return NULL;

	*new_tail = t;

	p = lp->tx_base;
	return p + (lp->tx_tail / LDC_PACKET_SIZE);
}

static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
{
	unsigned long orig_tail = lp->tx_tail;
	int limit = 1000;

	lp->tx_tail = tail;
	while (limit-- > 0) {
		unsigned long err;

		err = sun4v_ldc_tx_set_qtail(lp->id, tail);
		if (!err)
			return 0;

		if (err != HV_EWOULDBLOCK) {
			lp->tx_tail = orig_tail;
			return -EINVAL;
		}
		udelay(1);
	}

	lp->tx_tail = orig_tail;
	return -EBUSY;
}

/* This just updates the head value in the hypervisor using
 * a polling loop with a timeout.  The caller takes care of
 * upating software state representing the head change, if any.
 */
static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
{
	int limit = 1000;

	while (limit-- > 0) {
		unsigned long err;

		err = sun4v_ldc_rx_set_qhead(lp->id, head);
		if (!err)
			return 0;

		if (err != HV_EWOULDBLOCK)
			return -EINVAL;

		udelay(1);
	}

	return -EBUSY;
}

static int send_tx_packet(struct ldc_channel *lp,
			  struct ldc_packet *p,
			  unsigned long new_tail)
{
	BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));

	return set_tx_tail(lp, new_tail);
}

static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
						 u8 stype, u8 ctrl,
						 void *data, int dlen,
						 unsigned long *new_tail)
{
	struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);

	if (p) {
		memset(p, 0, sizeof(*p));
		p->type = LDC_CTRL;
		p->stype = stype;
		p->ctrl = ctrl;
		if (data)
			memcpy(p->u.u_data, data, dlen);
	}
	return p;
}

static int start_handshake(struct ldc_channel *lp)
{
	struct ldc_packet *p;
	struct ldc_version *ver;
	unsigned long new_tail;

	ver = &ver_arr[0];

	ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
	       ver->major, ver->minor);

	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
				   ver, sizeof(*ver), &new_tail);
	if (p) {
		int err = send_tx_packet(lp, p, new_tail);
		if (!err)
			lp->flags &= ~LDC_FLAG_RESET;
		return err;
	}
	return -EBUSY;
}

static int send_version_nack(struct ldc_channel *lp,
			     u16 major, u16 minor)
{
	struct ldc_packet *p;
	struct ldc_version ver;
	unsigned long new_tail;

	ver.major = major;
	ver.minor = minor;

	p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
				   &ver, sizeof(ver), &new_tail);
	if (p) {
		ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
		       ver.major, ver.minor);

		return send_tx_packet(lp, p, new_tail);
	}
	return -EBUSY;
}

static int send_version_ack(struct ldc_channel *lp,
			    struct ldc_version *vp)
{
	struct ldc_packet *p;
	unsigned long new_tail;

	p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
				   vp, sizeof(*vp), &new_tail);
	if (p) {
		ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
		       vp->major, vp->minor);

		return send_tx_packet(lp, p, new_tail);
	}
	return -EBUSY;
}

static int send_rts(struct ldc_channel *lp)
{
	struct ldc_packet *p;
	unsigned long new_tail;

	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
				   &new_tail);
	if (p) {
		p->env = lp->cfg.mode;
		p->seqid = 0;
		lp->rcv_nxt = 0;

		ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
		       p->env, p->seqid);

		return send_tx_packet(lp, p, new_tail);
	}
	return -EBUSY;
}

static int send_rtr(struct ldc_channel *lp)
{
	struct ldc_packet *p;
	unsigned long new_tail;

	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
				   &new_tail);
	if (p) {
		p->env = lp->cfg.mode;
		p->seqid = 0;

		ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
		       p->env, p->seqid);

		return send_tx_packet(lp, p, new_tail);
	}
	return -EBUSY;
}

static int send_rdx(struct ldc_channel *lp)
{
	struct ldc_packet *p;
	unsigned long new_tail;

	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
				   &new_tail);
	if (p) {
		p->env = 0;
		p->seqid = ++lp->snd_nxt;
		p->u.r.ackid = lp->rcv_nxt;

		ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
		       p->env, p->seqid, p->u.r.ackid);

		return send_tx_packet(lp, p, new_tail);
	}
	return -EBUSY;
}

static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
{
	struct ldc_packet *p;
	unsigned long new_tail;
	int err;

	p = data_get_tx_packet(lp, &new_tail);
	if (!p)
		return -EBUSY;
	memset(p, 0, sizeof(*p));
	p->type = data_pkt->type;
	p->stype = LDC_NACK;
	p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
	p->seqid = lp->snd_nxt + 1;
	p->u.r.ackid = lp->rcv_nxt;

	ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
	       p->type, p->ctrl, p->seqid, p->u.r.ackid);

	err = send_tx_packet(lp, p, new_tail);
	if (!err)
		lp->snd_nxt++;

	return err;
}

static int ldc_abort(struct ldc_channel *lp, const char *msg)
{
	unsigned long hv_err;

	ldcdbg(STATE, "ABORT[%s]\n", msg);
	ldc_print(lp);

	/* We report but do not act upon the hypervisor errors because
	 * there really isn't much we can do if they fail at this point.
	 */
	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
	if (hv_err)
		printk(KERN_ERR PFX "ldc_abort: "
		       "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
		       lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);

	hv_err = sun4v_ldc_tx_get_state(lp->id,
					&lp->tx_head,
					&lp->tx_tail,
					&lp->chan_state);
	if (hv_err)
		printk(KERN_ERR PFX "ldc_abort: "
		       "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
		       lp->id, hv_err);

	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
	if (hv_err)
		printk(KERN_ERR PFX "ldc_abort: "
		       "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
		       lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);

	/* Refetch the RX queue state as well, because we could be invoked
	 * here in the queue processing context.
	 */
	hv_err = sun4v_ldc_rx_get_state(lp->id,
					&lp->rx_head,
					&lp->rx_tail,
					&lp->chan_state);
	if (hv_err)
		printk(KERN_ERR PFX "ldc_abort: "
		       "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
		       lp->id, hv_err);

	return -ECONNRESET;
}

static struct ldc_version *find_by_major(u16 major)
{
	struct ldc_version *ret = NULL;
	int i;

	for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
		struct ldc_version *v = &ver_arr[i];
		if (v->major <= major) {
			ret = v;
			break;
		}
	}
	return ret;
}

static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
{
	struct ldc_version *vap;
	int err;

	ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
	       vp->major, vp->minor);

	if (lp->hs_state == LDC_HS_GOTVERS) {
		lp->hs_state = LDC_HS_OPEN;
		memset(&lp->ver, 0, sizeof(lp->ver));
	}

	vap = find_by_major(vp->major);
	if (!vap) {
		err = send_version_nack(lp, 0, 0);
	} else if (vap->major != vp->major) {
		err = send_version_nack(lp, vap->major, vap->minor);
	} else {
		struct ldc_version ver = *vp;
		if (ver.minor > vap->minor)
			ver.minor = vap->minor;
		err = send_version_ack(lp, &ver);
		if (!err) {
			lp->ver = ver;
			lp->hs_state = LDC_HS_GOTVERS;
		}
	}
	if (err)
		return LDC_ABORT(lp);

	return 0;
}

static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
{
	ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
	       vp->major, vp->minor);

	if (lp->hs_state == LDC_HS_GOTVERS) {
		if (lp->ver.major != vp->major ||
		    lp->ver.minor != vp->minor)
			return LDC_ABORT(lp);
	} else {
		lp->ver = *vp;
		lp->hs_state = LDC_HS_GOTVERS;
	}
	if (send_rts(lp))
		return LDC_ABORT(lp);
	return 0;
}

static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
{
	struct ldc_version *vap;
	struct ldc_packet *p;
	unsigned long new_tail;

	if (vp->major == 0 && vp->minor == 0)
		return LDC_ABORT(lp);

	vap = find_by_major(vp->major);
	if (!vap)
		return LDC_ABORT(lp);

	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
					   vap, sizeof(*vap),
					   &new_tail);
	if (!p)
		return LDC_ABORT(lp);

	return send_tx_packet(lp, p, new_tail);
}

static int process_version(struct ldc_channel *lp,
			   struct ldc_packet *p)
{
	struct ldc_version *vp;

	vp = (struct ldc_version *) p->u.u_data;

	switch (p->stype) {
	case LDC_INFO:
		return process_ver_info(lp, vp);

	case LDC_ACK:
		return process_ver_ack(lp, vp);

	case LDC_NACK:
		return process_ver_nack(lp, vp);

	default:
		return LDC_ABORT(lp);
	}
}

static int process_rts(struct ldc_channel *lp,
		       struct ldc_packet *p)
{
	ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
	       p->stype, p->seqid, p->env);

	if (p->stype     != LDC_INFO	   ||
	    lp->hs_state != LDC_HS_GOTVERS ||
	    p->env       != lp->cfg.mode)
		return LDC_ABORT(lp);

	lp->snd_nxt = p->seqid;
	lp->rcv_nxt = p->seqid;
	lp->hs_state = LDC_HS_SENTRTR;
	if (send_rtr(lp))
		return LDC_ABORT(lp);

	return 0;
}

static int process_rtr(struct ldc_channel *lp,
		       struct ldc_packet *p)
{
	ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
	       p->stype, p->seqid, p->env);

	if (p->stype     != LDC_INFO ||
	    p->env       != lp->cfg.mode)
		return LDC_ABORT(lp);

	lp->snd_nxt = p->seqid;
	lp->hs_state = LDC_HS_COMPLETE;
	ldc_set_state(lp, LDC_STATE_CONNECTED);
	send_rdx(lp);

	return LDC_EVENT_UP;
}

static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
{
	return lp->rcv_nxt + 1 == seqid;
}

static int process_rdx(struct ldc_channel *lp,
		       struct ldc_packet *p)
{
	ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
	       p->stype, p->seqid, p->env, p->u.r.ackid);

	if (p->stype != LDC_INFO ||
	    !(rx_seq_ok(lp, p->seqid)))
		return LDC_ABORT(lp);

	lp->rcv_nxt = p->seqid;

	lp->hs_state = LDC_HS_COMPLETE;
	ldc_set_state(lp, LDC_STATE_CONNECTED);

	return LDC_EVENT_UP;
}

static int process_control_frame(struct ldc_channel *lp,
				 struct ldc_packet *p)
{
	switch (p->ctrl) {
	case LDC_VERS:
		return process_version(lp, p);

	case LDC_RTS:
		return process_rts(lp, p);

	case LDC_RTR:
		return process_rtr(lp, p);

	case LDC_RDX:
		return process_rdx(lp, p);

	default:
		return LDC_ABORT(lp);
	}
}

static int process_error_frame(struct ldc_channel *lp,
			       struct ldc_packet *p)
{
	return LDC_ABORT(lp);
}

static int process_data_ack(struct ldc_channel *lp,
			    struct ldc_packet *ack)
{
	unsigned long head = lp->tx_acked;
	u32 ackid = ack->u.r.ackid;

	while (1) {
		struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);

		head = tx_advance(lp, head);

		if (p->seqid == ackid) {
			lp->tx_acked = head;
			return 0;
		}
		if (head == lp->tx_tail)
			return LDC_ABORT(lp);
	}

	return 0;
}

static void send_events(struct ldc_channel *lp, unsigned int event_mask)
{
	if (event_mask & LDC_EVENT_RESET)
		lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
	if (event_mask & LDC_EVENT_UP)
		lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
	if (event_mask & LDC_EVENT_DATA_READY)
		lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
}

static irqreturn_t ldc_rx(int irq, void *dev_id)
{
	struct ldc_channel *lp = dev_id;
	unsigned long orig_state, flags;
	unsigned int event_mask;

	spin_lock_irqsave(&lp->lock, flags);

	orig_state = lp->chan_state;

	/* We should probably check for hypervisor errors here and
	 * reset the LDC channel if we get one.
	 */
	sun4v_ldc_rx_get_state(lp->id,
			       &lp->rx_head,
			       &lp->rx_tail,
			       &lp->chan_state);

	ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
	       orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);

	event_mask = 0;

	if (lp->cfg.mode == LDC_MODE_RAW &&
	    lp->chan_state == LDC_CHANNEL_UP) {
		lp->hs_state = LDC_HS_COMPLETE;
		ldc_set_state(lp, LDC_STATE_CONNECTED);

		/*
		 * Generate an LDC_EVENT_UP event if the channel
		 * was not already up.
		 */
		if (orig_state != LDC_CHANNEL_UP) {
			event_mask |= LDC_EVENT_UP;
			orig_state = lp->chan_state;
		}
	}

	/* If we are in reset state, flush the RX queue and ignore
	 * everything.
	 */
	if (lp->flags & LDC_FLAG_RESET) {
		(void) ldc_rx_reset(lp);
		goto out;
	}

	/* Once we finish the handshake, we let the ldc_read()
	 * paths do all of the control frame and state management.
	 * Just trigger the callback.
	 */
	if (lp->hs_state == LDC_HS_COMPLETE) {
handshake_complete:
		if (lp->chan_state != orig_state) {
			unsigned int event = LDC_EVENT_RESET;

			if (lp->chan_state == LDC_CHANNEL_UP)
				event = LDC_EVENT_UP;

			event_mask |= event;
		}
		if (lp->rx_head != lp->rx_tail)
			event_mask |= LDC_EVENT_DATA_READY;

		goto out;
	}

	if (lp->chan_state != orig_state)
		goto out;

	while (lp->rx_head != lp->rx_tail) {
		struct ldc_packet *p;
		unsigned long new;
		int err;

		p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);

		switch (p->type) {
		case LDC_CTRL:
			err = process_control_frame(lp, p);
			if (err > 0)
				event_mask |= err;
			break;

		case LDC_DATA:
			event_mask |= LDC_EVENT_DATA_READY;
			err = 0;
			break;

		case LDC_ERR:
			err = process_error_frame(lp, p);
			break;

		default:
			err = LDC_ABORT(lp);
			break;
		}

		if (err < 0)
			break;

		new = lp->rx_head;
		new += LDC_PACKET_SIZE;
		if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
			new = 0;
		lp->rx_head = new;

		err = __set_rx_head(lp, new);
		if (err < 0) {
			(void) LDC_ABORT(lp);
			break;
		}
		if (lp->hs_state == LDC_HS_COMPLETE)
			goto handshake_complete;
	}

out:
	spin_unlock_irqrestore(&lp->lock, flags);

	send_events(lp, event_mask);

	return IRQ_HANDLED;
}

static irqreturn_t ldc_tx(int irq, void *dev_id)
{
	struct ldc_channel *lp = dev_id;
	unsigned long flags, orig_state;
	unsigned int event_mask = 0;

	spin_lock_irqsave(&lp->lock, flags);

	orig_state = lp->chan_state;

	/* We should probably check for hypervisor errors here and
	 * reset the LDC channel if we get one.
	 */
	sun4v_ldc_tx_get_state(lp->id,
			       &lp->tx_head,
			       &lp->tx_tail,
			       &lp->chan_state);

	ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
	       orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);

	if (lp->cfg.mode == LDC_MODE_RAW &&
	    lp->chan_state == LDC_CHANNEL_UP) {
		lp->hs_state = LDC_HS_COMPLETE;
		ldc_set_state(lp, LDC_STATE_CONNECTED);

		/*
		 * Generate an LDC_EVENT_UP event if the channel
		 * was not already up.
		 */
		if (orig_state != LDC_CHANNEL_UP) {
			event_mask |= LDC_EVENT_UP;
			orig_state = lp->chan_state;
		}
	}

	spin_unlock_irqrestore(&lp->lock, flags);

	send_events(lp, event_mask);

	return IRQ_HANDLED;
}

/* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
 * XXX that addition and removal from the ldc_channel_list has
 * XXX atomicity, otherwise the __ldc_channel_exists() check is
 * XXX totally pointless as another thread can slip into ldc_alloc()
 * XXX and add a channel with the same ID.  There also needs to be
 * XXX a spinlock for ldc_channel_list.
 */
static HLIST_HEAD(ldc_channel_list);

static int __ldc_channel_exists(unsigned long id)
{
	struct ldc_channel *lp;

	hlist_for_each_entry(lp, &ldc_channel_list, list) {
		if (lp->id == id)
			return 1;
	}
	return 0;
}

static int alloc_queue(const char *name, unsigned long num_entries,
		       struct ldc_packet **base, unsigned long *ra)
{
	unsigned long size, order;
	void *q;

	size = num_entries * LDC_PACKET_SIZE;
	order = get_order(size);

	q = (void *) __get_free_pages(GFP_KERNEL, order);
	if (!q) {
		printk(KERN_ERR PFX "Alloc of %s queue failed with "
		       "size=%lu order=%lu\n", name, size, order);
		return -ENOMEM;
	}

	memset(q, 0, PAGE_SIZE << order);

	*base = q;
	*ra = __pa(q);

	return 0;
}

static void free_queue(unsigned long num_entries, struct ldc_packet *q)
{
	unsigned long size, order;

	if (!q)
		return;

	size = num_entries * LDC_PACKET_SIZE;
	order = get_order(size);

	free_pages((unsigned long)q, order);
}

static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
{
	u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
	/* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */

	cookie &= ~COOKIE_PGSZ_CODE;

	return (cookie >> (13ULL + (szcode * 3ULL)));
}

static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
		      unsigned long entry, unsigned long npages)
{
	struct ldc_mtable_entry *base;
	unsigned long i, shift;

	shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
	base = iommu->page_table + entry;
	for (i = 0; i < npages; i++) {
		if (base->cookie)
			sun4v_ldc_revoke(id, cookie + (i << shift),
					 base->cookie);
		base->mte = 0;
	}
}

/* XXX Make this configurable... XXX */
#define LDC_IOTABLE_SIZE	(8 * 1024)

static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
{
	unsigned long sz, num_tsb_entries, tsbsize, order;
	struct ldc_iommu *ldc_iommu = &lp->iommu;
	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
	struct ldc_mtable_entry *table;
	unsigned long hv_err;
	int err;

	num_tsb_entries = LDC_IOTABLE_SIZE;
	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
	spin_lock_init(&ldc_iommu->lock);

	sz = num_tsb_entries / 8;
	sz = (sz + 7UL) & ~7UL;
	iommu->map = kzalloc(sz, GFP_KERNEL);
	if (!iommu->map) {
		printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
		return -ENOMEM;
	}
	iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
			    NULL, false /* no large pool */,
			    1 /* npools */,
			    true /* skip span boundary check */);

	order = get_order(tsbsize);

	table = (struct ldc_mtable_entry *)
		__get_free_pages(GFP_KERNEL, order);
	err = -ENOMEM;
	if (!table) {
		printk(KERN_ERR PFX "Alloc of MTE table failed, "
		       "size=%lu order=%lu\n", tsbsize, order);
		goto out_free_map;
	}

	memset(table, 0, PAGE_SIZE << order);

	ldc_iommu->page_table = table;

	hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
					 num_tsb_entries);
	err = -EINVAL;
	if (hv_err)
		goto out_free_table;

	return 0;

out_free_table:
	free_pages((unsigned long) table, order);
	ldc_iommu->page_table = NULL;

out_free_map:
	kfree(iommu->map);
	iommu->map = NULL;

	return err;
}

static void ldc_iommu_release(struct ldc_channel *lp)
{
	struct ldc_iommu *ldc_iommu = &lp->iommu;
	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
	unsigned long num_tsb_entries, tsbsize, order;

	(void) sun4v_ldc_set_map_table(lp->id, 0, 0);

	num_tsb_entries = iommu->poolsize * iommu->nr_pools;
	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
	order = get_order(tsbsize);

	free_pages((unsigned long) ldc_iommu->page_table, order);
	ldc_iommu->page_table = NULL;

	kfree(iommu->map);
	iommu->map = NULL;
}

struct ldc_channel *ldc_alloc(unsigned long id,
			      const struct ldc_channel_config *cfgp,
			      void *event_arg,
			      const char *name)
{
	struct ldc_channel *lp;
	const struct ldc_mode_ops *mops;
	unsigned long dummy1, dummy2, hv_err;
	u8 mss, *mssbuf;
	int err;

	err = -ENODEV;
	if (!ldom_domaining_enabled)
		goto out_err;

	err = -EINVAL;
	if (!cfgp)
		goto out_err;
	if (!name)
		goto out_err;

	switch (cfgp->mode) {
	case LDC_MODE_RAW:
		mops = &raw_ops;
		mss = LDC_PACKET_SIZE;
		break;

	case LDC_MODE_UNRELIABLE:
		mops = &nonraw_ops;
		mss = LDC_PACKET_SIZE - 8;
		break;

	case LDC_MODE_STREAM:
		mops = &stream_ops;
		mss = LDC_PACKET_SIZE - 8 - 8;
		break;

	default:
		goto out_err;
	}

	if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
		goto out_err;

	hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
	err = -ENODEV;
	if (hv_err == HV_ECHANNEL)
		goto out_err;

	err = -EEXIST;
	if (__ldc_channel_exists(id))
		goto out_err;

	mssbuf = NULL;

	lp = kzalloc(sizeof(*lp), GFP_KERNEL);
	err = -ENOMEM;
	if (!lp)
		goto out_err;

	spin_lock_init(&lp->lock);

	lp->id = id;

	err = ldc_iommu_init(name, lp);
	if (err)
		goto out_free_ldc;

	lp->mops = mops;
	lp->mss = mss;

	lp->cfg = *cfgp;
	if (!lp->cfg.mtu)
		lp->cfg.mtu = LDC_DEFAULT_MTU;

	if (lp->cfg.mode == LDC_MODE_STREAM) {
		mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
		if (!mssbuf) {
			err = -ENOMEM;
			goto out_free_iommu;
		}
		lp->mssbuf = mssbuf;
	}

	lp->event_arg = event_arg;

	/* XXX allow setting via ldc_channel_config to override defaults
	 * XXX or use some formula based upon mtu
	 */
	lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
	lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;

	err = alloc_queue("TX", lp->tx_num_entries,
			  &lp->tx_base, &lp->tx_ra);
	if (err)
		goto out_free_mssbuf;

	err = alloc_queue("RX", lp->rx_num_entries,
			  &lp->rx_base, &lp->rx_ra);
	if (err)
		goto out_free_txq;

	lp->flags |= LDC_FLAG_ALLOCED_QUEUES;

	lp->hs_state = LDC_HS_CLOSED;
	ldc_set_state(lp, LDC_STATE_INIT);

	INIT_HLIST_NODE(&lp->list);
	hlist_add_head(&lp->list, &ldc_channel_list);

	INIT_HLIST_HEAD(&lp->mh_list);

	snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
	snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);

	err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
			  lp->rx_irq_name, lp);
	if (err)
		goto out_free_txq;

	err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
			  lp->tx_irq_name, lp);
	if (err) {
		free_irq(lp->cfg.rx_irq, lp);
		goto out_free_txq;
	}

	return lp;

out_free_txq:
	free_queue(lp->tx_num_entries, lp->tx_base);

out_free_mssbuf:
	kfree(mssbuf);

out_free_iommu:
	ldc_iommu_release(lp);

out_free_ldc:
	kfree(lp);

out_err:
	return ERR_PTR(err);
}
EXPORT_SYMBOL(ldc_alloc);

void ldc_unbind(struct ldc_channel *lp)
{
	if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
		free_irq(lp->cfg.rx_irq, lp);
		free_irq(lp->cfg.tx_irq, lp);
		lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
	}

	if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
		sun4v_ldc_tx_qconf(lp->id, 0, 0);
		sun4v_ldc_rx_qconf(lp->id, 0, 0);
		lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
	}
	if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
		free_queue(lp->tx_num_entries, lp->tx_base);
		free_queue(lp->rx_num_entries, lp->rx_base);
		lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
	}

	ldc_set_state(lp, LDC_STATE_INIT);
}
EXPORT_SYMBOL(ldc_unbind);

void ldc_free(struct ldc_channel *lp)
{
	ldc_unbind(lp);
	hlist_del(&lp->list);
	kfree(lp->mssbuf);
	ldc_iommu_release(lp);

	kfree(lp);
}
EXPORT_SYMBOL(ldc_free);

/* Bind the channel.  This registers the LDC queues with
 * the hypervisor and puts the channel into a pseudo-listening
 * state.  This does not initiate a handshake, ldc_connect() does
 * that.
 */
int ldc_bind(struct ldc_channel *lp)
{
	unsigned long hv_err, flags;
	int err = -EINVAL;

	if (lp->state != LDC_STATE_INIT)
		return -EINVAL;

	spin_lock_irqsave(&lp->lock, flags);

	enable_irq(lp->cfg.rx_irq);
	enable_irq(lp->cfg.tx_irq);

	lp->flags |= LDC_FLAG_REGISTERED_IRQS;

	err = -ENODEV;
	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
	if (hv_err)
		goto out_free_irqs;

	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
	if (hv_err)
		goto out_free_irqs;

	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
	if (hv_err)
		goto out_unmap_tx;

	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
	if (hv_err)
		goto out_unmap_tx;

	lp->flags |= LDC_FLAG_REGISTERED_QUEUES;

	hv_err = sun4v_ldc_tx_get_state(lp->id,
					&lp->tx_head,
					&lp->tx_tail,
					&lp->chan_state);
	err = -EBUSY;
	if (hv_err)
		goto out_unmap_rx;

	lp->tx_acked = lp->tx_head;

	lp->hs_state = LDC_HS_OPEN;
	ldc_set_state(lp, LDC_STATE_BOUND);

	if (lp->cfg.mode == LDC_MODE_RAW) {
		/*
		 * There is no handshake in RAW mode, so handshake
		 * is completed.
		 */
		lp->hs_state = LDC_HS_COMPLETE;
	}

	spin_unlock_irqrestore(&lp->lock, flags);

	return 0;

out_unmap_rx:
	lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
	sun4v_ldc_rx_qconf(lp->id, 0, 0);

out_unmap_tx:
	sun4v_ldc_tx_qconf(lp->id, 0, 0);

out_free_irqs:
	lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
	free_irq(lp->cfg.tx_irq, lp);
	free_irq(lp->cfg.rx_irq, lp);

	spin_unlock_irqrestore(&lp->lock, flags);

	return err;
}
EXPORT_SYMBOL(ldc_bind);

int ldc_connect(struct ldc_channel *lp)
{
	unsigned long flags;
	int err;

	if (lp->cfg.mode == LDC_MODE_RAW)
		return -EINVAL;

	spin_lock_irqsave(&lp->lock, flags);

	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
	    lp->hs_state != LDC_HS_OPEN)
		err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
	else
		err = start_handshake(lp);

	spin_unlock_irqrestore(&lp->lock, flags);

	return err;
}
EXPORT_SYMBOL(ldc_connect);

int ldc_disconnect(struct ldc_channel *lp)
{
	unsigned long hv_err, flags;
	int err;

	if (lp->cfg.mode == LDC_MODE_RAW)
		return -EINVAL;

	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
		return -EINVAL;

	spin_lock_irqsave(&lp->lock, flags);

	err = -ENODEV;
	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
	if (hv_err)
		goto out_err;

	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
	if (hv_err)
		goto out_err;

	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
	if (hv_err)
		goto out_err;

	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
	if (hv_err)
		goto out_err;

	ldc_set_state(lp, LDC_STATE_BOUND);
	lp->hs_state = LDC_HS_OPEN;
	lp->flags |= LDC_FLAG_RESET;

	spin_unlock_irqrestore(&lp->lock, flags);

	return 0;

out_err:
	sun4v_ldc_tx_qconf(lp->id, 0, 0);
	sun4v_ldc_rx_qconf(lp->id, 0, 0);
	free_irq(lp->cfg.tx_irq, lp);
	free_irq(lp->cfg.rx_irq, lp);
	lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
		       LDC_FLAG_REGISTERED_QUEUES);
	ldc_set_state(lp, LDC_STATE_INIT);

	spin_unlock_irqrestore(&lp->lock, flags);

	return err;
}
EXPORT_SYMBOL(ldc_disconnect);

int ldc_state(struct ldc_channel *lp)
{
	return lp->state;
}
EXPORT_SYMBOL(ldc_state);

void ldc_set_state(struct ldc_channel *lp, u8 state)
{
	ldcdbg(STATE, "STATE (%s) --> (%s)\n",
	       state_to_str(lp->state),
	       state_to_str(state));

	lp->state = state;
}
EXPORT_SYMBOL(ldc_set_state);

int ldc_mode(struct ldc_channel *lp)
{
	return lp->cfg.mode;
}
EXPORT_SYMBOL(ldc_mode);

int ldc_rx_reset(struct ldc_channel *lp)
{
	return __set_rx_head(lp, lp->rx_tail);
}
EXPORT_SYMBOL(ldc_rx_reset);

void __ldc_print(struct ldc_channel *lp, const char *caller)
{
	pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
		"\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
		"\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
		"\trcv_nxt=%u snd_nxt=%u\n",
		caller, lp->id, lp->flags, state_to_str(lp->state),
		lp->chan_state, lp->hs_state,
		lp->rx_head, lp->rx_tail, lp->rx_num_entries,
		lp->tx_head, lp->tx_tail, lp->tx_num_entries,
		lp->rcv_nxt, lp->snd_nxt);
}
EXPORT_SYMBOL(__ldc_print);

static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
{
	struct ldc_packet *p;
	unsigned long new_tail, hv_err;
	int err;

	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
					&lp->chan_state);
	if (unlikely(hv_err))
		return -EBUSY;

	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
		return LDC_ABORT(lp);

	if (size > LDC_PACKET_SIZE)
		return -EMSGSIZE;

	p = data_get_tx_packet(lp, &new_tail);
	if (!p)
		return -EAGAIN;

	memcpy(p, buf, size);

	err = send_tx_packet(lp, p, new_tail);
	if (!err)
		err = size;

	return err;
}

static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
{
	struct ldc_packet *p;
	unsigned long hv_err, new;
	int err;

	if (size < LDC_PACKET_SIZE)
		return -EINVAL;

	hv_err = sun4v_ldc_rx_get_state(lp->id,
					&lp->rx_head,
					&lp->rx_tail,
					&lp->chan_state);
	if (hv_err)
		return LDC_ABORT(lp);

	if (lp->chan_state == LDC_CHANNEL_DOWN ||
	    lp->chan_state == LDC_CHANNEL_RESETTING)
		return -ECONNRESET;

	if (lp->rx_head == lp->rx_tail)
		return 0;

	p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
	memcpy(buf, p, LDC_PACKET_SIZE);

	new = rx_advance(lp, lp->rx_head);
	lp->rx_head = new;

	err = __set_rx_head(lp, new);
	if (err < 0)
		err = -ECONNRESET;
	else
		err = LDC_PACKET_SIZE;

	return err;
}

static const struct ldc_mode_ops raw_ops = {
	.write		=	write_raw,
	.read		=	read_raw,
};

static int write_nonraw(struct ldc_channel *lp, const void *buf,
			unsigned int size)
{
	unsigned long hv_err, tail;
	unsigned int copied;
	u32 seq;
	int err;

	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
					&lp->chan_state);
	if (unlikely(hv_err))
		return -EBUSY;

	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
		return LDC_ABORT(lp);

	if (!tx_has_space_for(lp, size))
		return -EAGAIN;

	seq = lp->snd_nxt;
	copied = 0;
	tail = lp->tx_tail;
	while (copied < size) {
		struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
		u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
			    p->u.u_data :
			    p->u.r.r_data);
		int data_len;

		p->type = LDC_DATA;
		p->stype = LDC_INFO;
		p->ctrl = 0;

		data_len = size - copied;
		if (data_len > lp->mss)
			data_len = lp->mss;

		BUG_ON(data_len > LDC_LEN);

		p->env = (data_len |
			  (copied == 0 ? LDC_START : 0) |
			  (data_len == size - copied ? LDC_STOP : 0));

		p->seqid = ++seq;

		ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
		       p->type,
		       p->stype,
		       p->ctrl,
		       p->env,
		       p->seqid);

		memcpy(data, buf, data_len);
		buf += data_len;
		copied += data_len;

		tail = tx_advance(lp, tail);
	}

	err = set_tx_tail(lp, tail);
	if (!err) {
		lp->snd_nxt = seq;
		err = size;
	}

	return err;
}

static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
		      struct ldc_packet *first_frag)
{
	int err;

	if (first_frag)
		lp->rcv_nxt = first_frag->seqid - 1;

	err = send_data_nack(lp, p);
	if (err)
		return err;

	err = ldc_rx_reset(lp);
	if (err < 0)
		return LDC_ABORT(lp);

	return 0;
}

static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
{
	if (p->stype & LDC_ACK) {
		int err = process_data_ack(lp, p);
		if (err)
			return err;
	}
	if (p->stype & LDC_NACK)
		return LDC_ABORT(lp);

	return 0;
}

static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
{
	unsigned long dummy;
	int limit = 1000;

	ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
	       cur_head, lp->rx_head, lp->rx_tail);
	while (limit-- > 0) {
		unsigned long hv_err;

		hv_err = sun4v_ldc_rx_get_state(lp->id,
						&dummy,
						&lp->rx_tail,
						&lp->chan_state);
		if (hv_err)
			return LDC_ABORT(lp);

		if (lp->chan_state == LDC_CHANNEL_DOWN ||
		    lp->chan_state == LDC_CHANNEL_RESETTING)
			return -ECONNRESET;

		if (cur_head != lp->rx_tail) {
			ldcdbg(DATA, "DATA WAIT DONE "
			       "head[%lx] tail[%lx] chan_state[%lx]\n",
			       dummy, lp->rx_tail, lp->chan_state);
			return 0;
		}

		udelay(1);
	}
	return -EAGAIN;
}

static int rx_set_head(struct ldc_channel *lp, unsigned long head)
{
	int err = __set_rx_head(lp, head);

	if (err < 0)
		return LDC_ABORT(lp);

	lp->rx_head = head;
	return 0;
}

static void send_data_ack(struct ldc_channel *lp)
{
	unsigned long new_tail;
	struct ldc_packet *p;

	p = data_get_tx_packet(lp, &new_tail);
	if (likely(p)) {
		int err;

		memset(p, 0, sizeof(*p));
		p->type = LDC_DATA;
		p->stype = LDC_ACK;
		p->ctrl = 0;
		p->seqid = lp->snd_nxt + 1;
		p->u.r.ackid = lp->rcv_nxt;

		err = send_tx_packet(lp, p, new_tail);
		if (!err)
			lp->snd_nxt++;
	}
}

static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
{
	struct ldc_packet *first_frag;
	unsigned long hv_err, new;
	int err, copied;

	hv_err = sun4v_ldc_rx_get_state(lp->id,
					&lp->rx_head,
					&lp->rx_tail,
					&lp->chan_state);
	if (hv_err)
		return LDC_ABORT(lp);

	if (lp->chan_state == LDC_CHANNEL_DOWN ||
	    lp->chan_state == LDC_CHANNEL_RESETTING)
		return -ECONNRESET;

	if (lp->rx_head == lp->rx_tail)
		return 0;

	first_frag = NULL;
	copied = err = 0;
	new = lp->rx_head;
	while (1) {
		struct ldc_packet *p;
		int pkt_len;

		BUG_ON(new == lp->rx_tail);
		p = lp->rx_base + (new / LDC_PACKET_SIZE);

		ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
		       "rcv_nxt[%08x]\n",
		       p->type,
		       p->stype,
		       p->ctrl,
		       p->env,
		       p->seqid,
		       p->u.r.ackid,
		       lp->rcv_nxt);

		if (unlikely(!rx_seq_ok(lp, p->seqid))) {
			err = rx_bad_seq(lp, p, first_frag);
			copied = 0;
			break;
		}

		if (p->type & LDC_CTRL) {
			err = process_control_frame(lp, p);
			if (err < 0)
				break;
			err = 0;
		}

		lp->rcv_nxt = p->seqid;

		/*
		 * If this is a control-only packet, there is nothing
		 * else to do but advance the rx queue since the packet
		 * was already processed above.
		 */
		if (!(p->type & LDC_DATA)) {
			new = rx_advance(lp, new);
			break;
		}
		if (p->stype & (LDC_ACK | LDC_NACK)) {
			err = data_ack_nack(lp, p);
			if (err)
				break;
		}
		if (!(p->stype & LDC_INFO)) {
			new = rx_advance(lp, new);
			err = rx_set_head(lp, new);
			if (err)
				break;
			goto no_data;
		}

		pkt_len = p->env & LDC_LEN;

		/* Every initial packet starts with the START bit set.
		 *
		 * Singleton packets will have both START+STOP set.
		 *
		 * Fragments will have START set in the first frame, STOP
		 * set in the last frame, and neither bit set in middle
		 * frames of the packet.
		 *
		 * Therefore if we are at the beginning of a packet and
		 * we don't see START, or we are in the middle of a fragmented
		 * packet and do see START, we are unsynchronized and should
		 * flush the RX queue.
		 */
		if ((first_frag == NULL && !(p->env & LDC_START)) ||
		    (first_frag != NULL &&  (p->env & LDC_START))) {
			if (!first_frag)
				new = rx_advance(lp, new);

			err = rx_set_head(lp, new);
			if (err)
				break;

			if (!first_frag)
				goto no_data;
		}
		if (!first_frag)
			first_frag = p;

		if (pkt_len > size - copied) {
			/* User didn't give us a big enough buffer,
			 * what to do?  This is a pretty serious error.
			 *
			 * Since we haven't updated the RX ring head to
			 * consume any of the packets, signal the error
			 * to the user and just leave the RX ring alone.
			 *
			 * This seems the best behavior because this allows
			 * a user of the LDC layer to start with a small
			 * RX buffer for ldc_read() calls and use -EMSGSIZE
			 * as a cue to enlarge it's read buffer.
			 */
			err = -EMSGSIZE;
			break;
		}

		/* Ok, we are gonna eat this one.  */
		new = rx_advance(lp, new);

		memcpy(buf,
		       (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
			p->u.u_data : p->u.r.r_data), pkt_len);
		buf += pkt_len;
		copied += pkt_len;

		if (p->env & LDC_STOP)
			break;

no_data:
		if (new == lp->rx_tail) {
			err = rx_data_wait(lp, new);
			if (err)
				break;
		}
	}

	if (!err)
		err = rx_set_head(lp, new);

	if (err && first_frag)
		lp->rcv_nxt = first_frag->seqid - 1;

	if (!err) {
		err = copied;
		if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
			send_data_ack(lp);
	}

	return err;
}

static const struct ldc_mode_ops nonraw_ops = {
	.write		=	write_nonraw,
	.read		=	read_nonraw,
};

static int write_stream(struct ldc_channel *lp, const void *buf,
			unsigned int size)
{
	if (size > lp->cfg.mtu)
		size = lp->cfg.mtu;
	return write_nonraw(lp, buf, size);
}

static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
{
	if (!lp->mssbuf_len) {
		int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
		if (err < 0)
			return err;

		lp->mssbuf_len = err;
		lp->mssbuf_off = 0;
	}

	if (size > lp->mssbuf_len)
		size = lp->mssbuf_len;
	memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);

	lp->mssbuf_off += size;
	lp->mssbuf_len -= size;

	return size;
}

static const struct ldc_mode_ops stream_ops = {
	.write		=	write_stream,
	.read		=	read_stream,
};

int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
{
	unsigned long flags;
	int err;

	if (!buf)
		return -EINVAL;

	if (!size)
		return 0;

	spin_lock_irqsave(&lp->lock, flags);

	if (lp->hs_state != LDC_HS_COMPLETE)
		err = -ENOTCONN;
	else
		err = lp->mops->write(lp, buf, size);

	spin_unlock_irqrestore(&lp->lock, flags);

	return err;
}
EXPORT_SYMBOL(ldc_write);

int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
{
	unsigned long flags;
	int err;

	ldcdbg(RX, "%s: entered size=%d\n", __func__, size);

	if (!buf)
		return -EINVAL;

	if (!size)
		return 0;

	spin_lock_irqsave(&lp->lock, flags);

	if (lp->hs_state != LDC_HS_COMPLETE)
		err = -ENOTCONN;
	else
		err = lp->mops->read(lp, buf, size);

	spin_unlock_irqrestore(&lp->lock, flags);

	ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
	       lp->cfg.mode, lp->rx_head, lp->rx_tail, err);

	return err;
}
EXPORT_SYMBOL(ldc_read);

static u64 pagesize_code(void)
{
	switch (PAGE_SIZE) {
	default:
	case (8ULL * 1024ULL):
		return 0;
	case (64ULL * 1024ULL):
		return 1;
	case (512ULL * 1024ULL):
		return 2;
	case (4ULL * 1024ULL * 1024ULL):
		return 3;
	case (32ULL * 1024ULL * 1024ULL):
		return 4;
	case (256ULL * 1024ULL * 1024ULL):
		return 5;
	}
}

static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
{
	return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
		(index << PAGE_SHIFT) |
		page_offset);
}


static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
					     unsigned long npages)
{
	long entry;

	entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
				      npages, NULL, (unsigned long)-1, 0);
	if (unlikely(entry == IOMMU_ERROR_CODE))
		return NULL;

	return iommu->page_table + entry;
}

static u64 perm_to_mte(unsigned int map_perm)
{
	u64 mte_base;

	mte_base = pagesize_code();

	if (map_perm & LDC_MAP_SHADOW) {
		if (map_perm & LDC_MAP_R)
			mte_base |= LDC_MTE_COPY_R;
		if (map_perm & LDC_MAP_W)
			mte_base |= LDC_MTE_COPY_W;
	}
	if (map_perm & LDC_MAP_DIRECT) {
		if (map_perm & LDC_MAP_R)
			mte_base |= LDC_MTE_READ;
		if (map_perm & LDC_MAP_W)
			mte_base |= LDC_MTE_WRITE;
		if (map_perm & LDC_MAP_X)
			mte_base |= LDC_MTE_EXEC;
	}
	if (map_perm & LDC_MAP_IO) {
		if (map_perm & LDC_MAP_R)
			mte_base |= LDC_MTE_IOMMU_R;
		if (map_perm & LDC_MAP_W)
			mte_base |= LDC_MTE_IOMMU_W;
	}

	return mte_base;
}

static int pages_in_region(unsigned long base, long len)
{
	int count = 0;

	do {
		unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;

		len -= (new - base);
		base = new;
		count++;
	} while (len > 0);

	return count;
}

struct cookie_state {
	struct ldc_mtable_entry		*page_table;
	struct ldc_trans_cookie		*cookies;
	u64				mte_base;
	u64				prev_cookie;
	u32				pte_idx;
	u32				nc;
};

static void fill_cookies(struct cookie_state *sp, unsigned long pa,
			 unsigned long off, unsigned long len)
{
	do {
		unsigned long tlen, new = pa + PAGE_SIZE;
		u64 this_cookie;

		sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;

		tlen = PAGE_SIZE;
		if (off)
			tlen = PAGE_SIZE - off;
		if (tlen > len)
			tlen = len;

		this_cookie = make_cookie(sp->pte_idx,
					  pagesize_code(), off);

		off = 0;

		if (this_cookie == sp->prev_cookie) {
			sp->cookies[sp->nc - 1].cookie_size += tlen;
		} else {
			sp->cookies[sp->nc].cookie_addr = this_cookie;
			sp->cookies[sp->nc].cookie_size = tlen;
			sp->nc++;
		}
		sp->prev_cookie = this_cookie + tlen;

		sp->pte_idx++;

		len -= tlen;
		pa = new;
	} while (len > 0);
}

static int sg_count_one(struct scatterlist *sg)
{
	unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
	long len = sg->length;

	if ((sg->offset | len) & (8UL - 1))
		return -EFAULT;

	return pages_in_region(base + sg->offset, len);
}

static int sg_count_pages(struct scatterlist *sg, int num_sg)
{
	int count;
	int i;

	count = 0;
	for (i = 0; i < num_sg; i++) {
		int err = sg_count_one(sg + i);
		if (err < 0)
			return err;
		count += err;
	}

	return count;
}

int ldc_map_sg(struct ldc_channel *lp,
	       struct scatterlist *sg, int num_sg,
	       struct ldc_trans_cookie *cookies, int ncookies,
	       unsigned int map_perm)
{
	unsigned long i, npages;
	struct ldc_mtable_entry *base;
	struct cookie_state state;
	struct ldc_iommu *iommu;
	int err;
	struct scatterlist *s;

	if (map_perm & ~LDC_MAP_ALL)
		return -EINVAL;

	err = sg_count_pages(sg, num_sg);
	if (err < 0)
		return err;

	npages = err;
	if (err > ncookies)
		return -EMSGSIZE;

	iommu = &lp->iommu;

	base = alloc_npages(iommu, npages);

	if (!base)
		return -ENOMEM;

	state.page_table = iommu->page_table;
	state.cookies = cookies;
	state.mte_base = perm_to_mte(map_perm);
	state.prev_cookie = ~(u64)0;
	state.pte_idx = (base - iommu->page_table);
	state.nc = 0;

	for_each_sg(sg, s, num_sg, i) {
		fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
			     s->offset, s->length);
	}

	return state.nc;
}
EXPORT_SYMBOL(ldc_map_sg);

int ldc_map_single(struct ldc_channel *lp,
		   void *buf, unsigned int len,
		   struct ldc_trans_cookie *cookies, int ncookies,
		   unsigned int map_perm)
{
	unsigned long npages, pa;
	struct ldc_mtable_entry *base;
	struct cookie_state state;
	struct ldc_iommu *iommu;

	if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
		return -EINVAL;

	pa = __pa(buf);
	if ((pa | len) & (8UL - 1))
		return -EFAULT;

	npages = pages_in_region(pa, len);

	iommu = &lp->iommu;

	base = alloc_npages(iommu, npages);

	if (!base)
		return -ENOMEM;

	state.page_table = iommu->page_table;
	state.cookies = cookies;
	state.mte_base = perm_to_mte(map_perm);
	state.prev_cookie = ~(u64)0;
	state.pte_idx = (base - iommu->page_table);
	state.nc = 0;
	fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
	BUG_ON(state.nc > ncookies);

	return state.nc;
}
EXPORT_SYMBOL(ldc_map_single);


static void free_npages(unsigned long id, struct ldc_iommu *iommu,
			u64 cookie, u64 size)
{
	unsigned long npages, entry;

	npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;

	entry = ldc_cookie_to_index(cookie, iommu);
	ldc_demap(iommu, id, cookie, entry, npages);
	iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
}

void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
	       int ncookies)
{
	struct ldc_iommu *iommu = &lp->iommu;
	int i;
	unsigned long flags;

	spin_lock_irqsave(&iommu->lock, flags);
	for (i = 0; i < ncookies; i++) {
		u64 addr = cookies[i].cookie_addr;
		u64 size = cookies[i].cookie_size;

		free_npages(lp->id, iommu, addr, size);
	}
	spin_unlock_irqrestore(&iommu->lock, flags);
}
EXPORT_SYMBOL(ldc_unmap);

int ldc_copy(struct ldc_channel *lp, int copy_dir,
	     void *buf, unsigned int len, unsigned long offset,
	     struct ldc_trans_cookie *cookies, int ncookies)
{
	unsigned int orig_len;
	unsigned long ra;
	int i;

	if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
		       lp->id, copy_dir);
		return -EINVAL;
	}

	ra = __pa(buf);
	if ((ra | len | offset) & (8UL - 1)) {
		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
		       "ra[%lx] len[%x] offset[%lx]\n",
		       lp->id, ra, len, offset);
		return -EFAULT;
	}

	if (lp->hs_state != LDC_HS_COMPLETE ||
	    (lp->flags & LDC_FLAG_RESET)) {
		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
		       "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
		return -ECONNRESET;
	}

	orig_len = len;
	for (i = 0; i < ncookies; i++) {
		unsigned long cookie_raddr = cookies[i].cookie_addr;
		unsigned long this_len = cookies[i].cookie_size;
		unsigned long actual_len;

		if (unlikely(offset)) {
			unsigned long this_off = offset;

			if (this_off > this_len)
				this_off = this_len;

			offset -= this_off;
			this_len -= this_off;
			if (!this_len)
				continue;
			cookie_raddr += this_off;
		}

		if (this_len > len)
			this_len = len;

		while (1) {
			unsigned long hv_err;

			hv_err = sun4v_ldc_copy(lp->id, copy_dir,
						cookie_raddr, ra,
						this_len, &actual_len);
			if (unlikely(hv_err)) {
				printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
				       "HV error %lu\n",
				       lp->id, hv_err);
				if (lp->hs_state != LDC_HS_COMPLETE ||
				    (lp->flags & LDC_FLAG_RESET))
					return -ECONNRESET;
				else
					return -EFAULT;
			}

			cookie_raddr += actual_len;
			ra += actual_len;
			len -= actual_len;
			if (actual_len == this_len)
				break;

			this_len -= actual_len;
		}

		if (!len)
			break;
	}

	/* It is caller policy what to do about short copies.
	 * For example, a networking driver can declare the
	 * packet a runt and drop it.
	 */

	return orig_len - len;
}
EXPORT_SYMBOL(ldc_copy);

void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
			  struct ldc_trans_cookie *cookies, int *ncookies,
			  unsigned int map_perm)
{
	void *buf;
	int err;

	if (len & (8UL - 1))
		return ERR_PTR(-EINVAL);

	buf = kzalloc(len, GFP_ATOMIC);
	if (!buf)
		return ERR_PTR(-ENOMEM);

	err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
	if (err < 0) {
		kfree(buf);
		return ERR_PTR(err);
	}
	*ncookies = err;

	return buf;
}
EXPORT_SYMBOL(ldc_alloc_exp_dring);

void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
			struct ldc_trans_cookie *cookies, int ncookies)
{
	ldc_unmap(lp, cookies, ncookies);
	kfree(buf);
}
EXPORT_SYMBOL(ldc_free_exp_dring);

static int __init ldc_init(void)
{
	unsigned long major, minor;
	struct mdesc_handle *hp;
	const u64 *v;
	int err;
	u64 mp;

	hp = mdesc_grab();
	if (!hp)
		return -ENODEV;

	mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
	err = -ENODEV;
	if (mp == MDESC_NODE_NULL)
		goto out;

	v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
	if (!v)
		goto out;

	major = 1;
	minor = 0;
	if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
		printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
		goto out;
	}

	printk(KERN_INFO "%s", version);

	if (!*v) {
		printk(KERN_INFO PFX "Domaining disabled.\n");
		goto out;
	}
	ldom_domaining_enabled = 1;
	err = 0;

out:
	mdesc_release(hp);
	return err;
}

core_initcall(ldc_init);