diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 89d1d0556b6e..630a3fcf65bc 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -124,6 +124,7 @@ struct xenvif { struct pending_tx_info pending_tx_info[MAX_PENDING_REQS]; grant_handle_t grant_tx_handle[MAX_PENDING_REQS]; + struct gnttab_copy tx_copy_ops[MAX_PENDING_REQS]; struct gnttab_map_grant_ref tx_map_ops[MAX_PENDING_REQS]; struct gnttab_unmap_grant_ref tx_unmap_ops[MAX_PENDING_REQS]; /* passed to gnttab_[un]map_refs with pages under (un)mapping */ diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index 4bb78862afc8..99c8f09b4654 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -940,35 +940,37 @@ static inline void xenvif_grant_handle_reset(struct xenvif *vif, static int xenvif_tx_check_gop(struct xenvif *vif, struct sk_buff *skb, - struct gnttab_map_grant_ref **gopp_map) + struct gnttab_map_grant_ref **gopp_map, + struct gnttab_copy **gopp_copy) { struct gnttab_map_grant_ref *gop_map = *gopp_map; u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx; struct skb_shared_info *shinfo = skb_shinfo(skb); - struct pending_tx_info *tx_info; int nr_frags = shinfo->nr_frags; - int i, err, start; + int i, err; struct sk_buff *first_skb = NULL; /* Check status of header. */ - err = gop_map->status; - if (unlikely(err)) + err = (*gopp_copy)->status; + (*gopp_copy)++; + if (unlikely(err)) { + if (net_ratelimit()) + netdev_dbg(vif->dev, + "Grant copy of header failed! status: %d pending_idx% %u ref: %u\n", + (*gopp_copy)->status, + pending_idx, + (*gopp_copy)->source.u.ref); xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR); - else - xenvif_grant_handle_set(vif, pending_idx , gop_map->handle); - - /* Skip first skb fragment if it is on same page as header fragment. */ - start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); + } check_frags: - for (i = start; i < nr_frags; i++) { + for (i = 0; i < nr_frags; i++, gop_map++) { int j, newerr; pending_idx = frag_get_pending_idx(&shinfo->frags[i]); - tx_info = &vif->pending_tx_info[pending_idx]; /* Check error status: if okay then remember grant handle. */ - newerr = (++gop_map)->status; + newerr = gop_map->status; if (likely(!newerr)) { xenvif_grant_handle_set(vif, @@ -981,18 +983,20 @@ check_frags: } /* Error on this fragment: respond to client with an error. */ + if (net_ratelimit()) + netdev_dbg(vif->dev, + "Grant map of %d. frag failed! status: %d pending_idx% %u ref: %u\n", + i, + gop_map->status, + pending_idx, + gop_map->ref); xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR); /* Not the first error? Preceding frags already invalidated. */ if (err) continue; - /* First error: invalidate header and preceding fragments. */ - if (!first_skb) - pending_idx = XENVIF_TX_CB(skb)->pending_idx; - else - pending_idx = XENVIF_TX_CB(skb)->pending_idx; - xenvif_idx_unmap(vif, pending_idx); - for (j = start; j < i; j++) { + /* First error: invalidate preceding fragments. */ + for (j = 0; j < i; j++) { pending_idx = frag_get_pending_idx(&shinfo->frags[j]); xenvif_idx_unmap(vif, pending_idx); } @@ -1006,7 +1010,6 @@ check_frags: skb = shinfo->frag_list; shinfo = skb_shinfo(skb); nr_frags = shinfo->nr_frags; - start = 0; goto check_frags; } @@ -1017,15 +1020,13 @@ check_frags: if (first_skb && err) { int j; shinfo = skb_shinfo(first_skb); - pending_idx = XENVIF_TX_CB(skb)->pending_idx; - start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx); - for (j = start; j < shinfo->nr_frags; j++) { + for (j = 0; j < shinfo->nr_frags; j++) { pending_idx = frag_get_pending_idx(&shinfo->frags[j]); xenvif_idx_unmap(vif, pending_idx); } } - *gopp_map = gop_map + 1; + *gopp_map = gop_map; return err; } @@ -1036,9 +1037,6 @@ static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb) int i; u16 prev_pending_idx = INVALID_PENDING_IDX; - if (skb_shinfo(skb)->destructor_arg) - prev_pending_idx = XENVIF_TX_CB(skb)->pending_idx; - for (i = 0; i < nr_frags; i++) { skb_frag_t *frag = shinfo->frags + i; struct xen_netif_tx_request *txp; @@ -1048,10 +1046,10 @@ static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb) pending_idx = frag_get_pending_idx(frag); /* If this is not the first frag, chain it to the previous*/ - if (unlikely(prev_pending_idx == INVALID_PENDING_IDX)) + if (prev_pending_idx == INVALID_PENDING_IDX) skb_shinfo(skb)->destructor_arg = &callback_param(vif, pending_idx); - else if (likely(pending_idx != prev_pending_idx)) + else callback_param(vif, prev_pending_idx).ctx = &callback_param(vif, pending_idx); @@ -1191,7 +1189,10 @@ static bool tx_credit_exceeded(struct xenvif *vif, unsigned size) return false; } -static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget) +static void xenvif_tx_build_gops(struct xenvif *vif, + int budget, + unsigned *copy_ops, + unsigned *map_ops) { struct gnttab_map_grant_ref *gop = vif->tx_map_ops, *request_gop; struct sk_buff *skb; @@ -1294,22 +1295,36 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget) } } - xenvif_tx_create_map_op(vif, pending_idx, &txreq, gop); - - gop++; - XENVIF_TX_CB(skb)->pending_idx = pending_idx; __skb_put(skb, data_len); + vif->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref; + vif->tx_copy_ops[*copy_ops].source.domid = vif->domid; + vif->tx_copy_ops[*copy_ops].source.offset = txreq.offset; + + vif->tx_copy_ops[*copy_ops].dest.u.gmfn = + virt_to_mfn(skb->data); + vif->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF; + vif->tx_copy_ops[*copy_ops].dest.offset = + offset_in_page(skb->data); + + vif->tx_copy_ops[*copy_ops].len = data_len; + vif->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref; + + (*copy_ops)++; skb_shinfo(skb)->nr_frags = ret; if (data_len < txreq.size) { skb_shinfo(skb)->nr_frags++; frag_set_pending_idx(&skb_shinfo(skb)->frags[0], pending_idx); + xenvif_tx_create_map_op(vif, pending_idx, &txreq, gop); + gop++; } else { frag_set_pending_idx(&skb_shinfo(skb)->frags[0], INVALID_PENDING_IDX); + memcpy(&vif->pending_tx_info[pending_idx].req, &txreq, + sizeof(txreq)); } vif->pending_cons++; @@ -1326,11 +1341,13 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget) vif->tx.req_cons = idx; - if ((gop-vif->tx_map_ops) >= ARRAY_SIZE(vif->tx_map_ops)) + if (((gop-vif->tx_map_ops) >= ARRAY_SIZE(vif->tx_map_ops)) || + (*copy_ops >= ARRAY_SIZE(vif->tx_copy_ops))) break; } - return gop - vif->tx_map_ops; + (*map_ops) = gop - vif->tx_map_ops; + return; } /* Consolidate skb with a frag_list into a brand new one with local pages on @@ -1402,6 +1419,7 @@ static int xenvif_handle_frag_list(struct xenvif *vif, struct sk_buff *skb) static int xenvif_tx_submit(struct xenvif *vif) { struct gnttab_map_grant_ref *gop_map = vif->tx_map_ops; + struct gnttab_copy *gop_copy = vif->tx_copy_ops; struct sk_buff *skb; int work_done = 0; @@ -1414,27 +1432,22 @@ static int xenvif_tx_submit(struct xenvif *vif) txp = &vif->pending_tx_info[pending_idx].req; /* Check the remap error code. */ - if (unlikely(xenvif_tx_check_gop(vif, skb, &gop_map))) { - netdev_dbg(vif->dev, "netback grant failed.\n"); + if (unlikely(xenvif_tx_check_gop(vif, skb, &gop_map, &gop_copy))) { skb_shinfo(skb)->nr_frags = 0; kfree_skb(skb); continue; } data_len = skb->len; - memcpy(skb->data, - (void *)(idx_to_kaddr(vif, pending_idx)|txp->offset), - data_len); callback_param(vif, pending_idx).ctx = NULL; if (data_len < txp->size) { /* Append the packet payload as a fragment. */ txp->offset += data_len; txp->size -= data_len; - skb_shinfo(skb)->destructor_arg = - &callback_param(vif, pending_idx); } else { /* Schedule a response immediately. */ - xenvif_idx_unmap(vif, pending_idx); + xenvif_idx_release(vif, pending_idx, + XEN_NETIF_RSP_OKAY); } if (txp->flags & XEN_NETTXF_csum_blank) @@ -1613,22 +1626,25 @@ static inline void xenvif_tx_dealloc_action(struct xenvif *vif) /* Called after netfront has transmitted */ int xenvif_tx_action(struct xenvif *vif, int budget) { - unsigned nr_mops; + unsigned nr_mops, nr_cops = 0; int work_done, ret; if (unlikely(!tx_work_todo(vif))) return 0; - nr_mops = xenvif_tx_build_gops(vif, budget); + xenvif_tx_build_gops(vif, budget, &nr_cops, &nr_mops); - if (nr_mops == 0) + if (nr_cops == 0) return 0; - ret = gnttab_map_refs(vif->tx_map_ops, - NULL, - vif->pages_to_map, - nr_mops); - BUG_ON(ret); + gnttab_batch_copy(vif->tx_copy_ops, nr_cops); + if (nr_mops != 0) { + ret = gnttab_map_refs(vif->tx_map_ops, + NULL, + vif->pages_to_map, + nr_mops); + BUG_ON(ret); + } work_done = xenvif_tx_submit(vif);