diff --git a/MAINTAINERS b/MAINTAINERS index b02b084cd755b..d037ddfd7b673 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5163,7 +5163,6 @@ F: Documentation/devicetree/bindings/net/brcm,unimac-mdio.yaml F: drivers/net/ethernet/broadcom/genet/ F: drivers/net/ethernet/broadcom/unimac.h F: drivers/net/mdio/mdio-bcm-unimac.c -F: include/linux/platform_data/bcmgenet.h F: include/linux/platform_data/mdio-bcm-unimac.h BROADCOM IPROC ARM ARCHITECTURE diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index fe15d684990fc..47e9ece4b37af 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -77,6 +77,7 @@ config BCMGENET select BCM7XXX_PHY select MDIO_BCM_UNIMAC select DIMLIB + select PAGE_POOL select BROADCOM_PHY if ARCH_BCM2835 help This driver supports the built-in Ethernet MACs found in the diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 8d54ca19a047c..9d9a3c749a39c 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -35,7 +35,6 @@ #include #include #include -#include #include @@ -49,9 +48,45 @@ #define GENET_Q0_TX_BD_CNT \ (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q) -#define RX_BUF_LENGTH 2048 #define SKB_ALIGNMENT 32 +/* RBUF and TBUF hand a frame to the DMA once the threshold is reached, so a + * longer frame arrives without an end of packet marker. Both registers are + * 8 bit in units of 16 bytes and want a multiple of the 256 byte burst size, + * so 0xf0 is the largest usable value. + */ +#define ENET_THLD_UNIT 16 +#define ENET_THLD_BURST 256 +#define ENET_THLD_DEFAULT 0x80 +#define ENET_THLD_MAX 0xf0 + +/* Page pool RX buffer layout: + * RSB(64) + pad(2) | frame data | skb_shared_info + * The HW writes the 64B RSB + 2B alignment padding before the frame. + */ +#define GENET_RBUF_ALIGN 2 +#define GENET_RSB_PAD (sizeof(struct status_64) + GENET_RBUF_ALIGN) + +/* A descriptor is one page, which also holds skb_shared_info behind the frame, + * so on 4K pages the page bounds the threshold before the register does. + */ +#define ENET_SHINFO_LEN SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +#define ENET_THLD_PAGE_LEN round_down(PAGE_SIZE - ENET_SHINFO_LEN - \ + sizeof(struct status_64), \ + ENET_THLD_BURST) +#define ENET_THLD_MAX_LEN min_t(unsigned int, \ + ENET_THLD_MAX * ENET_THLD_UNIT, \ + ENET_THLD_PAGE_LEN) + +/* Largest MTU that fits one descriptor, with room for a VLAN tag so a VLAN + * interface can use the parent MTU. + */ +#define ENET_MAX_MTU (ENET_THLD_MAX_LEN - GENET_RBUF_ALIGN - \ + ETH_HLEN - VLAN_HLEN) + +/* UMAC_MAX_FRAME_LEN is 14 bits wide and counts the FCS */ +#define ENET_MAX_JUMBO_MTU (GENMASK(13, 0) - ENET_FRAME_OVERHEAD) + /* Tx/Rx DMA register offset, skip 256 descriptors */ #define WORDS_PER_BD(p) (p->hw_params->words_per_bd) #define DMA_DESC_SIZE (WORDS_PER_BD(priv) * sizeof(u32)) @@ -744,8 +779,17 @@ static void bcmgenet_hfb_init(struct bcmgenet_priv *priv) INIT_LIST_HEAD(&priv->rxnfc_rules[i].list); priv->rxnfc_rules[i].state = BCMGENET_RXNFC_STATE_UNUSED; } +} + +static void bcmgenet_hfb_restore(struct bcmgenet_priv *priv) +{ + struct bcmgenet_rxnfc_rule *rule; bcmgenet_hfb_clear(priv); + + list_for_each_entry(rule, &priv->rxnfc_list, list) + if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) + bcmgenet_hfb_create_rxnfc_filter(priv, rule); } static int bcmgenet_begin(struct net_device *dev) @@ -1153,7 +1197,6 @@ static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = { UMAC_RBUF_ERR_CNT_V1), STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT), STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed), - STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed), STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed), STAT_GENET_SOFT_MIB("tx_realloc_tsb", mib.tx_realloc_tsb), STAT_GENET_SOFT_MIB("tx_realloc_tsb_failed", @@ -1632,6 +1675,13 @@ static int bcmgenet_get_num_flows(struct bcmgenet_priv *priv) return res; } +static u32 bcmgenet_get_rx_ring_count(struct net_device *dev) +{ + struct bcmgenet_priv *priv = netdev_priv(dev); + + return priv->hw_params->rx_queues ?: 1; +} + static int bcmgenet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, u32 *rule_locs) { @@ -1641,9 +1691,6 @@ static int bcmgenet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, int i = 0; switch (cmd->cmd) { - case ETHTOOL_GRXRINGS: - cmd->data = priv->hw_params->rx_queues ?: 1; - break; case ETHTOOL_GRXCLSRLCNT: cmd->rule_cnt = bcmgenet_get_num_flows(priv); cmd->data = MAX_NUM_OF_FS_RULES | RX_CLS_LOC_SPECIAL; @@ -1692,6 +1739,7 @@ static const struct ethtool_ops bcmgenet_ethtool_ops = { .get_ts_info = ethtool_op_get_ts_info, .get_rxnfc = bcmgenet_get_rxnfc, .set_rxnfc = bcmgenet_set_rxnfc, + .get_rx_ring_count = bcmgenet_get_rx_ring_count, .get_pauseparam = bcmgenet_get_pauseparam, .set_pauseparam = bcmgenet_set_pauseparam, }; @@ -1889,21 +1937,13 @@ static struct sk_buff *bcmgenet_free_tx_cb(struct device *dev, } /* Simple helper to free a receive control block's resources */ -static struct sk_buff *bcmgenet_free_rx_cb(struct device *dev, - struct enet_cb *cb) +static void bcmgenet_free_rx_cb(struct enet_cb *cb, + struct page_pool *pool) { - struct sk_buff *skb; - - skb = cb->skb; - cb->skb = NULL; - - if (dma_unmap_addr(cb, dma_addr)) { - dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr), - dma_unmap_len(cb, dma_len), DMA_FROM_DEVICE); - dma_unmap_addr_set(cb, dma_addr, 0); + if (cb->rx_page) { + page_pool_put_full_page(pool, cb->rx_page, false); + cb->rx_page = NULL; } - - return skb; } /* Unlocked version of the reclaim routine */ @@ -2137,6 +2177,20 @@ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) goto out; } + /* The MAC holds a frame to insert its checksum, but only up to the + * packet ready threshold. Longer frames are dropped silently. + */ + if (unlikely(skb->len > priv->tx_csum_max_len) && + skb->ip_summed == CHECKSUM_PARTIAL) { + if (skb_checksum_help(skb)) { + BCMGENET_STATS64_INC((&ring->stats64), dropped); + dev_kfree_skb_any(skb); + ret = NETDEV_TX_OK; + goto out; + } + nr_frags = skb_shinfo(skb)->nr_frags; + } + /* Retain how many bytes will be sent on the wire, without TSB inserted * by transmit checksum offload */ @@ -2237,46 +2291,64 @@ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) goto out; } -static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv, - struct enet_cb *cb) +static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring, + struct enet_cb *cb) { - struct device *kdev = &priv->pdev->dev; - struct sk_buff *skb; - struct sk_buff *rx_skb; + struct bcmgenet_priv *priv = ring->priv; dma_addr_t mapping; + struct page *page; - /* Allocate a new Rx skb */ - skb = __netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT, - GFP_ATOMIC | __GFP_NOWARN); - if (!skb) { + page = page_pool_alloc_pages(ring->page_pool, + GFP_ATOMIC); + if (!page) { priv->mib.alloc_rx_buff_failed++; netif_err(priv, rx_err, priv->dev, - "%s: Rx skb allocation failed\n", __func__); - return NULL; + "%s: Rx page allocation failed\n", __func__); + return -ENOMEM; } - /* DMA-map the new Rx skb */ - mapping = dma_map_single(kdev, skb->data, priv->rx_buf_len, - DMA_FROM_DEVICE); - if (dma_mapping_error(kdev, mapping)) { - priv->mib.rx_dma_failed++; - dev_kfree_skb_any(skb); - netif_err(priv, rx_err, priv->dev, - "%s: Rx skb DMA mapping failed\n", __func__); + /* page_pool handles DMA mapping via PP_FLAG_DMA_MAP */ + mapping = page_pool_get_dma_addr(page); + + cb->rx_page = page; + dmadesc_set_addr(priv, cb->bd_addr, mapping); + + return 0; +} + +static void bcmgenet_discard_frags(struct bcmgenet_rx_ring *ring) +{ + if (!ring->frag_head) + return; + + dev_kfree_skb_any(ring->frag_head); + ring->frag_head = NULL; +} + +/* A frame longer than the threshold arrives in several descriptors, each with + * its own status block. Only the first one carries a header, so hand the page + * of every later one to the frame already being collected. + */ +static struct sk_buff *bcmgenet_add_frag(struct bcmgenet_rx_ring *ring, + struct page *page, + unsigned int dma_flag, + unsigned int len) +{ + struct sk_buff *head = ring->frag_head; + + if (unlikely(skb_shinfo(head)->nr_frags >= MAX_SKB_FRAGS)) return NULL; - } - /* Grab the current Rx skb from the ring and DMA-unmap it */ - rx_skb = bcmgenet_free_rx_cb(kdev, cb); + skb_add_rx_frag(head, skb_shinfo(head)->nr_frags, page, + sizeof(struct status_64), + len - sizeof(struct status_64), PAGE_SIZE); - /* Put the new Rx skb on the ring */ - cb->skb = skb; - dma_unmap_addr_set(cb, dma_addr, mapping); - dma_unmap_len_set(cb, dma_len, priv->rx_buf_len); - dmadesc_set_addr(priv, cb->bd_addr, mapping); + if (!(dma_flag & DMA_EOP)) + return NULL; - /* Return the current Rx skb to caller */ - return rx_skb; + ring->frag_head = NULL; + + return head; } /* bcmgenet_desc_rx - descriptor based rx process. @@ -2328,25 +2400,30 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, while ((rxpktprocessed < rxpkttoprocess) && (rxpktprocessed < budget)) { struct status_64 *status; + struct page *rx_page; + void *hard_start; __be16 rx_csum; cb = &priv->rx_cbs[ring->read_ptr]; - skb = bcmgenet_rx_refill(priv, cb); - if (unlikely(!skb)) { + /* Save the received page before refilling */ + rx_page = cb->rx_page; + + if (bcmgenet_rx_refill(ring, cb)) { BCMGENET_STATS64_INC(stats, dropped); + bcmgenet_discard_frags(ring); goto next; } - status = (struct status_64 *)skb->data; + /* Sync the full buffer; the HW may have written anywhere + * up to priv->rx_buf_len. + */ + page_pool_dma_sync_for_cpu(ring->page_pool, rx_page, 0, + priv->rx_buf_len); + + hard_start = page_address(rx_page); + status = (struct status_64 *)hard_start; dma_length_status = status->length_status; - if (dev->features & NETIF_F_RXCSUM) { - rx_csum = (__force __be16)(status->rx_csum & 0xffff); - if (rx_csum) { - skb->csum = (__force __wsum)ntohs(rx_csum); - skb->ip_summed = CHECKSUM_COMPLETE; - } - } /* DMA flags and length are still valid no matter how * we got the Receive Status Vector (64B RSB or register) @@ -2359,18 +2436,27 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, __func__, p_index, ring->c_index, ring->read_ptr, dma_length_status); - if (unlikely(len > RX_BUF_LENGTH)) { - netif_err(priv, rx_status, dev, "oversized packet\n"); + /* Reject lengths that would underflow the SKB build path. */ + if (unlikely(len > priv->rx_buf_len || len < GENET_RSB_PAD)) { + netif_err(priv, rx_status, dev, + "invalid packet length %d\n", len); BCMGENET_STATS64_INC(stats, length_errors); - dev_kfree_skb_any(skb); + bcmgenet_discard_frags(ring); + page_pool_put_full_page(ring->page_pool, rx_page, + true); goto next; } - if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) { - netif_err(priv, rx_status, dev, - "dropping fragmented packet!\n"); + /* A new SOP resynchronizes after an incomplete frame */ + if (dma_flag & DMA_SOP) { + if (ring->frag_head) { + BCMGENET_STATS64_INC(stats, fragmented_errors); + bcmgenet_discard_frags(ring); + } + } else if (unlikely(!ring->frag_head)) { BCMGENET_STATS64_INC(stats, fragmented_errors); - dev_kfree_skb_any(skb); + page_pool_put_full_page(ring->page_pool, rx_page, + true); goto next; } @@ -2398,21 +2484,57 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, DMA_RX_RXER)) == DMA_RX_RXER) u64_stats_inc(&stats->errors); u64_stats_update_end(&stats->syncp); - dev_kfree_skb_any(skb); + bcmgenet_discard_frags(ring); + page_pool_put_full_page(ring->page_pool, rx_page, + true); goto next; } /* error packet */ - skb_put(skb, len); + if (!(dma_flag & DMA_SOP)) { + skb = bcmgenet_add_frag(ring, rx_page, dma_flag, len); + if (!skb) + goto next; + goto deliver; + } + + /* Build SKB from the page - data starts at hard_start, + * frame begins after RSB(64) + pad(2) = 66 bytes. + */ + skb = napi_build_skb(hard_start, PAGE_SIZE); + if (unlikely(!skb)) { + BCMGENET_STATS64_INC(stats, dropped); + page_pool_put_full_page(ring->page_pool, rx_page, + true); + goto next; + } + + skb_mark_for_recycle(skb); + + /* Reserve the RSB + pad, then set the data length */ + skb_reserve(skb, GENET_RSB_PAD); + __skb_put(skb, len - GENET_RSB_PAD); - /* remove RSB and hardware 2bytes added for IP alignment */ - skb_pull(skb, 66); - len -= 66; + if (unlikely(!(dma_flag & DMA_EOP))) { + ring->frag_head = skb; + goto next; + } + +deliver: if (priv->crc_fwd_en) { - skb_trim(skb, len - ETH_FCS_LEN); - len -= ETH_FCS_LEN; + skb_trim(skb, skb->len - ETH_FCS_LEN); + } + + /* Set up checksum offload */ + if (dev->features & NETIF_F_RXCSUM) { + rx_csum = (__force __be16)(status->rx_csum & 0xffff); + if (rx_csum) { + skb->csum = (__force __wsum)ntohs(rx_csum); + skb->ip_summed = CHECKSUM_COMPLETE; + } } + len = skb->len; bytes_processed += len; /*Finish setting up the received SKB and send it to the kernel*/ @@ -2484,12 +2606,11 @@ static void bcmgenet_dim_work(struct work_struct *work) dim->state = DIM_START_MEASURE; } -/* Assign skb to RX DMA descriptor. */ +/* Assign page_pool pages to RX DMA descriptors. */ static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv, struct bcmgenet_rx_ring *ring) { struct enet_cb *cb; - struct sk_buff *skb; int i; netif_dbg(priv, hw, priv->dev, "%s\n", __func__); @@ -2497,10 +2618,7 @@ static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv, /* loop here for each buffer needing assign */ for (i = 0; i < ring->size; i++) { cb = ring->cbs + i; - skb = bcmgenet_rx_refill(priv, cb); - if (skb) - dev_consume_skb_any(skb); - if (!cb->skb) + if (bcmgenet_rx_refill(ring, cb)) return -ENOMEM; } @@ -2509,16 +2627,18 @@ static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv, static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv) { - struct sk_buff *skb; + struct bcmgenet_rx_ring *ring; struct enet_cb *cb; - int i; - - for (i = 0; i < priv->num_rx_bds; i++) { - cb = &priv->rx_cbs[i]; + int q, i; - skb = bcmgenet_free_rx_cb(&priv->pdev->dev, cb); - if (skb) - dev_consume_skb_any(skb); + for (q = 0; q <= priv->hw_params->rx_queues; q++) { + ring = &priv->rx_rings[q]; + if (!ring->page_pool) + continue; + for (i = 0; i < ring->size; i++) { + cb = ring->cbs + i; + bcmgenet_free_rx_cb(cb, ring->page_pool); + } } } @@ -2588,6 +2708,44 @@ static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv) bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR); } +/* Threshold in register units. Covers the alignment bytes and the frame, but + * not the status block, which the hardware adds on top. + */ +static unsigned int bcmgenet_pkt_rdy_thld(unsigned int mtu) +{ + unsigned int len = GENET_RBUF_ALIGN + mtu + ETH_HLEN + VLAN_HLEN; + + len = round_up(len, ENET_THLD_BURST) / ENET_THLD_UNIT; + + /* Keep the reset default for the common MTUs */ + return clamp_t(unsigned int, len, ENET_THLD_DEFAULT, + ENET_THLD_MAX_LEN / ENET_THLD_UNIT); +} + +/* A buffer has to hold everything the threshold lets the hardware deliver */ +static unsigned int bcmgenet_rx_buf_len(unsigned int mtu) +{ + return sizeof(struct status_64) + + bcmgenet_pkt_rdy_thld(mtu) * ENET_THLD_UNIT; +} + +/* Program the MTU dependent registers. Call with the MAC disabled. */ +static void bcmgenet_set_mtu_regs(struct bcmgenet_priv *priv, unsigned int mtu) +{ + u32 thld = bcmgenet_pkt_rdy_thld(mtu); + + priv->tx_csum_max_len = thld * ENET_THLD_UNIT; + bcmgenet_umac_writel(priv, ENET_MAX_FRAME_LEN(mtu), UMAC_MAX_FRAME_LEN); + + /* GENET v1 maps other registers at these offsets */ + if (GENET_IS_V1(priv)) + return; + + bcmgenet_rbuf_writel(priv, thld, RBUF_PKT_RDY_THLD); + bcmgenet_writel(thld, priv->base + priv->hw_params->tbuf_offset + + TBUF_PKT_RDY_THLD); +} + static void init_umac(struct bcmgenet_priv *priv) { struct device *kdev = &priv->pdev->dev; @@ -2604,7 +2762,7 @@ static void init_umac(struct bcmgenet_priv *priv) UMAC_MIB_CTRL); bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL); - bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN); + bcmgenet_set_mtu_regs(priv, priv->dev->mtu); /* init tx registers, enable TSB */ reg = bcmgenet_tbuf_ctrl_get(priv); @@ -2710,7 +2868,7 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, /* Set flow period for ring != 0 */ if (index) - flow_period_val = ENET_MAX_MTU_SIZE << 16; + flow_period_val = ENET_MAX_FRAME_LEN(priv->dev->mtu) << 16; bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX); bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX); @@ -2720,7 +2878,7 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, TDMA_FLOW_PERIOD); bcmgenet_tdma_ring_writel(priv, index, ((size << DMA_RING_SIZE_SHIFT) | - RX_BUF_LENGTH), DMA_RING_BUF_SIZE); + priv->rx_buf_len), DMA_RING_BUF_SIZE); /* Set start and end address, read and write pointers */ bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd, @@ -2736,6 +2894,30 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, netif_napi_add_tx(priv->dev, &ring->napi, bcmgenet_tx_poll); } +static int bcmgenet_rx_ring_create_pool(struct bcmgenet_priv *priv, + struct bcmgenet_rx_ring *ring) +{ + struct page_pool_params pp_params = { + .order = 0, + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .pool_size = ring->size, + .nid = NUMA_NO_NODE, + .dev = &priv->pdev->dev, + .dma_dir = DMA_FROM_DEVICE, + .max_len = priv->rx_buf_len, + }; + int err; + + ring->page_pool = page_pool_create(&pp_params); + if (IS_ERR(ring->page_pool)) { + err = PTR_ERR(ring->page_pool); + ring->page_pool = NULL; + return err; + } + + return 0; +} + /* Initialize a RDMA ring */ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, unsigned int index, unsigned int size, @@ -2743,7 +2925,7 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, { struct bcmgenet_rx_ring *ring = &priv->rx_rings[index]; u32 words_per_bd = WORDS_PER_BD(priv); - int ret; + int ret, i; ring->priv = priv; ring->index = index; @@ -2754,10 +2936,19 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, ring->cb_ptr = start_ptr; ring->end_ptr = end_ptr - 1; - ret = bcmgenet_alloc_rx_buffers(priv, ring); + ret = bcmgenet_rx_ring_create_pool(priv, ring); if (ret) return ret; + ret = bcmgenet_alloc_rx_buffers(priv, ring); + if (ret) { + for (i = 0; i < ring->size; i++) + bcmgenet_free_rx_cb(ring->cbs + i, ring->page_pool); + page_pool_destroy(ring->page_pool); + ring->page_pool = NULL; + return ret; + } + bcmgenet_init_dim(ring, bcmgenet_dim_work); bcmgenet_init_rx_coalesce(ring); @@ -2768,7 +2959,7 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX); bcmgenet_rdma_ring_writel(priv, index, ((size << DMA_RING_SIZE_SHIFT) | - RX_BUF_LENGTH), DMA_RING_BUF_SIZE); + priv->rx_buf_len), DMA_RING_BUF_SIZE); bcmgenet_rdma_ring_writel(priv, index, (DMA_FC_THRESH_LO << DMA_XOFF_THRESHOLD_SHIFT) | @@ -2951,6 +3142,20 @@ static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv) } } +static void bcmgenet_destroy_rx_page_pools(struct bcmgenet_priv *priv) +{ + struct bcmgenet_rx_ring *ring; + unsigned int i; + + for (i = 0; i <= priv->hw_params->rx_queues; ++i) { + ring = &priv->rx_rings[i]; + if (ring->page_pool) { + page_pool_destroy(ring->page_pool); + ring->page_pool = NULL; + } + } +} + /* Initialize Rx queues * * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be @@ -3013,6 +3218,10 @@ static void bcmgenet_fini_dma(struct bcmgenet_priv *priv) struct netdev_queue *txq; int i; + /* An MTU change can fail with the rings already freed */ + if (!priv->rx_cbs) + return; + bcmgenet_fini_rx_napi(priv); bcmgenet_fini_tx_napi(priv); @@ -3022,8 +3231,11 @@ static void bcmgenet_fini_dma(struct bcmgenet_priv *priv) } bcmgenet_free_rx_buffers(priv); + bcmgenet_destroy_rx_page_pools(priv); kfree(priv->rx_cbs); + priv->rx_cbs = NULL; kfree(priv->tx_cbs); + priv->tx_cbs = NULL; } /* init_edma: Initialize DMA control register */ @@ -3083,6 +3295,7 @@ static int bcmgenet_init_dma(struct bcmgenet_priv *priv, bool flush_rx) GFP_KERNEL); if (!priv->tx_cbs) { kfree(priv->rx_cbs); + priv->rx_cbs = NULL; return -ENOMEM; } @@ -3100,8 +3313,11 @@ static int bcmgenet_init_dma(struct bcmgenet_priv *priv, bool flush_rx) if (ret) { netdev_err(priv->dev, "failed to initialize Rx queues\n"); bcmgenet_free_rx_buffers(priv); + bcmgenet_destroy_rx_page_pools(priv); kfree(priv->rx_cbs); + priv->rx_cbs = NULL; kfree(priv->tx_cbs); + priv->tx_cbs = NULL; return ret; } @@ -3270,7 +3486,7 @@ static void bcmgenet_get_hw_addr(struct bcmgenet_priv *priv, put_unaligned_be16(addr_tmp, &addr[4]); } -static void bcmgenet_netif_start(struct net_device *dev) +static void bcmgenet_netif_start(struct net_device *dev, bool start_phy) { struct bcmgenet_priv *priv = netdev_priv(dev); @@ -3287,7 +3503,8 @@ static void bcmgenet_netif_start(struct net_device *dev) /* Monitor link interrupts now */ bcmgenet_link_intr_enable(priv); - phy_start(dev->phydev); + if (start_phy) + phy_start(dev->phydev); } static int bcmgenet_open(struct net_device *dev) @@ -3318,8 +3535,8 @@ static int bcmgenet_open(struct net_device *dev) bcmgenet_set_hw_addr(priv, dev->dev_addr); - /* HFB init */ - bcmgenet_hfb_init(priv); + /* Restore the filters, the MAC was reset above */ + bcmgenet_hfb_restore(priv); /* Reinitialize TDMA and RDMA and SW housekeeping */ ret = bcmgenet_init_dma(priv, true); @@ -3350,8 +3567,9 @@ static int bcmgenet_open(struct net_device *dev) bcmgenet_phy_pause_set(dev, priv->rx_pause, priv->tx_pause); - bcmgenet_netif_start(dev); + bcmgenet_netif_start(dev, true); + priv->datapath_up = true; netif_tx_start_all_queues(dev); return 0; @@ -3374,6 +3592,8 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) { struct bcmgenet_priv *priv = netdev_priv(dev); + /* Stop completion polling before it can wake a stopped queue */ + bcmgenet_disable_tx_napi(priv); netif_tx_disable(dev); /* Disable MAC receive */ @@ -3388,7 +3608,6 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) /* Disable MAC transmit. TX DMA disabled must be done before this */ umac_enable_set(priv, CMD_TX_EN, false); - bcmgenet_disable_tx_napi(priv); bcmgenet_disable_rx_napi(priv); bcmgenet_intr_disable(priv); @@ -3409,7 +3628,11 @@ static int bcmgenet_close(struct net_device *dev) netif_dbg(priv, ifdown, dev, "bcmgenet_close\n"); - bcmgenet_netif_stop(dev, false); + /* A failed MTU change can have torn the datapath down already */ + if (priv->datapath_up) { + bcmgenet_netif_stop(dev, false); + priv->datapath_up = false; + } /* Really kill the PHY state machine and disconnect from it */ phy_disconnect(dev->phydev); @@ -3654,6 +3877,66 @@ static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier) return 0; } +static int bcmgenet_change_mtu(struct net_device *dev, int new_mtu) +{ + struct bcmgenet_priv *priv = netdev_priv(dev); + unsigned int old_mtu = dev->mtu; + int ret; + + if (!netif_running(dev)) { + WRITE_ONCE(dev->mtu, new_mtu); + priv->rx_buf_len = bcmgenet_rx_buf_len(new_mtu); + return 0; + } + + /* The watchdog trips on an idle queue once the rings are gone */ + netif_device_detach(dev); + + /* Only the buffers and the MTU registers change, leave the PHY up */ + bcmgenet_netif_stop(dev, false); + priv->datapath_up = false; + + WRITE_ONCE(dev->mtu, new_mtu); + priv->rx_buf_len = bcmgenet_rx_buf_len(new_mtu); + bcmgenet_set_mtu_regs(priv, new_mtu); + + ret = bcmgenet_init_dma(priv, true); + if (ret) { + /* Retry the size that was allocated a moment ago */ + WRITE_ONCE(dev->mtu, old_mtu); + priv->rx_buf_len = bcmgenet_rx_buf_len(old_mtu); + bcmgenet_set_mtu_regs(priv, old_mtu); + if (bcmgenet_init_dma(priv, true)) { + /* Nothing left to run on. Take the interface down so + * that close and suspend do not tear it down twice. + */ + netdev_err(dev, "failed to restore MTU %u, closing\n", + old_mtu); + netif_close(dev); + + /* Mark the device present again, __dev_open() + * refuses a detached one. The queues stay stopped + * because the interface is down by now. + */ + netif_device_attach(dev); + return ret; + } + } + + bcmgenet_hfb_restore(priv); + bcmgenet_netif_start(dev, false); + + /* bcmgenet_netif_start() only restores the link interrupt */ + if (bcmgenet_has_mdio_intr(priv)) + bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_MDIO_EVENT, + INTRL2_CPU_MASK_CLEAR); + + priv->datapath_up = true; + netif_device_attach(dev); + + return ret; +} + static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_open = bcmgenet_open, .ndo_stop = bcmgenet_close, @@ -3665,6 +3948,7 @@ static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_set_features = bcmgenet_set_features, .ndo_get_stats64 = bcmgenet_get_stats64, .ndo_change_carrier = bcmgenet_change_carrier, + .ndo_change_mtu = bcmgenet_change_mtu, }; /* GENET hardware parameters/characteristics */ @@ -3909,7 +4193,6 @@ MODULE_DEVICE_TABLE(of, bcmgenet_match); static int bcmgenet_probe(struct platform_device *pdev) { - struct bcmgenet_platform_data *pd = pdev->dev.platform_data; const struct bcmgenet_plat_data *pdata; struct bcmgenet_priv *priv; struct net_device *dev; @@ -3993,9 +4276,6 @@ static int bcmgenet_probe(struct platform_device *pdev) priv->version = pdata->version; priv->dma_max_burst_length = pdata->dma_max_burst_length; priv->flags = pdata->flags; - } else { - priv->version = pd->genet_version; - priv->dma_max_burst_length = DMA_MAX_BURST_LENGTH; } priv->clk = devm_clk_get_optional(&priv->pdev->dev, "enet"); @@ -4021,8 +4301,12 @@ static int bcmgenet_probe(struct platform_device *pdev) /* Mii wait queue */ init_waitqueue_head(&priv->wq); - /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */ - priv->rx_buf_len = RX_BUF_LENGTH; + bcmgenet_hfb_init(priv); + + /* v1 cannot program the thresholds, so it stays at the default MTU */ + priv->rx_buf_len = bcmgenet_rx_buf_len(dev->mtu); + if (!GENET_IS_V1(priv)) + dev->max_mtu = ENET_MAX_JUMBO_MTU; INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task); priv->clk_wol = devm_clk_get_optional(&priv->pdev->dev, "enet-wol"); @@ -4045,16 +4329,13 @@ static int bcmgenet_probe(struct platform_device *pdev) if (device_get_phy_mode(&pdev->dev) == PHY_INTERFACE_MODE_INTERNAL) bcmgenet_power_up(priv, GENET_POWER_PASSIVE); - if (pd && !IS_ERR_OR_NULL(pd->mac_address)) - eth_hw_addr_set(dev, pd->mac_address); - else - if (device_get_ethdev_address(&pdev->dev, dev)) - if (has_acpi_companion(&pdev->dev)) { - u8 addr[ETH_ALEN]; + if (device_get_ethdev_address(&pdev->dev, dev)) + if (has_acpi_companion(&pdev->dev)) { + u8 addr[ETH_ALEN]; - bcmgenet_get_hw_addr(priv, addr); - eth_hw_addr_set(dev, addr); - } + bcmgenet_get_hw_addr(priv, addr); + eth_hw_addr_set(dev, addr); + } if (!is_valid_ether_addr(dev->dev_addr)) { dev_warn(&pdev->dev, "using random Ethernet MAC\n"); @@ -4225,10 +4506,7 @@ static int bcmgenet_resume(struct device *d) bcmgenet_set_hw_addr(priv, dev->dev_addr); /* Restore hardware filters */ - bcmgenet_hfb_clear(priv); - list_for_each_entry(rule, &priv->rxnfc_list, list) - if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) - bcmgenet_hfb_create_rxnfc_filter(priv, rule); + bcmgenet_hfb_restore(priv); /* Reinitialize TDMA and RDMA and SW housekeeping */ ret = bcmgenet_init_dma(priv, false); @@ -4240,7 +4518,7 @@ static int bcmgenet_resume(struct device *d) if (!device_may_wakeup(d)) phy_resume(dev->phydev); - bcmgenet_netif_start(dev); + bcmgenet_netif_start(dev, true); netif_device_attach(dev); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 9e4110c7fdf6f..8bab6d6c773d0 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -15,6 +15,7 @@ #include #include #include +#include #include "../unimac.h" @@ -27,13 +28,14 @@ /* which ring is descriptor based */ #define DESC_INDEX 16 -/* Body(1500) + EH_SIZE(14) + VLANTAG(4) + BRCMTAG(6) + FCS(4) = 1528. - * 1536 is multiple of 256 bytes - */ #define ENET_BRCM_TAG_LEN 6 #define ENET_PAD 8 -#define ENET_MAX_MTU_SIZE (ETH_DATA_LEN + ETH_HLEN + VLAN_HLEN + \ - ENET_BRCM_TAG_LEN + ETH_FCS_LEN + ENET_PAD) + +/* Longest frame the MAC must accept for a given MTU */ +#define ENET_FRAME_OVERHEAD (ETH_HLEN + VLAN_HLEN + ENET_BRCM_TAG_LEN + \ + ETH_FCS_LEN + ENET_PAD) +#define ENET_MAX_FRAME_LEN(mtu) ((mtu) + ENET_FRAME_OVERHEAD) + #define DMA_MAX_BURST_LENGTH 0x10 /* misc. configuration */ @@ -149,7 +151,6 @@ struct bcmgenet_mib_counters { u32 rbuf_err_cnt; u32 mdf_err_cnt; u32 alloc_rx_buff_failed; - u32 rx_dma_failed; u32 tx_dma_failed; u32 tx_realloc_tsb; u32 tx_realloc_tsb_failed; @@ -219,6 +220,8 @@ struct bcmgenet_rx_stats64 { #define RBUF_ALIGN_2B (1 << 1) #define RBUF_BAD_DIS (1 << 2) +#define RBUF_PKT_RDY_THLD 0x08 + #define RBUF_STATUS 0x0C #define RBUF_STATUS_WOL (1 << 0) #define RBUF_STATUS_MPD_INTR_ACTIVE (1 << 1) @@ -249,6 +252,7 @@ struct bcmgenet_rx_stats64 { #define TBUF_CTRL 0x00 #define TBUF_64B_EN (1 << 0) #define TBUF_BP_MC 0x0C +#define TBUF_PKT_RDY_THLD 0x10 #define TBUF_ENERGY_CTRL 0x14 #define TBUF_EEE_EN (1 << 0) #define TBUF_PM_EN (1 << 1) @@ -469,6 +473,7 @@ struct bcmgenet_rx_stats64 { struct enet_cb { struct sk_buff *skb; + struct page *rx_page; void __iomem *bd_addr; DEFINE_DMA_UNMAP_ADDR(dma_addr); DEFINE_DMA_UNMAP_LEN(dma_len); @@ -572,9 +577,11 @@ struct bcmgenet_rx_ring { unsigned int cb_ptr; /* Rx ring initial CB ptr */ unsigned int end_ptr; /* Rx ring end CB ptr */ unsigned int old_discards; + struct sk_buff *frag_head; /* frame being reassembled */ struct bcmgenet_net_dim dim; u32 rx_max_coalesced_frames; u32 rx_coalesce_usecs; + struct page_pool *page_pool; struct bcmgenet_priv *priv; }; @@ -602,6 +609,7 @@ struct bcmgenet_priv { void __iomem *tx_bds; struct enet_cb *tx_cbs; unsigned int num_tx_bds; + unsigned int tx_csum_max_len; struct bcmgenet_tx_ring tx_rings[GENET_MAX_MQ_CNT + 1]; @@ -621,6 +629,7 @@ struct bcmgenet_priv { unsigned autoneg_pause:1; unsigned tx_pause:1; unsigned rx_pause:1; + unsigned datapath_up:1; /* MDIO bus variables */ wait_queue_head_t wq; diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 034f9b3599642..ec5f95b8daae6 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include "bcmgenet.h" @@ -434,23 +433,6 @@ static struct device_node *bcmgenet_mii_of_find_mdio(struct bcmgenet_priv *priv) return priv->mdio_dn; } -static void bcmgenet_mii_pdata_init(struct bcmgenet_priv *priv, - struct unimac_mdio_pdata *ppd) -{ - struct device *kdev = &priv->pdev->dev; - struct bcmgenet_platform_data *pd = kdev->platform_data; - - if (pd->phy_interface != PHY_INTERFACE_MODE_MOCA && pd->mdio_enabled) { - /* - * Internal or external PHY with MDIO access - */ - if (pd->phy_address >= 0 && pd->phy_address < PHY_MAX_ADDR) - ppd->phy_mask = 1 << pd->phy_address; - else - ppd->phy_mask = 0; - } -} - static int bcmgenet_mii_wait(void *wait_func_data) { struct bcmgenet_priv *priv = wait_func_data; @@ -465,7 +447,6 @@ static int bcmgenet_mii_wait(void *wait_func_data) static int bcmgenet_mii_register(struct bcmgenet_priv *priv) { struct platform_device *pdev = priv->pdev; - struct bcmgenet_platform_data *pdata = pdev->dev.platform_data; struct device_node *dn = pdev->dev.of_node; struct unimac_mdio_pdata ppd; struct platform_device *ppdev; @@ -509,8 +490,6 @@ static int bcmgenet_mii_register(struct bcmgenet_priv *priv) ppdev->dev.parent = &pdev->dev; if (dn) ppdev->dev.of_node = bcmgenet_mii_of_find_mdio(priv); - else if (pdata) - bcmgenet_mii_pdata_init(priv, &ppd); else ppd.phy_mask = ~0; @@ -592,58 +571,6 @@ static int bcmgenet_mii_of_init(struct bcmgenet_priv *priv) return 0; } -static int bcmgenet_mii_pd_init(struct bcmgenet_priv *priv) -{ - struct device *kdev = &priv->pdev->dev; - struct bcmgenet_platform_data *pd = kdev->platform_data; - char phy_name[MII_BUS_ID_SIZE + 3]; - char mdio_bus_id[MII_BUS_ID_SIZE]; - struct phy_device *phydev; - - snprintf(mdio_bus_id, MII_BUS_ID_SIZE, "%s-%d", - UNIMAC_MDIO_DRV_NAME, priv->pdev->id); - - if (pd->phy_interface != PHY_INTERFACE_MODE_MOCA && pd->mdio_enabled) { - snprintf(phy_name, MII_BUS_ID_SIZE, PHY_ID_FMT, - mdio_bus_id, pd->phy_address); - - /* - * Internal or external PHY with MDIO access - */ - phydev = phy_attach(priv->dev, phy_name, pd->phy_interface); - if (IS_ERR(phydev)) { - dev_err(kdev, "failed to register PHY device\n"); - return PTR_ERR(phydev); - } - } else { - /* - * MoCA port or no MDIO access. - * Use fixed PHY to represent the link layer. - */ - struct fixed_phy_status fphy_status = { - .link = 1, - .speed = pd->phy_speed, - .duplex = pd->phy_duplex, - .pause = 0, - .asym_pause = 0, - }; - - phydev = fixed_phy_register(&fphy_status, NULL); - if (IS_ERR(phydev)) { - dev_err(kdev, "failed to register fixed PHY device\n"); - return PTR_ERR(phydev); - } - - /* Make sure we initialize MoCA PHYs with a link down */ - phydev->link = 0; - - } - - priv->phy_interface = pd->phy_interface; - - return 0; -} - static int bcmgenet_mii_bus_init(struct bcmgenet_priv *priv) { struct device *kdev = &priv->pdev->dev; @@ -654,7 +581,7 @@ static int bcmgenet_mii_bus_init(struct bcmgenet_priv *priv) else if (has_acpi_companion(kdev)) return bcmgenet_phy_interface_init(priv); else - return bcmgenet_mii_pd_init(priv); + return -EINVAL; } int bcmgenet_mii_init(struct net_device *dev) diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index a4d8c4d6f19fb..a3f0f4eb4df0e 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -570,6 +570,13 @@ static int bcm54xx_config_init(struct phy_device *phydev) bcm54xx_ptp_config_init(phydev); + /* Transmit is limited to 4.5 KB without it, see the BCM54210PE + * datasheet section 5.4.12.14. + */ + err = bcm_phy_enable_jumbo(phydev); + if (err < 0) + return err; + /* Acknowledge any left over interrupt and charge the device for * wake-up. */ diff --git a/include/linux/platform_data/bcmgenet.h b/include/linux/platform_data/bcmgenet.h deleted file mode 100644 index d8f8738629d20..0000000000000 --- a/include/linux/platform_data/bcmgenet.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_PLATFORM_DATA_BCMGENET_H__ -#define __LINUX_PLATFORM_DATA_BCMGENET_H__ - -#include -#include -#include - -struct bcmgenet_platform_data { - bool mdio_enabled; - phy_interface_t phy_interface; - int phy_address; - int phy_speed; - int phy_duplex; - u8 mac_address[ETH_ALEN]; - int genet_version; -}; - -#endif