Skip to content

Commit dbadae9

Browse files
Gerhard Englederdavem330
Gerhard Engleder
authored and
davem330
committed
tsnep: Rework RX buffer allocation
Refill RX queue in batches of descriptors to improve performance. Refill is allowed to fail as long as a minimum number of descriptors is active. Thus, a limited number of failed RX buffer allocations is now allowed for normal operation. Previously every failed allocation resulted in a dropped frame. If the minimum number of active descriptors is reached, then RX buffers are still reused and frames are dropped. This ensures that the RX queue never runs empty and always continues to operate. Prework for future XDP support. Signed-off-by: Gerhard Engleder <[email protected]> Signed-off-by: David S. Miller <[email protected]>
1 parent d3dfe8d commit dbadae9

File tree

3 files changed

+140
-74
lines changed

3 files changed

+140
-74
lines changed

‎drivers/net/ethernet/engleder/tsnep.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
#define TSNEP "tsnep"
1919

2020
#define TSNEP_RING_SIZE 256
21+
#define TSNEP_RING_RX_REFILL 16
22+
#define TSNEP_RING_RX_REUSE (TSNEP_RING_SIZE - TSNEP_RING_SIZE / 4)
2123
#define TSNEP_RING_ENTRIES_PER_PAGE (PAGE_SIZE / TSNEP_DESC_SIZE)
2224
#define TSNEP_RING_PAGE_COUNT (TSNEP_RING_SIZE / TSNEP_RING_ENTRIES_PER_PAGE)
2325

@@ -110,6 +112,7 @@ struct tsnep_rx {
110112
dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT];
111113

112114
struct tsnep_rx_entry entry[TSNEP_RING_SIZE];
115+
int write;
113116
int read;
114117
u32 owner_counter;
115118
int increment_owner_counter;
@@ -119,6 +122,7 @@ struct tsnep_rx {
119122
u32 bytes;
120123
u32 dropped;
121124
u32 multicast;
125+
u32 alloc_failed;
122126
};
123127

124128
struct tsnep_queue {

‎drivers/net/ethernet/engleder/tsnep_ethtool.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ static const char tsnep_stats_strings[][ETH_GSTRING_LEN] = {
88
"rx_bytes",
99
"rx_dropped",
1010
"rx_multicast",
11+
"rx_alloc_failed",
1112
"rx_phy_errors",
1213
"rx_forwarded_phy_errors",
1314
"rx_invalid_frame_errors",
@@ -21,6 +22,7 @@ struct tsnep_stats {
2122
u64 rx_bytes;
2223
u64 rx_dropped;
2324
u64 rx_multicast;
25+
u64 rx_alloc_failed;
2426
u64 rx_phy_errors;
2527
u64 rx_forwarded_phy_errors;
2628
u64 rx_invalid_frame_errors;
@@ -36,6 +38,7 @@ static const char tsnep_rx_queue_stats_strings[][ETH_GSTRING_LEN] = {
3638
"rx_%d_bytes",
3739
"rx_%d_dropped",
3840
"rx_%d_multicast",
41+
"rx_%d_alloc_failed",
3942
"rx_%d_no_descriptor_errors",
4043
"rx_%d_buffer_too_small_errors",
4144
"rx_%d_fifo_overflow_errors",
@@ -47,6 +50,7 @@ struct tsnep_rx_queue_stats {
4750
u64 rx_bytes;
4851
u64 rx_dropped;
4952
u64 rx_multicast;
53+
u64 rx_alloc_failed;
5054
u64 rx_no_descriptor_errors;
5155
u64 rx_buffer_too_small_errors;
5256
u64 rx_fifo_overflow_errors;
@@ -178,6 +182,7 @@ static void tsnep_ethtool_get_ethtool_stats(struct net_device *netdev,
178182
tsnep_stats.rx_bytes += adapter->rx[i].bytes;
179183
tsnep_stats.rx_dropped += adapter->rx[i].dropped;
180184
tsnep_stats.rx_multicast += adapter->rx[i].multicast;
185+
tsnep_stats.rx_alloc_failed += adapter->rx[i].alloc_failed;
181186
}
182187
reg = ioread32(adapter->addr + ECM_STAT);
183188
tsnep_stats.rx_phy_errors =
@@ -200,6 +205,8 @@ static void tsnep_ethtool_get_ethtool_stats(struct net_device *netdev,
200205
tsnep_rx_queue_stats.rx_bytes = adapter->rx[i].bytes;
201206
tsnep_rx_queue_stats.rx_dropped = adapter->rx[i].dropped;
202207
tsnep_rx_queue_stats.rx_multicast = adapter->rx[i].multicast;
208+
tsnep_rx_queue_stats.rx_alloc_failed =
209+
adapter->rx[i].alloc_failed;
203210
reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
204211
TSNEP_RX_STATISTIC);
205212
tsnep_rx_queue_stats.rx_no_descriptor_errors =

‎drivers/net/ethernet/engleder/tsnep_main.c

Lines changed: 129 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -660,23 +660,6 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
660660
}
661661
}
662662

663-
static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx,
664-
struct tsnep_rx_entry *entry)
665-
{
666-
struct page *page;
667-
668-
page = page_pool_dev_alloc_pages(rx->page_pool);
669-
if (unlikely(!page))
670-
return -ENOMEM;
671-
672-
entry->page = page;
673-
entry->len = TSNEP_MAX_RX_BUF_SIZE;
674-
entry->dma = page_pool_get_dma_addr(entry->page);
675-
entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD);
676-
677-
return 0;
678-
}
679-
680663
static int tsnep_rx_ring_init(struct tsnep_rx *rx)
681664
{
682665
struct device *dmadev = rx->adapter->dmadev;
@@ -723,10 +706,6 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx)
723706
entry = &rx->entry[i];
724707
next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE];
725708
entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
726-
727-
retval = tsnep_rx_alloc_buffer(rx, entry);
728-
if (retval)
729-
goto failed;
730709
}
731710

732711
return 0;
@@ -736,6 +715,45 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx)
736715
return retval;
737716
}
738717

718+
static int tsnep_rx_desc_available(struct tsnep_rx *rx)
719+
{
720+
if (rx->read <= rx->write)
721+
return TSNEP_RING_SIZE - rx->write + rx->read - 1;
722+
else
723+
return rx->read - rx->write - 1;
724+
}
725+
726+
static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
727+
struct page *page)
728+
{
729+
entry->page = page;
730+
entry->len = TSNEP_MAX_RX_BUF_SIZE;
731+
entry->dma = page_pool_get_dma_addr(entry->page);
732+
entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD);
733+
}
734+
735+
static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
736+
{
737+
struct tsnep_rx_entry *entry = &rx->entry[index];
738+
struct page *page;
739+
740+
page = page_pool_dev_alloc_pages(rx->page_pool);
741+
if (unlikely(!page))
742+
return -ENOMEM;
743+
tsnep_rx_set_page(rx, entry, page);
744+
745+
return 0;
746+
}
747+
748+
static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
749+
{
750+
struct tsnep_rx_entry *entry = &rx->entry[index];
751+
struct tsnep_rx_entry *read = &rx->entry[rx->read];
752+
753+
tsnep_rx_set_page(rx, entry, read->page);
754+
read->page = NULL;
755+
}
756+
739757
static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
740758
{
741759
struct tsnep_rx_entry *entry = &rx->entry[index];
@@ -763,6 +781,48 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
763781
entry->desc->properties = __cpu_to_le32(entry->properties);
764782
}
765783

784+
static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
785+
{
786+
int index;
787+
bool alloc_failed = false;
788+
bool enable = false;
789+
int i;
790+
int retval;
791+
792+
for (i = 0; i < count && !alloc_failed; i++) {
793+
index = (rx->write + i) % TSNEP_RING_SIZE;
794+
795+
retval = tsnep_rx_alloc_buffer(rx, index);
796+
if (unlikely(retval)) {
797+
rx->alloc_failed++;
798+
alloc_failed = true;
799+
800+
/* reuse only if no other allocation was successful */
801+
if (i == 0 && reuse)
802+
tsnep_rx_reuse_buffer(rx, index);
803+
else
804+
break;
805+
}
806+
807+
tsnep_rx_activate(rx, index);
808+
809+
enable = true;
810+
}
811+
812+
if (enable) {
813+
rx->write = (rx->write + i) % TSNEP_RING_SIZE;
814+
815+
/* descriptor properties shall be valid before hardware is
816+
* notified
817+
*/
818+
dma_wmb();
819+
820+
iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
821+
}
822+
823+
return i;
824+
}
825+
766826
static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
767827
int length)
768828
{
@@ -798,23 +858,42 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
798858
int budget)
799859
{
800860
struct device *dmadev = rx->adapter->dmadev;
861+
int desc_available;
801862
int done = 0;
802863
enum dma_data_direction dma_dir;
803864
struct tsnep_rx_entry *entry;
804-
struct page *page;
805865
struct sk_buff *skb;
806866
int length;
807-
bool enable = false;
808-
int retval;
809867

868+
desc_available = tsnep_rx_desc_available(rx);
810869
dma_dir = page_pool_get_dma_dir(rx->page_pool);
811870

812-
while (likely(done < budget)) {
871+
while (likely(done < budget) && (rx->read != rx->write)) {
813872
entry = &rx->entry[rx->read];
814873
if ((__le32_to_cpu(entry->desc_wb->properties) &
815874
TSNEP_DESC_OWNER_COUNTER_MASK) !=
816875
(entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
817876
break;
877+
done++;
878+
879+
if (desc_available >= TSNEP_RING_RX_REFILL) {
880+
bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
881+
882+
desc_available -= tsnep_rx_refill(rx, desc_available,
883+
reuse);
884+
if (!entry->page) {
885+
/* buffer has been reused for refill to prevent
886+
* empty RX ring, thus buffer cannot be used for
887+
* RX processing
888+
*/
889+
rx->read = (rx->read + 1) % TSNEP_RING_SIZE;
890+
desc_available++;
891+
892+
rx->dropped++;
893+
894+
continue;
895+
}
896+
}
818897

819898
/* descriptor properties shall be read first, because valid data
820899
* is signaled there
@@ -826,49 +905,30 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
826905
TSNEP_DESC_LENGTH_MASK;
827906
dma_sync_single_range_for_cpu(dmadev, entry->dma, TSNEP_SKB_PAD,
828907
length, dma_dir);
829-
page = entry->page;
830908

831-
/* forward skb only if allocation is successful, otherwise
832-
* page is reused and frame dropped
833-
*/
834-
retval = tsnep_rx_alloc_buffer(rx, entry);
835-
if (!retval) {
836-
skb = tsnep_build_skb(rx, page, length);
837-
if (skb) {
838-
page_pool_release_page(rx->page_pool, page);
839-
840-
rx->packets++;
841-
rx->bytes += length -
842-
TSNEP_RX_INLINE_METADATA_SIZE;
843-
if (skb->pkt_type == PACKET_MULTICAST)
844-
rx->multicast++;
845-
846-
napi_gro_receive(napi, skb);
847-
} else {
848-
page_pool_recycle_direct(rx->page_pool, page);
909+
rx->read = (rx->read + 1) % TSNEP_RING_SIZE;
910+
desc_available++;
849911

850-
rx->dropped++;
851-
}
852-
done++;
853-
} else {
854-
rx->dropped++;
855-
}
912+
skb = tsnep_build_skb(rx, entry->page, length);
913+
if (skb) {
914+
page_pool_release_page(rx->page_pool, entry->page);
856915

857-
tsnep_rx_activate(rx, rx->read);
916+
rx->packets++;
917+
rx->bytes += length - TSNEP_RX_INLINE_METADATA_SIZE;
918+
if (skb->pkt_type == PACKET_MULTICAST)
919+
rx->multicast++;
858920

859-
enable = true;
921+
napi_gro_receive(napi, skb);
922+
} else {
923+
page_pool_recycle_direct(rx->page_pool, entry->page);
860924

861-
rx->read = (rx->read + 1) % TSNEP_RING_SIZE;
925+
rx->dropped++;
926+
}
927+
entry->page = NULL;
862928
}
863929

864-
if (enable) {
865-
/* descriptor properties shall be valid before hardware is
866-
* notified
867-
*/
868-
dma_wmb();
869-
870-
iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
871-
}
930+
if (desc_available)
931+
tsnep_rx_refill(rx, desc_available, false);
872932

873933
return done;
874934
}
@@ -877,11 +937,13 @@ static bool tsnep_rx_pending(struct tsnep_rx *rx)
877937
{
878938
struct tsnep_rx_entry *entry;
879939

880-
entry = &rx->entry[rx->read];
881-
if ((__le32_to_cpu(entry->desc_wb->properties) &
882-
TSNEP_DESC_OWNER_COUNTER_MASK) ==
883-
(entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
884-
return true;
940+
if (rx->read != rx->write) {
941+
entry = &rx->entry[rx->read];
942+
if ((__le32_to_cpu(entry->desc_wb->properties) &
943+
TSNEP_DESC_OWNER_COUNTER_MASK) ==
944+
(entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
945+
return true;
946+
}
885947

886948
return false;
887949
}
@@ -890,7 +952,6 @@ static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr,
890952
int queue_index, struct tsnep_rx *rx)
891953
{
892954
dma_addr_t dma;
893-
int i;
894955
int retval;
895956

896957
memset(rx, 0, sizeof(*rx));
@@ -908,13 +969,7 @@ static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr,
908969
rx->owner_counter = 1;
909970
rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
910971

911-
for (i = 0; i < TSNEP_RING_SIZE; i++)
912-
tsnep_rx_activate(rx, i);
913-
914-
/* descriptor properties shall be valid before hardware is notified */
915-
dma_wmb();
916-
917-
iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
972+
tsnep_rx_refill(rx, tsnep_rx_desc_available(rx), false);
918973

919974
return 0;
920975
}

0 commit comments

Comments
 (0)