/*
   forward-ethernet.c - Ethernet driver for SoftLab-NSK Forward boards

   Copyright (C) 2017 - 2024 SoftLab-NSK <forward@softlab.tv>

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; either version 2
   of the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#include "forward.h"
#include "forward-ethernet.h"
#include "forward-sdma.h"

#include <linux/module.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <linux/poll.h>
#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/etherdevice.h>
#include <linux/mii.h>
#include <linux/net_tstamp.h>

#define FORWARD_ETHERNET_FEATURES (NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_RXALL | NETIF_F_RXFCS)

#define FORWARD_ETHERNET_CMD_SOF (0x1)
#define FORWARD_ETHERNET_CMD_EOF (0x2)

#define FORWARD_ETHERNET_STAT_VALID (0x1)
#define FORWARD_ETHERNET_STAT_SOF (0x2)
#define FORWARD_ETHERNET_STAT_EOF (0x4)

#define FORWARD_ETHERNET_RX_PREROLL_SIZE (8192)
#define FORWARD_ETHERNET_TX_CHUNK_SIZE (64)

#define FORWARD_ETHERNET_BUF_SIZE (2048)
#define FORWARD_ETHERNET_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
#define FORWARD_ETHERNET_PAYLOAD_SIZE \
	(SKB_WITH_OVERHEAD(FORWARD_ETHERNET_BUF_SIZE) - FORWARD_ETHERNET_HEADROOM)

extern const struct forward_ethernet_ops fd2110_ops;
extern const struct ethtool_ops forward_ethtool_ops;

static int forward_hwtstamp_set(struct forward_ethernet *eth, struct ifreq *ifr)
{
	struct hwtstamp_config config;

	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
		return -EFAULT;

	config.rx_filter = HWTSTAMP_FILTER_ALL;

	eth->ts_config = config;

	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0;
}

static int forward_hwtstamp_get(struct forward_ethernet *eth, struct ifreq *ifr)
{
	return copy_to_user(ifr->ifr_data, &eth->ts_config, sizeof(eth->ts_config)) ? -EFAULT : 0;
}

static int forward_net_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
{
	struct forward_ethernet *eth = netdev_priv(dev);
	int retval;

	switch (cmd) {
	case SIOCSHWTSTAMP:
		retval = forward_hwtstamp_set(eth, ifr);
		break;
	case SIOCGHWTSTAMP:
		retval = forward_hwtstamp_get(eth, ifr);
		break;
	default:
		return -EOPNOTSUPP;
	}
	return retval;
}

static void forward_net_set_rx_mode(struct net_device *dev)
{
	struct forward_ethernet *eth = netdev_priv(dev);
	bool multicast = false, promisc = false;

	if (dev->flags & IFF_PROMISC)
		promisc = true;

	if ((dev->flags & IFF_ALLMULTI) || netdev_mc_count(dev))
		multicast = true;

	eth->ops->toggle_mcast(eth, multicast, promisc);
}

static void forward_ethernet_irq(void *p, const forward_irq_t *irq, u64 timestamp)
{
	struct forward_ethernet *eth = p;
	unsigned long sflags;
	struct timespec64 ts;
	bool rx = false, tx = false;

	eth->ops->irq_info(eth, irq, &rx, &tx, &ts);
	eth->ops->disable_irq(eth, rx, tx);

	spin_lock_irqsave(&eth->lock, sflags);
	eth->irq_time = ts;
	spin_unlock_irqrestore(&eth->lock, sflags);

	if (rx)
		napi_schedule(&eth->napi_rx);

	if (tx)
		napi_schedule(&eth->napi_tx);
}

static enum netdev_tx forward_net_queue_tx(struct forward_ethernet *eth,
					   struct forward_sdma_io_ring *ring, struct sk_buff *buf)
{
	dma_addr_t phys_addr;
	size_t len = skb_headlen(buf);
	int tx_hw_queued = ring->sw - ring->hw;
	u16 desc_pointer;
	sdma_desc_t desc;

	if (tx_hw_queued < 0)
		tx_hw_queued += ring->size;

	if (tx_hw_queued + eth->tx_queued + 1 >= ring->size)
		return NETDEV_TX_BUSY;

	desc_pointer = (ring->sw + eth->tx_queued) % ring->size;

	phys_addr = dma_map_single(eth->dev->parent_dev, buf->data, len, DMA_TO_DEVICE);

	if (dma_mapping_error(eth->dev->parent_dev, phys_addr))
		return NETDEV_TX_BUSY;

	len += 4;
	if (len < 64)
		len = 64;

	desc = forward_sdma_set_descriptor(ring, desc_pointer, phys_addr, len,
					   FORWARD_ETHERNET_CMD_SOF | FORWARD_ETHERNET_CMD_EOF,
					   buf);

	eth->tx_queued++;

	if (unlikely(skb_shinfo(buf)->tx_flags & SKBTX_HW_TSTAMP)) {
		skb_shinfo(buf)->tx_flags |= SKBTX_IN_PROGRESS;
	}
	skb_tx_timestamp(buf);

	if (!netdev_xmit_more() || (eth->tx_queued >= FORWARD_ETHERNET_TX_CHUNK_SIZE)) {
		dma_wmb();
		forward_sdma_advance(ring, eth->tx_queued);
		eth->tx_queued = 0;
	}

	return NETDEV_TX_OK;
}

static int forward_net_dequeue_tx(struct forward_ethernet *eth, struct forward_sdma_io_ring *ring,
				  int budget)
{
	struct forward_dev *dev = eth->dev;
	int transmitted_desc = (int)ring->hw - (int)eth->tx_pointer;
	struct timespec64 irq_time = eth->irq_time;
	bool first = true;
	u64 prev_ts = U64_MAX;
	int result;

	if (transmitted_desc == 0)
		return 0;

	if (transmitted_desc < 0)
		transmitted_desc += ring->size;

	transmitted_desc = (transmitted_desc > budget) ? budget : transmitted_desc;
	result = transmitted_desc;

	dma_rmb();

	while (transmitted_desc > 0) {
		sdma_desc_t desc;
		void *cookie;
		u16 len;
		u8 status;
		dma_addr_t dma;
		u64 timestamp;
		struct timespec64 time;
		struct sk_buff *skb;
		unsigned long sflags;

		desc = forward_sdma_get_descriptor(ring, eth->tx_pointer, &dma, &len, &status,
						   &timestamp, &cookie);
		skb = cookie;

		time.tv_nsec = timestamp & 0xFFFFFFFF;
		time.tv_sec = (timestamp >> 32) & 0xFFFF;

		if (unlikely(first)) {
			if (time.tv_sec > (irq_time.tv_sec & 0xFFFF))
				irq_time.tv_sec -= 0x10000;
			first = false;
			prev_ts = timestamp;
		}
		if (unlikely(timestamp < prev_ts)) {
			irq_time.tv_sec += 0x10000;
		}
		time.tv_sec |= (eth->irq_time.tv_sec) & ~(0xFFFF);
		prev_ts = timestamp;

		dma_unmap_single(dev->parent_dev, dma, skb_headlen(skb), DMA_TO_DEVICE);

		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
			struct skb_shared_hwtstamps shhwtstamps;
			shhwtstamps.hwtstamp = timespec64_to_ktime(time);
			skb_tstamp_tx(skb, &shhwtstamps);
		}

		eth->tx_pointer++;
		if (eth->tx_pointer >= ring->size)
			eth->tx_pointer -= ring->size;
		transmitted_desc--;

		spin_lock_irqsave(&eth->stats_lock, sflags);
		eth->stats.tx_packets++;
		eth->stats.tx_bytes += len - 4;
		spin_unlock_irqrestore(&eth->stats_lock, sflags);

		napi_consume_skb(skb, budget);
	}

	return result;
}

static void forward_net_queue_rx(struct forward_ethernet *eth, struct forward_sdma_io_ring *ring,
				 int count)
{
	struct forward_dev *dev = eth->dev;
	int queued_desc = 0;
	u16 wr_pointer = ring->sw;
	int new_desc = 0;

	if (count == 0)
		return;

	while (queued_desc < count) {
		void *buf = napi_alloc_frag(FORWARD_ETHERNET_BUF_SIZE);
		int len = SKB_WITH_OVERHEAD(FORWARD_ETHERNET_BUF_SIZE) - FORWARD_ETHERNET_HEADROOM;
		dma_addr_t dma;
		sdma_desc_t desc;

		dma = dma_map_single(dev->parent_dev, buf + FORWARD_ETHERNET_HEADROOM,
				     FORWARD_ETHERNET_PAYLOAD_SIZE, DMA_FROM_DEVICE);
		if (dma_mapping_error(dev->parent_dev, dma)) {
			skb_free_frag(buf);
			break;
		}

		desc = forward_sdma_set_descriptor(ring, wr_pointer, dma, len,
						   FORWARD_ETHERNET_CMD_SOF, buf);

		wr_pointer++;
		if (wr_pointer >= ring->size)
			wr_pointer -= ring->size;
		queued_desc++;
		new_desc++;
	}

	dma_wmb();
	forward_sdma_advance(ring, new_desc);
}

static int forward_net_dequeue_rx(struct forward_ethernet *eth, struct forward_sdma_io_ring *ring,
				  int budget)
{
	struct forward_dev *dev = eth->dev;
	int received_desc = (int)ring->hw - (int)eth->rx_pointer;
	struct timespec64 irq_time = eth->irq_time;
	bool first = true;
	u64 prev_ts = U64_MAX;
	int result;

	if (received_desc == 0)
		return 0;

	if (received_desc < 0)
		received_desc += ring->size;

	received_desc = (received_desc > budget) ? budget : received_desc;
	result = received_desc;

	dma_rmb();

	while (received_desc > 0) {
		sdma_desc_t desc;
		void *buf;
		u16 rxlen;
		u8 status;
		dma_addr_t dma;
		u64 timestamp;
		struct timespec64 time;
		struct sk_buff *skb;
		unsigned long sflags;

		desc = forward_sdma_get_descriptor(ring, eth->rx_pointer, &dma, &rxlen, &status,
						   &timestamp, &buf);

		time.tv_nsec = timestamp & 0xFFFFFFFF;
		time.tv_sec = (timestamp >> 32) & 0xFFFF;

		if (unlikely(first)) {
			if (time.tv_sec > (irq_time.tv_sec & 0xFFFF))
				irq_time.tv_sec -= 0x10000;
			first = false;
			prev_ts = timestamp;
		}
		if (unlikely(timestamp < prev_ts)) {
			irq_time.tv_sec += 0x10000;
		}
		time.tv_sec |= (eth->irq_time.tv_sec) & ~(0xFFFF);
		prev_ts = timestamp;

		dma_unmap_single(dev->parent_dev, dma, FORWARD_ETHERNET_PAYLOAD_SIZE,
				 DMA_FROM_DEVICE);

		eth->rx_pointer++;
		if (eth->rx_pointer >= ring->size)
			eth->rx_pointer -= ring->size;
		received_desc--;

		if (!(status & FORWARD_ETHERNET_STAT_VALID) ||
		    !(status & FORWARD_ETHERNET_STAT_SOF)) {
			skb_free_frag(buf);
			continue;
		}

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
		skb = napi_build_skb(buf, FORWARD_ETHERNET_BUF_SIZE);
#else
		skb = build_skb(buf, FORWARD_ETHERNET_BUF_SIZE);
#endif

		skb_reserve(skb, FORWARD_ETHERNET_HEADROOM);
		skb_put(skb, rxlen);

		skb->protocol = eth_type_trans(skb, eth->net);
		skb->ip_summed = CHECKSUM_UNNECESSARY;

		skb_hwtstamps(skb)->hwtstamp = timespec64_to_ktime(time);

		spin_lock_irqsave(&eth->stats_lock, sflags);
		eth->stats.rx_packets++;
		eth->stats.rx_bytes += rxlen - 4;
		spin_unlock_irqrestore(&eth->stats_lock, sflags);

		napi_gro_receive(&eth->napi_rx, skb);
	}

	return result;
}

static int forward_net_poll_rx(struct napi_struct *napi, int budget)
{
	struct forward_ethernet *eth = container_of(napi, struct forward_ethernet, napi_rx);
	struct forward_sdma_io_ring *rx = eth->rx;
	int received = 0;

	while (received < budget) {
		int packets;

		forward_sdma_update(rx);
		packets = forward_net_dequeue_rx(eth, rx, budget - received);
		if (packets == 0)
			break;

		forward_net_queue_rx(eth, rx, packets);
		received += packets;
	}

	if ((received != budget) && napi_complete_done(napi, received))
		eth->ops->enable_irq(eth, true, false);

	return received;
}

static int forward_net_poll_tx(struct napi_struct *napi, int budget)
{
	struct forward_ethernet *eth = container_of(napi, struct forward_ethernet, napi_tx);
	struct forward_sdma_io_ring *tx = eth->tx;
	int transmitted = 0;

	while (transmitted < budget) {
		int packets;

		forward_sdma_update(tx);
		packets = forward_net_dequeue_tx(eth, tx, budget - transmitted);

		if (packets == 0)
			break;

		transmitted += packets;
	}

	if ((transmitted != budget) && napi_complete_done(napi, transmitted))
		eth->ops->enable_irq(eth, false, true);

	return transmitted;
}

static int forward_net_open(struct net_device *dev)
{
	struct forward_ethernet *eth = netdev_priv(dev);

	forward_sdma_enable_ring(eth->rx);
	forward_sdma_reset(eth->rx);
	eth->rx_pointer = 0;

	forward_sdma_enable_ring(eth->tx);
	forward_sdma_reset(eth->tx);
	eth->tx_pointer = 0;
	eth->tx_queued = 0;

	forward_net_queue_rx(eth, eth->rx, FORWARD_ETHERNET_RX_PREROLL_SIZE);

	napi_enable(&eth->napi_rx);
	napi_enable(&eth->napi_tx);
	netif_start_queue(eth->net);

	schedule_delayed_work(&eth->watchdog, HZ / 100);

	eth->ops->enable_irq(eth, true, true);

	eth->ops->toggle_link(eth, true);

	return 0;
}

static int forward_net_close(struct net_device *dev)
{
	struct forward_ethernet *eth = netdev_priv(dev);

	eth->ops->disable_irq(eth, true, true);

	eth->ops->toggle_link(eth, false);

	napi_disable(&eth->napi_rx);
	napi_disable(&eth->napi_tx);

	forward_sdma_disable_ring(eth->rx);
	forward_sdma_reset(eth->rx);

	forward_sdma_disable_ring(eth->tx);
	forward_sdma_reset(eth->tx);

	cancel_delayed_work_sync(&eth->watchdog);

	return 0;
}

static void forward_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
{
	struct forward_ethernet *eth = netdev_priv(dev);
	unsigned long sflags;

	spin_lock_irqsave(&eth->stats_lock, sflags);
	stats->rx_bytes = eth->stats.rx_bytes;
	stats->rx_packets = eth->stats.rx_packets;
	stats->rx_missed_errors = eth->stats.rx_fifo_errors;
	stats->rx_crc_errors = eth->stats.rx_mac_errors;
	stats->tx_bytes = eth->stats.tx_bytes;
	stats->tx_packets = eth->stats.tx_packets;
	spin_unlock_irqrestore(&eth->stats_lock, sflags);
}

static void forward_net_watchdog(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct forward_ethernet *eth = container_of(dwork, struct forward_ethernet, watchdog);

	eth->ops->update_stats(eth);

	schedule_delayed_work(dwork, HZ / 100);
}

static netdev_tx_t forward_net_xmit_frame(struct sk_buff *skb, struct net_device *dev)
{
	struct forward_ethernet *eth = netdev_priv(dev);

	return forward_net_queue_tx(eth, eth->tx, skb);
}

static int forward_net_set_mac(struct net_device *dev, void *p)
{
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EADDRNOTAVAIL;

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
	eth_hw_addr_set(dev, addr->sa_data);
#else
	ether_addr_copy(dev->dev_addr, addr->sa_data);
#endif

	return 0;
}

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
static void forward_net_tx_timeout(struct net_device *dev, unsigned int queue)
#else
static void forward_net_tx_timeout(struct net_device *dev)
#endif
{
}

static int forward_net_change_mtu(struct net_device *dev, int new_mtu)
{
	return 0;
}

static netdev_features_t forward_net_fix_features(struct net_device *dev,
						  netdev_features_t features)
{
	return features & FORWARD_ETHERNET_FEATURES;
}

static int forward_net_set_features(struct net_device *dev, netdev_features_t features)
{
	return 0;
}

static const struct net_device_ops forward_netdev_ops = {
	.ndo_open = forward_net_open,
	.ndo_stop = forward_net_close,
	.ndo_get_stats64 = forward_net_get_stats64,
	.ndo_start_xmit = forward_net_xmit_frame,
	.ndo_set_mac_address = forward_net_set_mac,

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
	.ndo_eth_ioctl = forward_net_ioctl,
#else
	.ndo_do_ioctl = forward_net_ioctl,
#endif

	.ndo_set_rx_mode = forward_net_set_rx_mode,
	.ndo_tx_timeout = forward_net_tx_timeout,
	.ndo_change_mtu = forward_net_change_mtu,
	.ndo_validate_addr = eth_validate_addr,
	.ndo_fix_features = forward_net_fix_features,
	.ndo_set_features = forward_net_set_features,
};

static void forward_ethernet_iface_fini(struct forward_ethernet_board *board, int index)
{
	struct forward_ethernet *eth = board->eths[index];

	if (!eth)
		return;

	if (eth->irq.private == eth) {
		forward_irq_listener_remove(eth->board->dev, &eth->irq);
		eth->irq.private = NULL;
	}

	if (eth->net->reg_state == NETREG_REGISTERED)
		unregister_netdev(eth->net);

	board->eths[index] = NULL;
}

static int forward_ethernet_iface_init(struct forward_ethernet_board *board, int index)
{
	int result = 0;
	struct forward_ethernet *eth;
	struct net_device *net;

	net = devm_alloc_etherdev(board->dev->parent_dev, sizeof(struct forward_ethernet));
	if (!net)
		return -ENOMEM;

	eth = netdev_priv(net);
	eth->board = board;
	eth->dev = board->dev;
	eth->ops = board->ops;
	eth->index = index;
	eth->net = net;
	board->eths[index] = eth;

	spin_lock_init(&eth->lock);
	spin_lock_init(&eth->stats_lock);
	INIT_DELAYED_WORK(&eth->watchdog, forward_net_watchdog);

	SET_NETDEV_DEV(net, board->dev->parent_dev);
	net->dev_port = index + 1;
	net->netdev_ops = &forward_netdev_ops;
	net->ethtool_ops = &forward_ethtool_ops;
	net->watchdog_timeo = 5 * HZ;

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || \
	(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 4))
	netif_napi_add(net, &eth->napi_rx, forward_net_poll_rx);
	netif_napi_add_tx(net, &eth->napi_tx, forward_net_poll_tx);
#else
	netif_napi_add(net, &eth->napi_rx, forward_net_poll_rx, NAPI_POLL_WEIGHT);
	netif_napi_add(net, &eth->napi_tx, forward_net_poll_tx, NAPI_POLL_WEIGHT);
#endif

	net->hw_features = NETIF_F_SG | NETIF_F_RXALL | NETIF_F_RXFCS;
	net->min_mtu = ETH_ZLEN - ETH_HLEN;
	net->max_mtu = 1536 - (ETH_HLEN + ETH_FCS_LEN);

	result = eth->ops->init(eth);
	if (result)
		goto fail;

	eth->ops->disable_irq(eth, true, true);

	forward_irq_listener_init(&eth->irq);
	eth->irq.type = FORWARD_IRQ_LISTENER_CALLBACK;
	eth->irq.mask = eth->ops->irq_mask(eth);
	eth->irq.func = &forward_ethernet_irq;
	eth->irq.private = eth;
	forward_irq_listener_add(board->dev, &eth->irq);

	if (eth->rx) {
		forward_sdma_reset(eth->rx);
		eth->rx_pointer = 0;
	}

	if (eth->tx) {
		forward_sdma_reset(eth->tx);
		eth->tx_pointer = 0;
		eth->tx_queued = 0;
	}

	result = register_netdev(net);
	if (result)
		goto fail;

	return 0;

fail:
	forward_ethernet_iface_fini(board, index);

	return result;
}

static void forward_ethernet_remove(struct forward_dev *dev, void **private)
{
	struct forward_ethernet_board *board = *private;
	int i;

	forward_ethernet_ptp_remove(board);

	for (i = 0; i < board->num_eth; i++)
		forward_ethernet_iface_fini(board, i);

	*private = NULL;
	devm_kfree(dev->dev, board);
}

static int forward_ethernet_probe(struct forward_dev *dev, void **private)
{
	struct forward_ethernet_board *board;
	const struct forward_ethernet_ops *ops;
	int result = 0, i;

	if (!forward_has_mode(dev, FORWARD_ETHERNET_MODE_NAME))
		return -EOPNOTSUPP;

	if (dev->type == FORWARD_FD2110)
		ops = &fd2110_ops;
	else
		return -EOPNOTSUPP;

	board = devm_kzalloc(dev->dev, sizeof(struct forward_ethernet_board), GFP_KERNEL);
	if (!board)
		return -ENOMEM;

	board->dev = dev;
	board->ops = ops;
	board->num_eth = ops->num_eth;
	*private = board;

	for (i = 0; i < board->num_eth; i++) {
		result = forward_ethernet_iface_init(board, i);
		if (result)
			goto fail;
	}

	forward_ethernet_ptp_init(board);

	return 0;

fail:
	forward_ethernet_remove(dev, private);
	return result;
}

static int forward_ethernet_reprobe(struct forward_dev *dev, void **private)
{
	if (!forward_has_mode(dev, FORWARD_ETHERNET_MODE_NAME))
		return -EOPNOTSUPP;

	return 0;
}

static void forward_ethernet_io_changed(struct forward_dev *dev, struct forward_io *io,
					enum forward_io_state state, void **private)
{
}

static bool forward_ethernet_available(struct forward_dev *dev)
{
	switch (dev->type) {
	case FORWARD_FD2110:
		return true;
	default:
		return false;
	}
}

static struct forward_extension forward_ethernet_ext = {
	.name = FORWARD_ETHERNET_MODE_NAME,
	.version = FORWARD_VERSION_CODE,
	.available = forward_ethernet_available,
	.probe = forward_ethernet_probe,
	.reprobe = forward_ethernet_reprobe,
	.remove = forward_ethernet_remove,
	.io_changed = forward_ethernet_io_changed,
};

static int forward_ethernet_init(void)
{
	return forward_register_extension(&forward_ethernet_ext);
}

static void forward_ethernet_exit(void)
{
	forward_unregister_extension(&forward_ethernet_ext);
}

module_init(forward_ethernet_init);
module_exit(forward_ethernet_exit);

MODULE_LICENSE("GPL");
