/*
   forward-vdma.c - VideoDMA driver for SoftLab-NSK Forward video boards

   Copyright (C) 2017 - 2024 SoftLab-NSK <forward@softlab.tv>

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; either version 2
   of the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
*/

#include "forward-vdma.h"

#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/version.h>
#include <linux/bitmap.h>

#define FORWARD_VDMA_BUFFER_SIZE 1024

int forward_vdma_probe(struct forward_dev *dev)
{
	int i;
	struct forward_vdma *vdma = devm_kzalloc(dev->dev, sizeof(struct forward_vdma), GFP_KERNEL);
	size_t size = dev->cfg.vdma_size_mb * 1024ULL * 1024ULL;

	if (!vdma) {
		forward_err(dev, "vdma: failed to allocate VDMA context.\n");
		return -ENOMEM;
	}

	spin_lock_init(&vdma->lock);
	spin_lock_init(&vdma->rlock);
	INIT_LIST_HEAD(&vdma->regions);

	dev->cfg.disable_vdma(dev);

	vdma->dev = dev;

	if (!dma_set_mask(dev->parent_dev, DMA_BIT_MASK(44)))
		vdma->dma44bit = 1;
	else if (!dma_set_mask(dev->parent_dev, DMA_BIT_MASK(32)))
		vdma->dma44bit = 0;
	else {
		forward_err(dev, "vdma: failed to set VDMA mask.\n");
		return -ENODEV;
	}
	vdma->map_size = ((size - 1) >> PAGE_SHIFT) / FORWARD_VDMA_BUFFER_SIZE + 1;
	vdma->map = (u32 *)get_zeroed_page(GFP_DMA);
	vdma->map_addr = dma_map_single(dev->parent_dev, vdma->map, PAGE_SIZE, DMA_TO_DEVICE);

	vdma->dummy_wr = (u32 *)get_zeroed_page(GFP_DMA);
	vdma->dummy_wr_addr =
		dma_map_single(dev->parent_dev, vdma->dummy_wr, PAGE_SIZE, DMA_TO_DEVICE);

	vdma->dummy_rd = (u32 *)get_zeroed_page(GFP_DMA);
	vdma->dummy_rd_addr =
		dma_map_single(dev->parent_dev, vdma->dummy_rd, PAGE_SIZE, DMA_FROM_DEVICE);

	forward_dbg(dev, "vdma: map: 0x%08llx, h2d: 0x%08llx, d2h: 0x%08llx\n", (u64)vdma->map_addr,
		    (u64)vdma->dummy_wr_addr, (u64)vdma->dummy_rd_addr);

	vdma->descriptors = vzalloc(vdma->map_size * FORWARD_VDMA_BUFFER_SIZE * 4);
	vdma->pages = vzalloc(vdma->map_size * FORWARD_VDMA_BUFFER_SIZE * sizeof(struct page *));
	vdma->rw_bitmap = vzalloc(vdma->map_size * FORWARD_VDMA_BUFFER_SIZE / 8);
	vdma->user_bitmap = vzalloc(vdma->map_size * FORWARD_VDMA_BUFFER_SIZE / 8);

	for (i = 0; i < vdma->map_size; i++) {
		struct page *dp = vmalloc_to_page(((void *)vdma->descriptors) + (i << PAGE_SHIFT));
		vdma->map[i] = dma_map_page(dev->parent_dev, dp, 0, PAGE_SIZE, DMA_TO_DEVICE) >>
			       PAGE_SHIFT;
	}

	if (vdma->dma44bit)
		forward_info(dev, "vdma: using 44-bit VDMA with %d MiB region\n",
			     vdma->map_size * 4);
	else
		forward_info(dev, "vdma: using 32-bit VDMA with %d MiB region\n",
			     vdma->map_size * 4);

#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
	cpu_latency_qos_add_request(&vdma->qos, FORWARD_DMA_LATENCY_US);
#else
	pm_qos_add_request(&vdma->qos, PM_QOS_CPU_DMA_LATENCY, FORWARD_DMA_LATENCY_US);
#endif

	dev->vdma = vdma;

	return 0;
}

void forward_vdma_remove(struct forward_dev *dev)
{
	int i;
	struct forward_vdma *vdma = dev->vdma;

	if (!vdma)
		return;

	dev->vdma = NULL;

#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
	if (cpu_latency_qos_request_active(&vdma->qos))
		cpu_latency_qos_remove_request(&vdma->qos);
#else
	if (pm_qos_request_active(&vdma->qos))
		pm_qos_remove_request(&vdma->qos);
#endif

	forward_vdma_disable(dev);

	while (!list_empty(&vdma->regions))
		forward_vdma_put_all_regions(
			vdma,
			list_first_entry(&vdma->regions, struct forward_vdma_region, item)->owner);

	for (i = 0; i < vdma->map_size; i++) {
		dma_unmap_page(dev->parent_dev, vdma->map[i] << PAGE_SHIFT, PAGE_SIZE,
			       DMA_TO_DEVICE);
	}

	if (vdma->pages)
		vfree(vdma->pages);

	if (vdma->descriptors)
		vfree(vdma->descriptors);

	if (vdma->dummy_rd_addr)
		dma_unmap_single(dev->parent_dev, vdma->dummy_rd_addr, PAGE_SIZE, DMA_FROM_DEVICE);
	if (vdma->dummy_rd)
		free_page((unsigned long)vdma->dummy_rd);

	if (vdma->dummy_wr_addr)
		dma_unmap_single(dev->parent_dev, vdma->dummy_wr_addr, PAGE_SIZE, DMA_TO_DEVICE);
	if (vdma->dummy_wr)
		free_page((unsigned long)vdma->dummy_wr);

	if (vdma->map_addr)
		dma_unmap_single(dev->parent_dev, vdma->map_addr, PAGE_SIZE, DMA_TO_DEVICE);
	if (vdma->map)
		free_page((unsigned long)vdma->map);
}

int forward_vdma_init_region(struct forward_vdma *vdma, u64 vdma_address, size_t size, bool read)
{
	u32 vdma_page_address = (u32)((u64)vdma_address >> PAGE_SHIFT);
	u32 vdma_npages = (u32)((u64)size >> PAGE_SHIFT);
	unsigned int i;

	if ((u64)vdma_address & ~PAGE_MASK)
		return -EPROTO;

	if (size & ~PAGE_MASK)
		return -EPROTO;

	if ((vdma_page_address + vdma_npages) > vdma->map_size * FORWARD_VDMA_BUFFER_SIZE)
		return -EINVAL;

	for (i = vdma_page_address; i < vdma_page_address + vdma_npages; i++) {
		vdma->descriptors[i] = (read ? vdma->dummy_rd_addr : vdma->dummy_wr_addr) >>
				       PAGE_SHIFT;
		if (read)
			set_bit(i, vdma->rw_bitmap);
		else
			clear_bit(i, vdma->rw_bitmap);
	}
	return 0;
}
EXPORT_SYMBOL(forward_vdma_init_region);

inline bool forward_vdma_check_owner(struct forward_vdma *vdma, void *owner, u64 vdma_address,
				     size_t size)
{
	u32 start = vdma_address >> PAGE_SHIFT;
	u32 end = ((vdma_address + size) >> PAGE_SHIFT) - 1;
	struct forward_vdma_region *region;
	unsigned long sflags;
	bool result = false;

	spin_lock_irqsave(&vdma->rlock, sflags);
	list_for_each_entry (region, &vdma->regions, item) {
		u32 rstart = region->vdma_address, rend = region->vdma_address + region->size - 1;

		if ((region->owner == owner) && ((start >= rstart) && (start <= rend)) &&
		    ((end >= rstart) && (end <= rend))) {
			result = true;
			break;
		}
	}
	spin_unlock_irqrestore(&vdma->rlock, sflags);

	return result;
}

static int _forward_vdma_unmap_buf(struct forward_vdma *vdma, u32 vdma_page_address,
				   u32 vdma_npages)
{
	u32 i;
	unsigned long sflags;
	struct forward_dev *dev = vdma->dev;

	spin_lock_irqsave(&vdma->lock, sflags);
	for (i = vdma_page_address; i < (vdma_page_address + vdma_npages); i++) {
		bool was_read = test_bit(i, vdma->rw_bitmap);
		bool was_user = test_bit(i, vdma->user_bitmap);
		dma_addr_t pa = vdma->descriptors[i];

		if (!vdma->pages[i])
			continue;

		vdma->descriptors[i] = (was_read ? vdma->dummy_rd_addr : vdma->dummy_wr_addr) >>
				       PAGE_SHIFT;
		dma_unmap_page(dev->parent_dev, pa << PAGE_SHIFT, PAGE_SIZE,
			       was_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);

		if (was_read)
			set_page_dirty(vdma->pages[i]);

		if (was_user)
			put_page(vdma->pages[i]);

		vdma->pages[i] = 0;
	}
	spin_unlock_irqrestore(&vdma->lock, sflags);

	return 0;
}

int forward_vdma_unmap_buf(struct forward_vdma *vdma, void *owner, u64 vdma_address, size_t size)
{
	u32 vdma_page_address = (u32)((u64)vdma_address >> PAGE_SHIFT);
	u32 vdma_npages = (u32)((u64)size >> PAGE_SHIFT);

	if ((u64)vdma_address & ~PAGE_MASK)
		return -EINVAL;

	if (size & ~PAGE_MASK)
		return -EINVAL;

	if ((vdma_page_address + vdma_npages) > vdma->map_size * FORWARD_VDMA_BUFFER_SIZE)
		return -EINVAL;

	if (!forward_vdma_check_owner(vdma, owner, vdma_address, size))
		return -EINVAL;

	_forward_vdma_unmap_buf(vdma, vdma_page_address, vdma_npages);

	return 0;
}
EXPORT_SYMBOL(forward_vdma_unmap_buf);

int forward_vdma_get_region(struct forward_vdma *vdma, void *owner, u64 vdma_address, size_t size)
{
	struct forward_vdma_region *region, *nregion;
	unsigned long sflags;
	u32 start = vdma_address >> PAGE_SHIFT, end = ((vdma_address + size) >> PAGE_SHIFT) - 1;
	bool ok = true;

	if (vdma_address & ~PAGE_MASK)
		return -EPROTO;

	if (size & ~PAGE_MASK)
		return -EPROTO;

	if (end >= vdma->map_size * FORWARD_VDMA_BUFFER_SIZE)
		return -EINVAL;

	spin_lock_irqsave(&vdma->rlock, sflags);
	list_for_each_entry (region, &vdma->regions, item) {
		u32 rstart = region->vdma_address, rend = region->vdma_address + region->size - 1;

		if (((start >= rstart) && (start <= rend)) || ((end >= rstart) && (end <= rend))) {
			ok = false;
			break;
		}
	}
	spin_unlock_irqrestore(&vdma->rlock, sflags);

	if (!ok) {
		forward_warn(
			vdma->dev,
			"vdma: cannot get region [%08x:%08x] for client %p - interesects with [%08x:%08x], client %p\n",
			start, end, owner, region->vdma_address,
			region->vdma_address + region->size - 1, region->owner);
		return -EBUSY;
	}

	nregion = devm_kzalloc(vdma->dev->dev, sizeof(struct forward_vdma_region), GFP_KERNEL);
	if (!nregion)
		return -ENOMEM;

	nregion->owner = owner;
	nregion->vdma_address = vdma_address >> PAGE_SHIFT;
	nregion->size = size >> PAGE_SHIFT;

	spin_lock_irqsave(&vdma->rlock, sflags);
	list_add_tail(&nregion->item, &vdma->regions);
	spin_unlock_irqrestore(&vdma->rlock, sflags);

	return 0;
}
EXPORT_SYMBOL(forward_vdma_get_region);

int forward_vdma_put_region(struct forward_vdma *vdma, void *owner, u64 vdma_address)
{
	struct forward_vdma_region *region, *nregion;
	bool found = false;
	u32 start = vdma_address >> PAGE_SHIFT;
	unsigned long sflags;

	if (vdma_address & ~PAGE_MASK)
		return -EPROTO;

	spin_lock_irqsave(&vdma->rlock, sflags);
	list_for_each_entry_safe (region, nregion, &vdma->regions, item) {
		if ((region->owner == owner) && (region->vdma_address == start)) {
			list_del(&region->item);
			_forward_vdma_unmap_buf(vdma, region->vdma_address, region->size);
			devm_kfree(vdma->dev->dev, region);
			found = true;
			break;
		}
	}
	spin_unlock_irqrestore(&vdma->rlock, sflags);

	if (!found) {
		forward_warn(vdma->dev, "vdma: cannot find region %08x for client %p\n", start,
			     owner);
		return -EINVAL;
	}

	return 0;
}
EXPORT_SYMBOL(forward_vdma_put_region);

void forward_vdma_put_all_regions(struct forward_vdma *vdma, void *owner)
{
	struct forward_vdma_region *region, *nregion;
	unsigned long sflags;

	spin_lock_irqsave(&vdma->rlock, sflags);
	list_for_each_entry_safe (region, nregion, &vdma->regions, item) {
		if (region->owner == owner) {
			_forward_vdma_unmap_buf(vdma, region->vdma_address, region->size);
			list_del(&region->item);
			devm_kfree(vdma->dev->dev, region);
		}
	}
	spin_unlock_irqrestore(&vdma->rlock, sflags);
}
EXPORT_SYMBOL(forward_vdma_put_all_regions);

inline int forward_vdma_map_buf(struct forward_vdma *vdma, void *owner, void *address,
				u64 vdma_address, size_t size, bool read, bool user)
{
	u32 i;
	int ret;
	struct forward_dev *dev = vdma->dev;
	u32 vdma_page_address = (u32)((u64)vdma_address >> PAGE_SHIFT);
	u32 vdma_npages = (u32)((u64)size >> PAGE_SHIFT);
	unsigned long sflags;

	if ((u64)vdma_address & ~PAGE_MASK)
		return -EINVAL;

	if (size & ~PAGE_MASK)
		return -EINVAL;

	if ((vdma_page_address + vdma_npages) > vdma->map_size * FORWARD_VDMA_BUFFER_SIZE)
		return -EINVAL;

	if (!forward_vdma_check_owner(vdma, owner, vdma_address, size))
		return -EINVAL;

	_forward_vdma_unmap_buf(vdma, vdma_page_address, vdma_npages);

	if (!address)
		return 0;

	if (user) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)
		mmap_read_lock(current->mm);
#else
		down_read(&current->mm->mmap_sem);
#endif
#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)) || \
     (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 4)))
		ret = get_user_pages(((u64)address) & PAGE_MASK, vdma_npages, read,
				     &vdma->pages[vdma_page_address]);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
		ret = get_user_pages(((u64)address) & PAGE_MASK, vdma_npages, read,
				     &vdma->pages[vdma_page_address], NULL);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
		ret = get_user_pages(((u64)address) & PAGE_MASK, vdma_npages, read, 0,
				     &vdma->pages[vdma_page_address], NULL);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 168)
		ret = get_user_pages(current, current->mm, ((u64)address) & PAGE_MASK, vdma_npages,
				     read ? FOLL_WRITE : 0, &vdma->pages[vdma_page_address], NULL);
#else
		ret = get_user_pages(current, current->mm, ((u64)address) & PAGE_MASK, vdma_npages,
				     read, 0, &vdma->pages[vdma_page_address], NULL);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)
		mmap_read_unlock(current->mm);
#else
		up_read(&current->mm->mmap_sem);
#endif
	} else {
		for (i = 0; i < vdma_npages; i++)
			vdma->pages[vdma_page_address + i] =
				vmalloc_to_page((void *)((u8 *)address + (i << PAGE_SHIFT)));
	}

	spin_lock_irqsave(&vdma->lock, sflags);
	for (i = vdma_page_address; i < (vdma_page_address + vdma_npages); i++) {
		vdma->descriptors[i] = dma_map_page(dev->parent_dev, vdma->pages[i], 0, PAGE_SIZE,
						    read ? DMA_FROM_DEVICE : DMA_TO_DEVICE) >>
				       PAGE_SHIFT;

		if (read)
			set_bit(i, vdma->rw_bitmap);
		else
			clear_bit(i, vdma->rw_bitmap);

		if (user)
			set_bit(i, vdma->user_bitmap);
		else
			clear_bit(i, vdma->user_bitmap);
	}
	spin_unlock_irqrestore(&vdma->lock, sflags);

	return 0;
}

int forward_vdma_map_user_buf(struct forward_vdma *vdma, void *owner, void __user *address,
			      u64 vdma_address, size_t size, bool read)
{
	return forward_vdma_map_buf(vdma, owner, address, vdma_address, size, read, true);
}
EXPORT_SYMBOL(forward_vdma_map_user_buf);

int forward_vdma_map_kernel_buf(struct forward_vdma *vdma, void *owner, void *address,
				u64 vdma_address, size_t size, bool read)
{
	return forward_vdma_map_buf(vdma, owner, address, vdma_address, size, read, false);
}
EXPORT_SYMBOL(forward_vdma_map_kernel_buf);

void forward_vdma_sync_buf(struct forward_vdma *vdma, void *owner, u64 vdma_address, size_t size)
{
	int i;
	struct forward_dev *dev = vdma->dev;
	u32 vdma_page_address = (u32)((u64)vdma_address >> PAGE_SHIFT);
	u32 vdma_npages = (u32)((u64)size >> PAGE_SHIFT);
	bool read;
	unsigned long sflags;

	if ((u64)vdma_address & PAGE_MASK)
		return;

	if (size & PAGE_MASK)
		return;

	if ((vdma_page_address + vdma_npages) > vdma->map_size * FORWARD_VDMA_BUFFER_SIZE)
		return;

	spin_lock_irqsave(&vdma->lock, sflags);
	for (i = vdma_page_address; i < (vdma_page_address + vdma_npages); i++) {
		if (!vdma->pages[i])
			continue;

		read = test_bit(i, vdma->rw_bitmap);

		if (read) {
			dma_sync_single_for_cpu(dev->parent_dev, vdma->descriptors[i] << PAGE_SHIFT,
						PAGE_SIZE, DMA_FROM_DEVICE);
			set_page_dirty(vdma->pages[i]);
		} else
			dma_sync_single_for_device(dev->parent_dev,
						   vdma->descriptors[i] << PAGE_SHIFT, PAGE_SIZE,
						   DMA_TO_DEVICE);
	}
	spin_unlock_irqrestore(&vdma->lock, sflags);
}
EXPORT_SYMBOL(forward_vdma_sync_buf);

void forward_vdma_enable(struct forward_dev *dev)
{
	struct forward_vdma *vdma = dev->vdma;

	if (!vdma)
		return;

	dev->cfg.enable_vdma(dev, vdma->map_addr >> PAGE_SHIFT);
}
EXPORT_SYMBOL(forward_vdma_enable);

void forward_vdma_disable(struct forward_dev *dev)
{
	struct forward_vdma *vdma = dev->vdma;

	if (!vdma)
		return;

	dev->cfg.disable_vdma(dev);
}
EXPORT_SYMBOL(forward_vdma_disable);

void forward_vdma_debug_show_map(struct forward_vdma *vdma)
{
	unsigned int i, si = 0, end = vdma->map_size * FORWARD_VDMA_BUFFER_SIZE;
	u32 prev_desc = 0;
	u32 start = 0;

	forward_info(vdma->dev, "VDMA table adresses: \n");
	for (i = 0; i < vdma->map_size; i++) {
		u32 desc = vdma->map[i];

		if ((i != vdma->map_size - 1) &&
		    ((((int)desc - (int)prev_desc) == 1) || (((int)desc - (int)prev_desc) == -1) ||
		     (prev_desc == desc))) {
			prev_desc = desc;
			continue;
		}
		if (((start != 0) && (prev_desc != 0)) || (i == vdma->map_size - 1)) {
			forward_info(vdma->dev, "[%08x:%08x]: %016llx - %016llx\n",
				     si * FORWARD_VDMA_BUFFER_SIZE * 4096,
				     (i - 1) * FORWARD_VDMA_BUFFER_SIZE * 4096, (u64)start * 4096,
				     (u64)prev_desc * 4096);
		}
		start = desc;
		si = i;
		prev_desc = desc;
	}

	prev_desc = 0;
	start = 0;
	si = 0;
	end = vdma->map_size * FORWARD_VDMA_BUFFER_SIZE;

	forward_info(vdma->dev, "VDMA mapping: \n");
	for (i = 0; i < end; i++) {
		u32 desc = vdma->descriptors[i];

		if ((i != end - 1) &&
		    ((((int)desc - (int)prev_desc) == 1) || (((int)desc - (int)prev_desc) == -1) ||
		     (prev_desc == desc))) {
			prev_desc = desc;
			continue;
		}
		if (((start != 0) && (prev_desc != 0)) || (i == end - 1)) {
			forward_info(
				vdma->dev, "[%08x:%08x]: %016llx - %016llx (%016llx - %016llx)\n",
				si * 4096, (i - 1) * 4096, (u64)start * 4096, (u64)prev_desc * 4096,
				(u64)(vdma->pages[si] ? page_to_phys(vdma->pages[si]) : 0),
				(u64)(vdma->pages[i - 1] ? page_to_phys(vdma->pages[i - 1]) : 0));
		}
		start = desc;
		si = i;
		prev_desc = desc;
	}
}
EXPORT_SYMBOL(forward_vdma_debug_show_map);
