/*
 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */
#ifndef NVFS_BLK_H
#define NVFS_BLK_H

#include <linux/mm.h>
#include <linux/bio.h>
#include <linux/pci.h>
#include <linux/types.h>
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
#include <linux/version.h>
#ifdef HAVE_BLK_MQ_PCI_H
#include <linux/blk-mq-pci.h>
#endif
#include <linux/scatterlist.h>
#include <linux/page-flags.h>
#include <linux/dma-direction.h>

#ifdef HAVE_BLK_RQ_DMA_MAP_ITER_START
#include <linux/blk-mq-dma.h>
#include <linux/dma-mapping.h>
#endif

#ifdef HAVE_BLK_INTEGRITY_H
#include <linux/blk-integrity.h>
#endif

#include "nvfs-core.h"
#define NVFS_IO_ERR	-1
#define NVFS_BAD_REQ	-2

#ifndef SECTOR_SHIFT
#define SECTOR_SHIFT 12
#endif
#ifndef SECTOR_SIZE
#define SECTOR_SIZE (1 << SECTOR_SHIFT)
#endif

//These are same key values as the one's in the user space. When adding
//a new driver make sure to use the same values in user and kernel space.
#define NVFS_PROC_MOD_NVME_KEY          "nvme"
#define NVFS_PROC_MOD_NVME_RDMA_KEY     "nvme_rdma"
#define NVFS_PROC_MOD_SCSI_KEY          "scsi_mod"
#define NVFS_PROC_MOD_SCALEFLUX_CSD_KEY "sfxvdriver"
#define NVFS_PROC_MOD_NVMESH_KEY        "nvmeib_common"
#define NVFS_PROC_MOD_DDN_LUSTRE_KEY    "lnet"
#define NVFS_PROC_MOD_NTAP_BEEGFS_KEY   "beegfs"
#define NVFS_PROC_MOD_GPFS_KEY          "mmfslinux"
#define NVFS_PROC_MOD_NFS_KEY           "rpcrdma"
#define NVFS_PROC_MOD_WEKAFS_KEY        "wekafsio"
#define NVFS_PROC_MOD_SCATEFS_KEY       "scatefs"


struct nvfs_dma_rw_ops {
	unsigned long long ft_bmap; // feature bitmap

	int (*nvfs_blk_rq_map_sg) (struct request_queue *q,
                                   struct request *req, 
                                   struct scatterlist *sglist);

        int (*nvfs_dma_map_sg_attrs) (struct device *device,
                                      struct scatterlist *sglist,
			              int nents,
                                      enum dma_data_direction dma_dir,
                                      unsigned long attrs);

        int (*nvfs_dma_unmap_sg) (struct device *device,
                                   struct scatterlist *sglist,
                                   int nents,
                                   enum dma_data_direction dma_dir);

	bool (*nvfs_is_gpu_page) (struct page *page);

	unsigned int (*nvfs_gpu_index) (struct page *page);

	unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int gpu_index);
	
	int (*nvfs_get_gpu_sglist_rdma_info) (struct scatterlist *sglist,
					    int nents,
					    struct nvfs_rdma_info *rdma_infop);
};

// feature list for dma_ops, values indicate bit pos
enum ft_bits {
	nvfs_ft_prep_sglist         		= 1ULL << 0,
	nvfs_ft_map_sglist          		= 1ULL << 1,
	nvfs_ft_is_gpu_page         		= 1ULL << 2,
	nvfs_ft_device_priority     		= 1ULL << 3,
	nvfs_ft_get_gpu_sglist_rdma_info 	= 1ULL << 4,
	nvfs_ft_blk_dma_map_iter_start		= 1ULL << 5,	
	nvfs_ft_blk_dma_map_iter_next		= 1ULL << 6,	
};

// check features for use in registration with vendor drivers
#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops)         ((ops)->ft_bmap & nvfs_ft_prep_sglist)
#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops)          ((ops)->ft_bmap & nvfs_ft_map_sglist)
#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops)            ((ops)->ft_bmap & nvfs_ft_is_gpu_page)
#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops)     ((ops)->ft_bmap & nvfs_ft_device_priority)
#define NVIDIA_FS_CHECK_FT_GET_GPU_sglist_RDMA_INFO(ops)   \
						    ((ops)->ft_bmap & nvfs_ft_get_gpu_sglist_rdma_info)
#define NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_START(ops)    ((ops)->ft_bmap & nvfs_ft_blk_dma_map_iter_start)
#define NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_NEXT(ops)     ((ops)->ft_bmap & nvfs_ft_blk_dma_map_iter_next)
// publish features
#define NVIDIA_FS_SET_FT_ALL  (nvfs_ft_prep_sglist | nvfs_ft_map_sglist | nvfs_ft_is_gpu_page | nvfs_ft_device_priority | nvfs_ft_get_gpu_sglist_rdma_info)

typedef int (*nvfs_register_dma_ops_fn_t) (struct nvfs_dma_rw_ops *ops);
typedef void (*nvfs_unregister_dma_ops_fn_t) (void);

//Ops structure (for future extensibility) - only for kernels with iterator API
#ifdef HAVE_BLK_RQ_DMA_MAP_ITER_START
#define NVIDIA_FS_SET_BLK_DMA_MAP_ITER_FT_ALL  (nvfs_ft_is_gpu_page | nvfs_ft_device_priority | nvfs_ft_blk_dma_map_iter_start | nvfs_ft_blk_dma_map_iter_next)

// Forward declarations for blk_iter specific types
struct dma_iova_state;
struct blk_dma_iter;

struct nvfs_dma_rw_blk_iter_ops {
	unsigned long long ft_bmap; // feature bitmap

	int (*nvfs_blk_rq_dma_map_iter_start) (struct request *req,
                                               struct device *dma_dev,
                                               struct dma_iova_state *state,
                                               struct blk_dma_iter *iter,
					       void **cookie);

	int (*nvfs_blk_rq_dma_map_iter_next) (struct request *req,
                                              struct device *dma_dev,
                                              struct dma_iova_state *state,
                                              struct blk_dma_iter *iter);

	int (*nvfs_dma_unmap_page) (struct device *device,
				    void *cookie,
				    dma_addr_t addr,
				    size_t size,
				    enum dma_data_direction dir);

	bool (*nvfs_is_gpu_page) (struct page *page);

	unsigned int (*nvfs_gpu_index) (struct page *page);

	unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int gpu_index);
	
	int (*nvfs_get_gpu_sglist_rdma_info) (struct scatterlist *sglist,
					    int nents,
					    struct nvfs_rdma_info *rdma_infop);
};

typedef int (*nvfs_register_dma_ops_v2_fn_t) (struct nvfs_dma_rw_blk_iter_ops *ops);
#endif /* HAVE_BLK_RQ_DMA_MAP_ITER_START */

// Auto probing
struct module_entry {
	bool is_mod;   // scsi_mod is not built as a module
	bool found;
        const char *name;    // module owner
        const char *version; // module version number
        const char *reg_ksym; // registration symbol from symbol table above
        void *reg_func; // register function pointer (can be v1 or v2)
        const char *dreg_ksym; //deregister symbol
        nvfs_unregister_dma_ops_fn_t dreg_func; // deregister function pointer
        void *ops; // args (can be nvfs_dma_rw_ops or nvfs_dma_rw_blk_iter_ops)
};

int nr_modules(void);
int probe_module_list(void);
void cleanup_module_list(void);

int nvfs_blk_register_dma_ops(void);
void nvfs_blk_unregister_dma_ops(void);

//#define TEST_DISCONTIG_ADDR

#ifdef TEST_DISCONTIG_ADDR
void nvfs_init_simulated_address(void);
uint64_t nvfs_get_simulated_address(int key, int index);
int nvfs_get_simulated_key_index(void);
#endif

#endif /* NVFS_H */
