/* _NVRM_COPYRIGHT_BEGIN_
 *
 * Copyright 2001-2011 by NVIDIA Corporation.  All rights reserved.  All
 * information contained herein is proprietary and confidential to NVIDIA
 * Corporation.  Any use, reproduction, or disclosure without the written
 * permission of NVIDIA Corporation is prohibited.
 *
 * _NVRM_COPYRIGHT_END_
 */

#ifndef __NV_FREEBSD_H__
#define __NV_FREEBSD_H__

#ifdef TRUE
#undef TRUE
#endif

#ifdef FALSE
#undef FALSE
#endif

#include <sys/param.h>

#if __FreeBSD_version >= 1200000
#error This driver does not support FreeBSD 12.x/-CURRENT!
#endif
#if __FreeBSD_version >= 1100000 && __FreeBSD_version < 1100122
#error This driver requires FreeBSD 11.0 or later!
#endif
#if __FreeBSD_version >= 1000000 && __FreeBSD_version < 1000510
#error This driver requires FreeBSD 10.0 or later!
#endif
#if __FreeBSD_version >= 900000 && __FreeBSD_version < 900044
#error This driver requires FreeBSD 9.0 or later!
#endif
#if __FreeBSD_version >= 800000 && __FreeBSD_version < 800107
#error This driver requires FreeBSD 8.0 or later!
#endif
#if __FreeBSD_version < 702106
#error This driver requires FreeBSD 7.3 or later!
#endif

#include <sys/systm.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <sys/pciio.h>
#include <sys/vnode.h>

#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/ioccom.h>
#include <sys/malloc.h>
#include <sys/socket.h>
#include <sys/sysent.h>
#include <sys/ctype.h>
#include <sys/sysctl.h>

#include <machine/cpu.h>
#include <machine/resource.h>
#include <machine/clock.h>
#include <machine/bus.h>
#include <machine/specialreg.h>

#include <sys/conf.h>
#include <sys/rman.h>
#include <sys/proc.h>
#include <sys/lock.h>
#include <sys/mman.h>
#include <sys/file.h>
#include <sys/poll.h>
#include <sys/rwlock.h>

#include <sys/syscall.h>
#include <sys/bus.h>
#include <sys/memrange.h>
#include <sys/sysproto.h>
#include <sys/signalvar.h>

#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_extern.h>
#include <vm/vm_object.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_pager.h>
#include <vm/uma.h>

#if __FreeBSD_version < 800004
#include <pci/agpvar.h>
#else
#include <dev/agp/agpvar.h>
#endif
#include <sys/agpio.h>

#include <sys/smp.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <sys/kdb.h>
#include <sys/filedesc.h>

#include <sys/priv.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sx.h>
#include <sys/condvar.h>
#include <sys/sglist.h>
#include <sys/taskqueue.h>

#define CURTHREAD curthread

#if __FreeBSD_version >= 800049
#define suser(_td) priv_check((_td), PRIV_DRIVER)
#endif

#if __FreeBSD_version >= 900041
#include <sys/capability.h>
#if __FreeBSD_version < 1000053
#define cap_rights_init(rights, cap) (cap)
#endif
#else
#define fget(td, fd, rights, fp) fget(td, fd, fp)
#endif

#define __NV_ITHREAD() (curthread->td_pflags & TDP_ITHREAD)

/*
 * The NVIDIA kernel module's malloc identifier, needed for both tracking
 * and actual allocation/freeing purposes - declared here, but defined in
 * nvidia_os.c.
 */

MALLOC_DECLARE(M_NVIDIA);

/*
 * Enable/Disable support for FreeBSD's AGP GART driver. Please note that
 * agp.ko may need to be preloaded from loader.conf; this define does not
 * influence support for NVIDIA's builtin AGP GART driver.
 */

#undef NV_SUPPORT_OS_AGP

/*
 * This option decides if the driver will be built with support for Linux
 * or Linux 32-bit (FreeBSD/amd64) compatibility. This makes nvidia.ko
 * dependent on linux.ko; if you don't need Linux compatibility, then you
 * can safely unset this flag.
 */

#define NV_SUPPORT_LINUX_COMPAT

/*
 * Enable/Disable support for ACPI Power Management.
 */

#define NV_SUPPORT_ACPI_PM

/*
 * Enable/Disable heavy-weight cache-flush logic. By default, the driver
 * relies on the kernel to perform cache flushes using optimized
 * routines.
 */

#undef NV_USE_WBINVD

/*
 * The DMA memory allocation tracking structure. DMA memory alloctions on
 * FreeBSD are tracked with their base address and size. Since PCI memory
 * is allocated from kernel virtual memory and since AGP allocations have
 * a contiguous range of addresses in the AGP aperture, base and size are
 * sufficient to track allocations.
 */
typedef
struct nvidia_pte {
    vm_offset_t virtual_address;
    uint64_t physical_address;
} nvidia_pte_t;

typedef
struct nvidia_alloc {
    SLIST_ENTRY(nvidia_alloc) list;
    vm_memattr_t attr;
    uint32_t size;
    int alloc_type_contiguous;
    struct nvidia_pte *pte_array;
    struct sglist *sg_list;
    vm_object_t object;
} nvidia_alloc_t;

typedef
struct nvidia_event {
    STAILQ_ENTRY(nvidia_event) queue;
    nv_event_t event;
} nvidia_event_t;

typedef
struct nvidia_filep {
    nv_state_t *nv;
    STAILQ_HEAD(event_queue, nvidia_event) event_queue;
    struct mtx event_mtx;
    struct selinfo event_rsel;
} nvidia_filep_t;

typedef
struct nvidia_work {
    struct task task;
    void *data;
} nvidia_work_t;

typedef
struct nvidia_softc {
    device_t dev;
    device_t agp_dev;

    struct sglist *AGP_sg_list;
    vm_object_t AGP_object;

    struct sglist *UD_sg_list;
    vm_object_t UD_object;

    struct resource *BAR_recs[NV_GPU_NUM_BARS];
    int BAR_rids[NV_GPU_NUM_BARS];
    struct sglist *BAR_sg_lists[NV_GPU_NUM_BARS];
    vm_object_t BAR_objects[NV_GPU_NUM_BARS];

    struct resource *irq;
    void *irq_ih;
    int   irq_rid;

    nv_stack_t *attach_sp;
    nv_stack_t *api_sp;
    nv_stack_t *pci_cfgchk_sp;
    nv_stack_t *isr_sp;
    nv_stack_t *timer_sp;

    struct resource *iop;
    int iop_rid;

    bus_space_handle_t bs_handle;
    bus_space_tag_t bs_tag;

    struct cdev *cdev;
    nv_state_t *nv_state;

    struct sysctl_ctx_list sysctl_ctx;

    struct callout timer;

    uint64_t dma_mask;

    /* list of allocations */
    SLIST_HEAD(alloc_list, nvidia_alloc) alloc_list;

    uint32_t refcnt;

    struct sx api_sx;

} nvidia_softc_t;

#define CDEV_CTL_MINOR  255

extern devclass_t nvidia_devclass;

extern struct nvidia_softc nvidia_ctl_sc;
extern nv_state_t nvidia_ctl_state;

#define PCIR_CAP_LIST_ID   0x00
#define PCIR_CAP_LIST_NEXT 0x01
#define PCIR_CAP_ID_AGP    0x02
#define PCIR_CAP_ID_EXP    0x10

#if !defined(PCIS_DISPLAY_3D)
#define PCIS_DISPLAY_3D    0x002
#endif
#if !defined(PCIM_CMD_INTXDIS)
#define PCIM_CMD_INTXDIS   0x400
#endif

#if !defined(VM_MEMATTR_WEAK_UNCACHEABLE)
#define VM_MEMATTR_WEAK_UNCACHEABLE VM_MEMATTR_UNCACHED
#endif

#if !defined(PAT_UNCACHEABLE)
#define PAT_UNCACHEABLE         0x00
#endif
#if !defined(PAT_WRITE_COMBINING)
#define PAT_WRITE_COMBINING     0x01
#endif
#if !defined(PAT_WRITE_BACK)
#define PAT_WRITE_BACK          0x06
#endif

/*
 * These macros extract the encoded ioctl type and number from the
 * command; we inspect the type to verify that device/control ioctls
 * originate from NVIDIA RM clients and use the number to allow the
 * core resource manager's ioctl handler to be ignorant of operating
 * specific ioctl encodings.
 */

#define __NV_IOC_TYPE(_cmd) (((_cmd) >> 8) & 0xff)
#define __NV_IOC_NR(_cmd)   (((_cmd) >> 0) & 0xff)
#define __NV_IOC_SIZE(_cmd) (((_cmd) >> 16) & 0x1fff)

extern uma_zone_t nvidia_stack_t_zone;

#define NV_UMA_ZONE_ALLOC_STACK(ptr)                             \
    {                                                            \
        (ptr) = uma_zalloc(nvidia_stack_t_zone, M_WAITOK);       \
        if ((ptr) != NULL)                                       \
        {                                                        \
            (ptr)->size = sizeof((ptr)->stack);                  \
            (ptr)->top = (ptr)->stack + (ptr)->size;             \
        }                                                        \
    }

#define NV_UMA_ZONE_FREE_STACK(ptr)                              \
    {                                                            \
        uma_zfree(nvidia_stack_t_zone, (ptr));                   \
        (ptr) = NULL;                                            \
    }

/* nvidia_dev.c */
int    nvidia_dev_attach     (struct nvidia_softc *);
int    nvidia_dev_detach     (struct nvidia_softc *);

/* nvidia_ctl.c */
int    nvidia_ctl_attach     (void);
int    nvidia_ctl_detach     (void);

/* nvidia_pci.c */
void   nvidia_pci_check_config_space  (nv_stack_t *, device_t dev, BOOL, BOOL, BOOL);
int    nvidia_pci_setup_intr          (device_t dev);
int    nvidia_pci_teardown_intr       (device_t dev);
NvU8   nvidia_pci_find_capability     (device_t dev, NvU8);

#define NV_SHUTDOWN_ADAPTER(sp,nv)                                      \
    {                                                                   \
        rm_disable_adapter(sp, nv);                                     \
        rm_shutdown_adapter(sp, nv);                                    \
    }

#define NV_PCI_CHECK_CONFIG_SPACE(sp,nv,cb,as,mb)                       \
    {                                                                   \
        struct nvidia_softc *__sc;                                      \
        if (((nv)->flags & NV_FLAG_CONTROL) == 0) {                     \
            __sc = (nv)->os_state;                                      \
            nvidia_pci_check_config_space(sp, __sc->dev, cb, as, mb);   \
        }                                                               \
    }

/* nvidia_subr.c */
int    nvidia_attach         (device_t);
int    nvidia_detach         (device_t);
int    nvidia_suspend        (device_t);
int    nvidia_resume         (device_t);
int    nvidia_alloc_hardware (device_t);
void   nvidia_free_hardware  (device_t);
void   nvidia_intr           (void *);
int    nvidia_modevent       (module_t, int, void *);

void   nvidia_rc_timer       (void *);

void   nv_lock_api           (nv_state_t *);
void   nv_unlock_api         (nv_state_t *);

NvS32  nv_alloc_contig_pages (nv_state_t *, NvU32, NvU32, NvBool, NvU64 *, void **);
NvS32  nv_free_contig_pages  (nv_state_t *, void *);
NvS32  nv_alloc_system_pages (nv_state_t *, NvU32, NvU32, NvBool, NvU64 *, void **);
NvS32  nv_free_system_pages  (nv_state_t *, void *);

NvS32  nv_os_agp_init        (nv_stack_t *sp, nv_state_t *, NvU64 *, NvU64 *);
NvS32  nv_os_agp_teardown    (nv_stack_t *sp, nv_state_t *);
NvS32  nv_alloc_agp_pages    (nv_state_t *, NvU32, NvU32, void **);
NvS32  nv_free_agp_pages     (nv_state_t *, NvU32, void *, NvU32 *);

#define NV_SGLIST_FREE(sgl)             \
    {                                   \
        sglist_free(sgl);               \
        (sgl) = NULL;                   \
    }

#define NV_VM_OBJECT_DEALLOCATE(obj)    \
    {                                   \
        vm_object_deallocate(obj);      \
        (obj) = NULL;                   \
    }

#if __FreeBSD_version < 1000030
#define VM_OBJECT_WLOCK(object) VM_OBJECT_LOCK(object)
#define VM_OBJECT_WUNLOCK(object) VM_OBJECT_UNLOCK(object)
#endif

#if __FreeBSD_version < 1000055
#define vm_map_find(map, object, offset, addr, length, \
        max_addr, find_space, prot, max, cow) \
  vm_map_find(map, object, offset, addr, length, find_space, prot, max, cow)
#endif

#if __FreeBSD_version < 1000042
#define kmem_arena kernel_map
#endif

#if __FreeBSD_version >= 800000
#define NV_VM_PAGER_ALLOCATE(type, handle, size, prot, off, cred) \
  vm_pager_allocate(type, handle, size, prot, off, cred)
#else
#define NV_VM_PAGER_ALLOCATE(type, handle, size, prot, off, cred) \
  vm_pager_allocate(type, handle, size, prot, off)
#endif

#if __FreeBSD_version >= 800000
#define NV_KDB_ENTER(why,msg) kdb_enter(why, msg)
#else
#define NV_KDB_ENTER(why,msg) kdb_enter(msg)
#endif

/* nvidia_sysctl.c */
void   nvidia_sysctl_init    (void);
void   nvidia_sysctl_exit    (void);

int    nvidia_sysctl_gpu_model  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_gpu_uuid   (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_gpu_vbios  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_bus_type   (SYSCTL_HANDLER_ARGS);

int    nvidia_sysctl_registry_key     (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_registry_dwords  (SYSCTL_HANDLER_ARGS);

int    nvidia_sysctl_agp_rates      (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_fw         (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_sba        (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_registers  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_driver     (SYSCTL_HANDLER_ARGS);

int    nvidia_sysctl_agp_status       (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_rate_status  (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_fw_status    (SYSCTL_HANDLER_ARGS);
int    nvidia_sysctl_agp_sba_status   (SYSCTL_HANDLER_ARGS);

void   nv_sysctl_init        (nv_state_t *);
void   nv_sysctl_exit        (nv_state_t *);

/* nvidia_linux.c */
void   nvidia_linux_init     (void);
void   nvidia_linux_exit     (void);

/* nvidia_os_registry.c */

void   nvidia_update_registry (char *);

/* ioctl helpers */
int    nvidia_get_card_info  (void *, int);
int    nvidia_handle_ioctl   (nv_state_t *, struct nvidia_filep *, u_long, caddr_t);

/* device helpers */
int    nvidia_open_ctl       (nv_state_t *, struct nvidia_filep *);
int    nvidia_open_dev       (nv_state_t *, struct nvidia_filep *);
int    nvidia_close_ctl      (nv_state_t *, struct nvidia_filep *);
int    nvidia_close_dev      (nv_state_t *, struct nvidia_filep *);
int    nvidia_mmap_dev_single(nv_state_t *, vm_ooffset_t *, vm_size_t, vm_object_t *);

#endif /* __NV_FREEBSD_H__ */
