/* _NVRM_COPYRIGHT_BEGIN_
 *
 * Copyright 2001-2002 by NVIDIA Corporation.  All rights reserved.  All
 * information contained herein is proprietary and confidential to NVIDIA
 * Corporation.  Any use, reproduction, or disclosure without the written
 * permission of NVIDIA Corporation is prohibited.
 *
 * _NVRM_COPYRIGHT_END_
 */

#include "nv-misc.h"
#include "os-interface.h"
#include "nv.h"
#include "nv-freebsd.h"

#if defined(NVCPU_X86) && defined(NV_USE_OS_VM86_INT10CALL)
#include <machine/vm86.h>
#endif

uma_zone_t nvidia_stack_t_zone;
static nv_stack_t *__nvidia_init_sp = NULL;

static  void   *nv_find_alloc(struct nvidia_softc *sc, NvU64 address, NvU32 *page_index);

devclass_t nvidia_devclass;
nv_state_t nvidia_ctl_state;

static  void  *nv_find_alloc(
    struct nvidia_softc *sc,
    NvU64  address,
    NvU32 *page_index
)
{
    NvU32 i = 0;
    struct nvidia_alloc *at = NULL;

    SLIST_FOREACH(at, &sc->alloc_list, list) {
        for (i = 0; i < (at->size / PAGE_SIZE); i++) {
            if (!at->alloc_type_contiguous &&
                    (address == at->pte_array[i].physical_address))
                goto found;
            else if (at->alloc_type_contiguous &&
                     (address == ((i * PAGE_SIZE) +
                          at->pte_array[0].physical_address)))
                goto found;
        }
    }

found:
    if (at != NULL)
        *page_index = i;
    return at;
}

int nvidia_attach(device_t dev)
{
    int status;
    NvU32 i;
    struct nvidia_softc *sc;
    nv_state_t *nv;

    sc = device_get_softc(dev);
    nv = sc->nv_state;

    nv->os_state         = sc;
    nv->flags            = 0;
    nv->domain           = pci_get_domain(dev);
    nv->bus              = pci_get_bus(dev);
    nv->slot             = pci_get_slot(dev);
    nv->vendor_id        = pci_get_vendor(dev);
    nv->device_id        = pci_get_device(dev);
    nv->handle           = dev;

    for (i = 0; i < NV_GPU_NUM_BARS; i++) {
        if (sc->BAR_recs[i] != NULL) {
            nv->bars[i].address = rman_get_start(sc->BAR_recs[i]);
            nv->bars[i].strapped_size = rman_get_size(sc->BAR_recs[i]);
            nv->bars[i].size = nv->bars[i].strapped_size;
        }
    }

    nv->fb   = &nv->bars[NV_GPU_BAR_INDEX_FB];
    nv->regs = &nv->bars[NV_GPU_BAR_INDEX_REGS];

    pci_enable_io(dev, SYS_RES_MEMORY);

    if ((status = rm_is_supported_device(sc->attach_sp, nv)) != RM_OK) {
        nv_printf(NV_DBG_ERRORS,
            "NVRM: The NVIDIA GPU %02x:%02x (PCI ID: %04x:%04x) installed\n"
            "NVRM: in this system is not supported by the %s NVIDIA FreeBSD\n"
            "NVRM: graphics driver release.  Please see 'Appendix A -\n"
            "NVRM: Supported NVIDIA GPU Products' in this release's README,\n"
            "NVRM: available on the FreeBSD graphics driver download page at\n"
            "NVRM: www.nvidia.com.\n",
            nv->bus, nv->slot, nv->vendor_id, nv->device_id, NV_VERSION_STRING);
        return ENXIO;
    }

    for (i = 0; i < NV_GPU_NUM_BARS; i++) {
        if (sc->BAR_recs[i] != NULL) {
            sc->BAR_sg_lists[i] = sglist_alloc(1, M_WAITOK);
            if (!sc->BAR_sg_lists[i])
                goto failed;

            sglist_append_phys(sc->BAR_sg_lists[i],
                    nv->bars[i].address, nv->bars[i].size);

            sc->BAR_objects[i] = NV_VM_PAGER_ALLOCATE(OBJT_SG,
                    sc->BAR_sg_lists[i],
                    nv->bars[i].size, (VM_PROT_READ | VM_PROT_WRITE),
                    0, NULL);
            if (!sc->BAR_objects[i])
                goto failed;

            VM_OBJECT_WLOCK(sc->BAR_objects[i]);
            switch (i) {
                case NV_GPU_BAR_INDEX_FB:
                    vm_object_set_memattr(sc->BAR_objects[i],
                            VM_MEMATTR_WRITE_COMBINING);
                    break;
                case NV_GPU_BAR_INDEX_REGS:
                default:
                    vm_object_set_memattr(sc->BAR_objects[i],
                            VM_MEMATTR_UNCACHEABLE);
                    break;
            }
            VM_OBJECT_WUNLOCK(sc->BAR_objects[i]);
        }
    }

    sc->dma_mask = 0xffffffffULL;

    if ((status = nvidia_dev_attach(sc)) != 0)
        return status;

    if ((status = nvidia_ctl_attach()) != 0)
        return status;

    nv->interrupt_line = rman_get_start(sc->irq);

    nv_sysctl_init(nv);

    return 0;

failed:
    for (i = 0; i < NV_GPU_NUM_BARS; i++) {
        if (sc->BAR_recs[i] != NULL) {
            if (sc->BAR_objects[i])
                NV_VM_OBJECT_DEALLOCATE(sc->BAR_objects[i]);
            if (sc->BAR_sg_lists[i])
                NV_SGLIST_FREE(sc->BAR_sg_lists[i]);
        }
    }

    return ENOMEM;
}

int nvidia_detach(device_t dev)
{
    int status;
    struct nvidia_softc *sc;
    uint32_t i;

    sc = device_get_softc(dev);
    nv_sysctl_exit(sc->nv_state);

    status = nvidia_dev_detach(sc);
    if (status) {
        device_printf(dev, "NVRM: NVIDIA driver DEV detach failed.\n");
        goto failed;
    }

    status = nvidia_ctl_detach();
    if (status) {
        device_printf(dev, "NVRM: NVIDIA driver CTL detach failed.\n");
        goto failed;
    }

    for (i = 0; i < NV_GPU_NUM_BARS; i++) {
        if (sc->BAR_recs[i] != NULL) {
            if (sc->BAR_objects[i])
                NV_VM_OBJECT_DEALLOCATE(sc->BAR_objects[i]);
            if (sc->BAR_sg_lists[i])
                NV_SGLIST_FREE(sc->BAR_sg_lists[i]);
        }
    }

failed:
    /* XXX Fix me? (state) */
    return status;
}


#ifdef NV_SUPPORT_ACPI_PM
int nvidia_suspend(device_t dev)
{
    nv_stack_t *sp;
    struct nvidia_softc *sc;
    nv_state_t *nv;
    int status;

    /* Only if ACPI is running */
    if (devclass_get_softc(devclass_find("acpi"), 0) == NULL)
        return ENODEV;

    NV_UMA_ZONE_ALLOC_STACK(sp);
    if (sp == NULL)
        return ENOMEM;

    sc = device_get_softc(dev);
    nv = sc->nv_state;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, TRUE);
    status = rm_power_management(sp, nv, 0, NV_PM_ACPI_STANDBY);

    NV_UMA_ZONE_FREE_STACK(sp);

    return (status == RM_OK) ? 0 : EIO;
}

int nvidia_resume(device_t dev)
{
    nv_stack_t *sp;
    struct nvidia_softc *sc;
    nv_state_t *nv;
    int status;

    NV_UMA_ZONE_ALLOC_STACK(sp);
    if (sp == NULL)
        return ENOMEM;

    sc = device_get_softc(dev);
    nv = sc->nv_state;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, TRUE);
    status = rm_power_management(sp, nv, 0, NV_PM_ACPI_RESUME);

    NV_UMA_ZONE_FREE_STACK(sp);

    return (status == RM_OK) ? 0 : EIO;
}
#endif /* NV_SUPPORT_ACPI_PM */


int nvidia_alloc_hardware(device_t dev)
{
    int status = 0;
    struct nvidia_softc *sc;
    NvU32 flags, i;
    NvU32 enable_msi = 0;
    int count;
    nv_stack_t *sp;

    NV_UMA_ZONE_ALLOC_STACK(sp);
    if (sp == NULL)
        return ENOMEM;

    sc = device_get_softc(dev);
    sc->dev = dev;

    flags = 0; /* not RF_ACTIVE */
    for (i = 0; i < NV_GPU_NUM_BARS && sc->BAR_rids[i] != 0; i++) {
        struct resource *res;
        res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->BAR_rids[i], flags);
        if (res == NULL) {
            /*
             * The most likely reason for this failure is that the SBIOS failed
             * to assign a valid address range to this BAR; FreeBSD is unable to
             * correct the problem and fails this BUS resource allocation. We
             * trust the kernel with BAR validation at this point, but later try
             * to catch cases where the X server "corrects" "invalid" BAR's.
             *
             * Please see to nvidia_pci_check_config_space() in nvidia_pci.c for
             * additional information.
             */
            device_printf(dev,
                "NVRM: NVIDIA MEM resource alloc failed, BAR%d @ 0x%02x.\n",
                i, sc->nv_state->bars[i].offset);
            status = ENXIO;
            goto failed;
        }
        sc->BAR_recs[i] = res;
    }

    if ((rm_read_registry_dword(sp, NULL, "NVreg",
            "EnableMSI", &enable_msi) == RM_OK) && (enable_msi != 0)) {
        count = pci_msi_count(dev);
        if ((count == 1) && (pci_alloc_msi(dev, &count) == 0))
            sc->irq_rid = 1;
    }
    flags = RF_SHAREABLE | RF_ACTIVE;
    sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid, flags);
    if (sc->irq == NULL) {
        device_printf(dev, "NVRM: NVIDIA IRQ resource alloc failed.\n");
        status = ENXIO;
        goto failed;
    }

failed:
    NV_UMA_ZONE_FREE_STACK(sp);
    return status;
}

void nvidia_free_hardware(device_t dev)
{
    struct nvidia_softc *sc;
    NvU32 i;

    sc = device_get_softc(dev);

    for (i = 0; i < NV_GPU_NUM_BARS && sc->BAR_recs[i] != NULL; i++)
        bus_release_resource(dev, SYS_RES_MEMORY, sc->BAR_rids[i], sc->BAR_recs[i]);
    if (sc->irq != NULL)
        bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid, sc->irq);
    if (sc->irq_rid != 0)
        pci_release_msi(dev);
    if (sc->iop != NULL)
        bus_release_resource(dev, SYS_RES_IOPORT, sc->iop_rid, sc->iop);
}

void nvidia_intr(void *xsc)
{
    struct nvidia_softc *sc;
    nv_state_t *nv;
    NvU32 run_bottom_half = 0;
    nv_stack_t *sp;

    sc = (struct nvidia_softc *) xsc;
    nv = sc->nv_state;

    sp = sc->isr_sp;

    if (sp == NULL)
        return;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, FALSE);
    rm_isr(sp, nv, &run_bottom_half);

    if (run_bottom_half) {
        /* We're not executing in an HW ISR context */
        rm_isr_bh(sp, nv);
    }
}

int nvidia_get_card_info(void *args, int size)
{
    struct nv_ioctl_card_info *ci;
    struct nv_ioctl_rm_api_old_version *av;
    unsigned int i;
    struct nvidia_softc *sc;
    nv_state_t *nv;

    if (size < (sizeof(*ci) * NV_MAX_DEVICES))
        return EINVAL;

    av = args;
    switch (av->magic) {
        case NV_RM_API_OLD_VERSION_MAGIC_OVERRIDE_REQ:
        case NV_RM_API_OLD_VERSION_MAGIC_LAX_REQ:
        case NV_RM_API_OLD_VERSION_MAGIC_REQ:
            /*
             * the client is using the old major-minor-patch API
             * version check; reject it.
             */
            nv_printf(NV_DBG_ERRORS,
                      "NVRM: API mismatch: the client has the version %d.%d-%d, but\n"
                      "NVRM: this kernel module has the version %s.  Please\n"
                      "NVRM: make sure that this kernel module and all NVIDIA driver\n"
                      "NVRM: components have the same version.\n",
                      av->major, av->minor, av->patch,
                      NV_VERSION_STRING);
            return EINVAL;
        case NV_RM_API_OLD_VERSION_MAGIC_IGNORE:
            /*
             * the client is telling us to ignore the old version
             * scheme; it will do a version check via
             * NV_ESC_CHECK_VERSION_STR
             */
            break;
        default:
            return EINVAL;
    }

    ci = args;
    memset(ci, 0, sizeof(ci));

    for (i = 0; i < NV_MAX_DEVICES; i++) {
        sc = devclass_get_softc(nvidia_devclass, i);
        if (!sc)
            continue;
        nv = sc->nv_state;

        ci[i].flags          = NV_IOCTL_CARD_INFO_FLAG_PRESENT;
        ci[i].domain         = nv->domain;
        ci[i].bus            = nv->bus;
        ci[i].slot           = nv->slot;
        ci[i].vendor_id      = nv->vendor_id;
        ci[i].device_id      = nv->device_id;
        ci[i].gpu_id         = nv->gpu_id;
        ci[i].interrupt_line = nv->interrupt_line;
        ci[i].fb_address     = nv->fb->address;
        ci[i].fb_size        = nv->fb->size;
        ci[i].reg_address    = nv->regs->address;
        ci[i].reg_size       = nv->regs->size;
    }

    return 0;
}

int nvidia_handle_ioctl(
    nv_state_t *nv,
    struct nvidia_filep *filep,
    u_long cmd,
    caddr_t data
)
{
    struct nvidia_softc *sc;
    nv_stack_t *sp;
    void *args;
    nv_ioctl_xfer_t *xfer = NULL;
    int status;
    int nr, size;

    sc = nv->os_state;
    sp = sc->api_sp;

    size = __NV_IOC_SIZE(cmd);
    nr = __NV_IOC_NR(cmd);

    args = (void *)data;

    if (nr == NV_ESC_IOCTL_XFER_CMD) {
        if (__NV_IOC_SIZE(cmd) != sizeof(nv_ioctl_xfer_t))
            return EINVAL;

        xfer = args;
        size = xfer->size;

        if (size > NV_ABSOLUTE_MAX_IOCTL_SIZE)
            return EINVAL;

        args = malloc(size, M_NVIDIA, M_WAITOK);
        if (args == NULL)
            return ENOMEM;

        if (copyin(NvP64_VALUE(xfer->ptr), args, size) != 0) {
            free(args, M_NVIDIA);
            return EFAULT;
        }

        nr = xfer->cmd;
    }

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, TRUE);

    switch (nr) {
        case NV_ESC_CHECK_VERSION_STR:
            status = ((rm_perform_version_check(sp,
                            args, size) == RM_OK) ? 0 : EINVAL);
            break;

        case NV_ESC_CARD_INFO:
            status = nvidia_get_card_info(args, size);
            break;

        default:
            status = ((rm_ioctl(sp, nv, filep, nr,
                            args, size) == RM_OK) ? 0 : EINVAL);
            break;
    }

    if (args != (void *)data) {
        if (copyout(args, NvP64_VALUE(xfer->ptr), size) != 0)
            status = EFAULT;
        free(args, M_NVIDIA);
    }

    return status;
}

int nvidia_open_ctl(
    nv_state_t *nv,
    struct nvidia_filep *filep
)
{
    struct nvidia_softc *sc = nv->os_state;

    if (sc->refcnt == 0) {
        NV_UMA_ZONE_ALLOC_STACK(sc->api_sp);
        if (sc->api_sp == NULL)
            return ENOMEM;
        nv->flags |= (NV_FLAG_OPEN | NV_FLAG_CONTROL);
    }

    sc->refcnt++;

    return 0;
}

int nvidia_close_ctl(
    nv_state_t *nv,
    struct nvidia_filep *filep
)
{
    struct nvidia_softc *sc = nv->os_state;
    nv_stack_t *sp;

    sp = sc->api_sp;
    rm_free_unused_clients(sp, nv, filep);

    if (--sc->refcnt == 0) {
        NV_UMA_ZONE_FREE_STACK(sc->api_sp);
        nv->flags &= ~NV_FLAG_OPEN;
    }

    return 0;
}

int nvidia_open_dev(
    nv_state_t *nv,
    struct nvidia_filep *filep
)
{
    int status = ENOMEM;
    struct nvidia_softc *sc = nv->os_state;
    nv_stack_t *sp = NULL;

    if (sc->refcnt == 0) {
        NV_UMA_ZONE_ALLOC_STACK(sc->api_sp);
        if (sc->api_sp == NULL)
            goto failed;

        NV_UMA_ZONE_ALLOC_STACK(sc->pci_cfgchk_sp);
        if (sc->pci_cfgchk_sp == NULL)
            goto failed;

        NV_UMA_ZONE_ALLOC_STACK(sc->isr_sp);
        if (sc->isr_sp == NULL)
            goto failed;

        NV_UMA_ZONE_ALLOC_STACK(sc->timer_sp);
        if (sc->timer_sp == NULL)
            goto failed;
    }

    sp = sc->api_sp;
    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, TRUE);

    if (sc->refcnt == 0) {
        if (!rm_init_adapter(sp, nv)) {
            device_printf(sc->dev, "NVRM: rm_init_adapter() failed!\n");
            status = EIO;
            goto failed;
        }

        if (nv->ud.size != 0) {
            sc->UD_sg_list = sglist_alloc(1, M_WAITOK);
            if (!sc->UD_sg_list) {
                status = RM_ERR_NO_FREE_MEM;
                goto failed;
            }

            sglist_append_phys(sc->UD_sg_list, nv->ud.address, nv->ud.size);

            sc->UD_object = NV_VM_PAGER_ALLOCATE(OBJT_SG, sc->UD_sg_list,
                    nv->ud.size, (VM_PROT_READ | VM_PROT_WRITE),
                    0, NULL);
            if (!sc->UD_object) {
                status = RM_ERR_NO_FREE_MEM;
                goto failed;
            }

            VM_OBJECT_WLOCK(sc->UD_object);
            vm_object_set_memattr(sc->UD_object, VM_MEMATTR_UNCACHEABLE);
            VM_OBJECT_WUNLOCK(sc->UD_object);
        }

        nv->flags |= NV_FLAG_OPEN;
    }

    sc->refcnt++;

    return 0;

failed:
    if (sc->refcnt == 0) {
        if (sc->UD_object != NULL)
            NV_VM_OBJECT_DEALLOCATE(sc->UD_object);
        if (sc->UD_sg_list != NULL)
            NV_SGLIST_FREE(sc->UD_sg_list);

        NV_SHUTDOWN_ADAPTER(sp, nv);

        if (sc->timer_sp != NULL)
            NV_UMA_ZONE_FREE_STACK(sc->timer_sp);
        if (sc->isr_sp != NULL)
            NV_UMA_ZONE_FREE_STACK(sc->isr_sp);
        if (sc->pci_cfgchk_sp != NULL)
            NV_UMA_ZONE_FREE_STACK(sc->pci_cfgchk_sp);
        if (sc->api_sp != NULL)
            NV_UMA_ZONE_FREE_STACK(sc->api_sp);
    }

    return status;
}

int nvidia_close_dev(
    nv_state_t *nv,
    struct nvidia_filep *filep
)
{
    struct nvidia_softc *sc;
    nv_stack_t *sp;

    sc = nv->os_state;
    sp = sc->api_sp;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, TRUE);
    rm_free_unused_clients(sp, nv, filep);

    if (--sc->refcnt == 0) {
        if (sc->UD_object != NULL)
            NV_VM_OBJECT_DEALLOCATE(sc->UD_object);
        if (sc->UD_sg_list != NULL)
            NV_SGLIST_FREE(sc->UD_sg_list);

        NV_SHUTDOWN_ADAPTER(sp, nv);

        NV_UMA_ZONE_FREE_STACK(sc->timer_sp);
        NV_UMA_ZONE_FREE_STACK(sc->isr_sp);
        NV_UMA_ZONE_FREE_STACK(sc->pci_cfgchk_sp);
        NV_UMA_ZONE_FREE_STACK(sc->api_sp);

        nv->flags &= ~NV_FLAG_OPEN;
    }

    return 0;
}


int nvidia_modevent(
    module_t mod,
    int what,
    void *arg
)
{
    nv_state_t *nv;
    struct nvidia_softc *sc;
    nv_stack_t *sp;

    switch (what) {
        case MOD_LOAD:
            /*
             * The module load event. Our KLD has just been loaded and is
             * ready to initialize. We setup the core resource manager in
             * this routine, further initialization takes place at attach
             * time.
             */
            sc = &nvidia_ctl_sc;

            nvidia_stack_t_zone = uma_zcreate("nv_stack_t", sizeof(nv_stack_t),
                    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
            if (nvidia_stack_t_zone == NULL)
                return ENOMEM;

            NV_UMA_ZONE_ALLOC_STACK(sp);
            if (sp == NULL) {
                uma_zdestroy(nvidia_stack_t_zone);
                return ENOMEM;
            }

            bzero(sc, sizeof(nvidia_softc_t));

            if (!rm_init_rm(sp)) {
                printf("NVRM: rm_init_rm() failed!\n");
                NV_UMA_ZONE_FREE_STACK(sp);
                uma_zdestroy(nvidia_stack_t_zone);
                return EIO;
            }

            __nvidia_init_sp = sp;

            callout_init(&sc->timer, CALLOUT_MPSAFE);
            sx_init(&sc->api_sx, "ctl.api_sx");

            nvidia_ctl_state.os_state = sc;
            sc->nv_state = (void *)&nvidia_ctl_state;

            nvidia_sysctl_init();
            nvidia_linux_init();

            break;

        case MOD_UNLOAD:
            /*
             * Check if the control device is still open and reject the
             * unload request if it is. This event can occur even when the
             * module usage count is non-zero!
             */
            nv = &nvidia_ctl_state;
            sc = nv->os_state;

            nv_lock_api(nv);

            if (sc->refcnt != 0) { /* XXX Fix me? (refcnt) */
                nv_unlock_api(nv);
                return EBUSY;
            }

            nv_unlock_api(nv);
            sx_destroy(&sc->api_sx);

            sp = __nvidia_init_sp;
            rm_shutdown_rm(sp);

            NV_UMA_ZONE_FREE_STACK(sp);

            nvidia_sysctl_exit();
            nvidia_linux_exit();

            uma_zdestroy(nvidia_stack_t_zone);

            break;

        default:
            break;
    }

    return 0;
}


#ifdef NV_SUPPORT_OS_AGP
NvS32 nv_os_agp_init(
    nv_stack_t *sp,
    nv_state_t *nv,
    NvU64 *base,
    NvU64 *limit
)
{
    void *bitmap;
    struct nvidia_softc *sc = nv->os_state;
    struct agp_info ai;

    NvU32 mode = 0;
    NvU32 fw   = 0;
    NvU32 sba  = 0;
    NvU32 rate = (8 | 4 | 2 | 1);
    NvU32 size = 0;
    NvS32 memory_token = 0;

    sc->agp_dev = agp_find_device();
    if (!sc->agp_dev) {
        printf("NVRM: agp_find_device failed, chipset unsupported?\n");
        return ENODEV;
    }

    if (agp_acquire(sc->agp_dev) != 0)
        return EBUSY;

    agp_get_info(sc->agp_dev, &ai);
    mode = ai.ai_mode;

    if (os_set_mem_range(ai.ai_aperture_base, ai.ai_aperture_size,
                NV_MEMORY_WRITECOMBINED, &memory_token) != RM_OK) {
        /*
         * Failure to set a write-combining range for the AGP aperture is
         * not necessarily a fatal error condition; we don't know at this
         * point, however, and abort to prevent performance and stability
         * problems that may be hard to track down otherwise.
         */
        agp_release(sc->agp_dev);
        return EIO;
    }

    rm_read_registry_dword(sp, NULL, "NVreg", "ReqAGPRate", &rate);
    rm_read_registry_dword(sp, NULL, "NVreg", "EnableAGPFW", &fw);
    rm_read_registry_dword(sp, NULL, "NVreg", "EnableAGPSBA", &sba);

    if (AGP_MODE_GET_MODE_3(mode))
        rate = (rate >> 2) & 3;
    mode = AGP_MODE_SET_RATE(mode, AGP_MODE_GET_RATE(mode) & rate);
    mode |= 1 /* avoid 0x mode request */;

    if (AGP_MODE_GET_RATE(mode) & 2)
        mode = AGP_MODE_SET_RATE(mode, AGP_MODE_GET_RATE(mode) & ~1);
    if (AGP_MODE_GET_RATE(mode) & 4)
        mode = AGP_MODE_SET_RATE(mode, AGP_MODE_GET_RATE(mode) & ~2);

    mode = AGP_MODE_SET_FW(mode, fw);
    mode = AGP_MODE_SET_SBA(mode, sba);

    if (agp_enable(sc->agp_dev, mode) != 0) {
        agp_release(sc->agp_dev);
        os_unset_mem_range(ai.ai_aperture_base, ai.ai_aperture_size,
                           memory_token);
        return EIO;
    }

    size = ai.ai_aperture_size / RM_PAGE_SIZE / 8;

    if (os_alloc_mem((void **)&bitmap, size) != RM_OK) {
        agp_release(sc->agp_dev);
        os_unset_mem_range(ai.ai_aperture_base, ai.ai_aperture_size,
                           memory_token);
        return EIO;
    }

    os_mem_set(bitmap, 0xff, size);

    if (rm_set_agp_bitmap(sp, nv, bitmap) != RM_OK) {
        agp_release(sc->agp_dev);
        os_free_mem(bitmap);
        os_unset_mem_range(ai.ai_aperture_base, ai.ai_aperture_size,
                           memory_token);
        return EIO;
    }

    *base = ai.ai_aperture_base;
    *limit = (ai.ai_aperture_size - 1);

    return 0;
}

NvS32 nv_os_agp_teardown(
    nv_stack_t *sp,
    nv_state_t *nv
)
{
    struct nvidia_softc *sc = nv->os_state;
    void *bitmap;

    if (agp_release(sc->agp_dev) != 0)
        return EBUSY;

    rm_clear_agp_bitmap(sp, nv, &bitmap);
    os_free_mem(bitmap);

    /* Third parameter is a no-op on FreeBSD */
    os_unset_mem_range(nv->agp.address, nv->agp.size, 0);

    return 0;
}
#endif /* NV_SUPPORT_OS_AGP */

RM_STATUS NV_API_CALL nv_agp_init(
    nv_state_t *nv,
    NvU64 *base,
    NvU64 *limit,
    NvU32 config
)
{
    struct nvidia_softc *sc = nv->os_state;
    RM_STATUS status = RM_ERROR;
    nv_stack_t *sp;

    if (NV_AGP_ENABLED(nv))
        return RM_ERR_STATE_IN_USE;

    if (config == NVOS_AGP_CONFIG_DISABLE_AGP) {
        /*
         * Match the behavior on Linux, don't consider the attempt
         * to initialize AGP as 'disabled' an error.
         */
        nv->agp_config = NVOS_AGP_CONFIG_DISABLE_AGP;
        nv->agp_status = NV_AGP_STATUS_DISABLED;
        return RM_OK;
    }

    NV_UMA_ZONE_ALLOC_STACK(sp);
    if (sp == NULL)
        return RM_ERR_NO_FREE_MEM;

#ifdef NV_SUPPORT_OS_AGP
    if ((config & NVOS_AGP_CONFIG_OSAGP) != 0) {
        if (nv_os_agp_init(sp, nv, base, limit) == 0) {
            /*
             * If the operating system AGP GART driver successfully
             * configured its backend, apply chipset overrides.
             */
            rm_update_agp_config(sp, nv);
            NV_UMA_ZONE_FREE_STACK(sp);

            nv->agp_config = NVOS_AGP_CONFIG_OSAGP;
            nv->agp_status = NV_AGP_STATUS_ENABLED;
        }
    }
#endif /* NV_SUPPORT_OS_AGP */

    if (nv->agp_status != NV_AGP_STATUS_ENABLED) {
        if ((config & NVOS_AGP_CONFIG_NVAGP) == 0) {
            status = RM_ERR_NOT_SUPPORTED;
            goto failed;
        }

        /*
         * Make sure we don't try to use the internal GART driver when
         * the OS AGPGART driver (agp.ko) is attached. While that may
         * be perfectly fine on most systems, but is known to break on
         * some.
         * -------------------------------------------------------------
         * DO NOT REDISTRIBUTE THE DRIVER WITH THIS SANITY CHECK REMOVED!
         * -------------------------------------------------------------
         */
        if (devclass_get_softc(devclass_find("agp"), 0) != NULL) {
            printf("NVRM: detected agp.ko, aborting NVIDIA AGP setup!\n");
            goto failed;
        }

        status = rm_init_agp(sp, nv, base, limit);
        if (status != RM_OK)
            goto failed;
        else {
            NV_UMA_ZONE_FREE_STACK(sp);

            nv->agp_config = NVOS_AGP_CONFIG_NVAGP;
            nv->agp_status = NV_AGP_STATUS_ENABLED;
        }
    }

    sc->AGP_sg_list = sglist_alloc(1, M_WAITOK);
    if (!sc->AGP_sg_list) {
        nv_agp_teardown(nv);
        return RM_ERR_NO_FREE_MEM;
    }

    sglist_append_phys(sc->AGP_sg_list, *base, (*limit + 1));

    sc->AGP_object = NV_VM_PAGER_ALLOCATE(OBJT_SG, sc->AGP_sg_list,
            (*limit + 1), (VM_PROT_READ | VM_PROT_WRITE),
            0, NULL);
    if (!sc->AGP_object) {
        NV_SGLIST_FREE(sc->AGP_sg_list);
        nv_agp_teardown(nv);
        return RM_ERR_NO_FREE_MEM;
    }

    VM_OBJECT_WLOCK(sc->AGP_object);
    vm_object_set_memattr(sc->AGP_object, VM_MEMATTR_WRITE_COMBINING);
    VM_OBJECT_WUNLOCK(sc->AGP_object);

    return RM_OK;

failed:
    NV_UMA_ZONE_FREE_STACK(sp);

    nv->agp_config = NVOS_AGP_CONFIG_DISABLE_AGP;
    nv->agp_status = NV_AGP_STATUS_FAILED;

    return status;
}

RM_STATUS NV_API_CALL nv_agp_teardown(nv_state_t *nv)
{
    struct nvidia_softc *sc = nv->os_state;
    RM_STATUS status = RM_ERR_INVALID_STATE;
    nv_stack_t *sp;

    if (!NV_AGP_ENABLED(nv))
        return status;

    NV_UMA_ZONE_ALLOC_STACK(sp);
    if (sp == NULL)
        return RM_ERR_NO_FREE_MEM;

    NV_VM_OBJECT_DEALLOCATE(sc->AGP_object);
    NV_SGLIST_FREE(sc->AGP_sg_list);

#ifdef NV_SUPPORT_OS_AGP
    if (NV_OSAGP_ENABLED(nv)) {
        status = (nv_os_agp_teardown(sp, nv) == 0)
            ? RM_OK : RM_ERROR;
    }
#endif
    if (NV_NVAGP_ENABLED(nv))
        status = rm_teardown_agp(sp, nv);

    NV_UMA_ZONE_FREE_STACK(sp);

    nv->agp_config = NVOS_AGP_CONFIG_DISABLE_AGP;
    nv->agp_status = NV_AGP_STATUS_DISABLED;

    return status;
}

NvS32 NV_API_CALL nv_no_incoherent_mappings(void)
{
    return 1;
}

void nv_lock_api(nv_state_t *nv)
{
    struct nvidia_softc *sc = nv->os_state;
    sx_xlock(&sc->api_sx);
}

void nv_unlock_api(nv_state_t *nv)
{
    struct nvidia_softc *sc = nv->os_state;
    sx_xunlock(&sc->api_sx);
}

void NV_API_CALL nv_post_event(
    nv_state_t *nv,
    nv_event_t *event,
    NvU32 hObject,
    NvU32 index
)
{
    struct nvidia_filep *filep = event->file;
    struct nvidia_event *et;

    et = malloc(sizeof(nvidia_event_t), M_NVIDIA, M_NOWAIT);
    if (et == NULL)
        return;

    et->event = *event;
    et->event.hObject = hObject;
    et->event.index = index;

    mtx_lock(&filep->event_mtx);
    STAILQ_INSERT_TAIL(&filep->event_queue, et, queue);
    mtx_unlock(&filep->event_mtx);

    selwakeup(&filep->event_rsel);
}

NvS32 NV_API_CALL nv_get_event(
    nv_state_t *nv,
    void *file,
    nv_event_t *event,
    NvU32 *pending
)
{
    struct nvidia_filep *filep = file;
    struct nvidia_event *et;

    mtx_lock(&filep->event_mtx);
    et = STAILQ_FIRST(&filep->event_queue);

    if (et != NULL) {
        *event = et->event;

        STAILQ_REMOVE(&filep->event_queue, et, nvidia_event, queue);
        *pending = !STAILQ_EMPTY(&filep->event_queue);

        mtx_unlock(&filep->event_mtx);
        free(et, M_NVIDIA);

        return RM_OK;
    }

    mtx_unlock(&filep->event_mtx);
    return RM_ERROR;
}

void* NV_API_CALL nv_alloc_kernel_mapping(
    nv_state_t *nv,
    NvU64       address,
    NvU64       size,
    void      **private
)
{
    struct nvidia_alloc *at;
    struct nvidia_softc *sc = nv->os_state;
    uint32_t i;
    vm_offset_t offset, virtual_address;
    int status;

    offset = (vm_offset_t)address & PAGE_MASK;
    address &= ~PAGE_MASK;

    SLIST_FOREACH(at, &sc->alloc_list, list) {
        for (i = 0; i < (at->size / PAGE_SIZE); i++) {
            if (!at->alloc_type_contiguous &&
                    (address == at->pte_array[i].physical_address))
                goto found;
            else if (at->alloc_type_contiguous &&
                     (address == ((i * PAGE_SIZE) +
                          at->pte_array[0].physical_address)))
                goto found;
        }
    }

    return NULL;

found:
    if ((size + (i * PAGE_SIZE)) > at->size)
        return NULL;

    if (at->alloc_type_contiguous) {
        *private = NULL;
        return (void *)(NvUPtr)(at->pte_array[0].virtual_address +
                    (i * PAGE_SIZE) + offset);
    }

    size = (size + PAGE_MASK) & ~PAGE_MASK;

    vm_object_reference(at->object);
    virtual_address = vm_map_min(kernel_map);

    status = vm_map_find(kernel_map, at->object, (i * PAGE_SIZE),
            &virtual_address, size, 0, VMFS_ANY_SPACE,
            (VM_PROT_READ | VM_PROT_WRITE),
            (VM_PROT_READ | VM_PROT_WRITE), 0);
    if (status != KERN_SUCCESS) {
        NV_VM_OBJECT_DEALLOCATE(at->object);
        return NULL;
    }

    status = vm_map_wire(kernel_map, virtual_address,
            (virtual_address + size),
            (VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES));
    if (status != KERN_SUCCESS) {
        vm_map_remove(kernel_map, virtual_address,
            (virtual_address + size));
        return NULL;
    }

    *private = (void *)(NvUPtr)size;
    return (void *)(virtual_address + offset);
}

RM_STATUS NV_API_CALL nv_free_kernel_mapping(
    nv_state_t *nv,
    void       *address,
    void       *private,
    NvBool      guest_mapping
)
{
    vm_offset_t virtual_address;
    uint32_t size;

    if (private != NULL) {
        virtual_address = (vm_offset_t)address & ~PAGE_MASK;
        size = (NvUPtr)private;
        vm_map_remove(kernel_map, virtual_address,
            (virtual_address + size));
    }

    return RM_OK;
}

RM_STATUS NV_API_CALL nv_alloc_user_mapping(
    nv_state_t *nv,
    NvU64       address,
    NvU64       size,
    NvU32       protect,
    NvU64      *user_address,
    void      **private
)
{
    *user_address = address;
    return RM_OK;
}

RM_STATUS NV_API_CALL nv_free_user_mapping(
    nv_state_t *nv,
    NvU64       user_address,
    void       *priv_data
)
{
    return RM_OK;
}

NvS32 nv_alloc_contig_pages(
    nv_state_t *nv,
    NvU32       count,
    NvU32       cache_type,
    NvBool      zero,
    NvU64      *pte_array,
    void      **private
)
{
    struct nvidia_alloc *at;
    struct nvidia_softc *sc = nv->os_state;
    vm_memattr_t attr;
    vm_offset_t address;
    NvU32 size = (count * PAGE_SIZE);
    int flags = (zero ? M_ZERO : 0);
    int status;

    switch (cache_type) {
        case NV_MEMORY_UNCACHED:
            attr = VM_MEMATTR_UNCACHEABLE;
            break;
        case NV_MEMORY_DEFAULT:
        case NV_MEMORY_CACHED:
            attr = VM_MEMATTR_WRITE_BACK;
            break;
        case NV_MEMORY_UNCACHED_WEAK:
            attr = VM_MEMATTR_WEAK_UNCACHEABLE;
            break;
        case NV_MEMORY_WRITECOMBINED:
            attr = VM_MEMATTR_WRITE_COMBINING;
            break;
        default:
            nv_printf(NV_DBG_ERRORS,
                  "NVRM: unknown mode in nv_alloc_contig_pages()\n");
            return EINVAL;
    }

    at = malloc(sizeof(nvidia_alloc_t), M_NVIDIA, (M_WAITOK | M_ZERO));
    if (!at)
        return ENOMEM;

    at->size = size;
    at->alloc_type_contiguous = 1;
    at->attr = attr;

    at->pte_array = malloc(sizeof(nvidia_pte_t), M_NVIDIA,
            (M_WAITOK | M_ZERO));
    if (!at->pte_array) {
        free(at, M_NVIDIA);
        return ENOMEM;
    }

    address = kmem_alloc_contig(kmem_arena, size, flags, 0,
            sc->dma_mask, PAGE_SIZE, 0, attr);
    if (!address) {
        status = ENOMEM;
        goto failed;
    }
    malloc_type_allocated(M_NVIDIA, size);

    if (attr != VM_MEMATTR_WRITE_BACK)
        os_flush_cpu_cache();

    at->pte_array[0].virtual_address = address;
    at->pte_array[0].physical_address = (NvU64)vtophys(address);

    at->sg_list = sglist_alloc(1, M_WAITOK);
    if (!at->sg_list) {
        status = ENOMEM;
        goto failed;
    }

    pte_array[0] = at->pte_array[0].physical_address;
    sglist_append_phys(at->sg_list, pte_array[0], size);

    at->object = NV_VM_PAGER_ALLOCATE(OBJT_SG, at->sg_list, size,
            (VM_PROT_READ | VM_PROT_WRITE), 0, NULL);
    if (!at->object) {
        status = ENOMEM;
        goto failed;
    }

    VM_OBJECT_WLOCK(at->object);
    vm_object_set_memattr(at->object, attr);
    VM_OBJECT_WUNLOCK(at->object);

    *private = at;
    SLIST_INSERT_HEAD(&sc->alloc_list, at, list);

    return 0;

failed:
    if (at->object)
        NV_VM_OBJECT_DEALLOCATE(at->object);
    if (at->sg_list)
        NV_SGLIST_FREE(at->sg_list);

    if (attr != VM_MEMATTR_WRITE_BACK)
        os_flush_cpu_cache();

    if (at->pte_array[0].virtual_address != NULL) {
        kmem_free(kmem_arena, at->pte_array[0].virtual_address, at->size);
        malloc_type_freed(M_NVIDIA, at->size);
    }

    free(at->pte_array, M_NVIDIA);
    free(at, M_NVIDIA);

    return status;
}

NvS32 nv_free_contig_pages(
    nv_state_t *nv,
    void *private
)
{
    struct nvidia_alloc *at = private;
    struct nvidia_softc *sc = nv->os_state;

    SLIST_REMOVE(&sc->alloc_list, at, nvidia_alloc, list);

    NV_VM_OBJECT_DEALLOCATE(at->object);
    NV_SGLIST_FREE(at->sg_list);

    if (at->attr != VM_MEMATTR_WRITE_BACK)
        os_flush_cpu_cache();

    kmem_free(kmem_arena, at->pte_array[0].virtual_address, at->size);
    malloc_type_freed(M_NVIDIA, at->size);

    free(at->pte_array, M_NVIDIA);
    free(at, M_NVIDIA);

    return 0;
}

NvS32 nv_alloc_system_pages(
    nv_state_t  *nv,
    NvU32        count,
    NvU32        cache_type,
    NvBool       zero,
    NvU64       *pte_array,
    void       **private
)
{
    struct nvidia_alloc *at;
    struct nvidia_softc *sc = nv->os_state;
    vm_offset_t address;
    uint32_t i;
    vm_memattr_t attr;
    uint32_t size = (count * PAGE_SIZE);
    int flags = (zero ? M_ZERO : 0);
    int status;

    switch (cache_type) {
        case NV_MEMORY_UNCACHED:
            attr = VM_MEMATTR_UNCACHEABLE;
            break;
        case NV_MEMORY_DEFAULT:
        case NV_MEMORY_CACHED:
            attr = VM_MEMATTR_WRITE_BACK;
            break;
        case NV_MEMORY_UNCACHED_WEAK:
            attr = VM_MEMATTR_WEAK_UNCACHEABLE;
            break;
        case NV_MEMORY_WRITECOMBINED:
            attr = VM_MEMATTR_WRITE_COMBINING;
            break;
        default:
            nv_printf(NV_DBG_ERRORS,
                  "NVRM: unknown mode in nv_alloc_system_pages()\n");
            return EINVAL;
    }

    at = malloc(sizeof(nvidia_alloc_t), M_NVIDIA, (M_WAITOK | M_ZERO));
    if (!at)
        return ENOMEM;

    at->size = size;
    at->alloc_type_contiguous = 0;
    at->attr = attr;

    at->pte_array = malloc((sizeof(nvidia_pte_t) * count),
            M_NVIDIA, (M_WAITOK | M_ZERO));
    if (!at->pte_array) {
        free(at, M_NVIDIA);
        return ENOMEM;
    }

    for (i = 0; i < count; i++) {
        address = kmem_alloc_contig(kmem_arena, PAGE_SIZE, flags, 0,
                sc->dma_mask, PAGE_SIZE, 0, attr);
        if (!address) {
            status = ENOMEM;
            goto failed;
        }
        malloc_type_allocated(M_NVIDIA, PAGE_SIZE);

        at->pte_array[i].virtual_address = address;
        at->pte_array[i].physical_address = (NvU64)vtophys(address);
    }

    if (attr != VM_MEMATTR_WRITE_BACK)
        os_flush_cpu_cache();

    at->sg_list = sglist_alloc(count, M_WAITOK);
    if (!at->sg_list) {
        status = ENOMEM;
        goto failed;
    }

    for (i = 0; i < count; i++) {
        pte_array[i] = at->pte_array[i].physical_address;
        sglist_append_phys(at->sg_list, pte_array[i], PAGE_SIZE);
    }

    at->object = NV_VM_PAGER_ALLOCATE(OBJT_SG, at->sg_list, size,
            (VM_PROT_READ | VM_PROT_WRITE), 0, NULL);
    if (!at->object) {
        status = ENOMEM;
        goto failed;
    }

    VM_OBJECT_WLOCK(at->object);
    vm_object_set_memattr(at->object, attr);
    VM_OBJECT_WUNLOCK(at->object);

    *private = at;
    SLIST_INSERT_HEAD(&sc->alloc_list, at, list);

    return 0;

failed:
    if (at->object)
        NV_VM_OBJECT_DEALLOCATE(at->object);
    if (at->sg_list)
        NV_SGLIST_FREE(at->sg_list);

    if (attr != VM_MEMATTR_WRITE_BACK)
        os_flush_cpu_cache();

    for (i = 0; i < count; i++) {
        if (at->pte_array[i].virtual_address == 0)
            break;
        kmem_free(kmem_arena, at->pte_array[i].virtual_address, PAGE_SIZE);
        malloc_type_freed(M_NVIDIA, PAGE_SIZE);
    }

    free(at->pte_array, M_NVIDIA);
    free(at, M_NVIDIA);

    return status;
}

NvS32 nv_free_system_pages(
    nv_state_t *nv,
    void *private
)
{
    struct nvidia_alloc *at = private;
    struct nvidia_softc *sc = nv->os_state;
    uint32_t i, count;

    count = at->size / PAGE_SIZE;
    SLIST_REMOVE(&sc->alloc_list, at, nvidia_alloc, list);

    NV_VM_OBJECT_DEALLOCATE(at->object);
    NV_SGLIST_FREE(at->sg_list);

    if (at->attr != VM_MEMATTR_WRITE_BACK)
        os_flush_cpu_cache();

    for (i = 0; i < count; i++) {
        kmem_free(kmem_arena, at->pte_array[i].virtual_address, PAGE_SIZE);
        malloc_type_freed(M_NVIDIA, PAGE_SIZE);
    }

    free(at->pte_array, M_NVIDIA);
    free(at, M_NVIDIA);

    return 0;
}

#ifdef NV_SUPPORT_OS_AGP
NvS32 nv_alloc_agp_pages(
    nv_state_t *nv,
    NvU32 count,
    NvU32 offset,
    void **private
)
{
    void *handle;
    struct nvidia_softc *sc = nv->os_state;

    handle = agp_alloc_memory(sc->agp_dev, 0, count << PAGE_SHIFT);
    if (!handle) {
        /*
         * This is very unlikely to happen, the system's memory resources
         * would have to be nearly exhausted.
         */
        return ENOMEM;
    }

    os_flush_cpu_cache();

    if (agp_bind_memory(sc->agp_dev, handle, offset) != 0) {
        /*
         * This shouldn't happen, we claimed the AGP backend and are thus
         * using it exclusively; the resource manager manages AGP offsets
         * internally, we wouldn't have been called had we run out of AGP
         * aperture space.
         */
        os_dbg_breakpoint();

        agp_free_memory(sc->agp_dev, handle);
        return ENOMEM;
    }

    *private = handle;
    return 0;
}

NvS32 nv_free_agp_pages(
    nv_state_t *nv,
    NvU32 count,
    void *private,
    NvU32 *offset
)
{
    void *handle = private;
    struct nvidia_softc *sc = nv->os_state;
    struct agp_memory_info info;

    agp_memory_info(sc->agp_dev, handle, &info);
    *offset = info.ami_offset;

    if (agp_unbind_memory(sc->agp_dev, handle) != 0) {
        /*
         * This is the only place where previously bound AGP memory would
         * be freed. If we fail to unbind this memory now, something very
         * wrong must have happened.
         */
        os_dbg_breakpoint();
    }

    os_flush_cpu_cache();
    agp_free_memory(sc->agp_dev, handle);

    return 0;
}
#endif /* NV_SUPPORT_OS_AGP */

RM_STATUS NV_API_CALL nv_alias_pages(
    nv_state_t *nv,
    NvU32       count,
    NvU32       alloc_type_contiguous,
    NvU32       cache_type,
    NvU64       guest_id,
    NvU64      *pte_array,
    void      **priv_data
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_guest_pfn_list(
    nv_state_t  *nv,
    unsigned int key,
    unsigned int pfn_count,
    unsigned int offset_index,
    unsigned int *user_pfn_list
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_alloc_pages(
    nv_state_t *nv,
    NvU32       count,
    NvBool      alloc_type_agp,
    NvBool      alloc_type_contiguous,
    NvU32       cache_type,
    NvBool      alloc_type_zeroed,
    NvU64      *pte_array,
    void      **private
)
{
    NvU32 offset;
    RM_STATUS status = RM_ERR_NO_FREE_MEM;
    nv_stack_t *sp = NULL;
    NvBool zero = alloc_type_zeroed;

    if (alloc_type_agp) {
        if (!NV_AGP_ENABLED(nv))
            return RM_ERR_NOT_SUPPORTED;

        NV_UMA_ZONE_ALLOC_STACK(sp);
        if (sp == NULL)
            return RM_ERR_NO_FREE_MEM;

#ifdef NV_SUPPORT_OS_AGP
        if (NV_OSAGP_ENABLED(nv)) {
            status = rm_alloc_agp_bitmap(sp, nv, count, &offset);
            if (status != RM_OK)
                goto failed;

            if (nv_alloc_agp_pages(nv, count, (offset << PAGE_SHIFT),
                    private) != 0) {
                rm_free_agp_bitmap(sp, nv, count, offset);
                goto failed;
            }

            NV_UMA_ZONE_FREE_STACK(sp);

            pte_array[0] = (nv->agp.address + (offset << PAGE_SHIFT));
            return RM_OK;
        }
#endif /* NV_SUPPORT_OS_AGP */

        if (NV_NVAGP_ENABLED(nv)) {
            status = rm_alloc_agp_pages(sp, nv, count, private, &offset);
            if (status != RM_OK)
                goto failed;

            NV_UMA_ZONE_FREE_STACK(sp);

            pte_array[0] = (nv->agp.address + (offset << PAGE_SHIFT));
            return RM_OK;
        }
    } else {
        if (!alloc_type_contiguous) {
            if (nv_alloc_system_pages(nv, count, cache_type, zero,
                        pte_array, private)) {
                goto failed;
            }
        } else {
            if (nv_alloc_contig_pages(nv, count, cache_type, zero,
                        pte_array, private)) {
                goto failed;
            }
        }
        return RM_OK;
    }

failed:
    if (sp != NULL)
        NV_UMA_ZONE_FREE_STACK(sp);

    return status;
}

RM_STATUS NV_API_CALL nv_free_pages(
    nv_state_t *nv,
    NvU32 count,
    NvU32 alloc_type_agp,
    NvU32 alloc_type_contiguous,
    NvU32 cache_type,
    void *private
)
{
    RM_STATUS status = RM_ERROR;
    nv_stack_t *sp = NULL;

    if (alloc_type_agp) {
        if (!NV_AGP_ENABLED(nv))
            return RM_ERR_NOT_SUPPORTED;

        NV_UMA_ZONE_ALLOC_STACK(sp);
        if (sp == NULL)
            return RM_ERR_NO_FREE_MEM;

#ifdef NV_SUPPORT_OS_AGP
        if (NV_OSAGP_ENABLED(nv)) {
            NvU32 offset;

            if (nv_free_agp_pages(nv, count, private, &offset) != 0)
                goto failed;

            rm_free_agp_bitmap(sp, nv, count, (offset >> PAGE_SHIFT));
            NV_UMA_ZONE_FREE_STACK(sp);

            return RM_OK;
        }
#endif /* NV_SUPPORT_OS_AGP */

        if (NV_NVAGP_ENABLED(nv)) {
            if (rm_free_agp_pages(sp, nv, private) != RM_OK)
                goto failed;
        }

        NV_UMA_ZONE_FREE_STACK(sp);
    } else {
        if (!alloc_type_contiguous) {
            if (nv_free_system_pages(nv, private))
                goto failed;
        } else  {
            if (nv_free_contig_pages(nv, private))
                goto failed;
        }
    }

    return RM_OK;

failed:
    if (sp != NULL)
        NV_UMA_ZONE_FREE_STACK(sp);

    return status;
}

NvU64 NV_API_CALL nv_get_kern_phys_address(NvU64 address)
{
    vm_offset_t va = (vm_offset_t)address;

#if defined(NVCPU_X86_64)
    if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
        return DMAP_TO_PHYS(va);
#endif

    if (va < VM_MIN_KERNEL_ADDRESS) {
        os_dbg_breakpoint();
        return 0;
    }

    return vtophys(va);
}

NvU64 NV_API_CALL nv_get_user_phys_address(NvU64 address)
{
    struct vmspace *vm;
    vm_offset_t va = (vm_offset_t)address;

    if (va >= VM_MIN_KERNEL_ADDRESS) {
        os_dbg_breakpoint();
        return 0;
    }

    vm = curproc->p_vmspace;
    return pmap_extract(vmspace_pmap(vm), va);
}

int nvidia_mmap_dev_single(
    nv_state_t *nv,
    vm_ooffset_t *offset,
    vm_size_t size,
    vm_object_t *object
)
{
    struct nvidia_alloc *at;
    nv_stack_t *sp;
    RM_STATUS rmStatus;
    struct nvidia_softc *sc = nv->os_state;
    NvU32 i = 0;
    NvU32 prot;

    sp = sc->api_sp;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, TRUE);

    rmStatus = rm_validate_mmap_request(sp, nv, *offset, size, &prot);
    if ((rmStatus != RM_OK) && (NULL == nv_find_alloc(sc, *offset, &i)))
        return EINVAL;

    if (IS_UD_OFFSET(nv, *offset, size)) {
        *object = sc->UD_object;
        vm_object_reference(*object);
        *offset = (*offset - nv->ud.address);
        return 0;
    } else if (IS_FB_OFFSET(nv, *offset, size)) {
        *object = sc->BAR_objects[NV_GPU_BAR_INDEX_FB];
        vm_object_reference(*object);
        *offset = (*offset - nv->fb->address);
        return 0;
    } else if (IS_REG_OFFSET(nv, *offset, size)) {
        *object = sc->BAR_objects[NV_GPU_BAR_INDEX_REGS];
        vm_object_reference(*object);
        *offset = (*offset - nv->regs->address);
        return 0;
    } else if (IS_AGP_OFFSET(nv, *offset, size)) {
        *object = sc->AGP_object;
        vm_object_reference(*object);
        *offset = (*offset - nv->agp.address);
        return 0;
    }

    at = nv_find_alloc(sc, *offset, &i);
    if ((NULL == at) || (size + (i * PAGE_SIZE)) > at->size)
        return EINVAL;

    vm_object_reference(at->object);

    *object = at->object;
    *offset = (i * PAGE_SIZE);

    return 0;
}

void nvidia_rc_timer(void *data)
{
    nv_state_t *nv = data;
    struct nvidia_softc *sc = nv->os_state;
    nv_stack_t *sp;

    sp = sc->timer_sp;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, TRUE, TRUE, FALSE);

    if (rm_run_rc_callback(sp, nv) == RM_OK)
        callout_reset(&sc->timer, hz, nvidia_rc_timer, (void *)nv);
}

int NV_API_CALL nv_start_rc_timer(
    nv_state_t *nv
)
{
    struct nvidia_softc *sc = nv->os_state;

    if (nv->rc_timer_enabled != 0)
        return EBUSY;

    callout_reset(&sc->timer, hz, nvidia_rc_timer, (void *)nv);
    nv->rc_timer_enabled = 1;

    return 0;
}

int NV_API_CALL nv_stop_rc_timer(
    nv_state_t *nv
)
{
    struct nvidia_softc *sc = nv->os_state;

    if (nv->rc_timer_enabled == 0)
        return EIO;

    callout_drain(&sc->timer);
    nv->rc_timer_enabled = 0;

    return 0;
}

void NV_API_CALL nv_set_dma_address_size(
    nv_state_t *nv,
    NvU32 phys_addr_bits
)
{
    struct nvidia_softc *sc = nv->os_state;
#if defined(NVCPU_X86_64)
    sc->dma_mask = (((uint64_t)1) << phys_addr_bits) - 1;
#else
    sc->dma_mask = 0xffffffffULL;
#endif
}

void* NV_API_CALL nv_get_adapter_state(
    NvU32 domain,
    NvU16 bus,
    NvU16 slot
)
{
    unsigned int i;
    struct nvidia_softc *sc;
    nv_state_t *nv;

    for (i = 0; i < NV_MAX_DEVICES; i++) {
        sc = devclass_get_softc(nvidia_devclass, i);
        if (!sc)
            continue;
        nv = sc->nv_state;

        if ((nv->domain == domain) &&
                (nv->bus == bus) && (nv->slot == slot)) {
            return (void *) nv;
        }
    }

    if (bus == 255 && slot == 255) {
        nv = &nvidia_ctl_state;
        return (void *) nv;
    }

    return NULL;
}

void NV_API_CALL nv_verify_pci_config(
    nv_state_t *nv,
    BOOL check_the_bars
)
{
    nv_stack_t *sp;
    struct nvidia_softc *sc = nv->os_state;

    sp = sc->pci_cfgchk_sp;

    NV_PCI_CHECK_CONFIG_SPACE(sp, nv, check_the_bars, FALSE, FALSE);
}

void NV_API_CALL nv_acpi_methods_init(NvU32 *handlesPresent)
{
    *handlesPresent = 0;
}

void NV_API_CALL nv_acpi_methods_uninit(void)
{
    return;
}

RM_STATUS NV_API_CALL nv_acpi_method(
    NvU32 acpi_method,
    NvU32 function,
    NvU32 subFunction,
    void *inParams,
    NvU16 inParamSize,
    NvU32 *outStatus,
    void  *outData,
    NvU16 *outDataSize
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_acpi_dsm_method(
    nv_state_t *nv,
    NvU8 *pAcpiDsmGuid,
    NvU32 acpiDsmRev,
    NvU32 acpiDsmSubFunction,
    void  *pInParams,
    NvU16 inParamSize,
    NvU32 *outStatus,
    void  *pOutData,
    NvU16 *pSize
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_acpi_ddc_method(
    nv_state_t *nv,
    void *pEdidBuffer,
    NvU32 *pSize
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_acpi_rom_method(
    nv_state_t *nv,
    NvU32 *pInData,
    NvU32 *pOutData
)
{
    return RM_ERR_NOT_SUPPORTED;
}

void* NV_API_CALL nv_get_smu_state(void)
{
    return NULL;
}

RM_STATUS NV_API_CALL nv_lock_user_pages(
    nv_state_t *nv,
    void       *address,
    NvU64       count,
    NvU64      *pte_array,
    void      **private
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_unlock_user_pages(
    nv_state_t *nv,
    NvU64       count,
    NvU64      *pte_array,
    void       *private
)
{
    return RM_ERR_NOT_SUPPORTED;
}

RM_STATUS NV_API_CALL nv_log_error(
    nv_state_t *nv,
    NvU32       error_number,
    const char *format,
    va_list    ap
)
{
    return RM_OK;
}
