Lines Matching refs:aiocbe
342 static int aio_free_entry(struct aiocblist *aiocbe);
343 static void aio_process_rw(struct aiocblist *aiocbe);
344 static void aio_process_sync(struct aiocblist *aiocbe);
345 static void aio_process_mlock(struct aiocblist *aiocbe);
356 static void aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type);
641 aio_free_entry(struct aiocblist *aiocbe) in aio_free_entry() argument
647 p = aiocbe->userproc; in aio_free_entry()
653 MPASS(aiocbe->jobstate == JOBST_JOBFINISHED); in aio_free_entry()
660 TAILQ_REMOVE(&ki->kaio_done, aiocbe, plist); in aio_free_entry()
661 TAILQ_REMOVE(&ki->kaio_all, aiocbe, allist); in aio_free_entry()
663 lj = aiocbe->lio; in aio_free_entry()
680 knlist_delete(&aiocbe->klist, curthread, 1); in aio_free_entry()
682 sigqueue_take(&aiocbe->ksi); in aio_free_entry()
685 MPASS(aiocbe->bp == NULL); in aio_free_entry()
686 aiocbe->jobstate = JOBST_NULL; in aio_free_entry()
709 if (aiocbe->fd_file) in aio_free_entry()
710 fdrop(aiocbe->fd_file, curthread); in aio_free_entry()
711 crfree(aiocbe->cred); in aio_free_entry()
712 uma_zfree(aiocb_zone, aiocbe); in aio_free_entry()
817 struct aiocblist *aiocbe; in aio_selectjob() local
822 TAILQ_FOREACH(aiocbe, &aio_jobs, list) { in aio_selectjob()
823 userp = aiocbe->userproc; in aio_selectjob()
827 TAILQ_REMOVE(&aio_jobs, aiocbe, list); in aio_selectjob()
830 aiocbe->jobstate = JOBST_JOBRUNNING; in aio_selectjob()
834 return (aiocbe); in aio_selectjob()
872 aio_process_rw(struct aiocblist *aiocbe) in aio_process_rw() argument
886 KASSERT(aiocbe->uaiocb.aio_lio_opcode == LIO_READ || in aio_process_rw()
887 aiocbe->uaiocb.aio_lio_opcode == LIO_WRITE, in aio_process_rw()
888 ("%s: opcode %d", __func__, aiocbe->uaiocb.aio_lio_opcode)); in aio_process_rw()
892 td->td_ucred = aiocbe->cred; in aio_process_rw()
893 cb = &aiocbe->uaiocb; in aio_process_rw()
894 fp = aiocbe->fd_file; in aio_process_rw()
928 aiocbe->inputcharge = inblock_end - inblock_st; in aio_process_rw()
929 aiocbe->outputcharge = oublock_end - oublock_st; in aio_process_rw()
942 PROC_LOCK(aiocbe->userproc); in aio_process_rw()
943 kern_psignal(aiocbe->userproc, SIGPIPE); in aio_process_rw()
944 PROC_UNLOCK(aiocbe->userproc); in aio_process_rw()
956 aio_process_sync(struct aiocblist *aiocbe) in aio_process_sync() argument
960 struct aiocb *cb = &aiocbe->uaiocb; in aio_process_sync()
961 struct file *fp = aiocbe->fd_file; in aio_process_sync()
964 KASSERT(aiocbe->uaiocb.aio_lio_opcode == LIO_SYNC, in aio_process_sync()
965 ("%s: opcode %d", __func__, aiocbe->uaiocb.aio_lio_opcode)); in aio_process_sync()
967 td->td_ucred = aiocbe->cred; in aio_process_sync()
976 aio_process_mlock(struct aiocblist *aiocbe) in aio_process_mlock() argument
978 struct aiocb *cb = &aiocbe->uaiocb; in aio_process_mlock()
981 KASSERT(aiocbe->uaiocb.aio_lio_opcode == LIO_MLOCK, in aio_process_mlock()
982 ("%s: opcode %d", __func__, aiocbe->uaiocb.aio_lio_opcode)); in aio_process_mlock()
984 error = vm_mlock(aiocbe->userproc, aiocbe->cred, in aio_process_mlock()
991 aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type) in aio_bio_done_notify() argument
1000 lj = aiocbe->lio; in aio_bio_done_notify()
1008 aiocbe->jobflags |= AIOCBLIST_DONE; in aio_bio_done_notify()
1010 aiocbe->jobflags |= AIOCBLIST_BUFDONE; in aio_bio_done_notify()
1012 TAILQ_INSERT_TAIL(&ki->kaio_done, aiocbe, plist); in aio_bio_done_notify()
1013 aiocbe->jobstate = JOBST_JOBFINISHED; in aio_bio_done_notify()
1018 if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL || in aio_bio_done_notify()
1019 aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) in aio_bio_done_notify()
1020 aio_sendsig(userp, &aiocbe->uaiocb.aio_sigevent, &aiocbe->ksi); in aio_bio_done_notify()
1022 KNOTE_LOCKED(&aiocbe->klist, 1); in aio_bio_done_notify()
1039 if (aiocbe->jobflags & AIOCBLIST_CHECKSYNC) { in aio_bio_done_notify()
1041 if (aiocbe->fd_file == scb->fd_file && in aio_bio_done_notify()
1042 aiocbe->seqno < scb->seqno) { in aio_bio_done_notify()
1067 struct aiocblist *aiocbe; in aio_daemon() local
1119 while ((aiocbe = aio_selectjob(aiop)) != NULL) { in aio_daemon()
1121 userp = aiocbe->userproc; in aio_daemon()
1158 switch(aiocbe->uaiocb.aio_lio_opcode) { in aio_daemon()
1161 aio_process_rw(aiocbe); in aio_daemon()
1164 aio_process_sync(aiocbe); in aio_daemon()
1167 aio_process_mlock(aiocbe); in aio_daemon()
1177 TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist); in aio_daemon()
1178 aio_bio_done_notify(userp, aiocbe, DONE_QUEUE); in aio_daemon()
1294 aio_qphysio(struct proc *p, struct aiocblist *aiocbe) in aio_qphysio() argument
1308 cb = &aiocbe->uaiocb; in aio_qphysio()
1309 fp = aiocbe->fd_file; in aio_qphysio()
1354 aiocbe->bp = bp = g_alloc_bio(); in aio_qphysio()
1356 aiocbe->pbuf = pbuf = (struct buf *)getpbuf(NULL); in aio_qphysio()
1364 lj = aiocbe->lio; in aio_qphysio()
1367 TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist); in aio_qphysio()
1368 TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist); in aio_qphysio()
1369 aiocbe->jobstate = JOBST_JOBQBUF; in aio_qphysio()
1380 bp->bio_caller1 = (void *)aiocbe; in aio_qphysio()
1385 if ((aiocbe->npages = vm_fault_quick_hold_pages( in aio_qphysio()
1387 (vm_offset_t)bp->bio_data, bp->bio_length, prot, aiocbe->pages, in aio_qphysio()
1388 sizeof(aiocbe->pages)/sizeof(aiocbe->pages[0]))) < 0) { in aio_qphysio()
1394 aiocbe->pages, aiocbe->npages); in aio_qphysio()
1397 bp->bio_ma = aiocbe->pages; in aio_qphysio()
1398 bp->bio_ma_n = aiocbe->npages; in aio_qphysio()
1415 aiocbe->jobstate = JOBST_NULL; in aio_qphysio()
1416 TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist); in aio_qphysio()
1417 TAILQ_REMOVE(&ki->kaio_all, aiocbe, allist); in aio_qphysio()
1426 aiocbe->pbuf = NULL; in aio_qphysio()
1429 aiocbe->bp = NULL; in aio_qphysio()
1590 struct aiocblist *aiocbe, *cb; in aio_aqueue() local
1615 aiocbe = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO); in aio_aqueue()
1616 knlist_init_mtx(&aiocbe->klist, AIO_MTX(ki)); in aio_aqueue()
1618 error = ops->copyin(job, &aiocbe->uaiocb); in aio_aqueue()
1621 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1626 if (aiocbe->uaiocb.aio_nbytes > INT_MAX) { in aio_aqueue()
1627 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1631 if (aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT && in aio_aqueue()
1632 aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL && in aio_aqueue()
1633 aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID && in aio_aqueue()
1634 aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) { in aio_aqueue()
1636 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1640 if ((aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL || in aio_aqueue()
1641 aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) && in aio_aqueue()
1642 !_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) { in aio_aqueue()
1643 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1647 ksiginfo_init(&aiocbe->ksi); in aio_aqueue()
1650 aiocbe->uuaiocb = job; in aio_aqueue()
1654 aiocbe->uaiocb.aio_lio_opcode = type; in aio_aqueue()
1655 opcode = aiocbe->uaiocb.aio_lio_opcode; in aio_aqueue()
1665 fd = aiocbe->uaiocb.aio_fildes; in aio_aqueue()
1688 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1698 if (opcode != LIO_SYNC && aiocbe->uaiocb.aio_offset == -1LL) { in aio_aqueue()
1703 aiocbe->fd_file = fp; in aio_aqueue()
1707 aiocbe->seqno = jobseqno++; in aio_aqueue()
1714 aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jid; in aio_aqueue()
1718 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1722 if (aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT) in aio_aqueue()
1724 evflags = aiocbe->uaiocb.aio_sigevent.sigev_notify_kevent_flags; in aio_aqueue()
1729 kqfd = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue; in aio_aqueue()
1730 kev.ident = (uintptr_t)aiocbe->uuaiocb; in aio_aqueue()
1733 kev.data = (intptr_t)aiocbe; in aio_aqueue()
1734 kev.udata = (uint64_t)aiocbe->uaiocb.aio_sigevent.sigev_value.sival_ptr; in aio_aqueue()
1740 uma_zfree(aiocb_zone, aiocbe); in aio_aqueue()
1747 aiocbe->uaiocb._aiocb_private.error = EINPROGRESS; in aio_aqueue()
1748 aiocbe->userproc = p; in aio_aqueue()
1749 aiocbe->cred = crhold(td->td_ucred); in aio_aqueue()
1750 aiocbe->jobflags = 0; in aio_aqueue()
1751 aiocbe->lio = lj; in aio_aqueue()
1778 TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list); in aio_aqueue()
1782 TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist); in aio_aqueue()
1783 TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist); in aio_aqueue()
1784 aiocbe->jobstate = JOBST_JOBQSOCK; in aio_aqueue()
1797 if ((error = aio_qphysio(p, aiocbe)) == 0) in aio_aqueue()
1801 aiocbe->uaiocb._aiocb_private.error = error; in aio_aqueue()
1813 TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist); in aio_aqueue()
1814 TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist); in aio_aqueue()
1817 if (cb->fd_file == aiocbe->fd_file && in aio_aqueue()
1819 cb->seqno < aiocbe->seqno) { in aio_aqueue()
1821 aiocbe->pending++; in aio_aqueue()
1825 if (cb->fd_file == aiocbe->fd_file && in aio_aqueue()
1827 cb->seqno < aiocbe->seqno) { in aio_aqueue()
1829 aiocbe->pending++; in aio_aqueue()
1832 if (aiocbe->pending != 0) { in aio_aqueue()
1833 TAILQ_INSERT_TAIL(&ki->kaio_syncqueue, aiocbe, list); in aio_aqueue()
1834 aiocbe->jobstate = JOBST_JOBQSYNC; in aio_aqueue()
1840 TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list); in aio_aqueue()
1841 aiocbe->jobstate = JOBST_JOBQGLOBAL; in aio_aqueue()
2447 struct aiocblist *aiocbe = (struct aiocblist *)bp->bio_caller1; in aio_physwakeup() local
2453 if (aiocbe->pbuf) { in aio_physwakeup()
2454 pmap_qremove((vm_offset_t)aiocbe->pbuf->b_data, aiocbe->npages); in aio_physwakeup()
2455 relpbuf(aiocbe->pbuf, NULL); in aio_physwakeup()
2456 aiocbe->pbuf = NULL; in aio_physwakeup()
2459 vm_page_unhold_pages(aiocbe->pages, aiocbe->npages); in aio_physwakeup()
2461 bp = aiocbe->bp; in aio_physwakeup()
2462 aiocbe->bp = NULL; in aio_physwakeup()
2463 userp = aiocbe->userproc; in aio_physwakeup()
2466 aiocbe->uaiocb._aiocb_private.status -= bp->bio_resid; in aio_physwakeup()
2467 aiocbe->uaiocb._aiocb_private.error = 0; in aio_physwakeup()
2469 aiocbe->uaiocb._aiocb_private.error = bp->bio_error; in aio_physwakeup()
2470 nblks = btodb(aiocbe->uaiocb.aio_nbytes); in aio_physwakeup()
2471 if (aiocbe->uaiocb.aio_lio_opcode == LIO_WRITE) in aio_physwakeup()
2472 aiocbe->outputcharge += nblks; in aio_physwakeup()
2474 aiocbe->inputcharge += nblks; in aio_physwakeup()
2475 TAILQ_REMOVE(&userp->p_aioinfo->kaio_bufqueue, aiocbe, plist); in aio_physwakeup()
2477 aio_bio_done_notify(userp, aiocbe, DONE_BUF); in aio_physwakeup()
2600 struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata; in filt_aioattach() local
2609 kn->kn_ptr.p_aio = aiocbe; in filt_aioattach()
2612 knlist_add(&aiocbe->klist, kn, 0); in filt_aioattach()
2635 struct aiocblist *aiocbe = kn->kn_ptr.p_aio; in filt_aio() local
2637 kn->kn_data = aiocbe->uaiocb._aiocb_private.error; in filt_aio()
2638 if (aiocbe->jobstate != JOBST_JOBFINISHED) in filt_aio()