96     struct mps_command *cm);
99 static void mps_config_complete(struct mps_softc *sc, struct mps_command *cm);
102 static void mps_enqueue_request(struct mps_softc *sc, struct mps_command *cm);
738 	struct mps_command *cm;  in mps_iocfacts_free()  local
786 			cm = &sc->commands[i];  in mps_iocfacts_free()
787 			bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap);  in mps_iocfacts_free()
1095 mps_enqueue_request(struct mps_softc *sc, struct mps_command *cm)  in mps_enqueue_request()  argument
1100 	    cm->cm_desc.Default.SMID, cm, cm->cm_ccb);  in mps_enqueue_request()
1107 	rd.u.low = cm->cm_desc.Words.Low;  in mps_enqueue_request()
1108 	rd.u.high = cm->cm_desc.Words.High;  in mps_enqueue_request()
1111 	KASSERT(cm->cm_state == MPS_CM_STATE_BUSY,  in mps_enqueue_request()
1112 	    ("command not busy, state = %u\n", cm->cm_state));  in mps_enqueue_request()
1113 	cm->cm_state = MPS_CM_STATE_INQUEUE;  in mps_enqueue_request()
1424 	struct mps_command *cm;  in mps_alloc_requests()  local
1513 		cm = &sc->commands[i];  in mps_alloc_requests()
1514 		cm->cm_req = sc->req_frames + i * sc->reqframesz;  in mps_alloc_requests()
1515 		cm->cm_req_busaddr = sc->req_busaddr + i * sc->reqframesz;  in mps_alloc_requests()
1516 		cm->cm_sense = &sc->sense_frames[i];  in mps_alloc_requests()
1517 		cm->cm_sense_busaddr = sc->sense_busaddr + i * MPS_SENSE_LEN;  in mps_alloc_requests()
1518 		cm->cm_desc.Default.SMID = i;  in mps_alloc_requests()
1519 		cm->cm_sc = sc;  in mps_alloc_requests()
1520 		cm->cm_state = MPS_CM_STATE_BUSY;  in mps_alloc_requests()
1521 		TAILQ_INIT(&cm->cm_chain_list);  in mps_alloc_requests()
1522 		callout_init_mtx(&cm->cm_callout, &sc->mps_mtx, 0);  in mps_alloc_requests()
1525 		if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap) == 0)  in mps_alloc_requests()
1527 				mps_free_high_priority_command(sc, cm);  in mps_alloc_requests()
1529 				mps_free_command(sc, cm);  in mps_alloc_requests()
1950 	struct mps_command *cm;  in mps_dump_reqs()  local
1978 		cm = &sc->commands[i];  in mps_dump_reqs()
1979 		if ((sc->dump_reqs_alltypes == 0) && (cm->cm_state != state))  in mps_dump_reqs()
1982 		hdr.state = cm->cm_state;  in mps_dump_reqs()
1984 		hdr.deschi = cm->cm_desc.Words.High;  in mps_dump_reqs()
1985 		hdr.desclo = cm->cm_desc.Words.Low;  in mps_dump_reqs()
1986 		TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,  in mps_dump_reqs()
1990 		sbuf_bcat(sb, cm->cm_req, 128);  in mps_dump_reqs()
1991 		TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,  in mps_dump_reqs()
2230 mps_complete_command(struct mps_softc *sc, struct mps_command *cm)  in mps_complete_command()  argument
2234 	if (cm == NULL) {  in mps_complete_command()
2239 	KASSERT(cm->cm_state == MPS_CM_STATE_INQUEUE,  in mps_complete_command()
2240 	    ("command not inqueue, state = %u\n", cm->cm_state));  in mps_complete_command()
2241 	cm->cm_state = MPS_CM_STATE_BUSY;   in mps_complete_command()
2242 	if (cm->cm_flags & MPS_CM_FLAGS_POLLED)  in mps_complete_command()
2243 		cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;  in mps_complete_command()
2245 	if (cm->cm_complete != NULL) {  in mps_complete_command()
2248 			   __func__, cm, cm->cm_complete, cm->cm_complete_data,  in mps_complete_command()
2249 			   cm->cm_reply);  in mps_complete_command()
2250 		cm->cm_complete(sc, cm);  in mps_complete_command()
2253 	if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {  in mps_complete_command()
2254 		mps_dprint(sc, MPS_TRACE, "waking up %p\n", cm);  in mps_complete_command()
2255 		wakeup(cm);  in mps_complete_command()
2258 	if (cm->cm_sc->io_cmds_active != 0) {  in mps_complete_command()
2259 		cm->cm_sc->io_cmds_active--;  in mps_complete_command()
2373 	struct mps_command *cm = NULL;  in mps_intr_locked()  local
2386 		cm = NULL;  in mps_intr_locked()
2417 			cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)];  in mps_intr_locked()
2418 			cm->cm_reply = NULL;  in mps_intr_locked()
2490 				cm = &sc->commands[  in mps_intr_locked()
2492 				if (cm->cm_state == MPS_CM_STATE_INQUEUE) {  in mps_intr_locked()
2493 					cm->cm_reply = reply;  in mps_intr_locked()
2494 					cm->cm_reply_data = le32toh(  in mps_intr_locked()
2500 					    cm->cm_state, cm);  in mps_intr_locked()
2512 			cm = NULL;  in mps_intr_locked()
2517 		if (cm != NULL) {  in mps_intr_locked()
2519 			if (cm->cm_reply)  in mps_intr_locked()
2520 				mps_display_reply_info(sc,cm->cm_reply);  in mps_intr_locked()
2521 			mps_complete_command(sc, cm);  in mps_intr_locked()
2562 mps_reregister_events_complete(struct mps_softc *sc, struct mps_command *cm)  in mps_reregister_events_complete()  argument
2566 	if (cm->cm_reply)  in mps_reregister_events_complete()
2568 			(MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply);  in mps_reregister_events_complete()
2570 	mps_free_command(sc, cm);  in mps_reregister_events_complete()
2605 	struct mps_command *cm;  in mps_update_events()  local
2620 	if ((cm = mps_alloc_command(sc)) == NULL)  in mps_update_events()
2622 	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;  in mps_update_events()
2638 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;  in mps_update_events()
2639 	cm->cm_data = NULL;  in mps_update_events()
2641 	error = mps_wait_command(sc, &cm, 60, 0);  in mps_update_events()
2642 	if (cm != NULL)  in mps_update_events()
2643 		reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply;  in mps_update_events()
2653 	if (cm != NULL)  in mps_update_events()
2654 		mps_free_command(sc, cm);  in mps_update_events()
2662 	struct mps_command *cm;  in mps_reregister_events()  local
2678 	if ((cm = mps_alloc_command(sc)) == NULL)  in mps_reregister_events()
2680 	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;  in mps_reregister_events()
2696 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;  in mps_reregister_events()
2697 	cm->cm_data = NULL;  in mps_reregister_events()
2698 	cm->cm_complete = mps_reregister_events_complete;  in mps_reregister_events()
2700 	error = mps_map_command(sc, cm);  in mps_reregister_events()
2720 mps_add_chain(struct mps_command *cm)  in mps_add_chain()  argument
2726 	if (cm->cm_sglsize < MPS_SGC_SIZE)  in mps_add_chain()
2729 	chain = mps_alloc_chain(cm->cm_sc);  in mps_add_chain()
2733 	space = cm->cm_sc->reqframesz;  in mps_add_chain()
2739 	TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link);  in mps_add_chain()
2741 	sgc = (MPI2_SGE_CHAIN64 *)&cm->cm_sge->MpiChain;  in mps_add_chain()
2754 	cm->cm_sge = (MPI2_SGE_IO_UNION *)&chain->chain->MpiSimple;  in mps_add_chain()
2755 	cm->cm_sglsize = space;  in mps_add_chain()
2766 mps_push_sge(struct mps_command *cm, void *sgep, size_t len, int segsleft)  in mps_push_sge()  argument
2816 	if (cm->cm_sglsize < MPS_SGC_SIZE)  in mps_push_sge()
2819 	if (segsleft >= 1 && cm->cm_sglsize < len + MPS_SGC_SIZE) {  in mps_push_sge()
2826 		if ((error = mps_add_chain(cm)) != 0)  in mps_push_sge()
2831 	    cm->cm_sglsize < len + MPS_SGC_SIZE + MPS_SGE64_SIZE) {  in mps_push_sge()
2848 		cm->cm_sglsize -= len;  in mps_push_sge()
2849 		bcopy(sgep, cm->cm_sge, len);  in mps_push_sge()
2850 		cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);  in mps_push_sge()
2851 		return (mps_add_chain(cm));  in mps_push_sge()
2856 	if (segsleft == 1 && cm->cm_sglsize < len)  in mps_push_sge()
2858 		    cm->cm_sglsize, len);  in mps_push_sge()
2861 	if (segsleft == 2 && cm->cm_sglsize < len + MPS_SGE64_SIZE)  in mps_push_sge()
2863 		    cm->cm_sglsize, len);  in mps_push_sge()
2880 		if (cm->cm_out_len) {  in mps_push_sge()
2881 			sge->FlagsLength = htole32(cm->cm_out_len |  in mps_push_sge()
2887 			cm->cm_sglsize -= len;  in mps_push_sge()
2888 			bcopy(sgep, cm->cm_sge, len);  in mps_push_sge()
2889 			cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge  in mps_push_sge()
2899 		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN) {  in mps_push_sge()
2913 	cm->cm_sglsize -= len;  in mps_push_sge()
2914 	bcopy(sgep, cm->cm_sge, len);  in mps_push_sge()
2915 	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);  in mps_push_sge()
2923 mps_add_dmaseg(struct mps_command *cm, vm_paddr_t pa, size_t len, u_int flags,  in mps_add_dmaseg()  argument
2937 	return (mps_push_sge(cm, &sge, sizeof sge, segsleft));  in mps_add_dmaseg()
2944 	struct mps_command *cm;  in mps_data_cb()  local
2947 	cm = (struct mps_command *)arg;  in mps_data_cb()
2948 	sc = cm->cm_sc;  in mps_data_cb()
2954 	if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) {  in mps_data_cb()
2958 			   cm->cm_max_segs);  in mps_data_cb()
2966 	if (cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) {  in mps_data_cb()
2988 	} else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT) {  in mps_data_cb()
2995 		if ((cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) && (i != 0)) {  in mps_data_cb()
2998 		error = mps_add_dmaseg(cm, segs[i].ds_addr, segs[i].ds_len,  in mps_data_cb()
3005 			cm->cm_flags |= MPS_CM_FLAGS_CHAIN_FAILED;  in mps_data_cb()
3012 			cm->cm_state = MPS_CM_STATE_INQUEUE;  in mps_data_cb()
3013 			mps_complete_command(sc, cm);  in mps_data_cb()
3018 	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);  in mps_data_cb()
3019 	mps_enqueue_request(sc, cm);  in mps_data_cb()
3039 mps_map_command(struct mps_softc *sc, struct mps_command *cm)  in mps_map_command()  argument
3043 	if (cm->cm_flags & MPS_CM_FLAGS_USE_UIO) {  in mps_map_command()
3044 		error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap,  in mps_map_command()
3045 		    &cm->cm_uio, mps_data_cb2, cm, 0);  in mps_map_command()
3046 	} else if (cm->cm_flags & MPS_CM_FLAGS_USE_CCB) {  in mps_map_command()
3047 		error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap,  in mps_map_command()
3048 		    cm->cm_data, mps_data_cb, cm, 0);  in mps_map_command()
3049 	} else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) {  in mps_map_command()
3050 		error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap,  in mps_map_command()
3051 		    cm->cm_data, cm->cm_length, mps_data_cb, cm, 0);  in mps_map_command()
3054 		if (cm->cm_sge != NULL)  in mps_map_command()
3055 			mps_add_dmaseg(cm, 0, 0, 0, 1);  in mps_map_command()
3056 		mps_enqueue_request(sc, cm);	  in mps_map_command()
3073 	struct mps_command *cm = *cmp;  in mps_wait_command()  local
3078 	cm->cm_complete = NULL;  in mps_wait_command()
3079 	cm->cm_flags |= MPS_CM_FLAGS_POLLED;  in mps_wait_command()
3080 	error = mps_map_command(sc, cm);  in mps_wait_command()
3093 		cm->cm_flags |= MPS_CM_FLAGS_WAKEUP;  in mps_wait_command()
3094 		error = msleep(cm, &sc->mps_mtx, 0, "mpswait", timeout*hz);  in mps_wait_command()
3104 		while ((cm->cm_flags & MPS_CM_FLAGS_COMPLETE) == 0) {  in mps_wait_command()
3121 		if (cm->cm_timeout_handler == NULL) {  in mps_wait_command()
3129 			cm->cm_timeout_handler(sc, cm);  in mps_wait_command()
3150 	struct mps_command *cm;  in mps_read_config_page()  local
3157 	cm = mps_alloc_command(sc);  in mps_read_config_page()
3158 	if (cm == NULL) {  in mps_read_config_page()
3162 	req = (MPI2_CONFIG_REQUEST *)cm->cm_req;  in mps_read_config_page()
3188 	cm->cm_data = params->buffer;  in mps_read_config_page()
3189 	cm->cm_length = params->length;  in mps_read_config_page()
3190 	if (cm->cm_data != NULL) {  in mps_read_config_page()
3191 		cm->cm_sge = &req->PageBufferSGE;  in mps_read_config_page()
3192 		cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);  in mps_read_config_page()
3193 		cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE | MPS_CM_FLAGS_DATAIN;  in mps_read_config_page()
3195 		cm->cm_sge = NULL;  in mps_read_config_page()
3196 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;  in mps_read_config_page()
3198 	cm->cm_complete_data = params;  in mps_read_config_page()
3200 		cm->cm_complete = mps_config_complete;  in mps_read_config_page()
3201 		return (mps_map_command(sc, cm));  in mps_read_config_page()
3203 		error = mps_wait_command(sc, &cm, 0, CAN_SLEEP);  in mps_read_config_page()
3207 			if (cm != NULL)  in mps_read_config_page()
3208 				mps_free_command(sc, cm);  in mps_read_config_page()
3211 		mps_config_complete(sc, cm);  in mps_read_config_page()
3224 mps_config_complete(struct mps_softc *sc, struct mps_command *cm)  in mps_config_complete()  argument
3230 	params = cm->cm_complete_data;  in mps_config_complete()
3232 	if (cm->cm_data != NULL) {  in mps_config_complete()
3233 		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,  in mps_config_complete()
3235 		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);  in mps_config_complete()
3242 	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {  in mps_config_complete()
3247 	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;  in mps_config_complete()
3267 	mps_free_command(sc, cm);  in mps_config_complete()