510 static inline int use_cpu_reloc(const struct reloc_cache *cache,  in use_cpu_reloc()  argument
522           return (cache->has_llc ||  in use_cpu_reloc()
843 static void reloc_cache_init(struct reloc_cache *cache,  in reloc_cache_init()  argument
846           cache->page = -1;  in reloc_cache_init()
847           cache->vaddr = 0;  in reloc_cache_init()
849           cache->gen = INTEL_GEN(i915);  in reloc_cache_init()
850           cache->has_llc = HAS_LLC(i915);  in reloc_cache_init()
851           cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);  in reloc_cache_init()
852           cache->has_fence = cache->gen < 4;  in reloc_cache_init()
853           cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;  in reloc_cache_init()
854           cache->node.allocated = false;  in reloc_cache_init()
855           cache->rq = NULL;  in reloc_cache_init()
856           cache->rq_size = 0;  in reloc_cache_init()
871 static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)  in cache_to_ggtt()  argument
874                     container_of(cache, struct i915_execbuffer, reloc_cache)->i915;  in cache_to_ggtt()
878 static void reloc_gpu_flush(struct reloc_cache *cache)  in reloc_gpu_flush()  argument
880           GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32));  in reloc_gpu_flush()
881           cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;  in reloc_gpu_flush()
882           i915_gem_object_unpin_map(cache->rq->batch->obj);  in reloc_gpu_flush()
883           i915_gem_chipset_flush(cache->rq->i915);  in reloc_gpu_flush()
885           __i915_add_request(cache->rq, true);  in reloc_gpu_flush()
886           cache->rq = NULL;  in reloc_gpu_flush()
889 static void reloc_cache_reset(struct reloc_cache *cache)  in reloc_cache_reset()  argument
893           if (cache->rq)  in reloc_cache_reset()
894                     reloc_gpu_flush(cache);  in reloc_cache_reset()
896           if (!cache->vaddr)  in reloc_cache_reset()
899           vaddr = unmask_page(cache->vaddr);  in reloc_cache_reset()
900           if (cache->vaddr & KMAP) {  in reloc_cache_reset()
901                     if (cache->vaddr & CLFLUSH_AFTER)  in reloc_cache_reset()
905                     i915_gem_obj_finish_shmem_access((struct drm_i915_gem_object *)cache->node.mm);  in reloc_cache_reset()
909                     if (cache->node.allocated) {  in reloc_cache_reset()
910                               struct i915_ggtt *ggtt = cache_to_ggtt(cache);  in reloc_cache_reset()
913                                                          cache->node.start,  in reloc_cache_reset()
914                                                          cache->node.size);  in reloc_cache_reset()
915                               drm_mm_remove_node(&cache->node);  in reloc_cache_reset()
917                               i915_vma_unpin((struct i915_vma *)cache->node.mm);  in reloc_cache_reset()
921           cache->vaddr = 0;  in reloc_cache_reset()
922           cache->page = -1;  in reloc_cache_reset()
926                               struct reloc_cache *cache,  in reloc_kmap()  argument
931           if (cache->vaddr) {  in reloc_kmap()
932                     kunmap_atomic(unmask_page(cache->vaddr));  in reloc_kmap()
944                     cache->vaddr = flushes | KMAP;  in reloc_kmap()
945                     cache->node.mm = (void *)obj;  in reloc_kmap()
951           cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;  in reloc_kmap()
952           cache->page = page;  in reloc_kmap()
958                                struct reloc_cache *cache,  in reloc_iomap()  argument
961           struct i915_ggtt *ggtt = cache_to_ggtt(cache);  in reloc_iomap()
965           if (cache->vaddr) {  in reloc_iomap()
966                     io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));  in reloc_iomap()
971                     if (use_cpu_reloc(cache, obj))  in reloc_iomap()
983                               memset(&cache->node, 0, sizeof(cache->node));  in reloc_iomap()
985                                         (&ggtt->base.mm, &cache->node,  in reloc_iomap()
998                               cache->node.start = vma->node.start;  in reloc_iomap()
999                               cache->node.mm = (void *)vma;  in reloc_iomap()
1003           offset = cache->node.start;  in reloc_iomap()
1004           if (cache->node.allocated) {  in reloc_iomap()
1015           cache->page = page;  in reloc_iomap()
1016           cache->vaddr = (unsigned long)vaddr;  in reloc_iomap()
1022                                struct reloc_cache *cache,  in reloc_vaddr()  argument
1027           if (cache->page == page) {  in reloc_vaddr()
1028                     vaddr = unmask_page(cache->vaddr);  in reloc_vaddr()
1031                     if ((cache->vaddr & KMAP) == 0)  in reloc_vaddr()
1032                               vaddr = reloc_iomap(obj, cache, page);  in reloc_vaddr()
1034                               vaddr = reloc_kmap(obj, cache, page);  in reloc_vaddr()
1067           struct reloc_cache *cache = &eb->reloc_cache;  in __reloc_gpu_alloc()  local
1081                                               cache->has_llc ?  in __reloc_gpu_alloc()
1122                                                   cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);  in __reloc_gpu_alloc()
1140           cache->rq = rq;  in __reloc_gpu_alloc()
1141           cache->rq_cmd = cmd;  in __reloc_gpu_alloc()
1142           cache->rq_size = 0;  in __reloc_gpu_alloc()
1160           struct reloc_cache *cache = &eb->reloc_cache;  in reloc_gpu()  local
1163           if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))  in reloc_gpu()
1164                     reloc_gpu_flush(cache);  in reloc_gpu()
1166           if (unlikely(!cache->rq)) {  in reloc_gpu()
1181           cmd = cache->rq_cmd + cache->rq_size;  in reloc_gpu()
1182           cache->rq_size += len;  in reloc_gpu()