// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. */ #include #include #include #include #include #include "kgsl_device.h" #include "kgsl_pool.h" #include "kgsl_sharedmem.h" /* * The user can set this from debugfs to force failed memory allocations to * fail without trying OOM first. This is a debug setting useful for * stress applications that want to test failure cases without pushing the * system into unrecoverable OOM panics */ static bool sharedmem_noretry_flag; static DEFINE_MUTEX(kernel_map_global_lock); /* An attribute for showing per-process memory statistics */ struct kgsl_mem_entry_attribute { struct attribute attr; int memtype; ssize_t (*show)(struct kgsl_process_private *priv, int type, char *buf); }; #define to_mem_entry_attr(a) \ container_of(a, struct kgsl_mem_entry_attribute, attr) #define __MEM_ENTRY_ATTR(_type, _name, _show) \ { \ .attr = { .name = __stringify(_name), .mode = 0444 }, \ .memtype = _type, \ .show = _show, \ } /* * A structure to hold the attributes for a particular memory type. * For each memory type in each process we store the current and maximum * memory usage and display the counts in sysfs. This structure and * the following macro allow us to simplify the definition for those * adding new memory types */ struct mem_entry_stats { int memtype; struct kgsl_mem_entry_attribute attr; struct kgsl_mem_entry_attribute max_attr; }; #define MEM_ENTRY_STAT(_type, _name) \ { \ .memtype = _type, \ .attr = __MEM_ENTRY_ATTR(_type, _name, mem_entry_show), \ .max_attr = __MEM_ENTRY_ATTR(_type, _name##_max, \ mem_entry_max_show), \ } static ssize_t imported_mem_show(struct kgsl_process_private *priv, int type, char *buf) { struct kgsl_mem_entry *entry; uint64_t imported_mem = 0; int id = 0; spin_lock(&priv->mem_lock); for (entry = idr_get_next(&priv->mem_idr, &id); entry; id++, entry = idr_get_next(&priv->mem_idr, &id)) { int egl_surface_count = 0, egl_image_count = 0; struct kgsl_memdesc *m; if (kgsl_mem_entry_get(entry) == 0) continue; spin_unlock(&priv->mem_lock); m = &entry->memdesc; if (kgsl_memdesc_usermem_type(m) == KGSL_MEM_ENTRY_ION) { kgsl_get_egl_counts(entry, &egl_surface_count, &egl_image_count); if (kgsl_memdesc_get_memtype(m) == KGSL_MEMTYPE_EGL_SURFACE) imported_mem += m->size; else if (egl_surface_count == 0) { uint64_t size = m->size; do_div(size, (egl_image_count ? egl_image_count : 1)); imported_mem += size; } } kgsl_mem_entry_put(entry); spin_lock(&priv->mem_lock); } spin_unlock(&priv->mem_lock); return scnprintf(buf, PAGE_SIZE, "%llu\n", imported_mem); } static ssize_t gpumem_mapped_show(struct kgsl_process_private *priv, int type, char *buf) { return scnprintf(buf, PAGE_SIZE, "%lld\n", atomic64_read(&priv->gpumem_mapped)); } static ssize_t gpumem_unmapped_show(struct kgsl_process_private *priv, int type, char *buf) { u64 gpumem_total = atomic64_read(&priv->stats[type].cur); u64 gpumem_mapped = atomic64_read(&priv->gpumem_mapped); if (gpumem_mapped > gpumem_total) return -EIO; return scnprintf(buf, PAGE_SIZE, "%llu\n", gpumem_total - gpumem_mapped); } static struct kgsl_mem_entry_attribute debug_memstats[] = { __MEM_ENTRY_ATTR(0, imported_mem, imported_mem_show), __MEM_ENTRY_ATTR(0, gpumem_mapped, gpumem_mapped_show), __MEM_ENTRY_ATTR(KGSL_MEM_ENTRY_KERNEL, gpumem_unmapped, gpumem_unmapped_show), }; /** * Show the current amount of memory allocated for the given memtype */ static ssize_t mem_entry_show(struct kgsl_process_private *priv, int type, char *buf) { return scnprintf(buf, PAGE_SIZE, "%lld\n", atomic64_read(&priv->stats[type].cur)); } /** * Show the maximum memory allocated for the given memtype through the life of * the process */ static ssize_t mem_entry_max_show(struct kgsl_process_private *priv, int type, char *buf) { return scnprintf(buf, PAGE_SIZE, "%llu\n", priv->stats[type].max); } static ssize_t mem_entry_sysfs_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct kgsl_mem_entry_attribute *pattr = to_mem_entry_attr(attr); struct kgsl_process_private *priv; ssize_t ret; /* * kgsl_process_init_sysfs takes a refcount to the process_private, * which is put when the kobj is released. This implies that priv will * not be freed until this function completes, and no further locking * is needed. */ priv = kobj ? container_of(kobj, struct kgsl_process_private, kobj) : NULL; if (priv && pattr->show) ret = pattr->show(priv, pattr->memtype, buf); else ret = -EIO; return ret; } static void mem_entry_release(struct kobject *kobj) { struct kgsl_process_private *priv; priv = container_of(kobj, struct kgsl_process_private, kobj); /* Put the refcount we got in kgsl_process_init_sysfs */ kgsl_process_private_put(priv); } static const struct sysfs_ops mem_entry_sysfs_ops = { .show = mem_entry_sysfs_show, }; static struct kobj_type ktype_mem_entry = { .sysfs_ops = &mem_entry_sysfs_ops, .release = &mem_entry_release, }; static struct mem_entry_stats mem_stats[] = { MEM_ENTRY_STAT(KGSL_MEM_ENTRY_KERNEL, kernel), MEM_ENTRY_STAT(KGSL_MEM_ENTRY_USER, user), #ifdef CONFIG_ION MEM_ENTRY_STAT(KGSL_MEM_ENTRY_ION, ion), #endif }; void kgsl_process_uninit_sysfs(struct kgsl_process_private *private) { int i; for (i = 0; i < ARRAY_SIZE(mem_stats); i++) { sysfs_remove_file(&private->kobj, &mem_stats[i].attr.attr); sysfs_remove_file(&private->kobj, &mem_stats[i].max_attr.attr); } kobject_put(&private->kobj); } /** * kgsl_process_init_sysfs() - Initialize and create sysfs files for a process * * @device: Pointer to kgsl device struct * @private: Pointer to the structure for the process * * kgsl_process_init_sysfs() is called at the time of creating the * process struct when a process opens the kgsl device for the first time. * This function creates the sysfs files for the process. */ void kgsl_process_init_sysfs(struct kgsl_device *device, struct kgsl_process_private *private) { int i; /* Keep private valid until the sysfs enries are removed. */ kgsl_process_private_get(private); if (kobject_init_and_add(&private->kobj, &ktype_mem_entry, kgsl_driver.prockobj, "%d", pid_nr(private->pid))) { dev_err(device->dev, "Unable to add sysfs for process %d\n", pid_nr(private->pid)); return; } for (i = 0; i < ARRAY_SIZE(mem_stats); i++) { int ret; ret = sysfs_create_file(&private->kobj, &mem_stats[i].attr.attr); ret |= sysfs_create_file(&private->kobj, &mem_stats[i].max_attr.attr); if (ret) dev_err(device->dev, "Unable to create sysfs files for process %d\n", pid_nr(private->pid)); } for (i = 0; i < ARRAY_SIZE(debug_memstats); i++) { if (sysfs_create_file(&private->kobj, &debug_memstats[i].attr)) WARN(1, "Couldn't create sysfs file '%s'\n", debug_memstats[i].attr.name); } } static ssize_t memstat_show(struct device *dev, struct device_attribute *attr, char *buf) { uint64_t val = 0; if (!strcmp(attr->attr.name, "vmalloc")) val = atomic_long_read(&kgsl_driver.stats.vmalloc); else if (!strcmp(attr->attr.name, "vmalloc_max")) val = atomic_long_read(&kgsl_driver.stats.vmalloc_max); else if (!strcmp(attr->attr.name, "page_alloc")) val = atomic_long_read(&kgsl_driver.stats.page_alloc); else if (!strcmp(attr->attr.name, "page_alloc_max")) val = atomic_long_read(&kgsl_driver.stats.page_alloc_max); else if (!strcmp(attr->attr.name, "coherent")) val = atomic_long_read(&kgsl_driver.stats.coherent); else if (!strcmp(attr->attr.name, "coherent_max")) val = atomic_long_read(&kgsl_driver.stats.coherent_max); else if (!strcmp(attr->attr.name, "secure")) val = atomic_long_read(&kgsl_driver.stats.secure); else if (!strcmp(attr->attr.name, "secure_max")) val = atomic_long_read(&kgsl_driver.stats.secure_max); else if (!strcmp(attr->attr.name, "mapped")) val = atomic_long_read(&kgsl_driver.stats.mapped); else if (!strcmp(attr->attr.name, "mapped_max")) val = atomic_long_read(&kgsl_driver.stats.mapped_max); return scnprintf(buf, PAGE_SIZE, "%llu\n", val); } static ssize_t full_cache_threshold_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { int ret; unsigned int thresh = 0; ret = kgsl_sysfs_store(buf, &thresh); if (ret) return ret; kgsl_driver.full_cache_threshold = thresh; return count; } static ssize_t full_cache_threshold_show(struct device *dev, struct device_attribute *attr, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", kgsl_driver.full_cache_threshold); } static DEVICE_ATTR(vmalloc, 0444, memstat_show, NULL); static DEVICE_ATTR(vmalloc_max, 0444, memstat_show, NULL); static DEVICE_ATTR(page_alloc, 0444, memstat_show, NULL); static DEVICE_ATTR(page_alloc_max, 0444, memstat_show, NULL); static DEVICE_ATTR(coherent, 0444, memstat_show, NULL); static DEVICE_ATTR(coherent_max, 0444, memstat_show, NULL); static DEVICE_ATTR(secure, 0444, memstat_show, NULL); static DEVICE_ATTR(secure_max, 0444, memstat_show, NULL); static DEVICE_ATTR(mapped, 0444, memstat_show, NULL); static DEVICE_ATTR(mapped_max, 0444, memstat_show, NULL); static DEVICE_ATTR_RW(full_cache_threshold); static const struct attribute *drv_attr_list[] = { &dev_attr_vmalloc.attr, &dev_attr_vmalloc_max.attr, &dev_attr_page_alloc.attr, &dev_attr_page_alloc_max.attr, &dev_attr_coherent.attr, &dev_attr_coherent_max.attr, &dev_attr_secure.attr, &dev_attr_secure_max.attr, &dev_attr_mapped.attr, &dev_attr_mapped_max.attr, &dev_attr_full_cache_threshold.attr, NULL, }; void kgsl_sharedmem_uninit_sysfs(void) { sysfs_remove_files(&kgsl_driver.virtdev.kobj, drv_attr_list); } int kgsl_sharedmem_init_sysfs(void) { return sysfs_create_files(&kgsl_driver.virtdev.kobj, drv_attr_list); } static vm_fault_t kgsl_paged_vmfault(struct kgsl_memdesc *memdesc, struct vm_area_struct *vma, struct vm_fault *vmf) { int pgoff; pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; return vmf_insert_page(vma, vmf->address, memdesc->pages[pgoff]); } static void kgsl_paged_unmap_kernel(struct kgsl_memdesc *memdesc) { mutex_lock(&kernel_map_global_lock); if (!memdesc->hostptr) { /* If already unmapped the refcount should be 0 */ WARN_ON(memdesc->hostptr_count); goto done; } memdesc->hostptr_count--; if (memdesc->hostptr_count) goto done; vunmap(memdesc->hostptr); atomic_long_sub(memdesc->size, &kgsl_driver.stats.vmalloc); memdesc->hostptr = NULL; done: mutex_unlock(&kernel_map_global_lock); } #if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) #include static int lock_sgt(struct sg_table *sgt, u64 size) { struct scatterlist *sg; int dest_perms = PERM_READ | PERM_WRITE; int source_vm = VMID_HLOS; int dest_vm = VMID_CP_PIXEL; int ret; int i; do { ret = hyp_assign_table(sgt, &source_vm, 1, &dest_vm, &dest_perms, 1); } while (ret == -EAGAIN); if (ret) { /* * If returned error code is EADDRNOTAVAIL, then this * memory may no longer be in a usable state as security * state of the pages is unknown after this failure. This * memory can neither be added back to the pool nor buddy * system. */ if (ret == -EADDRNOTAVAIL) pr_err("Failure to lock secure GPU memory 0x%llx bytes will not be recoverable\n", size); return ret; } /* Set private bit for each sg to indicate that its secured */ for_each_sg(sgt->sgl, sg, sgt->nents, i) SetPagePrivate(sg_page(sg)); return 0; } static int unlock_sgt(struct sg_table *sgt) { int dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC; int source_vm = VMID_CP_PIXEL; int dest_vm = VMID_HLOS; int ret; struct sg_page_iter sg_iter; do { ret = hyp_assign_table(sgt, &source_vm, 1, &dest_vm, &dest_perms, 1); } while (ret == -EAGAIN); if (ret) return ret; for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) ClearPagePrivate(sg_page_iter_page(&sg_iter)); return 0; } #endif static int kgsl_paged_map_kernel(struct kgsl_memdesc *memdesc) { int ret = 0; /* Sanity check - don't map more than we could possibly chew */ if (memdesc->size > ULONG_MAX) return -ENOMEM; mutex_lock(&kernel_map_global_lock); if ((!memdesc->hostptr) && (memdesc->pages != NULL)) { pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL); memdesc->hostptr = vmap(memdesc->pages, memdesc->page_count, VM_IOREMAP, page_prot); if (memdesc->hostptr) KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.vmalloc, &kgsl_driver.stats.vmalloc_max); else ret = -ENOMEM; } if (memdesc->hostptr) memdesc->hostptr_count++; mutex_unlock(&kernel_map_global_lock); return ret; } static vm_fault_t kgsl_contiguous_vmfault(struct kgsl_memdesc *memdesc, struct vm_area_struct *vma, struct vm_fault *vmf) { unsigned long offset, pfn; offset = ((unsigned long) vmf->address - vma->vm_start) >> PAGE_SHIFT; pfn = (memdesc->physaddr >> PAGE_SHIFT) + offset; return vmf_insert_pfn(vma, vmf->address, pfn); } static void _dma_cache_op(struct device *dev, struct page *page, unsigned int op) { struct scatterlist sgl; sg_init_table(&sgl, 1); sg_set_page(&sgl, page, PAGE_SIZE, 0); sg_dma_address(&sgl) = page_to_phys(page); switch (op) { case KGSL_CACHE_OP_FLUSH: dma_sync_sg_for_device(dev, &sgl, 1, DMA_TO_DEVICE); dma_sync_sg_for_device(dev, &sgl, 1, DMA_FROM_DEVICE); break; case KGSL_CACHE_OP_CLEAN: dma_sync_sg_for_device(dev, &sgl, 1, DMA_TO_DEVICE); break; case KGSL_CACHE_OP_INV: dma_sync_sg_for_device(dev, &sgl, 1, DMA_FROM_DEVICE); break; } } int kgsl_cache_range_op(struct kgsl_memdesc *memdesc, uint64_t offset, uint64_t size, unsigned int op) { struct sg_table *sgt = NULL; struct sg_page_iter sg_iter; if (memdesc->flags & KGSL_MEMFLAGS_IOCOHERENT) return 0; if (size == 0 || size > UINT_MAX) return -EINVAL; /* Make sure that the offset + size does not overflow */ if ((offset + size < offset) || (offset + size < size)) return -ERANGE; /* Check that offset+length does not exceed memdesc->size */ if (offset + size > memdesc->size) return -ERANGE; if (memdesc->sgt != NULL) sgt = memdesc->sgt; else { if (memdesc->pages == NULL) return 0; sgt = kgsl_alloc_sgt_from_pages(memdesc); if (IS_ERR(sgt)) return PTR_ERR(sgt); } size += offset & PAGE_MASK; offset &= ~PAGE_MASK; for_each_sg_page(sgt->sgl, &sg_iter, PAGE_ALIGN(size) >> PAGE_SHIFT, offset >> PAGE_SHIFT) _dma_cache_op(memdesc->dev, sg_page_iter_page(&sg_iter), op); if (memdesc->sgt == NULL) kgsl_free_sgt(sgt); return 0; } void kgsl_memdesc_init(struct kgsl_device *device, struct kgsl_memdesc *memdesc, uint64_t flags) { struct kgsl_mmu *mmu = &device->mmu; unsigned int align; memset(memdesc, 0, sizeof(*memdesc)); /* Turn off SVM if the system doesn't support it */ if (!kgsl_mmu_use_cpu_map(mmu)) flags &= ~((uint64_t) KGSL_MEMFLAGS_USE_CPU_MAP); /* Secure memory disables advanced addressing modes */ if (flags & KGSL_MEMFLAGS_SECURE) flags &= ~((uint64_t) KGSL_MEMFLAGS_USE_CPU_MAP); /* Disable IO coherence if it is not supported on the chip */ if (!kgsl_mmu_has_feature(device, KGSL_MMU_IO_COHERENT)) { flags &= ~((uint64_t) KGSL_MEMFLAGS_IOCOHERENT); WARN_ONCE(IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT), "I/O coherency is not supported on this target\n"); } else if (IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT)) flags |= KGSL_MEMFLAGS_IOCOHERENT; /* * We can't enable I/O coherency on uncached surfaces because of * situations where hardware might snoop the cpu caches which can * have stale data. This happens primarily due to the limitations * of dma caching APIs available on arm64 */ if (!kgsl_cachemode_is_cached(flags)) flags &= ~((u64) KGSL_MEMFLAGS_IOCOHERENT); if (kgsl_mmu_has_feature(device, KGSL_MMU_NEED_GUARD_PAGE) || (flags & KGSL_MEMFLAGS_GUARD_PAGE)) memdesc->priv |= KGSL_MEMDESC_GUARD_PAGE; if (flags & KGSL_MEMFLAGS_SECURE) memdesc->priv |= KGSL_MEMDESC_SECURE; memdesc->flags = flags; memdesc->dev = &device->pdev->dev; align = max_t(unsigned int, (memdesc->flags & KGSL_MEMALIGN_MASK) >> KGSL_MEMALIGN_SHIFT, ilog2(PAGE_SIZE)); kgsl_memdesc_set_align(memdesc, align); spin_lock_init(&memdesc->lock); } void kgsl_sharedmem_free(struct kgsl_memdesc *memdesc) { if (!memdesc || !memdesc->size) return; /* Make sure the memory object has been unmapped */ kgsl_mmu_put_gpuaddr(memdesc); /* Assume if no operations were specified something went bad early */ if (!memdesc->ops || !memdesc->ops->free) return; memdesc->ops->free(memdesc); } #if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) void kgsl_free_secure_page(struct page *page) { struct sg_table sgt; struct scatterlist sgl; if (!page) return; sgt.sgl = &sgl; sgt.nents = 1; sgt.orig_nents = 1; sg_init_table(&sgl, 1); sg_set_page(&sgl, page, PAGE_SIZE, 0); unlock_sgt(&sgt); __free_page(page); } struct page *kgsl_alloc_secure_page(void) { struct page *page; struct sg_table sgt; struct scatterlist sgl; int status; page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_NORETRY | __GFP_HIGHMEM); if (!page) return NULL; sgt.sgl = &sgl; sgt.nents = 1; sgt.orig_nents = 1; sg_init_table(&sgl, 1); sg_set_page(&sgl, page, PAGE_SIZE, 0); status = lock_sgt(&sgt, PAGE_SIZE); if (status) { if (status == -EADDRNOTAVAIL) return NULL; __free_page(page); return NULL; } return page; } #else void kgsl_free_secure_page(struct page *page) { } struct page *kgsl_alloc_secure_page(void) { return NULL; } #endif int kgsl_sharedmem_readl(const struct kgsl_memdesc *memdesc, uint32_t *dst, uint64_t offsetbytes) { uint32_t *src; if (WARN_ON(memdesc == NULL || memdesc->hostptr == NULL || dst == NULL)) return -EINVAL; WARN_ON(offsetbytes % sizeof(uint32_t) != 0); if (offsetbytes % sizeof(uint32_t) != 0) return -EINVAL; WARN_ON(offsetbytes > (memdesc->size - sizeof(uint32_t))); if (offsetbytes > (memdesc->size - sizeof(uint32_t))) return -ERANGE; /* * We are reading shared memory between CPU and GPU. * Make sure reads before this are complete */ rmb(); src = (uint32_t *)(memdesc->hostptr + offsetbytes); *dst = *src; return 0; } void kgsl_sharedmem_writel(const struct kgsl_memdesc *memdesc, uint64_t offsetbytes, uint32_t src) { /* Quietly return if the memdesc isn't valid */ if (IS_ERR_OR_NULL(memdesc) || WARN_ON(!memdesc->hostptr)) return; if (WARN_ON(!IS_ALIGNED(offsetbytes, sizeof(u32)))) return; if (WARN_ON(offsetbytes > (memdesc->size - sizeof(u32)))) return; *((u32 *) (memdesc->hostptr + offsetbytes)) = src; /* Make sure the write is posted before continuing */ wmb(); } int kgsl_sharedmem_readq(const struct kgsl_memdesc *memdesc, uint64_t *dst, uint64_t offsetbytes) { uint64_t *src; if (WARN_ON(memdesc == NULL || memdesc->hostptr == NULL || dst == NULL)) return -EINVAL; WARN_ON(offsetbytes % sizeof(uint32_t) != 0); if (offsetbytes % sizeof(uint32_t) != 0) return -EINVAL; WARN_ON(offsetbytes > (memdesc->size - sizeof(uint32_t))); if (offsetbytes > (memdesc->size - sizeof(uint32_t))) return -ERANGE; /* * We are reading shared memory between CPU and GPU. * Make sure reads before this are complete */ rmb(); src = (uint64_t *)(memdesc->hostptr + offsetbytes); *dst = *src; return 0; } void kgsl_sharedmem_writeq(const struct kgsl_memdesc *memdesc, uint64_t offsetbytes, uint64_t src) { /* Quietly return if the memdesc isn't valid */ if (IS_ERR_OR_NULL(memdesc) || WARN_ON(!memdesc->hostptr)) return; if (WARN_ON(!IS_ALIGNED(offsetbytes, sizeof(u64)))) return; if (WARN_ON(offsetbytes > (memdesc->size - sizeof(u64)))) return; *((u64 *) (memdesc->hostptr + offsetbytes)) = src; /* Make sure the write is posted before continuing */ wmb(); } static const char * const memtype_str[] = { [KGSL_MEMTYPE_OBJECTANY] = "any(0)", [KGSL_MEMTYPE_FRAMEBUFFER] = "framebuffer", [KGSL_MEMTYPE_RENDERBUFFER] = "renderbuffer", [KGSL_MEMTYPE_ARRAYBUFFER] = "arraybuffer", [KGSL_MEMTYPE_ELEMENTARRAYBUFFER] = "elementarraybuffer", [KGSL_MEMTYPE_VERTEXARRAYBUFFER] = "vertexarraybuffer", [KGSL_MEMTYPE_TEXTURE] = "texture", [KGSL_MEMTYPE_SURFACE] = "surface", [KGSL_MEMTYPE_EGL_SURFACE] = "egl_surface", [KGSL_MEMTYPE_GL] = "gl", [KGSL_MEMTYPE_CL] = "cl", [KGSL_MEMTYPE_CL_BUFFER_MAP] = "cl_buffer_map", [KGSL_MEMTYPE_CL_BUFFER_NOMAP] = "cl_buffer_nomap", [KGSL_MEMTYPE_CL_IMAGE_MAP] = "cl_image_map", [KGSL_MEMTYPE_CL_IMAGE_NOMAP] = "cl_image_nomap", [KGSL_MEMTYPE_CL_KERNEL_STACK] = "cl_kernel_stack", [KGSL_MEMTYPE_COMMAND] = "command", [KGSL_MEMTYPE_2D] = "2d", [KGSL_MEMTYPE_EGL_IMAGE] = "egl_image", [KGSL_MEMTYPE_EGL_SHADOW] = "egl_shadow", [KGSL_MEMTYPE_MULTISAMPLE] = "egl_multisample", /* KGSL_MEMTYPE_KERNEL handled below, to avoid huge array */ }; void kgsl_get_memory_usage(char *name, size_t name_size, uint64_t memflags) { unsigned int type = MEMFLAGS(memflags, KGSL_MEMTYPE_MASK, KGSL_MEMTYPE_SHIFT); if (type == KGSL_MEMTYPE_KERNEL) strlcpy(name, "kernel", name_size); else if (type < ARRAY_SIZE(memtype_str) && memtype_str[type] != NULL) strlcpy(name, memtype_str[type], name_size); else snprintf(name, name_size, "VK/others(%3d)", type); } int kgsl_memdesc_sg_dma(struct kgsl_memdesc *memdesc, phys_addr_t addr, u64 size) { int ret; struct page *page = phys_to_page(addr); memdesc->sgt = kmalloc(sizeof(*memdesc->sgt), GFP_KERNEL); if (memdesc->sgt == NULL) return -ENOMEM; ret = sg_alloc_table(memdesc->sgt, 1, GFP_KERNEL); if (ret) { kfree(memdesc->sgt); memdesc->sgt = NULL; return ret; } sg_set_page(memdesc->sgt->sgl, page, (size_t) size, 0); return 0; } static void _kgsl_contiguous_free(struct kgsl_memdesc *memdesc) { dma_free_attrs(memdesc->dev, memdesc->size, memdesc->hostptr, memdesc->physaddr, memdesc->attrs); sg_free_table(memdesc->sgt); kfree(memdesc->sgt); memdesc->sgt = NULL; } static void kgsl_contiguous_free(struct kgsl_memdesc *memdesc) { if (!memdesc->hostptr) return; atomic_long_sub(memdesc->size, &kgsl_driver.stats.coherent); _kgsl_contiguous_free(memdesc); } #if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) static void kgsl_free_secure_system_pages(struct kgsl_memdesc *memdesc) { int i; struct scatterlist *sg; int ret = unlock_sgt(memdesc->sgt); if (ret) { /* * Unlock of the secure buffer failed. This buffer will * be stuck in secure side forever and is unrecoverable. * Give up on the buffer and don't return it to the * pool. */ pr_err("kgsl: secure buf unlock failed: gpuaddr: %llx size: %llx ret: %d\n", memdesc->gpuaddr, memdesc->size, ret); return; } atomic_long_sub(memdesc->size, &kgsl_driver.stats.secure); for_each_sg(memdesc->sgt->sgl, sg, memdesc->sgt->nents, i) { struct page *page = sg_page(sg); __free_pages(page, get_order(PAGE_SIZE)); } sg_free_table(memdesc->sgt); kfree(memdesc->sgt); memdesc->sgt = NULL; } static void kgsl_free_secure_pool_pages(struct kgsl_memdesc *memdesc) { int ret = unlock_sgt(memdesc->sgt); if (ret) { /* * Unlock of the secure buffer failed. This buffer will * be stuck in secure side forever and is unrecoverable. * Give up on the buffer and don't return it to the * pool. */ pr_err("kgsl: secure buf unlock failed: gpuaddr: %llx size: %llx ret: %d\n", memdesc->gpuaddr, memdesc->size, ret); return; } atomic_long_sub(memdesc->size, &kgsl_driver.stats.secure); kgsl_pool_free_sgt(memdesc->sgt); sg_free_table(memdesc->sgt); kfree(memdesc->sgt); memdesc->sgt = NULL; } #endif static void kgsl_free_pool_pages(struct kgsl_memdesc *memdesc) { kgsl_paged_unmap_kernel(memdesc); WARN_ON(memdesc->hostptr); atomic_long_sub(memdesc->size, &kgsl_driver.stats.page_alloc); kgsl_pool_free_pages(memdesc->pages, memdesc->page_count); memdesc->page_count = 0; kvfree(memdesc->pages); memdesc->pages = NULL; } static void kgsl_free_system_pages(struct kgsl_memdesc *memdesc) { int i; kgsl_paged_unmap_kernel(memdesc); WARN_ON(memdesc->hostptr); atomic_long_sub(memdesc->size, &kgsl_driver.stats.page_alloc); for (i = 0; i < memdesc->page_count; i++) __free_pages(memdesc->pages[i], get_order(PAGE_SIZE)); memdesc->page_count = 0; kvfree(memdesc->pages); memdesc->pages = NULL; } static struct kgsl_memdesc_ops kgsl_contiguous_ops = { .free = kgsl_contiguous_free, .vmflags = VM_DONTDUMP | VM_PFNMAP | VM_DONTEXPAND | VM_DONTCOPY, .vmfault = kgsl_contiguous_vmfault, }; #if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) static struct kgsl_memdesc_ops kgsl_secure_system_ops = { .free = kgsl_free_secure_system_pages, /* FIXME: Make sure vmflags / vmfault does the right thing here */ }; static struct kgsl_memdesc_ops kgsl_secure_pool_ops = { .free = kgsl_free_secure_pool_pages, /* FIXME: Make sure vmflags / vmfault does the right thing here */ }; #endif static struct kgsl_memdesc_ops kgsl_pool_ops = { .free = kgsl_free_pool_pages, .vmflags = VM_DONTDUMP | VM_DONTEXPAND | VM_DONTCOPY | VM_MIXEDMAP, .vmfault = kgsl_paged_vmfault, .map_kernel = kgsl_paged_map_kernel, .unmap_kernel = kgsl_paged_unmap_kernel, }; static struct kgsl_memdesc_ops kgsl_system_ops = { .free = kgsl_free_system_pages, .vmflags = VM_DONTDUMP | VM_DONTEXPAND | VM_DONTCOPY | VM_MIXEDMAP, .vmfault = kgsl_paged_vmfault, .map_kernel = kgsl_paged_map_kernel, .unmap_kernel = kgsl_paged_unmap_kernel, }; static int kgsl_system_alloc_pages(u64 size, struct page ***pages, struct device *dev) { struct scatterlist sg; struct page **local; int i, npages = size >> PAGE_SHIFT; local = kvcalloc(npages, sizeof(*pages), GFP_KERNEL | __GFP_NORETRY); if (!local) { pr_err("kgsl_system_alloc_pages, npages = %d\n", npages); return -ENOMEM; } for (i = 0; i < npages; i++) { gfp_t gfp = __GFP_ZERO | __GFP_HIGHMEM | GFP_KERNEL | __GFP_NORETRY; local[i] = alloc_pages(gfp, get_order(PAGE_SIZE)); if (!local[i]) { for (i = i - 1; i >= 0; i--) __free_pages(local[i], get_order(PAGE_SIZE)); kvfree(local); return -ENOMEM; } /* Make sure the cache is clean */ sg_init_table(&sg, 1); sg_set_page(&sg, local[i], PAGE_SIZE, 0); sg_dma_address(&sg) = page_to_phys(local[i]); dma_sync_sg_for_device(dev, &sg, 1, DMA_BIDIRECTIONAL); } *pages = local; return npages; } #if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) static int kgsl_alloc_secure_pages(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { struct page **pages; int count; struct sg_table *sgt; int ret; size = PAGE_ALIGN(size); if (!size || size > UINT_MAX) return -EINVAL; kgsl_memdesc_init(device, memdesc, flags); memdesc->priv |= priv; if (priv & KGSL_MEMDESC_SYSMEM) { memdesc->ops = &kgsl_secure_system_ops; count = kgsl_system_alloc_pages(size, &pages, device->dev); } else { memdesc->ops = &kgsl_secure_pool_ops; count = kgsl_pool_alloc_pages(size, &pages, device->dev); } if (count < 0) return count; sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) { kgsl_pool_free_pages(pages, count); return -ENOMEM; } ret = sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL); if (ret) { kfree(sgt); kgsl_pool_free_pages(pages, count); return ret; } /* Now that we've moved to a sg table don't need the pages anymore */ kvfree(pages); ret = lock_sgt(sgt, size); if (ret) { if (ret != -EADDRNOTAVAIL) kgsl_pool_free_sgt(sgt); sg_free_table(sgt); kfree(sgt); return ret; } memdesc->sgt = sgt; memdesc->size = size; KGSL_STATS_ADD(size, &kgsl_driver.stats.secure, &kgsl_driver.stats.secure_max); return 0; } #endif static int kgsl_alloc_pages(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { struct page **pages; int count; size = PAGE_ALIGN(size); if (!size || size > UINT_MAX) return -EINVAL; kgsl_memdesc_init(device, memdesc, flags); memdesc->priv |= priv; if (priv & KGSL_MEMDESC_SYSMEM) { memdesc->ops = &kgsl_system_ops; count = kgsl_system_alloc_pages(size, &pages, device->dev); } else { memdesc->ops = &kgsl_pool_ops; count = kgsl_pool_alloc_pages(size, &pages, device->dev); } if (count < 0) return count; memdesc->pages = pages; memdesc->size = size; memdesc->page_count = count; KGSL_STATS_ADD(size, &kgsl_driver.stats.page_alloc, &kgsl_driver.stats.page_alloc_max); return 0; } static int _kgsl_alloc_contiguous(struct device *dev, struct kgsl_memdesc *memdesc, u64 size, unsigned long attrs) { int ret; phys_addr_t phys; void *ptr; ptr = dma_alloc_attrs(dev, (size_t) size, &phys, GFP_KERNEL, attrs); if (!ptr) return -ENOMEM; memdesc->size = size; memdesc->dev = dev; memdesc->hostptr = ptr; memdesc->physaddr = phys; memdesc->gpuaddr = phys; memdesc->attrs = attrs; ret = kgsl_memdesc_sg_dma(memdesc, phys, size); if (ret) dma_free_attrs(dev, (size_t) size, ptr, phys, attrs); return ret; } static int kgsl_alloc_contiguous(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { int ret; size = PAGE_ALIGN(size); if (!size || size > UINT_MAX) return -EINVAL; kgsl_memdesc_init(device, memdesc, flags); memdesc->priv |= priv; memdesc->ops = &kgsl_contiguous_ops; ret = _kgsl_alloc_contiguous(&device->pdev->dev, memdesc, size, 0); if (!ret) KGSL_STATS_ADD(size, &kgsl_driver.stats.coherent, &kgsl_driver.stats.coherent_max); return ret; } #if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) static int kgsl_allocate_secure(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { return kgsl_alloc_secure_pages(device, memdesc, size, flags, priv); } #else static int kgsl_allocate_secure(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { return -ENODEV; } #endif int kgsl_allocate_user(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { if (device->mmu.type == KGSL_MMU_TYPE_NONE) return kgsl_alloc_contiguous(device, memdesc, size, flags, priv); else if (flags & KGSL_MEMFLAGS_SECURE) { int ret; ret = kgsl_allocate_secure(device, memdesc, size, flags, priv); if (ret != 0) pr_err("kgsl_allocate_secure failed\n"); return ret; } return kgsl_alloc_pages(device, memdesc, size, flags, priv); } int kgsl_allocate_kernel(struct kgsl_device *device, struct kgsl_memdesc *memdesc, u64 size, u64 flags, u32 priv) { int ret; ret = kgsl_allocate_user(device, memdesc, size, flags, priv); if (ret) return ret; if (memdesc->ops->map_kernel) { ret = memdesc->ops->map_kernel(memdesc); if (ret) { kgsl_sharedmem_free(memdesc); return ret; } } return 0; } struct kgsl_memdesc *kgsl_allocate_global_fixed(struct kgsl_device *device, const char *resource, const char *name) { struct kgsl_global_memdesc *md; u32 entry[2]; int ret; if (of_property_read_u32_array(device->pdev->dev.of_node, resource, entry, 2)) return ERR_PTR(-ENODEV); md = kzalloc(sizeof(*md), GFP_KERNEL); if (!md) return ERR_PTR(-ENOMEM); kgsl_memdesc_init(device, &md->memdesc, 0); md->memdesc.priv = KGSL_MEMDESC_GLOBAL; md->memdesc.physaddr = entry[0]; md->memdesc.size = entry[1]; ret = kgsl_memdesc_sg_dma(&md->memdesc, entry[0], entry[1]); if (ret) { kfree(md); return ERR_PTR(ret); } md->name = name; /* * No lock here, because this function is only called during probe/init * while the caller is holding the mutex */ list_add_tail(&md->node, &device->globals); kgsl_mmu_map_global(device, &md->memdesc); return &md->memdesc; } static struct kgsl_memdesc * kgsl_allocate_secure_global(struct kgsl_device *device, u64 size, u64 flags, u32 priv, const char *name) { struct kgsl_global_memdesc *md; int ret; md = kzalloc(sizeof(*md), GFP_KERNEL); if (!md) return ERR_PTR(-ENOMEM); /* Make sure that we get global memory from system memory */ priv |= KGSL_MEMDESC_GLOBAL | KGSL_MEMDESC_SYSMEM; ret = kgsl_allocate_secure(device, &md->memdesc, size, flags, priv); if (ret) { kfree(md); return ERR_PTR(ret); } md->name = name; /* * No lock here, because this function is only called during probe/init * while the caller is holding the mutex */ list_add_tail(&md->node, &device->globals); /* * No offset needed, we'll get an address inside of the pagetable * normally */ kgsl_mmu_map_global(device, &md->memdesc); return &md->memdesc; } struct kgsl_memdesc *kgsl_allocate_global(struct kgsl_device *device, u64 size, u64 flags, u32 priv, const char *name) { int ret; struct kgsl_global_memdesc *md; if (flags & KGSL_MEMFLAGS_SECURE) return kgsl_allocate_secure_global(device, size, flags, priv, name); md = kzalloc(sizeof(*md), GFP_KERNEL); if (!md) return ERR_PTR(-ENOMEM); /* * Make sure that we get global memory from system memory to keep from * taking up pool memory for the life of the driver */ priv |= KGSL_MEMDESC_GLOBAL | KGSL_MEMDESC_SYSMEM; ret = kgsl_allocate_kernel(device, &md->memdesc, size, flags, priv); if (ret) { kfree(md); return ERR_PTR(ret); } md->name = name; /* * No lock here, because this function is only called during probe/init * while the caller is holding the mute */ list_add_tail(&md->node, &device->globals); kgsl_mmu_map_global(device, &md->memdesc); return &md->memdesc; } void kgsl_free_globals(struct kgsl_device *device) { struct kgsl_global_memdesc *md, *tmp; list_for_each_entry_safe(md, tmp, &device->globals, node) { kgsl_sharedmem_free(&md->memdesc); list_del(&md->node); kfree(md); } } void kgsl_sharedmem_set_noretry(bool val) { sharedmem_noretry_flag = val; } bool kgsl_sharedmem_get_noretry(void) { return sharedmem_noretry_flag; }