/* iva_ctrl.c * * Copyright (C) 2016 Samsung Electronics Co.Ltd * Authors: * Ilkwan Kim * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef CONFIG_EXT_BOOTMEM #include #endif #ifdef CONFIG_EXYNOS_IOVMM #include #endif #ifdef CONFIG_EXYNOS_IMA #include #endif #include "iva_ctrl_ioctl.h" #include "regs/iva_base_addr.h" #include "iva_ipc_queue.h" #include "iva_ctrl.h" #include "iva_mbox.h" #include "iva_mcu.h" #include "iva_pmu.h" #include "iva_sh_mem.h" #include "iva_mem.h" #include "iva_sysfs.h" #include "iva_ipc_header.h" #include "iva_vdma.h" #undef ENABLE_MMAP_US #undef ENABLE_FPGA_REG_ACCESS #undef ENABLE_LOCAL_PLAT_DEVICE #undef ENABLE_MEASURE_IVA_PERF #undef ENABLE_IVA_PANIC_HANDLER #undef ENABLE_MCU_EXIT_AT_SHUTDOWN #undef ENABLE_POWER_OFF_AT_SHUTDOWN #undef ENABLE_SYSMMU_FAULT_REPORT /* depening on sysmmu driver impl. */ #ifdef MODULE #define IVA_CTRL_NAME "iva_ctrl" #define IVA_CTRL_MAJOR (63) #endif #define IVA_MBOX_IRQ_RSC_NAME "iva_mbox_irq" #ifdef CONFIG_ARCH_EXYNOS #if defined(CONFIG_SOC_EXYNOS8895) #define IVA_PM_QOS_IVA (PM_QOS_CAM_THROUGHPUT) #define IVA_PM_QOS_IVA_REQ (670000) /* L2, L0:(690000)*/ #elif defined(CONFIG_SOC_EXYNOS9810) #define IVA_PM_QOS_IVA (PM_QOS_IVA_THROUGHPUT) #define IVA_PM_QOS_IVA_REQ (534000) #define IVA_PM_IVA_MIN (34000) #else #undef IVA_PM_QOS_IVA #undef IVA_PM_QOS_IVA_REQ #define IVA_PM_IVA_MIN (10000) #endif #endif static struct iva_dev_data *g_iva_data; static int32_t __iva_ctrl_init(struct device *dev) { int ret; struct iva_dev_data *iva = dev_get_drvdata(dev); struct clk *iva_clk = iva->iva_clk; #ifdef ENABLE_FPGA_REG_ACCESS uint32_t val; void __iomem *peri_regs; /* TO DO : check again later */ /* enable the Qrequest otherwise IVA will start going into low-power mode. * this is required for all FPGA >= v29 * Also write 1 to bit 4 which is Qchannel_en (CMU model in FPGA). * Avail in FPGA >= 42b */ peri_regs = devm_ioremap(dev, PERI_PHY_BASE_ADDR, PERI_SIZE); if (!peri_regs) { dev_err(dev, "%s() fail to ioremap. peri phys(0x%x), size(0x%x)\n", __func__, PERI_PHY_BASE_ADDR, PERI_SIZE); return -ENOMEM; } writel(0x12, peri_regs + 0x1008); /* version register. if upper 8-bits are non-zero then "assume" to be in * 2-FPGA build environment where SCore FPGA version is contained in bits * 31:24 and IVA FPGA in 23:16 */ val = readl(peri_regs + 0x1000); if (val & 0xff000000) dev_info(dev, "%s() FPGA ver: IVA=%d SCore=%d\n", __func__, (val >> 16) & val, val >> 24); else dev_info(dev, "%s() FPGA ver: %d\n", __func__, val >> 16); devm_iounmap(dev, peri_regs); #endif dev_dbg(dev, "%s() init iva state(0x%lx) with hwa en(%d)\n", __func__, iva->state, iva->en_hwa_req); if (test_and_set_bit(IVA_ST_INIT_DONE, &iva->state)) { dev_warn(dev, "%s() already iva inited(0x%lx)\n", __func__, iva->state); return 0; } if (iva_clk) { ret = clk_prepare_enable(iva_clk); if (ret) { dev_err(dev, "fail to enable iva_clk(%d)\n", ret); return ret; } #if 0 /* confirm */ if (!__clk_is_enabled(iva_clk)) { dev_err(dev, "%s() req iva_clk on but off\n", __func__); return -EINVAL; } #endif dev_dbg(dev, "%s() success to enable iva_clk with rate(%ld)\n", __func__, clk_get_rate(iva_clk)); } #ifdef CONFIG_EXYNOS_IMA ima_host_begin(); #endif iva_pmu_init(iva, iva->en_hwa_req); /* iva mbox supports both of iva and score. */ iva_mbox_init(iva); return 0; } static int32_t __iva_ctrl_deinit(struct device *dev) { struct iva_dev_data *iva = dev_get_drvdata(dev); struct clk *iva_clk = iva->iva_clk; if (!test_and_clear_bit(IVA_ST_INIT_DONE, &iva->state)) { dev_warn(dev, "%s() already iva deinited(0x%lx)\n", __func__, iva->state); return 0; } dev_dbg(dev, "%s()\n", __func__); iva_mbox_deinit(iva); iva_pmu_deinit(iva); #ifdef CONFIG_EXYNOS_IMA ima_host_end(); #endif if (iva_clk) { clk_disable_unprepare(iva_clk); #if 1 /* check */ if (__clk_is_enabled(iva_clk)) { dev_err(dev, "%s() req iva_clk off but on\n", __func__); return -EINVAL; } #endif } return 0; } static int32_t iva_ctrl_init(struct iva_proc *proc, bool en_hwa) { int ret; struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; /* already init_ref > 0 */ if (kref_get_unless_zero(&proc->init_ref)) { dev_info(dev, "%s() already proc init requested\n", __func__); goto init_skip; } kref_init(&proc->init_ref); /* This process requests iva initialization, for the first time. */ if (kref_get_unless_zero(&iva->glob_init_ref)) { dev_info(dev, "%s() already iva initialized\n", __func__); goto init_skip; } /* initialize the iva globally, for the first time */ iva->en_hwa_req = en_hwa; #ifdef CONFIG_PM ret = pm_runtime_get_sync(dev); #else ret = __iva_ctrl_init(dev); #endif if (ret < 0) { dev_err(dev, "%s() fail to enable iva clk/pwr. ret(%d)\n", __func__, ret); atomic_set(&proc->init_ref.refcount, 0); return ret; } /* now initialize global krefs */ kref_init(&iva->glob_init_ref); dev_dbg(dev, "%s() rt ret(%d) iva clk rate(%ld)\n", __func__, ret, clk_get_rate(iva->iva_clk)); init_skip: dev_dbg(dev, "%s() glob_init_ref(%d) <- init_ref(%d)\n", __func__, atomic_read(&iva->glob_init_ref.refcount), atomic_read(&proc->init_ref.refcount)); return 0; } static inline int32_t iva_ctrl_init_usr(struct iva_proc *proc, uint32_t __user *en_hwa_usr) { uint32_t en_hwa; get_user(en_hwa, en_hwa_usr); return iva_ctrl_init(proc, en_hwa); } static void iva_ctrl_release_global_init_ref(struct kref *kref) { int ret; struct iva_dev_data *iva = container_of(kref, struct iva_dev_data, glob_init_ref); struct device *dev = iva->dev; if (!test_bit(IVA_ST_INIT_DONE, &iva->state)) { dev_info(dev, "%s() already iva deinited(0x%lx). ", __func__, iva->state); dev_info(dev, "plz check the pair of init/deinit\n"); return; } #ifdef CONFIG_PM ret = pm_runtime_put_sync(dev); #else ret = __iva_ctrl_deinit(dev); #endif if (ret < 0) { dev_err(dev, "%s() fail to deinit iva(%d)\n", __func__, ret); return; } dev_dbg(dev, "%s()\n", __func__); } static void iva_ctrl_release_proc_init_ref(struct kref *kref) { struct iva_proc *proc = container_of(kref, struct iva_proc, init_ref); struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; int ret; /* * if this is called, * this process want to deinit iva finally. * now this request is propagated globally. */ ret = kref_put(&iva->glob_init_ref, iva_ctrl_release_global_init_ref); if (!ret) { dev_info(dev, "%s() still global init requesters in a system. ", __func__); dev_info(dev, "glob_init_ref(%d)\n", atomic_read(&iva->glob_init_ref.refcount)); } } static int32_t iva_ctrl_deinit(struct iva_proc *proc, bool forced) { struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; int ret = 0; unsigned int put_cnt; put_cnt = atomic_read(&proc->init_ref.refcount); if (!put_cnt) return 0; if (forced) { dev_warn(dev, "%s() forced init requester removal,", __func__); dev_warn(dev, " put_cnt(%d) == init_ref(%d)\n", put_cnt, atomic_read(&proc->init_ref.refcount)); } else put_cnt = 1; ret = kref_sub(&proc->init_ref, put_cnt, iva_ctrl_release_proc_init_ref); if (!ret) { dev_info(dev, "%s() still init requesters in the process. ", __func__); dev_warn(dev, "glob_init_ref(%d) <- init_ref(%d)\n", atomic_read(&iva->glob_init_ref.refcount), atomic_read(&proc->init_ref.refcount)); } /* always return as successful */ return 0; } static void __maybe_unused iva_ctrl_deinit_forced(struct iva_dev_data *iva) { struct list_head *work_node; struct iva_proc *proc; list_for_each(work_node, &iva->proc_head) { proc = list_entry(work_node, struct iva_proc, proc_node); iva_ctrl_deinit(proc, true/*forced*/); } } static void iva_ctrl_mcu_boot_prepare_system(struct iva_dev_data *iva) { struct device *dev = iva->dev; #ifdef CONFIG_PM_SLEEP /* prevent the system to suspend */ if (!wake_lock_active(&iva->iva_wake_lock)) { wake_lock(&iva->iva_wake_lock); dev_dbg(dev, "%s() wake_lock, now(%d)\n", __func__, wake_lock_active(&iva->iva_wake_lock)); } #endif #ifdef IVA_PM_QOS_IVA if (!test_and_set_bit(IVA_ST_PM_QOS_IVA, &iva->state)) { dev_dbg(dev, "%s() request iva pm_qos(0x%lx), rate(%d)\n", __func__, iva->state, iva->iva_qos_rate); pm_qos_update_request(&iva->iva_qos_iva, iva->iva_qos_rate); } #endif dev_dbg(dev, "%s() iva_clk rate(%ld)\n", __func__, clk_get_rate(iva->iva_clk)); } static void iva_ctrl_mcu_boot_unprepare_system(struct iva_dev_data *iva) { struct device *dev = iva->dev; #ifdef IVA_PM_QOS_IVA if (test_and_clear_bit(IVA_ST_PM_QOS_IVA, &iva->state)) { pm_qos_update_request(&iva->iva_qos_iva, 0); clear_bit(IVA_ST_PM_QOS_IVA, &iva->state); } #endif #ifdef CONFIG_PM_SLEEP if (wake_lock_active(&iva->iva_wake_lock)) { wake_unlock(&iva->iva_wake_lock); dev_dbg(dev, "%s() wake_inlock, now(%d)\n", __func__, wake_lock_active(&iva->iva_wake_lock)); } #endif dev_dbg(dev, "%s() iva_clk rate(%ld)\n", __func__, clk_get_rate(iva->iva_clk)); } static int32_t iva_ctrl_mcu_boot_file(struct iva_proc *proc, const char __user *mcu_file_u, int32_t wait_ready) { struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; int name_len; char mcu_file[IVA_CTRL_BOOT_FILE_LEN]; int32_t ret; if (!iva_ctrl_is_iva_on(iva)) { dev_dbg(dev, "%s() mcu boot request w/o iva init(0x%lx)\n", __func__, iva->state); return -EIO; } name_len = strncpy_from_user(mcu_file, mcu_file_u, sizeof(mcu_file) - 1); mcu_file[sizeof(mcu_file) - 1] = 0x0; if (name_len <= 0) { dev_err(dev, "%s() unexpected name_len(%d)\n", __func__, name_len); return name_len; } /* already boot_ref > 0 */ if (kref_get_unless_zero(&proc->boot_ref)) { dev_info(dev, "%s() already proc boot requested\n", __func__); goto boot_skip; } kref_init(&proc->boot_ref); /* This process requests iva boot, for the first time. */ if (kref_get_unless_zero(&iva->glob_boot_ref)) { dev_info(dev, "%s() already iva booted.\n", __func__); goto boot_skip; } iva_ctrl_mcu_boot_prepare_system(iva); ret = iva_mcu_boot_file(iva, mcu_file, wait_ready); if (ret) { /* error */ iva_ctrl_mcu_boot_unprepare_system(iva); atomic_set(&proc->boot_ref.refcount, 0); return ret; } /* now initialize kref */ kref_init(&iva->glob_boot_ref); dev_dbg(dev, "%s() success to boot mcu\n", __func__); boot_skip: dev_dbg(dev, "%s() glob_init_ref(%d) <- boot_ref(%d)\n", __func__, atomic_read(&iva->glob_init_ref.refcount), atomic_read(&proc->init_ref.refcount)); return 0; } static void iva_ctrl_release_global_boot_ref(struct kref *kref) { struct iva_dev_data *iva = container_of(kref, struct iva_dev_data, glob_boot_ref); struct device *dev = iva->dev; iva_mcu_exit(iva); iva_ctrl_mcu_boot_unprepare_system(iva); dev_dbg(dev, "%s()\n", __func__); } static void iva_ctrl_release_proc_boot_ref(struct kref *kref) { struct iva_proc *proc = container_of(kref, struct iva_proc, boot_ref); struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; int ret; /* * if this is called, this process want to exit mcu finally. * now this request is propagated globally. */ ret = kref_put(&iva->glob_boot_ref, iva_ctrl_release_global_boot_ref); if (!ret) { dev_info(dev, "%s() still global boot requesters in a system.", __func__); dev_info(dev, "glob_boot_ref(%d)\n", atomic_read(&iva->glob_boot_ref.refcount)); } } static int32_t iva_ctrl_mcu_exit(struct iva_proc *proc, bool forced) { struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; int ret = 0; unsigned int put_cnt; put_cnt = atomic_read(&proc->boot_ref.refcount); if (!put_cnt) return 0; if (forced) { dev_warn(dev, "%s() forced mcu exit requester removal, ", __func__); dev_warn(dev, "put_cnt(%d) == boot_ref(%d)\n", put_cnt, atomic_read(&proc->boot_ref.refcount)); } else put_cnt = 1; ret = kref_sub(&proc->boot_ref, put_cnt, iva_ctrl_release_proc_boot_ref); if (!ret) { dev_info(dev, "%s() still mcu boot requesters in the process.", __func__); dev_info(dev, "glob_boot_ref(%d) <- boot_ref(%d)\n", atomic_read(&iva->glob_boot_ref.refcount), atomic_read(&proc->boot_ref.refcount)); } /* always return as successful */ return 0; } static void __maybe_unused iva_ctrl_mcu_exit_forced(struct iva_dev_data *iva) { struct list_head *work_node; struct iva_proc *proc; list_for_each(work_node, &iva->proc_head) { proc = list_entry(work_node, struct iva_proc, proc_node); iva_ctrl_mcu_exit(proc, true/*forced*/); } } bool iva_ctrl_is_iva_on(struct iva_dev_data *iva) { return !!atomic_read(&iva->glob_init_ref.refcount); } bool iva_ctrl_is_boot(struct iva_dev_data *iva) { return !!atomic_read(&iva->glob_boot_ref.refcount); } static int32_t iva_ctrl_get_iva_status_usr(struct iva_dev_data *iva, struct iva_status_param __user *st_usr) { if (!st_usr) { dev_err(iva->dev, "no usr pointer provided.\n"); return -EINVAL; } if (put_user((uint32_t)iva->state, (uint32_t *) &st_usr->status)) return -EFAULT; if (put_user((uint32_t)iva->iva_qos_rate, (uint32_t *) &st_usr->clk_rate)) return -EFAULT; return 0; } static int32_t iva_ctrl_set_clk_usr(struct iva_dev_data *iva, uint32_t __user *usr_clk_rate) { struct device *dev = &iva->iva_dvfs_dev->dev; int32_t iva_clk_rate; bool hit = false; get_user(iva_clk_rate, usr_clk_rate); if ((iva_clk_rate < IVA_PM_IVA_MIN) || (iva_clk_rate > IVA_PM_QOS_IVA_REQ)){ dev_err(iva->dev, "invalid usr clk rate provided.\n"); return -EINVAL; } if (iva->iva_dvfs_dev) { struct dev_pm_opp *opp; unsigned long freq = 0; rcu_read_lock(); do { opp = dev_pm_opp_find_freq_ceil(dev, &freq); if (IS_ERR(opp)) break; if (freq == iva_clk_rate) { hit = true; break; } freq++; } while (1); rcu_read_unlock(); } if (hit) { iva->iva_qos_rate = iva_clk_rate; pm_qos_update_request(&iva->iva_qos_iva, iva->iva_qos_rate); } else { dev_warn(dev, "fail to set qos rate, req(%dKhz), cur(%dKhz)\n", iva_clk_rate, iva->iva_qos_rate); } dev_dbg(dev, "%s() iva_clk rate(%ld)\n", __func__, clk_get_rate(iva->iva_clk)); return 0; } const char *iva_ctrl_get_ioctl_str(unsigned int cmd) { #define IOCTL_STR_ENTRY(cmd_nr, cmd_str)[cmd_nr] = cmd_str static const char * const iva_ioctl_str[] = { IOCTL_STR_ENTRY(_IOC_NR(IVA_CTRL_INIT_IVA), "INIT_IVA"), IOCTL_STR_ENTRY(_IOC_NR(IVA_CTRL_DEINIT_IVA), "DEINIT_IVA"), IOCTL_STR_ENTRY(_IOC_NR(IVA_CTRL_BOOT_MCU), "BOOT_MCU"), IOCTL_STR_ENTRY(_IOC_NR(IVA_CTRL_EXIT_MCU), "EXIT_MCU"), IOCTL_STR_ENTRY(_IOC_NR(IVA_CTRL_GET_STATUS), "GET_IVA_STATUS"), IOCTL_STR_ENTRY(_IOC_NR(IVA_IPC_QUEUE_SEND_CMD), "IPCQ_SEND"), IOCTL_STR_ENTRY(_IOC_NR(IVA_IPC_QUEUE_RECEIVE_RSP), "IPCQ_RECEIVE"), IOCTL_STR_ENTRY(_IOC_NR(IVA_ION_GET_IOVA), "ION_GET_IOVA"), IOCTL_STR_ENTRY(_IOC_NR(IVA_ION_PUT_IOVA), "ION_PUT_IOVA"), IOCTL_STR_ENTRY(_IOC_NR(IVA_ION_ALLOC), "ION_ALLOC"), IOCTL_STR_ENTRY(_IOC_NR(IVA_ION_FREE), "ION_FREE"), IOCTL_STR_ENTRY(_IOC_NR(IVA_ION_SYNC_FOR_CPU), "ION_SYNC_FOR_CPU"), IOCTL_STR_ENTRY(_IOC_NR(IVA_ION_SYNC_FOR_DEVICE), "ION_SYNC_FOR_DEVICE"), IOCTL_STR_ENTRY(_IOC_NR(IVA_CTRL_SET_CLK), "SET_IVA_CLK"), }; unsigned int cmd_nr = _IOC_NR(cmd); if (cmd_nr >= (unsigned int) ARRAY_SIZE(iva_ioctl_str)) return "UNKNOWN IOCTL"; return iva_ioctl_str[cmd_nr]; } static inline s32 __maybe_unused iva_timeval_diff_us(struct timeval *a, struct timeval *b) { return (a->tv_sec - b->tv_sec) * USEC_PER_SEC + (a->tv_usec - b->tv_usec); } static long iva_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { void __user *p = (void __user *)arg; struct device *dev; int ret; struct iva_dev_data *iva; struct iva_proc *proc; #ifdef ENABLE_MEASURE_IVA_PERF struct timeval before, after; do_gettimeofday(&before); #endif proc = (struct iva_proc *) filp->private_data; iva = proc->iva_data; dev = iva->dev; dev_dbg(dev, "%s() cmd(%s) arg(0x%p)\n", __func__, iva_ctrl_get_ioctl_str(cmd), p); if (_IOC_TYPE(cmd) != IVA_CTRL_MAGIC || !p) { dev_warn(dev, "%s() check the your ioctl cmd(0x%x) arg(0x%p)\n", __func__, cmd, p); return -EINVAL; } switch (cmd) { case IVA_CTRL_INIT_IVA: ret = iva_ctrl_init_usr(proc, (uint32_t __user *) p); break; case IVA_CTRL_DEINIT_IVA: ret = iva_ctrl_deinit(proc, false); break; case IVA_CTRL_BOOT_MCU: ret = iva_ctrl_mcu_boot_file(proc, (const char __user *) p, false); break; case IVA_CTRL_EXIT_MCU: ret = iva_ctrl_mcu_exit(proc, false); break; case IVA_CTRL_GET_STATUS: ret = iva_ctrl_get_iva_status_usr(iva, (struct iva_status_param __user *) p); break; case IVA_IPC_QUEUE_SEND_CMD: ret = iva_ipcq_send_cmd_usr(proc, (struct ipc_cmd_param __user *) p); break; case IVA_IPC_QUEUE_RECEIVE_RSP: ret = iva_ipcq_wait_res_usr(proc, (struct ipc_res_param __user *) p); break; case IVA_CTRL_SET_CLK: ret = iva_ctrl_set_clk_usr(iva, (uint32_t __user *) p); break; case IVA_ION_GET_IOVA: case IVA_ION_PUT_IOVA: case IVA_ION_ALLOC: case IVA_ION_FREE: /* This may not be used */ case IVA_ION_SYNC_FOR_CPU: case IVA_ION_SYNC_FOR_DEVICE: ret = iva_mem_ioctl(proc, cmd, p); break; default: dev_warn(dev, "%s() unhandled cmd 0x%x", __func__, cmd); ret = -ENOTTY; break; } #ifdef ENABLE_MEASURE_IVA_PERF do_gettimeofday(&after); dev_info(dev, "%s(%d usec)\n", iva_ctrl_get_ioctl_str(cmd), iva_timeval_diff_us(&after, &before)); #endif return ret; } static int iva_ctrl_mmap(struct file *filp, struct vm_area_struct *vma) { #ifndef MODULE struct iva_proc *proc = (struct iva_proc *) filp->private_data; struct iva_dev_data *iva = proc->iva_data; struct device *dev = iva->dev; #else struct iva_dev_data *iva = filp->private_data; struct device *dev = iva->dev; #endif #ifdef ENABLE_MMAP_US int ret; unsigned long req_size = vma->vm_end - vma->vm_start; unsigned long phys_iva = iva->iva_res->start; unsigned long req_phys_iva = vma->vm_pgoff << PAGE_SHIFT; if ((req_phys_iva < phys_iva) || ((req_phys_iva + req_size) > (phys_iva + resource_size(iva->iva_res)))) { dev_warn(dev, "%s() not allowed to map the region(0x%lx-0x%lx)", __func__, req_phys_iva, req_phys_iva + req_size); return -EPERM; } vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, req_size, vma->vm_page_prot); if (ret) { dev_err(dev, "%s() fail to remap, ret(%d), start(0x%lx), ", __func__, ret, vma->vm_start); dev_err(dev, "end(0x%lx), pgoff(0x%lx)\n", vma->vm_end, vma->vm_pgoff); return ret; } dev_dbg(dev, "%s() success!!! start(0x%lx), end(0x%lx), pgoff(0x%lx)\n", __func__, vma->vm_start, vma->vm_end, vma->vm_pgoff); return 0; #else dev_err(dev, "%s() not allowed to map with start(0x%lx), ", __func__, vma->vm_start); dev_err(dev, "end(0x%lx), pgoff(0x%lx)\n", vma->vm_end, vma->vm_pgoff); return -EPERM; /* error: not allowed */ #endif } static int iva_ctrl_open(struct inode *inode, struct file *filp) { struct device *dev; struct iva_dev_data *iva; struct iva_proc *proc; #ifndef MODULE struct miscdevice *miscdev; miscdev = filp->private_data; dev = miscdev->parent; iva = dev_get_drvdata(dev); #else /* smarter */ iva = filp->private_data = g_iva_data; dev = iva->dev; #endif proc = devm_kmalloc(dev, sizeof(*proc), GFP_KERNEL); if (!proc) { dev_err(dev, "%s() fail to alloc mem for struct iva_proc\n", __func__); return -ENOMEM; } /* assume only one open() per process */ get_task_struct(current); proc->tsk = current; proc->pid = current->group_leader->pid; proc->tid = current->pid; proc->iva_data = iva; iva_mem_init_proc_mem(proc); atomic_set(&proc->init_ref.refcount, 0); atomic_set(&proc->boot_ref.refcount, 0); mutex_lock(&iva->proc_mutex); list_add(&proc->proc_node, &iva->proc_head); mutex_unlock(&iva->proc_mutex); /* update file->private to hold struct iva_proc */ filp->private_data = (void *) proc; dev_dbg(dev, "%s() succeed to open from (%s, %d, %d)\n", __func__, proc->tsk->comm, proc->pid, proc->tid); return 0; } static int iva_ctrl_release(struct inode *inode, struct file *filp) { struct device *dev; struct iva_dev_data *iva; struct iva_proc *proc; proc = (struct iva_proc *) filp->private_data; iva = proc->iva_data; dev = iva->dev; mutex_lock(&iva->proc_mutex); list_del(&proc->proc_node); mutex_unlock(&iva->proc_mutex); dev_dbg(dev, "%s() try to close from (%s, %d, %d)\n", __func__, proc->tsk->comm, proc->pid, proc->tid); /* to do : per process */ dev_dbg(dev, "%s() state(0x%lx)\n", __func__, iva->state); /* forced to release all iva mcu exit requests from this process */ iva_ctrl_mcu_exit(proc, true); /* forced to release all iva init requests from this process */ iva_ctrl_deinit(proc, true); iva_mem_deinit_proc_mem(proc); put_task_struct(proc->tsk); devm_kfree(dev, proc); return 0; } static const struct file_operations iva_ctl_fops = { .owner = THIS_MODULE, .open = iva_ctrl_open, .release = iva_ctrl_release, .unlocked_ioctl = iva_ctrl_ioctl, .compat_ioctl = iva_ctrl_ioctl, .mmap = iva_ctrl_mmap }; #ifdef CONFIG_EXYNOS_IOVMM static int iva_iommu_fault_handler(struct iommu_domain *domain, struct device *dev, unsigned long fault_addr, int fault_flags, void *token) { struct iva_dev_data *iva = token; dev_err(dev, "%s() is called with fault addr(0x%lx), err_cnt(%d)\n", __func__, fault_addr, atomic_read(&iva->mcu_err_cnt)); #ifdef ENABLE_SYSMMU_FAULT_REPORT iva_mcu_handle_error_k(iva, mcu_err_from_irq, 35 /* SYSMMU_FAULT */); #else iva_mcu_show_status(iva); iva_mem_show_global_mapped_list_nolock(iva); #endif return -EAGAIN; } #endif #ifdef ENABLE_IVA_PANIC_HANDLER static int iva_ctrl_catch_panic_event(struct notifier_block *nb, unsigned long event, void *data) { struct iva_dev_data *iva = container_of(nb, struct iva_dev_data, k_panic_nb); dev_err(iva->dev, "%s() iva state(0x%lx)\n", __func__, iva->state); return NOTIFY_DONE; } static int iva_ctrl_register_panic_handler(struct iva_dev_data *iva) { struct notifier_block *iva_nb = &iva->k_panic_nb; iva_nb->notifier_call = iva_ctrl_catch_panic_event; iva_nb->next = NULL; iva_nb->priority = 0; return atomic_notifier_chain_register(&panic_notifier_list, iva_nb); } static int iva_ctrl_unregister_panic_handler(struct iva_dev_data *iva) { struct notifier_block *iva_nb = &iva->k_panic_nb; atomic_notifier_chain_unregister(&panic_notifier_list, iva_nb); iva_nb->notifier_call = NULL; iva_nb->next = NULL; iva_nb->priority = 0; return 0; } #endif static void iva_ctl_parse_of_dt(struct device *dev, struct iva_dev_data *iva) { struct device_node *of_node = dev->of_node; struct device_node *child_node; /* overwrite parameters from of */ if (!of_node) { dev_info(dev, "of node not found.\n"); return; } child_node = of_find_node_by_name(of_node, "mcu-info"); if (child_node) { of_property_read_u32(child_node, "mem_size", &iva->mcu_mem_size); of_property_read_u32(child_node, "shmem_size", &iva->mcu_shmem_size); of_property_read_u32(child_node, "print_delay", &iva->mcu_print_delay); dev_dbg(dev, "%s() of mem_size(0x%x) shmem_size(0x%x) print_delay(%d)\n", __func__, iva->mcu_mem_size, iva->mcu_shmem_size, iva->mcu_print_delay); } child_node = of_parse_phandle(of_node, "dvfs-dev", 0); iva->iva_dvfs_dev = of_find_device_by_node(child_node); if (!iva->iva_dvfs_dev) dev_warn(dev, "fail to get iva dvfs dev\n"); if (of_property_read_s32(of_node, "qos_rate", &iva->iva_qos_rate)) iva->iva_qos_rate = IVA_PM_QOS_IVA_REQ; /* default */ } static int iva_ctl_probe(struct platform_device *pdev) { int ret = 0; struct device *dev = &pdev->dev; struct iva_dev_data *iva; struct resource *res_iva; #ifdef USE_NAMED_IRQ_RSC struct resource *res_mbox_irq; #endif int irq_nr = IVA_MBOX_IRQ_NOT_DEFINED; /* iva */ res_iva = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res_iva) { dev_err(dev, "failed to find memory resource for iva sfr\n"); return -ENXIO; } dev_dbg(dev, "resource 0 %p (%x..%x)\n", res_iva, (uint32_t) res_iva->start, (uint32_t) res_iva->end); /* mbox irq */ #ifdef USE_NAMED_IRQ_RSC res_mbox_irq = platform_get_resource_byname(pdev, IORESOURCE_IRQ, IVA_MBOX_IRQ_RSC_NAME); if (res_mbox_irq) irq_nr = res_mbox_irq->start; else dev_warn(dev, "not defined irq resource for iva mbox\n"); #else irq_nr = platform_get_irq(pdev, 0); #endif dev_dbg(dev, "irq nr(%d)\n", irq_nr); iva = dev_get_drvdata(dev); if (!iva) { iva = devm_kzalloc(dev, sizeof(struct iva_dev_data), GFP_KERNEL); if (!iva) { dev_err(dev, "%s() fail to alloc mem for plat data\n", __func__); return -ENOMEM; } dev_dbg(dev, "%s() success to alloc mem for iva ctrl(%p)\n", __func__, iva); iva->flags = IVA_CTL_ALLOC_LOCALLY_F; dev_set_drvdata(dev, iva); } /* fill data */ #ifndef MODULE iva->iva_clk = devm_clk_get(dev, "clk_iva"); if (IS_ERR(iva->iva_clk)) { dev_err(dev, "%s() fail to get clk_iva\n", __func__); ret = PTR_ERR(iva->iva_clk); goto err_iva_clk; } #endif iva->dev = dev; iva->misc.minor = MISC_DYNAMIC_MINOR; iva->misc.name = "iva_ctl"; iva->misc.fops = &iva_ctl_fops; iva->misc.parent = dev; mutex_init(&iva->proc_mutex); INIT_LIST_HEAD(&iva->proc_head); iva_ctl_parse_of_dt(dev, iva); if (!iva->mcu_mem_size || !iva->mcu_shmem_size) { dev_warn(dev, "%s() forced set to mcu dft value\n", __func__); iva->mcu_mem_size = VMCU_MEM_SIZE; iva->mcu_shmem_size = SHMEM_SIZE; } /* reserve the resources */ iva->mbox_irq_nr = irq_nr; iva->iva_res = devm_request_mem_region(dev, res_iva->start, resource_size(res_iva), res_iva->name ? : dev_name(dev)); if (!iva->iva_res) { dev_err(dev, "%s() fail to request resources. (%p)\n", __func__, iva->iva_res); ret = -EINVAL; goto err_req_iva_res; } iva->iva_va = devm_ioremap(dev, iva->iva_res->start, resource_size(iva->iva_res)); if (!iva->iva_va) { dev_err(dev, "%s() fail to ioremap resources.(0x%llx--0x%llx)\n", __func__, (u64) iva->iva_res->start, (u64) iva->iva_res->end); ret = -EINVAL; goto err_io_remap; } ret = iva_mem_create_ion_client(iva); if (ret) { dev_err(dev, "%s() fail to create ion client ret(%d)\n", __func__, ret); goto err_ion_client; } ret = iva_pmu_probe(iva); if (ret) { dev_err(dev, "%s() fail to probe iva pmu. ret(%d)\n", __func__, ret); goto err_pmu_prb; } ret = iva_mcu_probe(iva); if (ret) { dev_err(dev, "%s() fail to probe iva mcu. ret(%d)\n", __func__, ret); goto err_mcu_prb; } #ifdef MODULE ret = register_chrdev(IVA_CTRL_MAJOR, IVA_CTRL_NAME, &iva_ctl_fops); if (ret < 0) { dev_err(dev, "%s() fail to register. ret(%d)\n", __func__, ret); goto err_misc_reg; } #else ret = misc_register(&iva->misc); if (ret) { dev_err(dev, "%s() failed to register misc device\n", __func__); goto err_misc_reg; } #endif /* enable runtime pm */ pm_runtime_enable(dev); #ifdef CONFIG_EXYNOS_IOVMM iovmm_set_fault_handler(dev, iva_iommu_fault_handler, iva); /* automatically controlled by rpm */ ret = iovmm_activate(dev); if (ret) { dev_err(dev, "%s() fail to activate iovmm ret(%d)\n", __func__, ret); goto err_iovmm; } #endif /* set sysfs */ ret = iva_sysfs_init(dev); if (ret) { dev_err(dev, "%s() sysfs registeration is failed(%d)\n", __func__, ret); } #ifdef ENABLE_IVA_PANIC_HANDLER /* register panic handler for debugging */ iva_ctrl_register_panic_handler(iva); #endif #ifdef IVA_PM_QOS_IVA pm_qos_add_request(&iva->iva_qos_iva, IVA_PM_QOS_IVA, 0); #else iva->iva_qos_rate = IVA_PM_IVA_MIN; #endif #ifdef CONFIG_PM_SLEEP /* initialize the iva wake lock */ wake_lock_init(&iva->iva_wake_lock, WAKE_LOCK_SUSPEND, "iva_run_wlock"); #endif /* just for easy debug */ g_iva_data = iva; dev_dbg(dev, "%s() probed successfully\n", __func__); return 0; #ifdef CONFIG_EXYNOS_IOVMM err_iovmm: pm_runtime_disable(dev); #endif err_misc_reg: iva_mcu_remove(iva); err_mcu_prb: iva_pmu_remove(iva); err_pmu_prb: iva_mem_destroy_ion_client(iva); err_ion_client: devm_iounmap(dev, iva->iva_va); err_io_remap: if (iva->iva_res) { devm_release_mem_region(dev, iva->iva_res->start, resource_size(iva->iva_res)); } err_req_iva_res: if (iva->iva_clk) devm_clk_put(dev, iva->iva_clk); #ifndef MODULE err_iva_clk: #endif if (iva->flags & IVA_CTL_ALLOC_LOCALLY_F) devm_kfree(dev, iva); dev_set_drvdata(dev, NULL); return ret; } static int iva_ctl_remove(struct platform_device *pdev) { int ret = 0; struct device *dev = &pdev->dev; struct iva_dev_data *iva = platform_get_drvdata(pdev); dev_info(dev, "%s()\n", __func__); g_iva_data = NULL; #ifdef CONFIG_PM_SLEEP wake_lock_destroy(&iva->iva_wake_lock); #endif #ifdef ENABLE_IVA_PANIC_HANDLER iva_ctrl_unregister_panic_handler(iva); #endif iva_sysfs_deinit(dev); #ifdef CONFIG_EXYNOS_IOVMM iovmm_deactivate(dev); #endif pm_runtime_disable(dev); #ifdef MODULE unregister_chrdev(IVA_CTRL_MAJOR, IVA_CTRL_NAME); #else misc_deregister(&iva->misc); #endif iva_mcu_remove(iva); iva_pmu_remove(iva); iva_mem_destroy_ion_client(iva); devm_iounmap(dev, iva->iva_va); devm_release_mem_region(dev, iva->iva_res->start, resource_size(iva->iva_res)); if (iva->iva_clk) { devm_clk_put(dev, iva->iva_clk); iva->iva_clk = NULL; } if (iva->flags & IVA_CTL_ALLOC_LOCALLY_F) { devm_kfree(dev, iva); platform_set_drvdata(pdev, NULL); } return ret; } static void iva_ctl_shutdown(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct iva_dev_data *iva = platform_get_drvdata(pdev); unsigned long state_org = iva->state; #ifdef ENABLE_MCU_EXIT_AT_SHUTDOWN iva_ctrl_mcu_exit_forced(iva); #endif #ifdef ENABLE_POWER_OFF_AT_SHUTDOWN iva_ctrl_deinit_forced(iva); #endif pm_runtime_disable(dev); dev_info(dev, "%s() iva state(0x%lx -> 0x%lx)\n", __func__, state_org, iva->state); } #ifdef CONFIG_PM_SLEEP static int iva_ctl_suspend(struct device *dev) { /* to do */ dev_info(dev, "%s()\n", __func__); return 0; } static int iva_ctl_resume(struct device *dev) { /* to do */ dev_info(dev, "%s()\n", __func__); return 0; } #endif #ifdef CONFIG_PM static int iva_ctl_runtime_suspend(struct device *dev) { return __iva_ctrl_deinit(dev); } static int iva_ctl_runtime_resume(struct device *dev) { return __iva_ctrl_init(dev); } #endif static const struct dev_pm_ops iva_ctl_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(iva_ctl_suspend, iva_ctl_resume) SET_RUNTIME_PM_OPS(iva_ctl_runtime_suspend, iva_ctl_runtime_resume, NULL) }; #ifdef CONFIG_OF static const struct of_device_id iva_ctl_match[] = { { .compatible = "samsung,iva", .data = (void *)0 }, {}, }; MODULE_DEVICE_TABLE(of, iva_ctl_match); #endif static struct platform_driver iva_ctl_driver = { .probe = iva_ctl_probe, .remove = iva_ctl_remove, .shutdown = iva_ctl_shutdown, .driver = { .owner = THIS_MODULE, .name = "iva_ctl", .pm = &iva_ctl_pm_ops, .of_match_table = of_match_ptr(iva_ctl_match), }, }; #ifdef ENABLE_LOCAL_PLAT_DEVICE static struct resource iva_res[] = { { /* mcu mbox irq */ .name = IVA_MBOX_IRQ_RSC_NAME, .start = IVA_MBOX_IRQ_NOT_DEFINED, .end = IVA_MBOX_IRQ_NOT_DEFINED, .flags = IORESOURCE_IRQ, }, { .name = "iva_sfr", .start = IVA_CFG_PHY_BASE_ADDR .end = IVA_CFG_PHY_BASE_ADDR + IVA_CFG_SIZE - 1, .flags = IORESOURCE_MEM, }, }; static struct iva_dev_data iva_pdata = { .mcu_mem_size = VMCU_MEM_SIZE, .mcu_shmem_size = SHMEM_SIZE, .mcu_print_delay = 50, }; static struct platform_device *iva_ctl_pdev; #endif static int __init iva_ctrl_module_init(void) { #ifdef ENABLE_LOCAL_PLAT_DEVICE iva_ctl_pdev = platform_device_register_resndata(NULL, "iva_ctl", -1, iva_res, (unsigned int) ARRAY_SIZE(iva_res), &iva_pdata, sizeof(iva_pdata)); #endif return platform_driver_register(&iva_ctl_driver); } static void __exit iva_ctrl_module_exit(void) { platform_driver_unregister(&iva_ctl_driver); #ifdef ENABLE_LOCAL_PLAT_DEVICE platform_device_unregister(iva_ctl_pdev); #endif } #ifdef MODULE module_init(iva_ctrl_module_init); #else /* dummy ion */ device_initcall_sync(iva_ctrl_module_init); #endif module_exit(iva_ctrl_module_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("ilkwan.kim@samsung.com");