drivers/video/rockchip/vcodec/vcodec_service.c

   1 /**
   2  * Copyright (C) 2015 Fuzhou Rockchip Electronics Co., Ltd
   3  * author: chenhengming, chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *         Jung Zhao, jung.zhao@rock-chips.com
   6  *
   7  * This software is licensed under the terms of the GNU General Public
   8  * License version 2, as published by the Free Software Foundation, and
   9  * may be copied, distributed, and modified under those terms.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/clk.h>
  21 #include <linux/compat.h>
  22 #include <linux/delay.h>
  23 #include <linux/init.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/mm.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/reset.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/wakelock.h>
  33 #include <linux/cdev.h>
  34 #include <linux/of.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/regmap.h>
  38 #include <linux/mfd/syscon.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/pm_runtime.h>
  42 #include <linux/iopoll.h>
  43
  44 #include <linux/rockchip/cru.h>
  45 #include <linux/rockchip/pmu.h>
  46 #include <linux/rockchip/grf.h>
  47
  48 #include <linux/dma-buf.h>
  49 #include <linux/rockchip-iovmm.h>
  50
  51 #include "vcodec_hw_info.h"
  52 #include "vcodec_hw_vpu.h"
  53 #include "vcodec_hw_rkv.h"
  54 #include "vcodec_hw_vpu2.h"
  55
  56 #include "vcodec_service.h"
  57
  58 #include "vcodec_iommu_ops.h"
  59
  60 /*
  61  * debug flag usage:
  62  * +------+-------------------+
  63  * | 8bit |      24bit        |
  64  * +------+-------------------+
  65  *  0~23 bit is for different information type
  66  * 24~31 bit is for information print format
  67  */
  68
  69 #define DEBUG_POWER                             0x00000001
  70 #define DEBUG_CLOCK                             0x00000002
  71 #define DEBUG_IRQ_STATUS                        0x00000004
  72 #define DEBUG_IOMMU                             0x00000008
  73 #define DEBUG_IOCTL                             0x00000010
  74 #define DEBUG_FUNCTION                          0x00000020
  75 #define DEBUG_REGISTER                          0x00000040
  76 #define DEBUG_EXTRA_INFO                        0x00000080
  77 #define DEBUG_TIMING                            0x00000100
  78 #define DEBUG_TASK_INFO                         0x00000200
  79
  80 #define DEBUG_SET_REG                           0x00001000
  81 #define DEBUG_GET_REG                           0x00002000
  82 #define DEBUG_PPS_FILL                          0x00004000
  83 #define DEBUG_IRQ_CHECK                         0x00008000
  84 #define DEBUG_CACHE_32B                         0x00010000
  85
  86 #define PRINT_FUNCTION                          0x80000000
  87 #define PRINT_LINE                              0x40000000
  88
  89 #define MHZ                                     (1000 * 1000)
  90 #define SIZE_REG(reg)                           ((reg) * 4)
  91
  92 #define VCODEC_CLOCK_ENABLE     1
  93 #define EXTRA_INFO_MAGIC        0x4C4A46
  94
  95 static int debug;
  96 module_param(debug, int, S_IRUGO | S_IWUSR);
  97 MODULE_PARM_DESC(debug, "bit switch for vcodec_service debug information");
  98 /*
  99  * hardware information organization
 100  *
 101  * In order to support multiple hardware with different version the hardware
 102  * information is organized as follow:
 103  *
 104  * 1. First, index hardware by register size / position.
 105  *    These information is fix for each hardware and do not relate to runtime
 106  *    work flow. It only related to resource allocation.
 107  *    Descriptor: struct vpu_hw_info
 108  *
 109  * 2. Then, index hardware by runtime configuration
 110  *    These information is related to runtime setting behave including enable
 111  *    register, irq register and other key control flag
 112  *    Descriptor: struct vpu_task_info
 113  *
 114  * 3. Final, on iommu case the fd translation is required
 115  *    Descriptor: struct vpu_trans_info
 116  */
 117
 118 enum VPU_FREQ {
 119         VPU_FREQ_200M,
 120         VPU_FREQ_266M,
 121         VPU_FREQ_300M,
 122         VPU_FREQ_400M,
 123         VPU_FREQ_500M,
 124         VPU_FREQ_600M,
 125         VPU_FREQ_DEFAULT,
 126         VPU_FREQ_BUT,
 127 };
 128
 129 struct extra_info_elem {
 130         u32 index;
 131         u32 offset;
 132 };
 133
 134
 135 struct extra_info_for_iommu {
 136         u32 magic;
 137         u32 cnt;
 138         struct extra_info_elem elem[20];
 139 };
 140
 141 static const struct vcodec_info vcodec_info_set[] = {
 142         {
 143                 .hw_id          = VPU_ID_8270,
 144                 .hw_info        = &hw_vpu_8270,
 145                 .task_info      = task_vpu,
 146                 .trans_info     = trans_vpu,
 147         },
 148         {
 149                 .hw_id          = VPU_ID_4831,
 150                 .hw_info        = &hw_vpu_4831,
 151                 .task_info      = task_vpu,
 152                 .trans_info     = trans_vpu,
 153         },
 154         {
 155                 .hw_id          = VPU_DEC_ID_9190,
 156                 .hw_info        = &hw_vpu_9190,
 157                 .task_info      = task_vpu,
 158                 .trans_info     = trans_vpu,
 159         },
 160         {
 161                 .hw_id          = HEVC_ID,
 162                 .hw_info        = &hw_rkhevc,
 163                 .task_info      = task_rkv,
 164                 .trans_info     = trans_rkv,
 165         },
 166         {
 167                 .hw_id          = RKV_DEC_ID,
 168                 .hw_info        = &hw_rkvdec,
 169                 .task_info      = task_rkv,
 170                 .trans_info     = trans_rkv,
 171         },
 172         {
 173                 .hw_id          = VPU2_ID,
 174                 .hw_info        = &hw_vpu2,
 175                 .task_info      = task_vpu2,
 176                 .trans_info     = trans_vpu2,
 177         },
 178         {
 179                 .hw_id          = RKV_DEC_ID2,
 180                 .hw_info        = &hw_rkvdec,
 181                 .task_info      = task_rkv,
 182                 .trans_info     = trans_rkv,
 183         },
 184 };
 185
 186 /* Both VPU1 and VPU2 */
 187 static const struct vcodec_device_info vpu_device_info = {
 188         .device_type = VCODEC_DEVICE_TYPE_VPUX,
 189         .name = "vpu-service",
 190 };
 191
 192 static const struct vcodec_device_info vpu_combo_device_info = {
 193         .device_type = VCODEC_DEVICE_TYPE_VPUC,
 194         .name = "vpu-combo",
 195 };
 196
 197 static const struct vcodec_device_info hevc_device_info = {
 198         .device_type = VCODEC_DEVICE_TYPE_HEVC,
 199         .name = "hevc-service",
 200 };
 201
 202 static const struct vcodec_device_info rkvd_device_info = {
 203         .device_type = VCODEC_DEVICE_TYPE_RKVD,
 204         .name = "rkvdec",
 205 };
 206
 207 #define DEBUG
 208 #ifdef DEBUG
 209 #define vpu_debug_func(type, fmt, args...)                      \
 210         do {                                                    \
 211                 if (unlikely(debug & type)) {                   \
 212                         pr_info("%s:%d: " fmt,                  \
 213                                  __func__, __LINE__, ##args);   \
 214                 }                                               \
 215         } while (0)
 216 #define vpu_debug(type, fmt, args...)                           \
 217         do {                                                    \
 218                 if (unlikely(debug & type)) {                   \
 219                         pr_info(fmt, ##args);                   \
 220                 }                                               \
 221         } while (0)
 222 #else
 223 #define vpu_debug_func(level, fmt, args...)
 224 #define vpu_debug(level, fmt, args...)
 225 #endif
 226
 227 #define vpu_debug_enter() vpu_debug_func(DEBUG_FUNCTION, "enter\n")
 228 #define vpu_debug_leave() vpu_debug_func(DEBUG_FUNCTION, "leave\n")
 229
 230 #define vpu_err(fmt, args...)                           \
 231                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 232
 233 enum VPU_DEC_FMT {
 234         VPU_DEC_FMT_H264,
 235         VPU_DEC_FMT_MPEG4,
 236         VPU_DEC_FMT_H263,
 237         VPU_DEC_FMT_JPEG,
 238         VPU_DEC_FMT_VC1,
 239         VPU_DEC_FMT_MPEG2,
 240         VPU_DEC_FMT_MPEG1,
 241         VPU_DEC_FMT_VP6,
 242         VPU_DEC_FMT_RESERV0,
 243         VPU_DEC_FMT_VP7,
 244         VPU_DEC_FMT_VP8,
 245         VPU_DEC_FMT_AVS,
 246         VPU_DEC_FMT_RES
 247 };
 248
 249 /**
 250  * struct for process session which connect to vpu
 251  *
 252  * @author ChenHengming (2011-5-3)
 253  */
 254 struct vpu_session {
 255         enum VPU_CLIENT_TYPE type;
 256         /* a linked list of data so we can access them for debugging */
 257         struct list_head list_session;
 258         /* a linked list of register data waiting for process */
 259         struct list_head waiting;
 260         /* a linked list of register data in processing */
 261         struct list_head running;
 262         /* a linked list of register data processed */
 263         struct list_head done;
 264         wait_queue_head_t wait;
 265         pid_t pid;
 266         atomic_t task_running;
 267 };
 268
 269 /**
 270  * struct for process register set
 271  *
 272  * @author ChenHengming (2011-5-4)
 273  */
 274 struct vpu_reg {
 275         enum VPU_CLIENT_TYPE type;
 276         enum VPU_FREQ freq;
 277         struct vpu_session *session;
 278         struct vpu_subdev_data *data;
 279         struct vpu_task_info *task;
 280         const struct vpu_trans_info *trans;
 281
 282         /* link to vpu service session */
 283         struct list_head session_link;
 284         /* link to register set list */
 285         struct list_head status_link;
 286
 287         unsigned long size;
 288         struct list_head mem_region_list;
 289         u32 dec_base;
 290         u32 *reg;
 291 };
 292
 293 struct vpu_device {
 294         atomic_t irq_count_codec;
 295         atomic_t irq_count_pp;
 296         unsigned int iosize;
 297         u32 *regs;
 298 };
 299
 300 enum vcodec_device_id {
 301         VCODEC_DEVICE_ID_VPU,
 302         VCODEC_DEVICE_ID_HEVC,
 303         VCODEC_DEVICE_ID_COMBO,
 304         VCODEC_DEVICE_ID_RKVDEC,
 305         VCODEC_DEVICE_ID_BUTT
 306 };
 307
 308 enum VCODEC_RUNNING_MODE {
 309         VCODEC_RUNNING_MODE_NONE = -1,
 310         VCODEC_RUNNING_MODE_VPU,
 311         VCODEC_RUNNING_MODE_HEVC,
 312         VCODEC_RUNNING_MODE_RKVDEC
 313 };
 314
 315 struct vcodec_mem_region {
 316         struct list_head srv_lnk;
 317         struct list_head reg_lnk;
 318         struct list_head session_lnk;
 319         unsigned long iova;     /* virtual address for iommu */
 320         unsigned long len;
 321         u32 reg_idx;
 322         int hdl;
 323 };
 324
 325 enum vpu_ctx_state {
 326         MMU_ACTIVATED   = BIT(0)
 327 };
 328
 329 struct vpu_subdev_data {
 330         struct cdev cdev;
 331         dev_t dev_t;
 332         struct class *cls;
 333         struct device *child_dev;
 334
 335         int irq_enc;
 336         int irq_dec;
 337         struct vpu_service_info *pservice;
 338
 339         u32 *regs;
 340         enum VCODEC_RUNNING_MODE mode;
 341         struct list_head lnk_service;
 342
 343         struct device *dev;
 344
 345         struct vpu_device enc_dev;
 346         struct vpu_device dec_dev;
 347
 348         enum VPU_HW_ID hw_id;
 349         struct vpu_hw_info *hw_info;
 350         struct vpu_task_info *task_info;
 351         const struct vpu_trans_info *trans_info;
 352
 353         u32 reg_size;
 354         unsigned long state;
 355
 356 #ifdef CONFIG_DEBUG_FS
 357         struct dentry *debugfs_dir;
 358         struct dentry *debugfs_file_regs;
 359 #endif
 360
 361         struct device *mmu_dev;
 362         struct vcodec_iommu_info *iommu_info;
 363         struct work_struct set_work;
 364 };
 365
 366 struct vpu_service_info {
 367         struct wake_lock wake_lock;
 368         struct delayed_work power_off_work;
 369         struct wake_lock set_wake_lock;
 370         struct workqueue_struct *set_workq;
 371         ktime_t last; /* record previous power-on time */
 372         /* vpu service structure global lock */
 373         struct mutex lock;
 374         /* link to link_reg in struct vpu_reg */
 375         struct list_head waiting;
 376         /* link to link_reg in struct vpu_reg */
 377         struct list_head running;
 378         /* link to link_reg in struct vpu_reg */
 379         struct list_head done;
 380         /* link to list_session in struct vpu_session */
 381         struct list_head session;
 382         atomic_t total_running;
 383         atomic_t enabled;
 384         atomic_t power_on_cnt;
 385         atomic_t power_off_cnt;
 386         atomic_t service_on;
 387         struct mutex shutdown_lock;
 388         struct vpu_reg *reg_codec;
 389         struct vpu_reg *reg_pproc;
 390         struct vpu_reg *reg_resev;
 391         struct vpu_dec_config dec_config;
 392         struct vpu_enc_config enc_config;
 393
 394         bool auto_freq;
 395         bool bug_dec_addr;
 396         atomic_t freq_status;
 397
 398         struct clk *aclk_vcodec;
 399         struct clk *hclk_vcodec;
 400         struct clk *clk_core;
 401         struct clk *clk_cabac;
 402         struct clk *pd_video;
 403
 404 #ifdef CONFIG_RESET_CONTROLLER
 405         struct reset_control *rst_a;
 406         struct reset_control *rst_h;
 407         struct reset_control *rst_v;
 408 #endif
 409         struct device *dev;
 410
 411         u32 irq_status;
 412         atomic_t reset_request;
 413         struct list_head mem_region_list;
 414
 415         enum vcodec_device_id dev_id;
 416
 417         enum VCODEC_RUNNING_MODE curr_mode;
 418         u32 prev_mode;
 419
 420         struct delayed_work simulate_work;
 421
 422         u32 mode_bit;
 423         u32 mode_ctrl;
 424         u32 *reg_base;
 425         u32 ioaddr;
 426         struct regmap *grf;
 427         u32 *grf_base;
 428
 429         char *name;
 430
 431         u32 subcnt;
 432         struct list_head subdev_list;
 433
 434         u32 alloc_type;
 435 };
 436
 437 struct vpu_request {
 438         u32 *req;
 439         u32 size;
 440 };
 441
 442 #ifdef CONFIG_COMPAT
 443 struct compat_vpu_request {
 444         compat_uptr_t req;
 445         u32 size;
 446 };
 447 #endif
 448
 449 #define VDPU_SOFT_RESET_REG     101
 450 #define VDPU_CLEAN_CACHE_REG    516
 451 #define VEPU_CLEAN_CACHE_REG    772
 452 #define HEVC_CLEAN_CACHE_REG    260
 453
 454 #define VPU_REG_ENABLE(base, reg)       writel_relaxed(1, base + reg)
 455
 456 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 457 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 458 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 459 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 460
 461 #define VPU_POWER_OFF_DELAY             (4 * HZ) /* 4s */
 462 #define VPU_TIMEOUT_DELAY               (2 * HZ) /* 2s */
 463
 464 static void *vcodec_get_drv_data(struct platform_device *pdev);
 465
 466 static void vpu_service_power_on(struct vpu_subdev_data *data,
 467                                  struct vpu_service_info *pservice);
 468
 469 static void time_record(struct vpu_task_info *task, int is_end)
 470 {
 471         if (unlikely(debug & DEBUG_TIMING) && task)
 472                 do_gettimeofday((is_end) ? (&task->end) : (&task->start));
 473 }
 474
 475 static void time_diff(struct vpu_task_info *task)
 476 {
 477         vpu_debug(DEBUG_TIMING, "%s task: %ld ms\n", task->name,
 478                   (task->end.tv_sec  - task->start.tv_sec)  * 1000 +
 479                   (task->end.tv_usec - task->start.tv_usec) / 1000);
 480 }
 481
 482 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 483 {
 484         int bits;
 485         u32 raw = 0;
 486         struct vpu_service_info *pservice = data->pservice;
 487         struct vpu_subdev_data *subdata, *n;
 488
 489         if (pservice->subcnt < 2) {
 490                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 491                         set_bit(MMU_ACTIVATED, &data->state);
 492
 493                         if (atomic_read(&pservice->enabled)) {
 494                                 if (vcodec_iommu_attach(data->iommu_info))
 495                                         dev_err(data->dev,
 496                                                 "vcodec service attach failed\n"
 497                                                 );
 498                                 else
 499                                         BUG_ON(
 500                                                !atomic_read(&pservice->enabled)
 501                                                );
 502                         }
 503                 }
 504                 return;
 505         }
 506
 507         if (pservice->curr_mode == data->mode)
 508                 return;
 509
 510         vpu_debug(DEBUG_IOMMU, "vcodec enter mode %d\n", data->mode);
 511         list_for_each_entry_safe(subdata, n,
 512                                  &pservice->subdev_list, lnk_service) {
 513                 if (data != subdata && subdata->mmu_dev &&
 514                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 515                         clear_bit(MMU_ACTIVATED, &subdata->state);
 516                         vcodec_iommu_detach(subdata->iommu_info);
 517                 }
 518         }
 519         bits = 1 << pservice->mode_bit;
 520 #ifdef CONFIG_MFD_SYSCON
 521         if (pservice->grf) {
 522                 regmap_read(pservice->grf, pservice->mode_ctrl, &raw);
 523
 524                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 525                         regmap_write(pservice->grf, pservice->mode_ctrl,
 526                                      raw | bits | (bits << 16));
 527                 else
 528                         regmap_write(pservice->grf, pservice->mode_ctrl,
 529                                      (raw & (~bits)) | (bits << 16));
 530         } else if (pservice->grf_base) {
 531                 u32 *grf_base = pservice->grf_base;
 532
 533                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 534                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 535                         writel_relaxed(raw | bits | (bits << 16),
 536                                        grf_base + pservice->mode_ctrl / 4);
 537                 else
 538                         writel_relaxed((raw & (~bits)) | (bits << 16),
 539                                        grf_base + pservice->mode_ctrl / 4);
 540         } else {
 541                 vpu_err("no grf resource define, switch decoder failed\n");
 542                 return;
 543         }
 544 #else
 545         if (pservice->grf_base) {
 546                 u32 *grf_base = pservice->grf_base;
 547
 548                 raw = readl_relaxed(grf_base + pservice->mode_ctrl / 4);
 549                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 550                         writel_relaxed(raw | bits | (bits << 16),
 551                                        grf_base + pservice->mode_ctrl / 4);
 552                 else
 553                         writel_relaxed((raw & (~bits)) | (bits << 16),
 554                                        grf_base + pservice->mode_ctrl / 4);
 555         } else {
 556                 vpu_err("no grf resource define, switch decoder failed\n");
 557                 return;
 558         }
 559 #endif
 560         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 561                 set_bit(MMU_ACTIVATED, &data->state);
 562                 if (atomic_read(&pservice->enabled))
 563                         vcodec_iommu_attach(data->iommu_info);
 564                 else
 565                         /* FIXME BUG_ON should not be used in mass produce */
 566                         BUG_ON(!atomic_read(&pservice->enabled));
 567         }
 568
 569         pservice->prev_mode = pservice->curr_mode;
 570         pservice->curr_mode = data->mode;
 571 }
 572
 573 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 574 {
 575         /*
 576          * In case of VPU Combo, it require HW switch its running mode
 577          * before the other HW component start work. set current HW running
 578          * mode to none, can ensure HW switch to its reqired mode properly.
 579          */
 580         data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 581 }
 582
 583 static int vpu_get_clk(struct vpu_service_info *pservice)
 584 {
 585 #if VCODEC_CLOCK_ENABLE
 586         struct device *dev = pservice->dev;
 587
 588         switch (pservice->dev_id) {
 589         case VCODEC_DEVICE_ID_HEVC:
 590                 pservice->pd_video = devm_clk_get(dev, "pd_hevc");
 591                 if (IS_ERR(pservice->pd_video)) {
 592                         pservice->pd_video = NULL;
 593                         dev_info(dev, "failed on clk_get pd_hevc\n");
 594                 }
 595         case VCODEC_DEVICE_ID_COMBO:
 596         case VCODEC_DEVICE_ID_RKVDEC:
 597                 pservice->clk_cabac = devm_clk_get(dev, "clk_cabac");
 598                 if (IS_ERR(pservice->clk_cabac)) {
 599                         dev_err(dev, "failed on clk_get clk_cabac\n");
 600                         pservice->clk_cabac = NULL;
 601                 }
 602                 pservice->clk_core = devm_clk_get(dev, "clk_core");
 603                 if (IS_ERR(pservice->clk_core)) {
 604                         dev_err(dev, "failed on clk_get clk_core\n");
 605                         pservice->clk_core = NULL;
 606                         return -1;
 607                 }
 608         case VCODEC_DEVICE_ID_VPU:
 609                 pservice->aclk_vcodec = devm_clk_get(dev, "aclk_vcodec");
 610                 if (IS_ERR(pservice->aclk_vcodec)) {
 611                         dev_err(dev, "failed on clk_get aclk_vcodec\n");
 612                         pservice->aclk_vcodec = NULL;
 613                         return -1;
 614                 }
 615
 616                 pservice->hclk_vcodec = devm_clk_get(dev, "hclk_vcodec");
 617                 if (IS_ERR(pservice->hclk_vcodec)) {
 618                         dev_err(dev, "failed on clk_get hclk_vcodec\n");
 619                         pservice->hclk_vcodec = NULL;
 620                         return -1;
 621                 }
 622                 if (pservice->pd_video == NULL) {
 623                         pservice->pd_video = devm_clk_get(dev, "pd_video");
 624                         if (IS_ERR(pservice->pd_video)) {
 625                                 pservice->pd_video = NULL;
 626                                 dev_info(dev, "do not have pd_video\n");
 627                         }
 628                 }
 629                 break;
 630         default:
 631                 break;
 632         }
 633
 634         return 0;
 635 #else
 636         return 0;
 637 #endif
 638 }
 639
 640 static void _vpu_reset(struct vpu_subdev_data *data)
 641 {
 642         struct vpu_service_info *pservice = data->pservice;
 643         enum pmu_idle_req type = IDLE_REQ_VIDEO;
 644
 645         if (pservice->dev_id == VCODEC_DEVICE_ID_HEVC)
 646                 type = IDLE_REQ_HEVC;
 647
 648         dev_info(pservice->dev, "resetting...\n");
 649         WARN_ON(pservice->reg_codec != NULL);
 650         WARN_ON(pservice->reg_pproc != NULL);
 651         WARN_ON(pservice->reg_resev != NULL);
 652         pservice->reg_codec = NULL;
 653         pservice->reg_pproc = NULL;
 654         pservice->reg_resev = NULL;
 655
 656 #ifdef CONFIG_RESET_CONTROLLER
 657         dev_info(pservice->dev, "for 3288/3368...");
 658         if (of_machine_is_compatible("rockchip,rk3288"))
 659                 rockchip_pmu_idle_request(pservice->dev, true);
 660         if (pservice->rst_a && pservice->rst_h) {
 661                 dev_info(pservice->dev, "vpu reset in\n");
 662
 663                 if (pservice->rst_v)
 664                         reset_control_assert(pservice->rst_v);
 665                 reset_control_assert(pservice->rst_a);
 666                 reset_control_assert(pservice->rst_h);
 667                 udelay(5);
 668
 669                 reset_control_deassert(pservice->rst_h);
 670                 reset_control_deassert(pservice->rst_a);
 671                 if (pservice->rst_v)
 672                         reset_control_deassert(pservice->rst_v);
 673         } else if (pservice->rst_v) {
 674                 dev_info(pservice->dev, "hevc reset in\n");
 675                 reset_control_assert(pservice->rst_v);
 676                 udelay(5);
 677
 678                 reset_control_deassert(pservice->rst_v);
 679         }
 680         if (of_machine_is_compatible("rockchip,rk3288"))
 681                 rockchip_pmu_idle_request(pservice->dev, false);
 682 #endif
 683 }
 684
 685 static void vpu_reset(struct vpu_subdev_data *data)
 686 {
 687         struct vpu_service_info *pservice = data->pservice;
 688
 689         _vpu_reset(data);
 690         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 691                 clear_bit(MMU_ACTIVATED, &data->state);
 692                 if (atomic_read(&pservice->enabled)) {
 693                         /* Need to reset iommu */
 694                         vcodec_iommu_detach(data->iommu_info);
 695                 } else {
 696                         /* FIXME BUG_ON should not be used in mass produce */
 697                         BUG_ON(!atomic_read(&pservice->enabled));
 698                 }
 699         }
 700
 701         atomic_set(&pservice->reset_request, 0);
 702         dev_info(pservice->dev, "reset done\n");
 703 }
 704
 705 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg);
 706 static void vpu_service_session_clear(struct vpu_subdev_data *data,
 707                                       struct vpu_session *session)
 708 {
 709         struct vpu_reg *reg, *n;
 710
 711         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 712                 reg_deinit(data, reg);
 713         }
 714         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 715                 reg_deinit(data, reg);
 716         }
 717         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 718                 reg_deinit(data, reg);
 719         }
 720 }
 721
 722 static void vpu_service_clear(struct vpu_subdev_data *data)
 723 {
 724         struct vpu_reg *reg, *n;
 725         struct vpu_session *session, *s;
 726         struct vpu_service_info *pservice = data->pservice;
 727
 728         list_for_each_entry_safe(reg, n, &pservice->waiting, status_link) {
 729                 reg_deinit(reg->data, reg);
 730         }
 731
 732         /* wake up session wait event to prevent the timeout hw reset
 733          * during reboot procedure.
 734          */
 735         list_for_each_entry_safe(session, s,
 736                                  &pservice->session, list_session)
 737                 wake_up(&session->wait);
 738 }
 739
 740 static void vpu_service_dump(struct vpu_service_info *pservice)
 741 {
 742 }
 743
 744
 745 static void vpu_service_power_off(struct vpu_service_info *pservice)
 746 {
 747         int total_running;
 748         struct vpu_subdev_data *data = NULL, *n;
 749         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 750
 751         if (!ret)
 752                 return;
 753
 754         total_running = atomic_read(&pservice->total_running);
 755         if (total_running) {
 756                 pr_alert("alert: power off when %d task running!!\n",
 757                          total_running);
 758                 mdelay(50);
 759                 pr_alert("alert: delay 50 ms for running task\n");
 760                 vpu_service_dump(pservice);
 761         }
 762
 763         dev_dbg(pservice->dev, "power off...\n");
 764
 765         udelay(5);
 766
 767         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 768                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 769                         clear_bit(MMU_ACTIVATED, &data->state);
 770                         vcodec_iommu_detach(data->iommu_info);
 771                 }
 772         }
 773         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 774         pm_runtime_put(pservice->dev);
 775 #if VCODEC_CLOCK_ENABLE
 776                 if (pservice->pd_video)
 777                         clk_disable_unprepare(pservice->pd_video);
 778                 if (pservice->hclk_vcodec)
 779                         clk_disable_unprepare(pservice->hclk_vcodec);
 780                 if (pservice->aclk_vcodec)
 781                         clk_disable_unprepare(pservice->aclk_vcodec);
 782                 if (pservice->clk_core)
 783                         clk_disable_unprepare(pservice->clk_core);
 784                 if (pservice->clk_cabac)
 785                         clk_disable_unprepare(pservice->clk_cabac);
 786 #endif
 787
 788         atomic_add(1, &pservice->power_off_cnt);
 789         wake_unlock(&pservice->wake_lock);
 790         dev_dbg(pservice->dev, "power off done\n");
 791 }
 792
 793 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 794 {
 795         queue_delayed_work(system_wq, &pservice->power_off_work,
 796                            VPU_POWER_OFF_DELAY);
 797 }
 798
 799 static void vpu_power_off_work(struct work_struct *work_s)
 800 {
 801         struct delayed_work *dlwork = container_of(work_s,
 802                         struct delayed_work, work);
 803         struct vpu_service_info *pservice = container_of(dlwork,
 804                         struct vpu_service_info, power_off_work);
 805
 806         if (mutex_trylock(&pservice->lock)) {
 807                 vpu_service_power_off(pservice);
 808                 mutex_unlock(&pservice->lock);
 809         } else {
 810                 /* Come back later if the device is busy... */
 811                 vpu_queue_power_off_work(pservice);
 812         }
 813 }
 814
 815 static void vpu_service_power_on(struct vpu_subdev_data *data,
 816                                  struct vpu_service_info *pservice)
 817 {
 818         int ret;
 819         ktime_t now = ktime_get();
 820
 821         if (ktime_to_ns(ktime_sub(now, pservice->last)) > NSEC_PER_SEC ||
 822             atomic_read(&pservice->power_on_cnt)) {
 823                 /* NSEC_PER_SEC */
 824                 cancel_delayed_work_sync(&pservice->power_off_work);
 825                 vpu_queue_power_off_work(pservice);
 826                 pservice->last = now;
 827         }
 828         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 829         if (!ret)
 830                 return;
 831
 832         dev_dbg(pservice->dev, "power on\n");
 833
 834 #define BIT_VCODEC_CLK_SEL      (1<<10)
 835         if (of_machine_is_compatible("rockchip,rk3126"))
 836                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1)
 837                         | BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 838                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 839
 840 #if VCODEC_CLOCK_ENABLE
 841         if (pservice->aclk_vcodec)
 842                 clk_prepare_enable(pservice->aclk_vcodec);
 843         if (pservice->hclk_vcodec)
 844                 clk_prepare_enable(pservice->hclk_vcodec);
 845         if (pservice->clk_core)
 846                 clk_prepare_enable(pservice->clk_core);
 847         if (pservice->clk_cabac)
 848                 clk_prepare_enable(pservice->clk_cabac);
 849         if (pservice->pd_video)
 850                 clk_prepare_enable(pservice->pd_video);
 851 #endif
 852         pm_runtime_get_sync(pservice->dev);
 853
 854         udelay(5);
 855         atomic_add(1, &pservice->power_on_cnt);
 856         wake_lock(&pservice->wake_lock);
 857 }
 858
 859 static inline bool reg_check_interlace(struct vpu_reg *reg)
 860 {
 861         u32 type = (reg->reg[3] & (1 << 23));
 862
 863         return (type > 0);
 864 }
 865
 866 static inline enum VPU_DEC_FMT reg_check_fmt(struct vpu_reg *reg)
 867 {
 868         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] >> 28) & 0xf);
 869
 870         return type;
 871 }
 872
 873 static inline int reg_probe_width(struct vpu_reg *reg)
 874 {
 875         int width_in_mb = reg->reg[4] >> 23;
 876
 877         return width_in_mb * 16;
 878 }
 879
 880 static inline int reg_probe_hevc_y_stride(struct vpu_reg *reg)
 881 {
 882         int y_virstride = reg->reg[8];
 883
 884         return y_virstride;
 885 }
 886
 887 static int vcodec_fd_to_iova(struct vpu_subdev_data *data,
 888                 struct vpu_session *session,
 889                 struct vpu_reg *reg,
 890                 int fd)
 891 {
 892         int hdl;
 893         int ret = 0;
 894         struct vcodec_mem_region *mem_region;
 895
 896         hdl = vcodec_iommu_import(data->iommu_info, session, fd);
 897         if (hdl < 0)
 898                 return hdl;
 899
 900         mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
 901         if (mem_region == NULL) {
 902                 vpu_err("allocate memory for iommu memory region failed\n");
 903                 vcodec_iommu_free(data->iommu_info, session, hdl);
 904                 return -ENOMEM;
 905         }
 906
 907         mem_region->hdl = hdl;
 908         ret = vcodec_iommu_map_iommu(data->iommu_info, session, mem_region->hdl,
 909                                      &mem_region->iova, &mem_region->len);
 910         if (ret < 0) {
 911                 vpu_err("fd %d ion map iommu failed\n", fd);
 912                 kfree(mem_region);
 913                 vcodec_iommu_free(data->iommu_info, session, hdl);
 914
 915                 return -EFAULT;
 916         }
 917         INIT_LIST_HEAD(&mem_region->reg_lnk);
 918         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 919         return mem_region->iova;
 920 }
 921
 922 /*
 923  * NOTE: rkvdec/rkhevc put scaling list address in pps buffer hardware will read
 924  * it by pps id in video stream data.
 925  *
 926  * So we need to translate the address in iommu case. The address data is also
 927  * 10bit fd + 22bit offset mode.
 928  * Because userspace decoder do not give the pps id in the register file sets
 929  * kernel driver need to translate each scaling list address in pps buffer which
 930  * means 256 pps for H.264, 64 pps for H.265.
 931  *
 932  * In order to optimize the performance kernel driver ask userspace decoder to
 933  * set all scaling list address in pps buffer to the same one which will be used
 934  * on current decoding task. Then kernel driver can only translate the first
 935  * address then copy it all pps buffer.
 936  */
 937 static int fill_scaling_list_addr_in_pps(
 938                 struct vpu_subdev_data *data,
 939                 struct vpu_reg *reg,
 940                 char *pps,
 941                 int pps_info_count,
 942                 int pps_info_size,
 943                 int scaling_list_addr_offset)
 944 {
 945         int base = scaling_list_addr_offset;
 946         int scaling_fd = 0;
 947         u32 scaling_offset;
 948
 949         scaling_offset  = (u32)pps[base + 0];
 950         scaling_offset += (u32)pps[base + 1] << 8;
 951         scaling_offset += (u32)pps[base + 2] << 16;
 952         scaling_offset += (u32)pps[base + 3] << 24;
 953
 954         scaling_fd = scaling_offset & 0x3ff;
 955         scaling_offset = scaling_offset >> 10;
 956
 957         if (scaling_fd > 0) {
 958                 int i = 0;
 959                 u32 tmp = vcodec_fd_to_iova(data, reg->session, reg,
 960                                             scaling_fd);
 961
 962                 if (IS_ERR_VALUE(tmp))
 963                         return -1;
 964                 tmp += scaling_offset;
 965
 966                 for (i = 0; i < pps_info_count; i++, base += pps_info_size) {
 967                         pps[base + 0] = (tmp >>  0) & 0xff;
 968                         pps[base + 1] = (tmp >>  8) & 0xff;
 969                         pps[base + 2] = (tmp >> 16) & 0xff;
 970                         pps[base + 3] = (tmp >> 24) & 0xff;
 971                 }
 972         }
 973
 974         return 0;
 975 }
 976
 977 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data,
 978                                 struct vpu_session *session,
 979                                 const u8 *tbl,
 980                                 int size, struct vpu_reg *reg,
 981                                 struct extra_info_for_iommu *ext_inf)
 982 {
 983         struct vpu_service_info *pservice = data->pservice;
 984         struct vpu_task_info *task = reg->task;
 985         enum FORMAT_TYPE type;
 986         int hdl;
 987         int ret = 0;
 988         struct vcodec_mem_region *mem_region;
 989         int i;
 990         int offset = 0;
 991
 992         if (tbl == NULL || size <= 0) {
 993                 dev_err(pservice->dev, "input arguments invalidate\n");
 994                 return -EINVAL;
 995         }
 996
 997         if (task->get_fmt)
 998                 type = task->get_fmt(reg->reg);
 999         else {
1000                 dev_err(pservice->dev, "invalid task with NULL get_fmt\n");
1001                 return -EINVAL;
1002         }
1003
1004         for (i = 0; i < size; i++) {
1005                 int usr_fd = reg->reg[tbl[i]] & 0x3FF;
1006
1007                 /* if userspace do not set the fd at this register, skip */
1008                 if (usr_fd == 0)
1009                         continue;
1010
1011                 /*
1012                  * for avoiding cache sync issue, we need to map/unmap
1013                  * input buffer every time. FIX ME, if it is unnecessary
1014                  */
1015                 if (task->reg_rlc == tbl[i])
1016                         vcodec_iommu_free_fd(data->iommu_info, session, usr_fd);
1017                 /*
1018                  * special offset scale case
1019                  *
1020                  * This translation is for fd + offset translation.
1021                  * One register has 32bits. We need to transfer both buffer file
1022                  * handle and the start address offset so we packet file handle
1023                  * and offset together using below format.
1024                  *
1025                  *  0~9  bit for buffer file handle range 0 ~ 1023
1026                  * 10~31 bit for offset range 0 ~ 4M
1027                  *
1028                  * But on 4K case the offset can be larger the 4M
1029                  * So on H.264 4K vpu/vpu2 decoder we scale the offset by 16
1030                  * But MPEG4 will use the same register for colmv and it do not
1031                  * need scale.
1032                  *
1033                  * RKVdec do not have this issue.
1034                  */
1035                 if ((type == FMT_H264D || type == FMT_VP9D) &&
1036                     task->reg_dir_mv > 0 && task->reg_dir_mv == tbl[i])
1037                         offset = reg->reg[tbl[i]] >> 10 << 4;
1038                 else
1039                         offset = reg->reg[tbl[i]] >> 10;
1040
1041                 vpu_debug(DEBUG_IOMMU, "pos %3d fd %3d offset %10d i %d\n",
1042                           tbl[i], usr_fd, offset, i);
1043
1044                 hdl = vcodec_iommu_import(data->iommu_info, session, usr_fd);
1045
1046                 if (task->reg_pps > 0 && task->reg_pps == tbl[i]) {
1047                         int pps_info_offset;
1048                         int pps_info_count;
1049                         int pps_info_size;
1050                         int scaling_list_addr_offset;
1051
1052                         switch (type) {
1053                         case FMT_H264D: {
1054                                 pps_info_offset = offset;
1055                                 pps_info_count = 256;
1056                                 pps_info_size = 32;
1057                                 scaling_list_addr_offset = 23;
1058                         } break;
1059                         case FMT_H265D: {
1060                                 pps_info_offset = 0;
1061                                 pps_info_count = 64;
1062                                 pps_info_size = 80;
1063                                 scaling_list_addr_offset = 74;
1064                         } break;
1065                         default: {
1066                                 pps_info_offset = 0;
1067                                 pps_info_count = 0;
1068                                 pps_info_size = 0;
1069                                 scaling_list_addr_offset = 0;
1070                         } break;
1071                         }
1072
1073                         vpu_debug(DEBUG_PPS_FILL,
1074                                   "scaling list filling parameter:\n");
1075                         vpu_debug(DEBUG_PPS_FILL,
1076                                   "pps_info_offset %d\n", pps_info_offset);
1077                         vpu_debug(DEBUG_PPS_FILL,
1078                                   "pps_info_count  %d\n", pps_info_count);
1079                         vpu_debug(DEBUG_PPS_FILL,
1080                                   "pps_info_size   %d\n", pps_info_size);
1081                         vpu_debug(DEBUG_PPS_FILL,
1082                                   "scaling_list_addr_offset %d\n",
1083                                   scaling_list_addr_offset);
1084
1085                         if (pps_info_count) {
1086                                 u8 *pps;
1087
1088                                 pps = vcodec_iommu_map_kernel
1089                                         (data->iommu_info, session, hdl);
1090
1091                                 vpu_debug(DEBUG_PPS_FILL,
1092                                           "scaling list setting pps %p\n", pps);
1093                                 pps += pps_info_offset;
1094
1095                                 fill_scaling_list_addr_in_pps
1096                                         (data, reg, pps, pps_info_count,
1097                                          pps_info_size,
1098                                          scaling_list_addr_offset);
1099
1100                                 vcodec_iommu_unmap_kernel
1101                                         (data->iommu_info, session, hdl);
1102                         }
1103                 }
1104
1105                 mem_region = kzalloc(sizeof(*mem_region), GFP_KERNEL);
1106
1107                 if (!mem_region) {
1108                         vcodec_iommu_free(data->iommu_info, session, hdl);
1109                         return -ENOMEM;
1110                 }
1111
1112                 mem_region->hdl = hdl;
1113                 mem_region->reg_idx = tbl[i];
1114
1115                 ret = vcodec_iommu_map_iommu(data->iommu_info, session,
1116                                              mem_region->hdl, &mem_region->iova,
1117                                              &mem_region->len);
1118                 if (ret < 0) {
1119                         dev_err(pservice->dev,
1120                                 "reg %d fd %d ion map iommu failed\n",
1121                                 tbl[i], usr_fd);
1122                         kfree(mem_region);
1123                         vcodec_iommu_free(data->iommu_info, session, hdl);
1124                         return ret;
1125                 }
1126
1127                 /*
1128                  * special for vpu dec num 12: record decoded length
1129                  * hacking for decoded length
1130                  * NOTE: not a perfect fix, the fd is not recorded
1131                  */
1132                 if (task->reg_len > 0 && task->reg_len == tbl[i]) {
1133                         reg->dec_base = mem_region->iova + offset;
1134                         vpu_debug(DEBUG_REGISTER, "dec_set %08x\n",
1135                                   reg->dec_base);
1136                 }
1137
1138                 reg->reg[tbl[i]] = mem_region->iova + offset;
1139                 INIT_LIST_HEAD(&mem_region->reg_lnk);
1140                 list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
1141         }
1142
1143         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
1144                 for (i = 0; i < ext_inf->cnt; i++) {
1145                         vpu_debug(DEBUG_IOMMU, "reg[%d] + offset %d\n",
1146                                   ext_inf->elem[i].index,
1147                                   ext_inf->elem[i].offset);
1148                         reg->reg[ext_inf->elem[i].index] +=
1149                                 ext_inf->elem[i].offset;
1150                 }
1151         }
1152
1153         return 0;
1154 }
1155
1156 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
1157                                         struct vpu_session *session,
1158                                         struct vpu_reg *reg,
1159                                         struct extra_info_for_iommu *ext_inf)
1160 {
1161         struct vpu_service_info *pservice = data->pservice;
1162         enum FORMAT_TYPE type = reg->task->get_fmt(reg->reg);
1163
1164         if (type < FMT_TYPE_BUTT) {
1165                 const struct vpu_trans_info *info = &reg->trans[type];
1166                 const u8 *tbl = info->table;
1167                 int size = info->count;
1168
1169                 return vcodec_bufid_to_iova(data, session, tbl, size, reg,
1170                                             ext_inf);
1171         }
1172
1173         dev_err(pservice->dev, "found invalid format type!\n");
1174         return -EINVAL;
1175 }
1176
1177 static void get_reg_freq(struct vpu_subdev_data *data, struct vpu_reg *reg)
1178 {
1179
1180         if (!of_machine_is_compatible("rockchip,rk2928g")) {
1181                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1182                         if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1183                                 if (reg_probe_width(reg) > 3200) {
1184                                         /*raise frequency for 4k avc.*/
1185                                         reg->freq = VPU_FREQ_600M;
1186                                 }
1187                         } else {
1188                                 if (reg_check_interlace(reg))
1189                                         reg->freq = VPU_FREQ_400M;
1190                         }
1191                 }
1192                 if (data->hw_id == HEVC_ID) {
1193                         if (reg_probe_hevc_y_stride(reg) > 60000)
1194                                 reg->freq = VPU_FREQ_400M;
1195                 }
1196                 if (reg->type == VPU_PP)
1197                         reg->freq = VPU_FREQ_400M;
1198         }
1199 }
1200
1201 static struct vpu_reg *reg_init(struct vpu_subdev_data *data,
1202                                 struct vpu_session *session,
1203                                 void __user *src, u32 size)
1204 {
1205         struct vpu_service_info *pservice = data->pservice;
1206         int extra_size = 0;
1207         struct extra_info_for_iommu extra_info;
1208         struct vpu_reg *reg = kzalloc(sizeof(*reg) + data->reg_size,
1209                                       GFP_KERNEL);
1210
1211         vpu_debug_enter();
1212
1213         if (!reg) {
1214                 vpu_err("error: kzalloc failed\n");
1215                 return NULL;
1216         }
1217
1218         if (size > data->reg_size) {
1219                 extra_size = size - data->reg_size;
1220                 size = data->reg_size;
1221         }
1222         reg->session = session;
1223         reg->data = data;
1224         reg->type = session->type;
1225         reg->size = size;
1226         reg->freq = VPU_FREQ_DEFAULT;
1227         reg->task = &data->task_info[session->type];
1228         reg->trans = data->trans_info;
1229         reg->reg = (u32 *)&reg[1];
1230         INIT_LIST_HEAD(&reg->session_link);
1231         INIT_LIST_HEAD(&reg->status_link);
1232
1233         INIT_LIST_HEAD(&reg->mem_region_list);
1234
1235         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1236                 vpu_err("error: copy_from_user failed\n");
1237                 kfree(reg);
1238                 return NULL;
1239         }
1240
1241         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1242                 vpu_err("error: copy_from_user failed\n");
1243                 kfree(reg);
1244                 return NULL;
1245         }
1246
1247         if (vcodec_reg_address_translate(data, session, reg, &extra_info) < 0) {
1248                 int i = 0;
1249
1250                 vpu_err("error: translate reg address failed, dumping regs\n");
1251                 for (i = 0; i < size >> 2; i++)
1252                         dev_err(pservice->dev, "reg[%02d]: %08x\n",
1253                                 i, *((u32 *)src + i));
1254
1255                 kfree(reg);
1256                 return NULL;
1257         }
1258
1259         mutex_lock(&pservice->lock);
1260         list_add_tail(&reg->status_link, &pservice->waiting);
1261         list_add_tail(&reg->session_link, &session->waiting);
1262         mutex_unlock(&pservice->lock);
1263
1264         if (pservice->auto_freq)
1265                 get_reg_freq(data, reg);
1266
1267         vpu_debug_leave();
1268
1269         return reg;
1270 }
1271
1272 static void reg_deinit(struct vpu_subdev_data *data, struct vpu_reg *reg)
1273 {
1274         struct vpu_service_info *pservice = data->pservice;
1275         struct vcodec_mem_region *mem_region = NULL, *n;
1276
1277         list_del_init(&reg->session_link);
1278         list_del_init(&reg->status_link);
1279         if (reg == pservice->reg_codec)
1280                 pservice->reg_codec = NULL;
1281         if (reg == pservice->reg_pproc)
1282                 pservice->reg_pproc = NULL;
1283
1284         /* release memory region attach to this registers table. */
1285         list_for_each_entry_safe(mem_region, n,
1286                         &reg->mem_region_list, reg_lnk) {
1287                 vcodec_iommu_unmap_iommu(data->iommu_info, reg->session,
1288                                          mem_region->hdl);
1289                 vcodec_iommu_free(data->iommu_info, reg->session,
1290                                   mem_region->hdl);
1291                 list_del_init(&mem_region->reg_lnk);
1292                 kfree(mem_region);
1293         }
1294
1295         kfree(reg);
1296 }
1297
1298 static void reg_from_wait_to_run(struct vpu_service_info *pservice,
1299                                  struct vpu_reg *reg)
1300 {
1301         vpu_debug_enter();
1302         list_del_init(&reg->status_link);
1303         list_add_tail(&reg->status_link, &pservice->running);
1304
1305         list_del_init(&reg->session_link);
1306         list_add_tail(&reg->session_link, &reg->session->running);
1307         vpu_debug_leave();
1308 }
1309
1310 static void reg_copy_from_hw(struct vpu_reg *reg, u32 *src, u32 count)
1311 {
1312         int i;
1313         u32 *dst = reg->reg;
1314
1315         vpu_debug_enter();
1316         for (i = 0; i < count; i++, src++)
1317                 *dst++ = readl_relaxed(src);
1318
1319         dst = (u32 *)&reg->reg[0];
1320         for (i = 0; i < count; i++)
1321                 vpu_debug(DEBUG_GET_REG, "get reg[%02d] %08x\n", i, dst[i]);
1322
1323         vpu_debug_leave();
1324 }
1325
1326 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1327                                  struct vpu_reg *reg)
1328 {
1329         struct vpu_service_info *pservice = data->pservice;
1330         struct vpu_hw_info *hw_info = data->hw_info;
1331         struct vpu_task_info *task = reg->task;
1332
1333         vpu_debug_enter();
1334
1335         list_del_init(&reg->status_link);
1336         list_add_tail(&reg->status_link, &pservice->done);
1337
1338         list_del_init(&reg->session_link);
1339         list_add_tail(&reg->session_link, &reg->session->done);
1340
1341         switch (reg->type) {
1342         case VPU_ENC: {
1343                 pservice->reg_codec = NULL;
1344                 reg_copy_from_hw(reg, data->enc_dev.regs, hw_info->enc_reg_num);
1345                 reg->reg[task->reg_irq] = pservice->irq_status;
1346         } break;
1347         case VPU_DEC: {
1348                 pservice->reg_codec = NULL;
1349                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1350
1351                 /* revert hack for decoded length */
1352                 if (task->reg_len > 0) {
1353                         int reg_len = task->reg_len;
1354                         u32 dec_get = reg->reg[reg_len];
1355                         s32 dec_length = dec_get - reg->dec_base;
1356
1357                         vpu_debug(DEBUG_REGISTER,
1358                                   "dec_get %08x dec_length %d\n",
1359                                   dec_get, dec_length);
1360                         reg->reg[reg_len] = dec_length << 10;
1361                 }
1362
1363                 reg->reg[task->reg_irq] = pservice->irq_status;
1364         } break;
1365         case VPU_PP: {
1366                 pservice->reg_pproc = NULL;
1367                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1368                 writel_relaxed(0, data->dec_dev.regs + task->reg_irq);
1369         } break;
1370         case VPU_DEC_PP: {
1371                 u32 pipe_mode;
1372                 u32 *regs = data->dec_dev.regs;
1373
1374                 pservice->reg_codec = NULL;
1375                 pservice->reg_pproc = NULL;
1376
1377                 reg_copy_from_hw(reg, data->dec_dev.regs, hw_info->dec_reg_num);
1378
1379                 /* NOTE: remove pp pipeline mode flag first */
1380                 pipe_mode = readl_relaxed(regs + task->reg_pipe);
1381                 pipe_mode &= ~task->pipe_mask;
1382                 writel_relaxed(pipe_mode, regs + task->reg_pipe);
1383
1384                 /* revert hack for decoded length */
1385                 if (task->reg_len > 0) {
1386                         int reg_len = task->reg_len;
1387                         u32 dec_get = reg->reg[reg_len];
1388                         s32 dec_length = dec_get - reg->dec_base;
1389
1390                         vpu_debug(DEBUG_REGISTER,
1391                                   "dec_get %08x dec_length %d\n",
1392                                   dec_get, dec_length);
1393                         reg->reg[reg_len] = dec_length << 10;
1394                 }
1395
1396                 reg->reg[task->reg_irq] = pservice->irq_status;
1397         } break;
1398         default: {
1399                 vpu_err("error: copy reg from hw with unknown type %d\n",
1400                         reg->type);
1401         } break;
1402         }
1403         vcodec_exit_mode(data);
1404
1405         atomic_sub(1, &reg->session->task_running);
1406         atomic_sub(1, &pservice->total_running);
1407         wake_up(&reg->session->wait);
1408
1409         vpu_debug_leave();
1410 }
1411
1412 static void vpu_service_set_freq(struct vpu_service_info *pservice,
1413                                  struct vpu_reg *reg)
1414 {
1415         enum VPU_FREQ curr = atomic_read(&pservice->freq_status);
1416
1417         if (curr == reg->freq)
1418                 return;
1419
1420         atomic_set(&pservice->freq_status, reg->freq);
1421         switch (reg->freq) {
1422         case VPU_FREQ_200M: {
1423                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1424         } break;
1425         case VPU_FREQ_266M: {
1426                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1427         } break;
1428         case VPU_FREQ_300M: {
1429                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1430         } break;
1431         case VPU_FREQ_400M: {
1432                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1433         } break;
1434         case VPU_FREQ_500M: {
1435                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1436         } break;
1437         case VPU_FREQ_600M: {
1438                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1439         } break;
1440         default: {
1441                 unsigned long rate = 300*MHZ;
1442
1443                 if (of_machine_is_compatible("rockchip,rk2928g"))
1444                         rate = 400*MHZ;
1445
1446                 clk_set_rate(pservice->aclk_vcodec, rate);
1447         } break;
1448         }
1449 }
1450
1451 static void reg_copy_to_hw(struct vpu_subdev_data *data, struct vpu_reg *reg)
1452 {
1453         struct vpu_service_info *pservice = data->pservice;
1454         struct vpu_task_info *task = reg->task;
1455         struct vpu_hw_info *hw_info = data->hw_info;
1456         int i;
1457         u32 *src = (u32 *)&reg->reg[0];
1458         u32 enable_mask = task->enable_mask;
1459         u32 gating_mask = task->gating_mask;
1460         u32 reg_en = task->reg_en;
1461
1462         vpu_debug_enter();
1463
1464         atomic_add(1, &pservice->total_running);
1465         atomic_add(1, &reg->session->task_running);
1466
1467         if (pservice->auto_freq)
1468                 vpu_service_set_freq(pservice, reg);
1469
1470         vcodec_enter_mode(data);
1471
1472         switch (reg->type) {
1473         case VPU_ENC: {
1474                 u32 *dst = data->enc_dev.regs;
1475                 u32 base = 0;
1476                 u32 end  = hw_info->enc_reg_num;
1477                 /* u32 reg_gating = task->reg_gating; */
1478
1479                 pservice->reg_codec = reg;
1480
1481                 vpu_debug(DEBUG_TASK_INFO,
1482                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1483                           base, end, reg_en, enable_mask, gating_mask);
1484
1485                 VEPU_CLEAN_CACHE(dst);
1486
1487                 if (debug & DEBUG_SET_REG)
1488                         for (i = base; i < end; i++)
1489                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1490                                           i, src[i]);
1491
1492                 /*
1493                  * NOTE: encoder need to setup mode first
1494                  */
1495                 writel_relaxed(src[reg_en] & enable_mask, dst + reg_en);
1496
1497                 /* NOTE: encoder gating is not on enable register */
1498                 /* src[reg_gating] |= gating_mask; */
1499
1500                 for (i = base; i < end; i++) {
1501                         if (i != reg_en)
1502                                 writel_relaxed(src[i], dst + i);
1503                 }
1504
1505                 writel(src[reg_en], dst + reg_en);
1506                 dsb(sy);
1507
1508                 time_record(reg->task, 0);
1509         } break;
1510         case VPU_DEC: {
1511                 u32 *dst = data->dec_dev.regs;
1512                 u32 len = hw_info->dec_reg_num;
1513                 u32 base = hw_info->base_dec;
1514                 u32 end  = hw_info->end_dec;
1515
1516                 pservice->reg_codec = reg;
1517
1518                 vpu_debug(DEBUG_TASK_INFO,
1519                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1520                           base, end, reg_en, enable_mask, gating_mask);
1521
1522                 VDPU_CLEAN_CACHE(dst);
1523
1524                 /* on rkvdec set cache size to 64byte */
1525                 if (pservice->dev_id == VCODEC_DEVICE_ID_RKVDEC) {
1526                         u32 *cache_base = dst + 0x100;
1527                         u32 val = (debug & DEBUG_CACHE_32B) ? (0x3) : (0x13);
1528                         writel_relaxed(val, cache_base + 0x07);
1529                         writel_relaxed(val, cache_base + 0x17);
1530                 }
1531
1532                 if (debug & DEBUG_SET_REG)
1533                         for (i = 0; i < len; i++)
1534                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1535                                           i, src[i]);
1536                 /*
1537                  * NOTE: The end register is invalid. Do NOT write to it
1538                  *       Also the base register must be written
1539                  */
1540                 for (i = base; i < end; i++) {
1541                         if (i != reg_en)
1542                                 writel_relaxed(src[i], dst + i);
1543                 }
1544
1545                 writel(src[reg_en] | gating_mask, dst + reg_en);
1546                 dsb(sy);
1547
1548                 time_record(reg->task, 0);
1549         } break;
1550         case VPU_PP: {
1551                 u32 *dst = data->dec_dev.regs;
1552                 u32 base = hw_info->base_pp;
1553                 u32 end  = hw_info->end_pp;
1554
1555                 pservice->reg_pproc = reg;
1556
1557                 vpu_debug(DEBUG_TASK_INFO,
1558                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1559                           base, end, reg_en, enable_mask, gating_mask);
1560
1561                 if (debug & DEBUG_SET_REG)
1562                         for (i = base; i < end; i++)
1563                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1564                                           i, src[i]);
1565
1566                 for (i = base; i < end; i++) {
1567                         if (i != reg_en)
1568                                 writel_relaxed(src[i], dst + i);
1569                 }
1570
1571                 writel(src[reg_en] | gating_mask, dst + reg_en);
1572                 dsb(sy);
1573
1574                 time_record(reg->task, 0);
1575         } break;
1576         case VPU_DEC_PP: {
1577                 u32 *dst = data->dec_dev.regs;
1578                 u32 base = hw_info->base_dec_pp;
1579                 u32 end  = hw_info->end_dec_pp;
1580
1581                 pservice->reg_codec = reg;
1582                 pservice->reg_pproc = reg;
1583
1584                 vpu_debug(DEBUG_TASK_INFO,
1585                           "reg: base %3d end %d en %2d mask: en %x gate %x\n",
1586                           base, end, reg_en, enable_mask, gating_mask);
1587
1588                 /* VDPU_SOFT_RESET(dst); */
1589                 VDPU_CLEAN_CACHE(dst);
1590
1591                 if (debug & DEBUG_SET_REG)
1592                         for (i = base; i < end; i++)
1593                                 vpu_debug(DEBUG_SET_REG, "set reg[%02d] %08x\n",
1594                                           i, src[i]);
1595
1596                 for (i = base; i < end; i++) {
1597                         if (i != reg_en)
1598                                 writel_relaxed(src[i], dst + i);
1599                 }
1600
1601                 /* NOTE: dec output must be disabled */
1602
1603                 writel(src[reg_en] | gating_mask, dst + reg_en);
1604                 dsb(sy);
1605
1606                 time_record(reg->task, 0);
1607         } break;
1608         default: {
1609                 vpu_err("error: unsupport session type %d", reg->type);
1610                 atomic_sub(1, &pservice->total_running);
1611                 atomic_sub(1, &reg->session->task_running);
1612         } break;
1613         }
1614
1615         vpu_debug_leave();
1616 }
1617
1618 static void try_set_reg(struct vpu_subdev_data *data)
1619 {
1620         struct vpu_service_info *pservice = data->pservice;
1621
1622         vpu_debug_enter();
1623
1624         mutex_lock(&pservice->shutdown_lock);
1625         if (atomic_read(&pservice->service_on) == 0) {
1626                 mutex_unlock(&pservice->shutdown_lock);
1627                 return;
1628         }
1629         if (!list_empty(&pservice->waiting)) {
1630                 struct vpu_reg *reg_codec = pservice->reg_codec;
1631                 struct vpu_reg *reg_pproc = pservice->reg_pproc;
1632                 int can_set = 0;
1633                 bool change_able = (reg_codec == NULL) && (reg_pproc == NULL);
1634                 int reset_request = atomic_read(&pservice->reset_request);
1635                 struct vpu_reg *reg = list_entry(pservice->waiting.next,
1636                                 struct vpu_reg, status_link);
1637
1638                 vpu_service_power_on(data, pservice);
1639
1640                 if (change_able || !reset_request) {
1641                         switch (reg->type) {
1642                         case VPU_ENC: {
1643                                 if (change_able)
1644                                         can_set = 1;
1645                         } break;
1646                         case VPU_DEC: {
1647                                 if (reg_codec == NULL)
1648                                         can_set = 1;
1649                                 if (pservice->auto_freq && (reg_pproc != NULL))
1650                                         can_set = 0;
1651                         } break;
1652                         case VPU_PP: {
1653                                 if (reg_codec == NULL) {
1654                                         if (reg_pproc == NULL)
1655                                                 can_set = 1;
1656                                 } else {
1657                                         if ((reg_codec->type == VPU_DEC) &&
1658                                             (reg_pproc == NULL))
1659                                                 can_set = 1;
1660
1661                                         /*
1662                                          * NOTE:
1663                                          * can not charge frequency
1664                                          * when vpu is working
1665                                          */
1666                                         if (pservice->auto_freq)
1667                                                 can_set = 0;
1668                                 }
1669                         } break;
1670                         case VPU_DEC_PP: {
1671                                 if (change_able)
1672                                         can_set = 1;
1673                                 } break;
1674                         default: {
1675                                 dev_err(pservice->dev,
1676                                         "undefined reg type %d\n",
1677                                         reg->type);
1678                         } break;
1679                         }
1680                 }
1681
1682                 /* then check reset request */
1683                 if (reset_request && !change_able)
1684                         reset_request = 0;
1685
1686                 /* do reset before setting registers */
1687                 if (reset_request)
1688                         vpu_reset(data);
1689
1690                 if (can_set) {
1691                         reg_from_wait_to_run(pservice, reg);
1692                         reg_copy_to_hw(reg->data, reg);
1693                 }
1694         }
1695
1696         mutex_unlock(&pservice->shutdown_lock);
1697         vpu_debug_leave();
1698 }
1699
1700 static void vpu_set_register_work(struct work_struct *work_s)
1701 {
1702         struct vpu_subdev_data *data = container_of(work_s,
1703                                                     struct vpu_subdev_data,
1704                                                     set_work);
1705         struct vpu_service_info *pservice = data->pservice;
1706
1707         mutex_lock(&pservice->lock);
1708         try_set_reg(data);
1709         mutex_unlock(&pservice->lock);
1710 }
1711
1712 static int return_reg(struct vpu_subdev_data *data,
1713                       struct vpu_reg *reg, u32 __user *dst)
1714 {
1715         struct vpu_hw_info *hw_info = data->hw_info;
1716         size_t size = reg->size;
1717         u32 base;
1718
1719         vpu_debug_enter();
1720         switch (reg->type) {
1721         case VPU_ENC: {
1722                 base = 0;
1723         } break;
1724         case VPU_DEC: {
1725                 base = hw_info->base_dec_pp;
1726         } break;
1727         case VPU_PP: {
1728                 base = hw_info->base_pp;
1729         } break;
1730         case VPU_DEC_PP: {
1731                 base = hw_info->base_dec_pp;
1732         } break;
1733         default: {
1734                 vpu_err("error: copy reg to user with unknown type %d\n",
1735                         reg->type);
1736                 return -EFAULT;
1737         } break;
1738         }
1739
1740         if (copy_to_user(dst, &reg->reg[base], size)) {
1741                 vpu_err("error: copy_to_user failed\n");
1742                 return -EFAULT;
1743         }
1744
1745         reg_deinit(data, reg);
1746         vpu_debug_leave();
1747         return 0;
1748 }
1749
1750 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1751                               unsigned long arg)
1752 {
1753         struct vpu_subdev_data *data =
1754                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1755                              struct vpu_subdev_data, cdev);
1756         struct vpu_service_info *pservice = data->pservice;
1757         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1758
1759         vpu_debug_enter();
1760         if (NULL == session)
1761                 return -EINVAL;
1762
1763         switch (cmd) {
1764         case VPU_IOC_SET_CLIENT_TYPE: {
1765                 session->type = (enum VPU_CLIENT_TYPE)arg;
1766                 vpu_debug(DEBUG_IOCTL, "pid %d set client type %d\n",
1767                           session->pid, session->type);
1768         } break;
1769         case VPU_IOC_GET_HW_FUSE_STATUS: {
1770                 struct vpu_request req;
1771
1772                 vpu_debug(DEBUG_IOCTL, "pid %d get hw status %d\n",
1773                           session->pid, session->type);
1774                 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) {
1775                         vpu_err("error: get hw status copy_from_user failed\n");
1776                         return -EFAULT;
1777                 } else {
1778                         void *config = (session->type != VPU_ENC) ?
1779                                        ((void *)&pservice->dec_config) :
1780                                        ((void *)&pservice->enc_config);
1781                         size_t size = (session->type != VPU_ENC) ?
1782                                       (sizeof(struct vpu_dec_config)) :
1783                                       (sizeof(struct vpu_enc_config));
1784                         if (copy_to_user((void __user *)req.req,
1785                                          config, size)) {
1786                                 vpu_err("error: get hw status copy_to_user failed type %d\n",
1787                                         session->type);
1788                                 return -EFAULT;
1789                         }
1790                 }
1791         } break;
1792         case VPU_IOC_SET_REG: {
1793                 struct vpu_request req;
1794                 struct vpu_reg *reg;
1795
1796                 vpu_debug(DEBUG_IOCTL, "pid %d set reg type %d\n",
1797                           session->pid, session->type);
1798                 if (copy_from_user(&req, (void __user *)arg,
1799                                    sizeof(struct vpu_request))) {
1800                         vpu_err("error: set reg copy_from_user failed\n");
1801                         return -EFAULT;
1802                 }
1803
1804                 reg = reg_init(data, session, (void __user *)req.req, req.size);
1805                 if (NULL == reg) {
1806                         return -EFAULT;
1807                 } else {
1808                         queue_work(pservice->set_workq, &data->set_work);
1809                 }
1810         } break;
1811         case VPU_IOC_GET_REG: {
1812                 struct vpu_request req;
1813                 struct vpu_reg *reg;
1814                 int ret;
1815
1816                 vpu_debug(DEBUG_IOCTL, "pid %d get reg type %d\n",
1817                           session->pid, session->type);
1818                 if (copy_from_user(&req, (void __user *)arg,
1819                                    sizeof(struct vpu_request))) {
1820                         vpu_err("error: get reg copy_from_user failed\n");
1821                         return -EFAULT;
1822                 }
1823
1824                 ret = wait_event_timeout(session->wait,
1825                                          !list_empty(&session->done),
1826                                          VPU_TIMEOUT_DELAY);
1827
1828                 if (!list_empty(&session->done)) {
1829                         if (ret < 0)
1830                                 vpu_err("warning: pid %d wait task error ret %d\n",
1831                                         session->pid, ret);
1832                         ret = 0;
1833                 } else {
1834                         if (unlikely(ret < 0)) {
1835                                 vpu_err("error: pid %d wait task ret %d\n",
1836                                         session->pid, ret);
1837                         } else if (ret == 0) {
1838                                 vpu_err("error: pid %d wait %d task done timeout\n",
1839                                         session->pid,
1840                                         atomic_read(&session->task_running));
1841                                 ret = -ETIMEDOUT;
1842                         }
1843                 }
1844
1845                 if (ret < 0) {
1846                         int task_running = atomic_read(&session->task_running);
1847
1848                         mutex_lock(&pservice->lock);
1849                         vpu_service_dump(pservice);
1850                         if (task_running) {
1851                                 atomic_set(&session->task_running, 0);
1852                                 atomic_sub(task_running,
1853                                            &pservice->total_running);
1854                                 dev_err(pservice->dev,
1855                                         "%d task is running but not return, reset hardware...",
1856                                         task_running);
1857                                 vpu_reset(data);
1858                                 dev_err(pservice->dev, "done\n");
1859                         }
1860                         vpu_service_session_clear(data, session);
1861                         mutex_unlock(&pservice->lock);
1862
1863                         return ret;
1864                 }
1865                 mutex_lock(&pservice->lock);
1866                 reg = list_entry(session->done.next,
1867                                  struct vpu_reg, session_link);
1868                 return_reg(data, reg, (u32 __user *)req.req);
1869                 mutex_unlock(&pservice->lock);
1870         } break;
1871         case VPU_IOC_PROBE_IOMMU_STATUS: {
1872                 int iommu_enable = 1;
1873
1874                 vpu_debug(DEBUG_IOCTL, "VPU_IOC_PROBE_IOMMU_STATUS\n");
1875
1876                 if (copy_to_user((void __user *)arg,
1877                                  &iommu_enable, sizeof(int))) {
1878                         vpu_err("error: iommu status copy_to_user failed\n");
1879                         return -EFAULT;
1880                 }
1881         } break;
1882         default: {
1883                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1884         } break;
1885         }
1886         vpu_debug_leave();
1887         return 0;
1888 }
1889
1890 #ifdef CONFIG_COMPAT
1891 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1892                                      unsigned long arg)
1893 {
1894         struct vpu_subdev_data *data =
1895                 container_of(filp->f_path.dentry->d_inode->i_cdev,
1896                              struct vpu_subdev_data, cdev);
1897         struct vpu_service_info *pservice = data->pservice;
1898         struct vpu_session *session = (struct vpu_session *)filp->private_data;
1899
1900         vpu_debug_enter();
1901         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1902                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1903         if (NULL == session)
1904                 return -EINVAL;
1905
1906         switch (cmd) {
1907         case COMPAT_VPU_IOC_SET_CLIENT_TYPE: {
1908                 session->type = (enum VPU_CLIENT_TYPE)arg;
1909                 vpu_debug(DEBUG_IOCTL, "compat set client type %d\n",
1910                           session->type);
1911         } break;
1912         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS: {
1913                 struct compat_vpu_request req;
1914
1915                 vpu_debug(DEBUG_IOCTL, "compat get hw status %d\n",
1916                           session->type);
1917                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1918                                    sizeof(struct compat_vpu_request))) {
1919                         vpu_err("error: compat get hw status copy_from_user failed\n");
1920                         return -EFAULT;
1921                 } else {
1922                         void *config = (session->type != VPU_ENC) ?
1923                                        ((void *)&pservice->dec_config) :
1924                                        ((void *)&pservice->enc_config);
1925                         size_t size = (session->type != VPU_ENC) ?
1926                                       (sizeof(struct vpu_dec_config)) :
1927                                       (sizeof(struct vpu_enc_config));
1928
1929                         if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1930                                          config, size)) {
1931                                 vpu_err("error: compat get hw status copy_to_user failed type %d\n",
1932                                         session->type);
1933                                 return -EFAULT;
1934                         }
1935                 }
1936         } break;
1937         case COMPAT_VPU_IOC_SET_REG: {
1938                 struct compat_vpu_request req;
1939                 struct vpu_reg *reg;
1940
1941                 vpu_debug(DEBUG_IOCTL, "compat set reg type %d\n",
1942                           session->type);
1943                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1944                                    sizeof(struct compat_vpu_request))) {
1945                         vpu_err("compat set_reg copy_from_user failed\n");
1946                         return -EFAULT;
1947                 }
1948                 reg = reg_init(data, session,
1949                                compat_ptr((compat_uptr_t)req.req), req.size);
1950                 if (NULL == reg) {
1951                         return -EFAULT;
1952                 } else {
1953                         queue_work(pservice->set_workq, &data->set_work);
1954                 }
1955         } break;
1956         case COMPAT_VPU_IOC_GET_REG: {
1957                 struct compat_vpu_request req;
1958                 struct vpu_reg *reg;
1959                 int ret;
1960
1961                 vpu_debug(DEBUG_IOCTL, "compat get reg type %d\n",
1962                           session->type);
1963                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1964                                    sizeof(struct compat_vpu_request))) {
1965                         vpu_err("compat get reg copy_from_user failed\n");
1966                         return -EFAULT;
1967                 }
1968
1969                 ret = wait_event_timeout(session->wait,
1970                                          !list_empty(&session->done),
1971                                          VPU_TIMEOUT_DELAY);
1972
1973                 if (!list_empty(&session->done)) {
1974                         if (ret < 0)
1975                                 vpu_err("warning: pid %d wait task error ret %d\n",
1976                                         session->pid, ret);
1977                         ret = 0;
1978                 } else {
1979                         if (unlikely(ret < 0)) {
1980                                 vpu_err("error: pid %d wait task ret %d\n",
1981                                         session->pid, ret);
1982                         } else if (ret == 0) {
1983                                 vpu_err("error: pid %d wait %d task done timeout\n",
1984                                         session->pid,
1985                                         atomic_read(&session->task_running));
1986                                 ret = -ETIMEDOUT;
1987                         }
1988                 }
1989
1990                 if (ret < 0) {
1991                         int task_running = atomic_read(&session->task_running);
1992
1993                         mutex_lock(&pservice->lock);
1994                         vpu_service_dump(pservice);
1995                         if (task_running) {
1996                                 atomic_set(&session->task_running, 0);
1997                                 atomic_sub(task_running,
1998                                            &pservice->total_running);
1999                                 dev_err(pservice->dev,
2000                                         "%d task is running but not return, reset hardware...",
2001                                         task_running);
2002                                 vpu_reset(data);
2003                                 dev_err(pservice->dev, "done\n");
2004                         }
2005                         vpu_service_session_clear(data, session);
2006                         mutex_unlock(&pservice->lock);
2007                         return ret;
2008                 }
2009
2010                 mutex_lock(&pservice->lock);
2011                 reg = list_entry(session->done.next,
2012                                  struct vpu_reg, session_link);
2013                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
2014                 mutex_unlock(&pservice->lock);
2015         } break;
2016         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS: {
2017                 int iommu_enable = 1;
2018
2019                 vpu_debug(DEBUG_IOCTL, "COMPAT_VPU_IOC_PROBE_IOMMU_STATUS\n");
2020
2021                 if (copy_to_user(compat_ptr((compat_uptr_t)arg),
2022                                  &iommu_enable, sizeof(int))) {
2023                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
2024                         return -EFAULT;
2025                 }
2026         } break;
2027         default: {
2028                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
2029         } break;
2030         }
2031         vpu_debug_leave();
2032         return 0;
2033 }
2034 #endif
2035
2036 static int vpu_service_check_hw(struct vpu_subdev_data *data)
2037 {
2038         struct vpu_service_info *pservice = data->pservice;
2039         int ret = -EINVAL, i = 0;
2040         u32 hw_id = readl_relaxed(data->regs);
2041
2042         hw_id = (hw_id >> 16) & 0xFFFF;
2043         dev_info(pservice->dev, "checking hw id %x\n", hw_id);
2044         data->hw_info = NULL;
2045
2046         for (i = 0; i < ARRAY_SIZE(vcodec_info_set); i++) {
2047                 const struct vcodec_info *info = &vcodec_info_set[i];
2048
2049                 if (hw_id == info->hw_id) {
2050                         data->hw_id = info->hw_id;
2051                         data->hw_info = info->hw_info;
2052                         data->task_info = info->task_info;
2053                         data->trans_info = info->trans_info;
2054                         ret = 0;
2055                         break;
2056                 }
2057         }
2058         return ret;
2059 }
2060
2061 static int vpu_service_open(struct inode *inode, struct file *filp)
2062 {
2063         struct vpu_subdev_data *data = container_of(
2064                         inode->i_cdev, struct vpu_subdev_data, cdev);
2065         struct vpu_service_info *pservice = data->pservice;
2066         struct vpu_session *session = NULL;
2067
2068         vpu_debug_enter();
2069
2070         session = kzalloc(sizeof(*session), GFP_KERNEL);
2071         if (!session) {
2072                 vpu_err("error: unable to allocate memory for vpu_session.");
2073                 return -ENOMEM;
2074         }
2075
2076         data->iommu_info->debug_level = debug;
2077
2078         session->type   = VPU_TYPE_BUTT;
2079         session->pid    = current->pid;
2080         INIT_LIST_HEAD(&session->waiting);
2081         INIT_LIST_HEAD(&session->running);
2082         INIT_LIST_HEAD(&session->done);
2083         INIT_LIST_HEAD(&session->list_session);
2084         init_waitqueue_head(&session->wait);
2085         atomic_set(&session->task_running, 0);
2086         mutex_lock(&pservice->lock);
2087         list_add_tail(&session->list_session, &pservice->session);
2088         filp->private_data = (void *)session;
2089         mutex_unlock(&pservice->lock);
2090
2091         dev_dbg(pservice->dev, "dev opened\n");
2092         vpu_debug_leave();
2093         return nonseekable_open(inode, filp);
2094 }
2095
2096 static int vpu_service_release(struct inode *inode, struct file *filp)
2097 {
2098         struct vpu_subdev_data *data = container_of(
2099                         inode->i_cdev, struct vpu_subdev_data, cdev);
2100         struct vpu_service_info *pservice = data->pservice;
2101         int task_running;
2102         struct vpu_session *session = (struct vpu_session *)filp->private_data;
2103
2104         vpu_debug_enter();
2105         if (NULL == session)
2106                 return -EINVAL;
2107
2108         task_running = atomic_read(&session->task_running);
2109         if (task_running) {
2110                 dev_err(pservice->dev,
2111                         "error: session %d still has %d task running when closing\n",
2112                         session->pid, task_running);
2113                 msleep(50);
2114         }
2115         wake_up(&session->wait);
2116
2117         mutex_lock(&pservice->lock);
2118         /* remove this filp from the asynchronusly notified filp's */
2119         list_del_init(&session->list_session);
2120         vpu_service_session_clear(data, session);
2121         vcodec_iommu_clear(data->iommu_info, session);
2122         kfree(session);
2123         filp->private_data = NULL;
2124         mutex_unlock(&pservice->lock);
2125
2126         dev_info(pservice->dev, "closed\n");
2127         vpu_debug_leave();
2128         return 0;
2129 }
2130
2131 static const struct file_operations vpu_service_fops = {
2132         .unlocked_ioctl = vpu_service_ioctl,
2133         .open           = vpu_service_open,
2134         .release        = vpu_service_release,
2135 #ifdef CONFIG_COMPAT
2136         .compat_ioctl   = compat_vpu_service_ioctl,
2137 #endif
2138 };
2139
2140 static irqreturn_t vdpu_irq(int irq, void *dev_id);
2141 static irqreturn_t vdpu_isr(int irq, void *dev_id);
2142 static irqreturn_t vepu_irq(int irq, void *dev_id);
2143 static irqreturn_t vepu_isr(int irq, void *dev_id);
2144 static void get_hw_info(struct vpu_subdev_data *data);
2145
2146 static struct device *rockchip_get_sysmmu_dev(const char *compt)
2147 {
2148         struct device_node *dn = NULL;
2149         struct platform_device *pd = NULL;
2150         struct device *ret = NULL;
2151
2152         dn = of_find_compatible_node(NULL, NULL, compt);
2153         if (!dn) {
2154                 pr_err("can't find device node %s \r\n", compt);
2155                 return NULL;
2156         }
2157
2158         pd = of_find_device_by_node(dn);
2159         if (!pd) {
2160                 pr_err("can't find platform device in device node %s\n", compt);
2161                 return  NULL;
2162         }
2163         ret = &pd->dev;
2164
2165         return ret;
2166 }
2167
2168 #ifdef CONFIG_IOMMU_API
2169 static inline void platform_set_sysmmu(struct device *iommu,
2170                                        struct device *dev)
2171 {
2172         dev->archdata.iommu = iommu;
2173 }
2174 #else
2175 static inline void platform_set_sysmmu(struct device *iommu,
2176                                        struct device *dev)
2177 {
2178 }
2179 #endif
2180
2181 int vcodec_sysmmu_fault_hdl(struct device *dev,
2182                             enum rk_iommu_inttype itype,
2183                             unsigned long pgtable_base,
2184                             unsigned long fault_addr, unsigned int status)
2185 {
2186         struct platform_device *pdev;
2187         struct vpu_service_info *pservice;
2188         struct vpu_subdev_data *data;
2189
2190         vpu_debug_enter();
2191
2192         if (dev == NULL) {
2193                 pr_err("invalid NULL dev\n");
2194                 return 0;
2195         }
2196
2197         pdev = container_of(dev, struct platform_device, dev);
2198         if (pdev == NULL) {
2199                 pr_err("invalid NULL platform_device\n");
2200                 return 0;
2201         }
2202
2203         data = platform_get_drvdata(pdev);
2204         if (data == NULL) {
2205                 pr_err("invalid NULL vpu_subdev_data\n");
2206                 return 0;
2207         }
2208
2209         pservice = data->pservice;
2210         if (pservice == NULL) {
2211                 pr_err("invalid NULL vpu_service_info\n");
2212                 return 0;
2213         }
2214
2215         if (pservice->reg_codec) {
2216                 struct vpu_reg *reg = pservice->reg_codec;
2217                 struct vcodec_mem_region *mem, *n;
2218                 int i = 0;
2219
2220                 pr_err("vcodec, fault addr 0x%08lx\n", fault_addr);
2221                 if (!list_empty(&reg->mem_region_list)) {
2222                         list_for_each_entry_safe(mem, n, &reg->mem_region_list,
2223                                                  reg_lnk) {
2224                                 pr_err("vcodec, reg[%02u] mem region [%02d] 0x%lx %lx\n",
2225                                        mem->reg_idx, i, mem->iova, mem->len);
2226                                 i++;
2227                         }
2228                 } else {
2229                         pr_err("no memory region mapped\n");
2230                 }
2231
2232                 if (reg->data) {
2233                         struct vpu_subdev_data *data = reg->data;
2234                         u32 *base = (u32 *)data->dec_dev.regs;
2235                         u32 len = data->hw_info->dec_reg_num;
2236
2237                         pr_err("current errror register set:\n");
2238
2239                         for (i = 0; i < len; i++)
2240                                 pr_err("reg[%02d] %08x\n",
2241                                        i, readl_relaxed(base + i));
2242                 }
2243
2244                 pr_alert("vcodec, page fault occur, reset hw\n");
2245
2246                 /* reg->reg[101] = 1; */
2247                 _vpu_reset(data);
2248         }
2249
2250         return 0;
2251 }
2252
2253 static int vcodec_subdev_probe(struct platform_device *pdev,
2254                                struct vpu_service_info *pservice)
2255 {
2256         uint8_t *regs = NULL;
2257         int32_t ret = 0;
2258         uint32_t ioaddr = 0;
2259         struct resource *res = NULL;
2260         struct vpu_hw_info *hw_info = NULL;
2261         struct device *dev = &pdev->dev;
2262         struct device_node *np = pdev->dev.of_node;
2263         struct vpu_subdev_data *data = NULL;
2264         struct platform_device *sub_dev = NULL;
2265         struct device_node *sub_np = NULL;
2266         const char *name  = np->name;
2267         char mmu_dev_dts_name[40];
2268
2269         dev_info(dev, "probe device");
2270
2271         data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
2272         if (!data)
2273                 return -ENOMEM;
2274
2275         data->pservice = pservice;
2276         data->dev = dev;
2277
2278         INIT_WORK(&data->set_work, vpu_set_register_work);
2279         of_property_read_u32(np, "dev_mode", (u32 *)&data->mode);
2280
2281         if (pservice->reg_base == 0) {
2282                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2283                 data->regs = devm_ioremap_resource(dev, res);
2284                 if (IS_ERR(data->regs)) {
2285                         ret = PTR_ERR(data->regs);
2286                         goto err;
2287                 }
2288                 ioaddr = res->start;
2289         } else {
2290                 data->regs = pservice->reg_base;
2291                 ioaddr = pservice->ioaddr;
2292         }
2293
2294         sub_np = of_parse_phandle(np, "iommus", 0);
2295         if (sub_np) {
2296                 sub_dev = of_find_device_by_node(sub_np);
2297                 data->mmu_dev = &sub_dev->dev;
2298         }
2299
2300         /* Back to legacy iommu probe */
2301         if (!data->mmu_dev) {
2302                 switch (data->mode) {
2303                 case VCODEC_RUNNING_MODE_VPU:
2304                         sprintf(mmu_dev_dts_name,
2305                                 VPU_IOMMU_COMPATIBLE_NAME);
2306                         break;
2307                 case VCODEC_RUNNING_MODE_RKVDEC:
2308                         sprintf(mmu_dev_dts_name,
2309                                 VDEC_IOMMU_COMPATIBLE_NAME);
2310                         break;
2311                 case VCODEC_RUNNING_MODE_HEVC:
2312                 default:
2313                         sprintf(mmu_dev_dts_name,
2314                                 HEVC_IOMMU_COMPATIBLE_NAME);
2315                         break;
2316                 }
2317
2318                 data->mmu_dev =
2319                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2320                 if (data->mmu_dev)
2321                         platform_set_sysmmu(data->mmu_dev, dev);
2322
2323                 rockchip_iovmm_set_fault_handler
2324                         (dev, vcodec_sysmmu_fault_hdl);
2325         }
2326
2327         dev_info(dev, "vpu mmu dec %p\n", data->mmu_dev);
2328
2329         clear_bit(MMU_ACTIVATED, &data->state);
2330         vpu_service_power_on(data, pservice);
2331
2332         of_property_read_u32(np, "allocator", (u32 *)&pservice->alloc_type);
2333         data->iommu_info = vcodec_iommu_info_create(dev, data->mmu_dev,
2334                                                     pservice->alloc_type);
2335         dev_info(dev, "allocator is %s\n", pservice->alloc_type == 1 ? "drm" :
2336                 (pservice->alloc_type == 2 ? "ion" : "null"));
2337         vcodec_enter_mode(data);
2338         ret = vpu_service_check_hw(data);
2339         if (ret < 0) {
2340                 vpu_err("error: hw info check faild\n");
2341                 goto err;
2342         }
2343         vcodec_exit_mode(data);
2344
2345         hw_info = data->hw_info;
2346         regs = (u8 *)data->regs;
2347
2348         if (hw_info->dec_reg_num) {
2349                 data->dec_dev.iosize = hw_info->dec_io_size;
2350                 data->dec_dev.regs = (u32 *)(regs + hw_info->dec_offset);
2351         }
2352
2353         if (hw_info->enc_reg_num) {
2354                 data->enc_dev.iosize = hw_info->enc_io_size;
2355                 data->enc_dev.regs = (u32 *)(regs + hw_info->enc_offset);
2356         }
2357
2358         data->reg_size = max(hw_info->dec_io_size, hw_info->enc_io_size);
2359
2360         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2361         if (data->irq_enc > 0) {
2362                 ret = devm_request_threaded_irq(dev, data->irq_enc,
2363                                                 vepu_irq, vepu_isr,
2364                                                 IRQF_SHARED, dev_name(dev),
2365                                                 (void *)data);
2366                 if (ret) {
2367                         dev_err(dev, "error: can't request vepu irq %d\n",
2368                                 data->irq_enc);
2369                         goto err;
2370                 }
2371         }
2372         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2373         if (data->irq_dec > 0) {
2374                 ret = devm_request_threaded_irq(dev, data->irq_dec,
2375                                                 vdpu_irq, vdpu_isr,
2376                                                 IRQF_SHARED, dev_name(dev),
2377                                                 (void *)data);
2378                 if (ret) {
2379                         dev_err(dev, "error: can't request vdpu irq %d\n",
2380                                 data->irq_dec);
2381                         goto err;
2382                 }
2383         }
2384         atomic_set(&data->dec_dev.irq_count_codec, 0);
2385         atomic_set(&data->dec_dev.irq_count_pp, 0);
2386         atomic_set(&data->enc_dev.irq_count_codec, 0);
2387         atomic_set(&data->enc_dev.irq_count_pp, 0);
2388
2389         get_hw_info(data);
2390         pservice->auto_freq = true;
2391
2392         /* create device node */
2393         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2394         if (ret) {
2395                 dev_err(dev, "alloc dev_t failed\n");
2396                 goto err;
2397         }
2398
2399         cdev_init(&data->cdev, &vpu_service_fops);
2400
2401         data->cdev.owner = THIS_MODULE;
2402         data->cdev.ops = &vpu_service_fops;
2403
2404         ret = cdev_add(&data->cdev, data->dev_t, 1);
2405
2406         if (ret) {
2407                 dev_err(dev, "add dev_t failed\n");
2408                 goto err;
2409         }
2410
2411         data->cls = class_create(THIS_MODULE, name);
2412
2413         if (IS_ERR(data->cls)) {
2414                 ret = PTR_ERR(data->cls);
2415                 dev_err(dev, "class_create err:%d\n", ret);
2416                 goto err;
2417         }
2418
2419         data->child_dev = device_create(data->cls, dev,
2420                 data->dev_t, NULL, "%s", name);
2421
2422         platform_set_drvdata(pdev, data);
2423
2424         INIT_LIST_HEAD(&data->lnk_service);
2425         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2426
2427         return 0;
2428 err:
2429         if (data->child_dev) {
2430                 device_destroy(data->cls, data->dev_t);
2431                 cdev_del(&data->cdev);
2432                 unregister_chrdev_region(data->dev_t, 1);
2433         }
2434
2435         if (data->cls)
2436                 class_destroy(data->cls);
2437         return -1;
2438 }
2439
2440 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2441 {
2442         struct vpu_service_info *pservice = data->pservice;
2443
2444         vcodec_iommu_info_destroy(data->iommu_info);
2445         data->iommu_info = NULL;
2446
2447         mutex_lock(&pservice->lock);
2448         cancel_delayed_work_sync(&pservice->power_off_work);
2449         vpu_service_power_off(pservice);
2450         mutex_unlock(&pservice->lock);
2451
2452         device_destroy(data->cls, data->dev_t);
2453         class_destroy(data->cls);
2454         cdev_del(&data->cdev);
2455         unregister_chrdev_region(data->dev_t, 1);
2456
2457 #ifdef CONFIG_DEBUG_FS
2458         if (!IS_ERR_OR_NULL(data->debugfs_dir))
2459                 debugfs_remove_recursive(data->debugfs_dir);
2460 #endif
2461 }
2462
2463 static void vcodec_read_property(struct device_node *np,
2464                                  struct vpu_service_info *pservice)
2465 {
2466         pservice->mode_bit = 0;
2467         pservice->mode_ctrl = 0;
2468         pservice->subcnt = 0;
2469         pservice->grf_base = NULL;
2470
2471         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2472
2473         if (pservice->subcnt > 1) {
2474                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2475                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2476         }
2477 #ifdef CONFIG_MFD_SYSCON
2478         pservice->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2479         if (IS_ERR_OR_NULL(pservice->grf)) {
2480                 pservice->grf = NULL;
2481 #ifdef CONFIG_ARM
2482                 pservice->grf_base = RK_GRF_VIRT;
2483 #else
2484                 vpu_err("can't find vpu grf property\n");
2485                 return;
2486 #endif
2487         }
2488 #else
2489 #ifdef CONFIG_ARM
2490         pservice->grf_base = RK_GRF_VIRT;
2491 #else
2492         vpu_err("can't find vpu grf property\n");
2493         return;
2494 #endif
2495 #endif
2496
2497 #ifdef CONFIG_RESET_CONTROLLER
2498         pservice->rst_a = devm_reset_control_get(pservice->dev, "video_a");
2499         pservice->rst_h = devm_reset_control_get(pservice->dev, "video_h");
2500         pservice->rst_v = devm_reset_control_get(pservice->dev, "video");
2501
2502         if (IS_ERR_OR_NULL(pservice->rst_a)) {
2503                 dev_warn(pservice->dev, "No aclk reset resource define\n");
2504                 pservice->rst_a = NULL;
2505         }
2506
2507         if (IS_ERR_OR_NULL(pservice->rst_h)) {
2508                 dev_warn(pservice->dev, "No hclk reset resource define\n");
2509                 pservice->rst_h = NULL;
2510         }
2511
2512         if (IS_ERR_OR_NULL(pservice->rst_v)) {
2513                 dev_warn(pservice->dev, "No core reset resource define\n");
2514                 pservice->rst_v = NULL;
2515         }
2516 #endif
2517
2518         of_property_read_string(np, "name", (const char **)&pservice->name);
2519 }
2520
2521 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2522 {
2523         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2524         pservice->curr_mode = -1;
2525
2526         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2527         INIT_LIST_HEAD(&pservice->waiting);
2528         INIT_LIST_HEAD(&pservice->running);
2529         mutex_init(&pservice->lock);
2530         mutex_init(&pservice->shutdown_lock);
2531         atomic_set(&pservice->service_on, 1);
2532
2533         INIT_LIST_HEAD(&pservice->done);
2534         INIT_LIST_HEAD(&pservice->session);
2535         INIT_LIST_HEAD(&pservice->subdev_list);
2536
2537         pservice->reg_pproc     = NULL;
2538         atomic_set(&pservice->total_running, 0);
2539         atomic_set(&pservice->enabled,       0);
2540         atomic_set(&pservice->power_on_cnt,  0);
2541         atomic_set(&pservice->power_off_cnt, 0);
2542         atomic_set(&pservice->reset_request, 0);
2543
2544         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2545         pservice->last.tv64 = 0;
2546
2547         pservice->alloc_type = 0;
2548 }
2549
2550 static int vcodec_probe(struct platform_device *pdev)
2551 {
2552         int i;
2553         int ret = 0;
2554         struct resource *res = NULL;
2555         struct device *dev = &pdev->dev;
2556         struct device_node *np = pdev->dev.of_node;
2557         struct vpu_service_info *pservice = NULL;
2558         struct vcodec_device_info *driver_data;
2559
2560         pservice = devm_kzalloc(dev, sizeof(struct vpu_service_info),
2561                                 GFP_KERNEL);
2562         if (!pservice)
2563                 return -ENOMEM;
2564         pservice->dev = dev;
2565
2566         pservice->set_workq = create_singlethread_workqueue("vcodec");
2567         if (!pservice->set_workq) {
2568                 dev_err(dev, "failed to create workqueue\n");
2569                 return -ENOMEM;
2570         }
2571
2572         driver_data = vcodec_get_drv_data(pdev);
2573         if (!driver_data)
2574                 return -EINVAL;
2575
2576         vcodec_read_property(np, pservice);
2577         vcodec_init_drvdata(pservice);
2578
2579         /* Underscore for label, hyphens for name */
2580         switch (driver_data->device_type) {
2581         case VCODEC_DEVICE_TYPE_VPUX:
2582                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2583                 break;
2584         case VCODEC_DEVICE_TYPE_VPUC:
2585                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2586                 break;
2587         case VCODEC_DEVICE_TYPE_HEVC:
2588                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2589                 break;
2590         case VCODEC_DEVICE_TYPE_RKVD:
2591                 pservice->dev_id = VCODEC_DEVICE_ID_RKVDEC;
2592                 break;
2593         default:
2594                 dev_err(dev, "unsupported device type\n");
2595                 return -ENODEV;
2596         }
2597
2598         if (0 > vpu_get_clk(pservice))
2599                 goto err;
2600
2601         if (of_property_read_bool(np, "reg")) {
2602                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2603
2604                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2605                 if (IS_ERR(pservice->reg_base)) {
2606                         vpu_err("ioremap registers base failed\n");
2607                         ret = PTR_ERR(pservice->reg_base);
2608                         goto err;
2609                 }
2610                 pservice->ioaddr = res->start;
2611         } else {
2612                 pservice->reg_base = 0;
2613         }
2614
2615         pm_runtime_enable(dev);
2616
2617         if (of_property_read_bool(np, "subcnt")) {
2618                 struct vpu_subdev_data *data = NULL;
2619
2620                 data = devm_kzalloc(dev, sizeof(struct vpu_subdev_data),
2621                                     GFP_KERNEL);
2622                 if (!data)
2623                         return -ENOMEM;
2624
2625                 for (i = 0; i < pservice->subcnt; i++) {
2626                         struct device_node *sub_np;
2627                         struct platform_device *sub_pdev;
2628
2629                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2630                         sub_pdev = of_find_device_by_node(sub_np);
2631
2632                         vcodec_subdev_probe(sub_pdev, pservice);
2633                 }
2634                 data->pservice = pservice;
2635                 platform_set_drvdata(pdev, data);
2636         } else {
2637                 vcodec_subdev_probe(pdev, pservice);
2638         }
2639
2640         vpu_service_power_off(pservice);
2641
2642         dev_info(dev, "init success\n");
2643
2644         return 0;
2645
2646 err:
2647         dev_info(dev, "init failed\n");
2648         vpu_service_power_off(pservice);
2649         destroy_workqueue(pservice->set_workq);
2650         wake_lock_destroy(&pservice->wake_lock);
2651
2652         return ret;
2653 }
2654
2655 static int vcodec_remove(struct platform_device *pdev)
2656 {
2657         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2658
2659         vcodec_subdev_remove(data);
2660
2661         pm_runtime_disable(data->pservice->dev);
2662
2663         return 0;
2664 }
2665
2666 static void vcodec_shutdown(struct platform_device *pdev)
2667 {
2668         struct vpu_subdev_data *data = platform_get_drvdata(pdev);
2669         struct vpu_service_info *pservice = data->pservice;
2670         struct device_node *np = pdev->dev.of_node;
2671         int val;
2672         int ret;
2673         int i;
2674
2675         dev_info(&pdev->dev, "vcodec shutdown");
2676
2677         mutex_lock(&pservice->shutdown_lock);
2678         atomic_set(&pservice->service_on, 0);
2679         mutex_unlock(&pservice->shutdown_lock);
2680
2681         ret = readx_poll_timeout(atomic_read,
2682                                  &pservice->total_running,
2683                                  val, val == 0, 20000, 200000);
2684         if (ret == -ETIMEDOUT)
2685                 dev_err(&pdev->dev, "wait total running time out\n");
2686
2687         vcodec_exit_mode(data);
2688         vpu_service_clear(data);
2689         if (of_property_read_bool(np, "subcnt")) {
2690                 for (i = 0; i < pservice->subcnt; i++) {
2691                         struct device_node *sub_np;
2692                         struct platform_device *sub_pdev;
2693
2694                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2695                         sub_pdev = of_find_device_by_node(sub_np);
2696                         vcodec_subdev_remove(platform_get_drvdata(sub_pdev));
2697                 }
2698
2699         } else {
2700                 vcodec_subdev_remove(data);
2701         }
2702
2703         pm_runtime_disable(&pdev->dev);
2704 }
2705
2706 static const struct of_device_id vcodec_service_dt_ids[] = {
2707         {
2708                 .compatible = "rockchip,vpu_service",
2709                 .data = &vpu_device_info,
2710         },
2711         {
2712                 .compatible = "rockchip,hevc_service",
2713                 .data = &hevc_device_info,
2714         },
2715         {
2716                 .compatible = "rockchip,vpu_combo",
2717                 .data = &vpu_combo_device_info,
2718         },
2719         {
2720                 .compatible = "rockchip,rkvdec",
2721                 .data = &rkvd_device_info,
2722         },
2723         {},
2724 };
2725
2726 MODULE_DEVICE_TABLE(of, vcodec_service_dt_ids);
2727
2728 static void *vcodec_get_drv_data(struct platform_device *pdev)
2729 {
2730         struct vcodec_device_info *driver_data = NULL;
2731         const struct of_device_id *match;
2732
2733         match = of_match_node(vcodec_service_dt_ids, pdev->dev.of_node);
2734         if (match)
2735                 driver_data = (struct vcodec_device_info *)match->data;
2736
2737         return driver_data;
2738 }
2739
2740 static struct platform_driver vcodec_driver = {
2741         .probe = vcodec_probe,
2742         .remove = vcodec_remove,
2743         .shutdown = vcodec_shutdown,
2744         .driver = {
2745                 .name = "rk-vcodec",
2746                 .owner = THIS_MODULE,
2747                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2748         },
2749 };
2750
2751 static void get_hw_info(struct vpu_subdev_data *data)
2752 {
2753         struct vpu_service_info *pservice = data->pservice;
2754         struct vpu_dec_config *dec = &pservice->dec_config;
2755         struct vpu_enc_config *enc = &pservice->enc_config;
2756
2757         if (of_machine_is_compatible("rockchip,rk2928") ||
2758                         of_machine_is_compatible("rockchip,rk3036") ||
2759                         of_machine_is_compatible("rockchip,rk3066") ||
2760                         of_machine_is_compatible("rockchip,rk3126") ||
2761                         of_machine_is_compatible("rockchip,rk3188"))
2762                 dec->max_dec_pic_width = 1920;
2763         else
2764                 dec->max_dec_pic_width = 4096;
2765
2766         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2767                 dec->h264_support = 3;
2768                 dec->jpeg_support = 1;
2769                 dec->mpeg4_support = 2;
2770                 dec->vc1_support = 3;
2771                 dec->mpeg2_support = 1;
2772                 dec->pp_support = 1;
2773                 dec->sorenson_support = 1;
2774                 dec->ref_buf_support = 3;
2775                 dec->vp6_support = 1;
2776                 dec->vp7_support = 1;
2777                 dec->vp8_support = 1;
2778                 dec->avs_support = 1;
2779                 dec->jpeg_ext_support = 0;
2780                 dec->custom_mpeg4_support = 1;
2781                 dec->reserve = 0;
2782                 dec->mvc_support = 1;
2783
2784                 if (!of_machine_is_compatible("rockchip,rk3036")) {
2785                         u32 config_reg = readl_relaxed(data->enc_dev.regs + 63);
2786
2787                         enc->max_encoded_width = config_reg & ((1 << 11) - 1);
2788                         enc->h264_enabled = 1;
2789                         enc->mpeg4_enabled = (config_reg >> 26) & 1;
2790                         enc->jpeg_enabled = 1;
2791                         enc->vs_enabled = (config_reg >> 24) & 1;
2792                         enc->rgb_enabled = (config_reg >> 28) & 1;
2793                         enc->reg_size = data->reg_size;
2794                         enc->reserv[0] = 0;
2795                         enc->reserv[1] = 0;
2796                 }
2797
2798                 pservice->auto_freq = true;
2799                 vpu_debug(DEBUG_EXTRA_INFO,
2800                           "vpu_service set to auto frequency mode\n");
2801                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2802
2803                 pservice->bug_dec_addr = of_machine_is_compatible
2804                         ("rockchip,rk30xx");
2805         } else if (data->mode == VCODEC_RUNNING_MODE_RKVDEC) {
2806                 pservice->auto_freq = true;
2807                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2808         } else {
2809                 /* disable frequency switch in hevc.*/
2810                 pservice->auto_freq = false;
2811         }
2812 }
2813
2814 static bool check_irq_err(struct vpu_task_info *task, u32 irq_status)
2815 {
2816         vpu_debug(DEBUG_IRQ_CHECK, "task %s status %08x mask %08x\n",
2817                   task->name, irq_status, task->error_mask);
2818
2819         return (task->error_mask & irq_status) ? true : false;
2820 }
2821
2822 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2823 {
2824         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2825         struct vpu_service_info *pservice = data->pservice;
2826         struct vpu_task_info *task = NULL;
2827         struct vpu_device *dev = &data->dec_dev;
2828         u32 hw_id = data->hw_info->hw_id;
2829         u32 raw_status;
2830         u32 dec_status;
2831
2832         task = &data->task_info[TASK_DEC];
2833
2834         raw_status = readl_relaxed(dev->regs + task->reg_irq);
2835         dec_status = raw_status;
2836
2837         vpu_debug(DEBUG_TASK_INFO,
2838                   "vdpu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2839                   task->reg_irq, dec_status,
2840                   task->irq_mask, task->ready_mask, task->error_mask);
2841
2842         if (dec_status & task->irq_mask) {
2843                 time_record(task, 1);
2844                 vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq dec status %08x\n",
2845                           dec_status);
2846                 if ((dec_status & 0x40001) == 0x40001) {
2847                         do {
2848                                 dec_status =
2849                                         readl_relaxed(dev->regs +
2850                                                 task->reg_irq);
2851                         } while ((dec_status & 0x40001) == 0x40001);
2852                 }
2853
2854                 if (check_irq_err(task, dec_status))
2855                         atomic_add(1, &pservice->reset_request);
2856
2857                 writel_relaxed(0, dev->regs + task->reg_irq);
2858
2859                 /* set clock gating to save power */
2860                 writel(task->gating_mask, dev->regs + task->reg_en);
2861
2862                 atomic_add(1, &dev->irq_count_codec);
2863                 time_diff(task);
2864         }
2865
2866         task = &data->task_info[TASK_PP];
2867         if (hw_id != HEVC_ID && hw_id != RKV_DEC_ID) {
2868                 u32 pp_status = readl_relaxed(dev->regs + task->irq_mask);
2869
2870                 if (pp_status & task->irq_mask) {
2871                         time_record(task, 1);
2872                         vpu_debug(DEBUG_IRQ_STATUS, "vdpu_irq pp status %08x\n",
2873                                   pp_status);
2874
2875                         if (check_irq_err(task, dec_status))
2876                                 atomic_add(1, &pservice->reset_request);
2877
2878                         /* clear pp IRQ */
2879                         writel_relaxed(pp_status & (~task->reg_irq),
2880                                        dev->regs + task->irq_mask);
2881                         atomic_add(1, &dev->irq_count_pp);
2882                         time_diff(task);
2883                 }
2884         }
2885
2886         pservice->irq_status = raw_status;
2887
2888         if (atomic_read(&dev->irq_count_pp) ||
2889             atomic_read(&dev->irq_count_codec))
2890                 return IRQ_WAKE_THREAD;
2891         else
2892                 return IRQ_NONE;
2893 }
2894
2895 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2896 {
2897         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2898         struct vpu_service_info *pservice = data->pservice;
2899         struct vpu_device *dev = &data->dec_dev;
2900
2901         mutex_lock(&pservice->lock);
2902         if (atomic_read(&dev->irq_count_codec)) {
2903                 atomic_sub(1, &dev->irq_count_codec);
2904                 if (pservice->reg_codec == NULL) {
2905                         vpu_err("error: dec isr with no task waiting\n");
2906                 } else {
2907                         reg_from_run_to_done(data, pservice->reg_codec);
2908                         /* avoid vpu timeout and can't recover problem */
2909                         if (data->mode == VCODEC_RUNNING_MODE_VPU)
2910                                 VDPU_SOFT_RESET(data->regs);
2911                 }
2912         }
2913
2914         if (atomic_read(&dev->irq_count_pp)) {
2915                 atomic_sub(1, &dev->irq_count_pp);
2916                 if (pservice->reg_pproc == NULL)
2917                         vpu_err("error: pp isr with no task waiting\n");
2918                 else
2919                         reg_from_run_to_done(data, pservice->reg_pproc);
2920         }
2921
2922         queue_work(pservice->set_workq, &data->set_work);
2923         mutex_unlock(&pservice->lock);
2924         return IRQ_HANDLED;
2925 }
2926
2927 static irqreturn_t vepu_irq(int irq, void *dev_id)
2928 {
2929         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2930         struct vpu_service_info *pservice = data->pservice;
2931         struct vpu_task_info *task = &data->task_info[TASK_ENC];
2932         struct vpu_device *dev = &data->enc_dev;
2933         u32 irq_status;
2934
2935         irq_status = readl_relaxed(dev->regs + task->reg_irq);
2936
2937         vpu_debug(DEBUG_TASK_INFO,
2938                   "vepu_irq reg %d status %x mask: irq %x ready %x error %0x\n",
2939                   task->reg_irq, irq_status,
2940                   task->irq_mask, task->ready_mask, task->error_mask);
2941
2942         vpu_debug(DEBUG_IRQ_STATUS, "vepu_irq enc status %08x\n", irq_status);
2943
2944         if (likely(irq_status & task->irq_mask)) {
2945                 time_record(task, 1);
2946
2947                 if (check_irq_err(task, irq_status))
2948                         atomic_add(1, &pservice->reset_request);
2949
2950                 /* clear enc IRQ */
2951                 writel_relaxed(irq_status & (~task->irq_mask),
2952                                dev->regs + task->reg_irq);
2953
2954                 atomic_add(1, &dev->irq_count_codec);
2955                 time_diff(task);
2956         }
2957
2958         pservice->irq_status = irq_status;
2959
2960         if (atomic_read(&dev->irq_count_codec))
2961                 return IRQ_WAKE_THREAD;
2962         else
2963                 return IRQ_NONE;
2964 }
2965
2966 static irqreturn_t vepu_isr(int irq, void *dev_id)
2967 {
2968         struct vpu_subdev_data *data = (struct vpu_subdev_data *)dev_id;
2969         struct vpu_service_info *pservice = data->pservice;
2970         struct vpu_device *dev = &data->enc_dev;
2971
2972         mutex_lock(&pservice->lock);
2973         if (atomic_read(&dev->irq_count_codec)) {
2974                 atomic_sub(1, &dev->irq_count_codec);
2975                 if (NULL == pservice->reg_codec)
2976                         vpu_err("error: enc isr with no task waiting\n");
2977                 else
2978                         reg_from_run_to_done(data, pservice->reg_codec);
2979         }
2980         queue_work(pservice->set_workq, &data->set_work);
2981         mutex_unlock(&pservice->lock);
2982
2983         return IRQ_HANDLED;
2984 }
2985
2986 module_platform_driver(vcodec_driver);
2987 MODULE_LICENSE("GPL v2");