arch/arm/mach-rockchip/vcodec_service.c

   1 /**
   2  * Copyright (C) 2014 ROCKCHIP, Inc.
   3  * author: chenhengming chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/clk.h>
  20 #include <linux/delay.h>
  21 #include <linux/init.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/io.h>
  24 #include <linux/kernel.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/ioport.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/mm.h>
  30 #include <linux/poll.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/sched.h>
  33 #include <linux/slab.h>
  34 #include <linux/wakelock.h>
  35 #include <linux/cdev.h>
  36 #include <linux/of.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/of_irq.h>
  39 #include <linux/rockchip/cpu.h>
  40 #include <linux/rockchip/cru.h>
  41 #ifdef CONFIG_MFD_SYSCON
  42 #include <linux/regmap.h>
  43 #endif
  44 #include <linux/mfd/syscon.h>
  45
  46 #include <asm/cacheflush.h>
  47 #include <linux/uaccess.h>
  48 #include <linux/rockchip/grf.h>
  49
  50 #if defined(CONFIG_ION_ROCKCHIP)
  51 #include <linux/rockchip_ion.h>
  52 #endif
  53
  54 #if defined(CONFIG_ROCKCHIP_IOMMU) & defined(CONFIG_ION_ROCKCHIP)
  55 #define CONFIG_VCODEC_MMU
  56 #endif
  57
  58 #ifdef CONFIG_VCODEC_MMU
  59 #include <linux/rockchip-iovmm.h>
  60 #include <linux/dma-buf.h>
  61 #endif
  62
  63 #ifdef CONFIG_DEBUG_FS
  64 #include <linux/debugfs.h>
  65 #endif
  66
  67 #if defined(CONFIG_ARCH_RK319X)
  68 #include <mach/grf.h>
  69 #endif
  70
  71 #include "vcodec_service.h"
  72
  73 static int debug;
  74 module_param(debug, int, S_IRUGO | S_IWUSR);
  75 MODULE_PARM_DESC(debug,
  76                  "Debug level - higher value produces more verbose messages");
  77
  78 #define HEVC_TEST_ENABLE        0
  79 #define VCODEC_CLOCK_ENABLE     1
  80
  81 typedef enum {
  82         VPU_DEC_ID_9190         = 0x6731,
  83         VPU_ID_8270             = 0x8270,
  84         VPU_ID_4831             = 0x4831,
  85         HEVC_ID                 = 0x6867,
  86 } VPU_HW_ID;
  87
  88 enum VPU_HW_SPEC {
  89         VPU_TYPE_VPU,
  90         VPU_TYPE_HEVC,
  91         VPU_TYPE_COMBO_NOENC,
  92         VPU_TYPE_COMBO
  93 };
  94
  95 typedef enum {
  96         VPU_DEC_TYPE_9190       = 0,
  97         VPU_ENC_TYPE_8270       = 0x100,
  98         VPU_ENC_TYPE_4831       ,
  99 } VPU_HW_TYPE_E;
 100
 101 typedef enum VPU_FREQ {
 102         VPU_FREQ_200M,
 103         VPU_FREQ_266M,
 104         VPU_FREQ_300M,
 105         VPU_FREQ_400M,
 106         VPU_FREQ_500M,
 107         VPU_FREQ_600M,
 108         VPU_FREQ_DEFAULT,
 109         VPU_FREQ_BUT,
 110 } VPU_FREQ;
 111
 112 typedef struct {
 113         VPU_HW_ID               hw_id;
 114         unsigned long           hw_addr;
 115         unsigned long           enc_offset;
 116         unsigned long           enc_reg_num;
 117         unsigned long           enc_io_size;
 118         unsigned long           dec_offset;
 119         unsigned long           dec_reg_num;
 120         unsigned long           dec_io_size;
 121 } VPU_HW_INFO_E;
 122
 123 struct extra_info_elem {
 124         u32 index;
 125         u32 offset;
 126 };
 127
 128 #define EXTRA_INFO_MAGIC        0x4C4A46
 129
 130 struct extra_info_for_iommu {
 131         u32 magic;
 132         u32 cnt;
 133         struct extra_info_elem elem[20];
 134 };
 135
 136 #define VPU_SERVICE_SHOW_TIME                   0
 137
 138 #if VPU_SERVICE_SHOW_TIME
 139 static struct timeval enc_start, enc_end;
 140 static struct timeval dec_start, dec_end;
 141 static struct timeval pp_start,  pp_end;
 142 #endif
 143
 144 #define MHZ                                     (1000*1000)
 145
 146 #define REG_NUM_9190_DEC                        (60)
 147 #define REG_NUM_9190_PP                         (41)
 148 #define REG_NUM_9190_DEC_PP                     (REG_NUM_9190_DEC+REG_NUM_9190_PP)
 149
 150 #define REG_NUM_DEC_PP                          (REG_NUM_9190_DEC+REG_NUM_9190_PP)
 151
 152 #define REG_NUM_ENC_8270                        (96)
 153 #define REG_SIZE_ENC_8270                       (0x200)
 154 #define REG_NUM_ENC_4831                        (164)
 155 #define REG_SIZE_ENC_4831                       (0x400)
 156
 157 #define REG_NUM_HEVC_DEC                        (68)
 158
 159 #define SIZE_REG(reg)                           ((reg)*4)
 160
 161 static VPU_HW_INFO_E vpu_hw_set[] = {
 162         [0] = {
 163                 .hw_id          = VPU_ID_8270,
 164                 .hw_addr        = 0,
 165                 .enc_offset     = 0x0,
 166                 .enc_reg_num    = REG_NUM_ENC_8270,
 167                 .enc_io_size    = REG_NUM_ENC_8270 * 4,
 168                 .dec_offset     = REG_SIZE_ENC_8270,
 169                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 170                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 171         },
 172         [1] = {
 173                 .hw_id          = VPU_ID_4831,
 174                 .hw_addr        = 0,
 175                 .enc_offset     = 0x0,
 176                 .enc_reg_num    = REG_NUM_ENC_4831,
 177                 .enc_io_size    = REG_NUM_ENC_4831 * 4,
 178                 .dec_offset     = REG_SIZE_ENC_4831,
 179                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 180                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 181         },
 182         [2] = {
 183                 .hw_id          = HEVC_ID,
 184                 .hw_addr        = 0,
 185                 .dec_offset     = 0x0,
 186                 .dec_reg_num    = REG_NUM_HEVC_DEC,
 187                 .dec_io_size    = REG_NUM_HEVC_DEC * 4,
 188         },
 189         [3] = {
 190                 .hw_id          = VPU_DEC_ID_9190,
 191                 .hw_addr        = 0,
 192                 .enc_offset     = 0x0,
 193                 .enc_reg_num    = 0,
 194                 .enc_io_size    = 0,
 195                 .dec_offset     = 0,
 196                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 197                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 198         },
 199 };
 200
 201
 202 #define DEC_INTERRUPT_REGISTER                  1
 203 #define PP_INTERRUPT_REGISTER                   60
 204 #define ENC_INTERRUPT_REGISTER                  1
 205
 206 #define DEC_INTERRUPT_BIT                       0x100
 207 #define DEC_BUFFER_EMPTY_BIT                    0x4000
 208 #define PP_INTERRUPT_BIT                        0x100
 209 #define ENC_INTERRUPT_BIT                       0x1
 210
 211 #define HEVC_DEC_INT_RAW_BIT                    0x200
 212 #define HEVC_DEC_STR_ERROR_BIT                  0x4000
 213 #define HEVC_DEC_BUS_ERROR_BIT                  0x2000
 214 #define HEVC_DEC_BUFFER_EMPTY_BIT               0x10000
 215
 216 #define VPU_REG_EN_ENC                          14
 217 #define VPU_REG_ENC_GATE                        2
 218 #define VPU_REG_ENC_GATE_BIT                    (1<<4)
 219
 220 #define VPU_REG_EN_DEC                          1
 221 #define VPU_REG_DEC_GATE                        2
 222 #define VPU_REG_DEC_GATE_BIT                    (1<<10)
 223 #define VPU_REG_EN_PP                           0
 224 #define VPU_REG_PP_GATE                         1
 225 #define VPU_REG_PP_GATE_BIT                     (1<<8)
 226 #define VPU_REG_EN_DEC_PP                       1
 227 #define VPU_REG_DEC_PP_GATE                     61
 228 #define VPU_REG_DEC_PP_GATE_BIT                 (1<<8)
 229
 230 #define DEBUG
 231 #ifdef DEBUG
 232 #define vpu_debug(level, fmt, args...)                          \
 233         do {                                                    \
 234                 if (debug >= level)                             \
 235                         pr_info("%s:%d: " fmt,                  \
 236                                  __func__, __LINE__, ##args);   \
 237         } while (0)
 238 #else
 239 #define vpu_debug(level, fmt, args...)
 240 #endif
 241
 242 #define vpu_debug_enter() vpu_debug(4, "enter\n")
 243 #define vpu_debug_leave() vpu_debug(4, "leave\n")
 244
 245 #define vpu_err(fmt, args...)                           \
 246                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 247
 248 #if defined(CONFIG_VCODEC_MMU)
 249 static u8 addr_tbl_vpu_h264dec[] = {
 250         12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
 251         25, 26, 27, 28, 29, 40, 41
 252 };
 253
 254 static u8 addr_tbl_vpu_vp8dec[] = {
 255         10, 12, 13, 14, 18, 19, 22, 23, 24, 25, 26, 27, 28, 29, 40
 256 };
 257
 258 static u8 addr_tbl_vpu_vp6dec[] = {
 259         12, 13, 14, 18, 27, 40
 260 };
 261
 262 static u8 addr_tbl_vpu_vc1dec[] = {
 263         12, 13, 14, 15, 16, 17, 27, 41
 264 };
 265
 266 static u8 addr_tbl_vpu_jpegdec[] = {
 267         12, 40, 66, 67
 268 };
 269
 270 static u8 addr_tbl_vpu_defaultdec[] = {
 271         12, 13, 14, 15, 16, 17, 40, 41
 272 };
 273
 274 static u8 addr_tbl_vpu_enc[] = {
 275         5, 6, 7, 8, 9, 10, 11, 12, 13, 51
 276 };
 277
 278 static u8 addr_tbl_hevc_dec[] = {
 279         4, 6, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
 280         21, 22, 23, 24, 42, 43
 281 };
 282 #endif
 283
 284 enum VPU_DEC_FMT {
 285         VPU_DEC_FMT_H264,
 286         VPU_DEC_FMT_MPEG4,
 287         VPU_DEC_FMT_H263,
 288         VPU_DEC_FMT_JPEG,
 289         VPU_DEC_FMT_VC1,
 290         VPU_DEC_FMT_MPEG2,
 291         VPU_DEC_FMT_MPEG1,
 292         VPU_DEC_FMT_VP6,
 293         VPU_DEC_FMT_RV,
 294         VPU_DEC_FMT_VP7,
 295         VPU_DEC_FMT_VP8,
 296         VPU_DEC_FMT_AVS,
 297         VPU_DEC_FMT_SVC,
 298         VPU_DEC_FMT_VC2,
 299         VPU_DEC_FMT_MVC,
 300         VPU_DEC_FMT_THEORA,
 301         VPU_DEC_FMT_RES
 302 };
 303
 304 /**
 305  * struct for process session which connect to vpu
 306  *
 307  * @author ChenHengming (2011-5-3)
 308  */
 309 typedef struct vpu_session {
 310         enum VPU_CLIENT_TYPE type;
 311         /* a linked list of data so we can access them for debugging */
 312         struct list_head list_session;
 313         /* a linked list of register data waiting for process */
 314         struct list_head waiting;
 315         /* a linked list of register data in processing */
 316         struct list_head running;
 317         /* a linked list of register data processed */
 318         struct list_head done;
 319         wait_queue_head_t wait;
 320         pid_t pid;
 321         atomic_t task_running;
 322 } vpu_session;
 323
 324 /**
 325  * struct for process register set
 326  *
 327  * @author ChenHengming (2011-5-4)
 328  */
 329 typedef struct vpu_reg {
 330         enum VPU_CLIENT_TYPE type;
 331         VPU_FREQ freq;
 332         vpu_session *session;
 333         struct vpu_subdev_data *data;
 334         struct list_head session_link;          /* link to vpu service session */
 335         struct list_head status_link;           /* link to register set list */
 336         unsigned long size;
 337 #if defined(CONFIG_VCODEC_MMU)
 338         struct list_head mem_region_list;
 339 #endif
 340         u32 *reg;
 341 } vpu_reg;
 342
 343 typedef struct vpu_device {
 344         atomic_t                irq_count_codec;
 345         atomic_t                irq_count_pp;
 346         unsigned long           iobaseaddr;
 347         unsigned int            iosize;
 348         volatile u32            *hwregs;
 349 } vpu_device;
 350
 351 enum vcodec_device_id {
 352         VCODEC_DEVICE_ID_VPU,
 353         VCODEC_DEVICE_ID_HEVC,
 354         VCODEC_DEVICE_ID_COMBO
 355 };
 356
 357 enum VCODEC_RUNNING_MODE {
 358         VCODEC_RUNNING_MODE_NONE = -1,
 359         VCODEC_RUNNING_MODE_VPU,
 360         VCODEC_RUNNING_MODE_HEVC,
 361 };
 362
 363 struct vcodec_mem_region {
 364         struct list_head srv_lnk;
 365         struct list_head reg_lnk;
 366         struct list_head session_lnk;
 367         unsigned long iova;     /* virtual address for iommu */
 368         unsigned long len;
 369         u32 reg_idx;
 370         struct ion_handle *hdl;
 371 };
 372
 373 enum vpu_ctx_state {
 374         MMU_ACTIVATED   = BIT(0)
 375 };
 376
 377 struct vpu_subdev_data {
 378         struct cdev cdev;
 379         dev_t dev_t;
 380         struct class *cls;
 381         struct device *child_dev;
 382
 383         int irq_enc;
 384         int irq_dec;
 385         struct vpu_service_info *pservice;
 386
 387         u32 *regs;
 388         enum VCODEC_RUNNING_MODE mode;
 389         struct list_head lnk_service;
 390
 391         struct device *dev;
 392
 393         vpu_device enc_dev;
 394         vpu_device dec_dev;
 395         VPU_HW_INFO_E *hw_info;
 396
 397         u32 reg_size;
 398         unsigned long state;
 399
 400 #ifdef CONFIG_DEBUG_FS
 401         struct dentry *debugfs_dir;
 402         struct dentry *debugfs_file_regs;
 403 #endif
 404
 405 #if defined(CONFIG_VCODEC_MMU)
 406         struct device *mmu_dev;
 407 #endif
 408 };
 409
 410 typedef struct vpu_service_info {
 411         struct wake_lock wake_lock;
 412         struct delayed_work power_off_work;
 413         struct mutex lock;
 414         struct list_head waiting;               /* link to link_reg in struct vpu_reg */
 415         struct list_head running;               /* link to link_reg in struct vpu_reg */
 416         struct list_head done;                  /* link to link_reg in struct vpu_reg */
 417         struct list_head session;               /* link to list_session in struct vpu_session */
 418         atomic_t total_running;
 419         bool enabled;
 420         vpu_reg *reg_codec;
 421         vpu_reg *reg_pproc;
 422         vpu_reg *reg_resev;
 423         struct vpu_dec_config dec_config;
 424         struct vpu_enc_config enc_config;
 425
 426         bool auto_freq;
 427         bool bug_dec_addr;
 428         atomic_t freq_status;
 429
 430         struct clk *aclk_vcodec;
 431         struct clk *hclk_vcodec;
 432         struct clk *clk_core;
 433         struct clk *clk_cabac;
 434         struct clk *pd_video;
 435
 436         struct device *dev;
 437
 438         u32 irq_status;
 439 #if defined(CONFIG_VCODEC_MMU)
 440         struct ion_client *ion_client;
 441         struct list_head mem_region_list;
 442 #endif
 443
 444         enum vcodec_device_id dev_id;
 445
 446         enum VCODEC_RUNNING_MODE curr_mode;
 447         u32 prev_mode;
 448
 449         struct delayed_work simulate_work;
 450
 451         u32 mode_bit;
 452         u32 mode_ctrl;
 453         u32 *reg_base;
 454         u32 ioaddr;
 455 #ifdef CONFIG_MFD_SYSCON
 456         struct regmap *grf_base;
 457 #else
 458         u32 *grf_base;
 459 #endif
 460         char *name;
 461
 462         u32 subcnt;
 463         struct list_head subdev_list;
 464 } vpu_service_info;
 465
 466 struct vcodec_combo {
 467         struct vpu_service_info *vpu_srv;
 468         struct vpu_service_info *hevc_srv;
 469         struct list_head waiting;
 470         struct list_head running;
 471         struct mutex run_lock;
 472         vpu_reg *reg_codec;
 473         enum vcodec_device_id current_hw_mode;
 474 };
 475
 476 typedef struct vpu_request {
 477         u64 req;
 478         u32 size;
 479 } vpu_request;
 480
 481 /* debugfs root directory for all device (vpu, hevc).*/
 482 static struct dentry *parent;
 483
 484 #ifdef CONFIG_DEBUG_FS
 485 static int vcodec_debugfs_init(void);
 486 static void vcodec_debugfs_exit(void);
 487 static struct dentry* vcodec_debugfs_create_device_dir(char *dirname, struct dentry *parent);
 488 static int debug_vcodec_open(struct inode *inode, struct file *file);
 489
 490 static const struct file_operations debug_vcodec_fops = {
 491         .open = debug_vcodec_open,
 492         .read = seq_read,
 493         .llseek = seq_lseek,
 494         .release = single_release,
 495 };
 496 #endif
 497
 498 #define VDPU_SOFT_RESET_REG     101
 499 #define VDPU_CLEAN_CACHE_REG    516
 500 #define VEPU_CLEAN_CACHE_REG    772
 501 #define HEVC_CLEAN_CACHE_REG    260
 502
 503 #define VPU_REG_ENABLE(base, reg)       do { \
 504                                                 base[reg] = 1; \
 505                                         } while (0)
 506
 507 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 508 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 509 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 510 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 511
 512 #define VPU_POWER_OFF_DELAY             4*HZ /* 4s */
 513 #define VPU_TIMEOUT_DELAY               2*HZ /* 2s */
 514
 515 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 516 {
 517         int bits;
 518         u32 raw = 0;
 519         struct vpu_service_info *pservice = data->pservice;
 520         struct vpu_subdev_data *subdata, *n;
 521         if (pservice->subcnt < 2) {
 522 #if defined(CONFIG_VCODEC_MMU)
 523                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 524                         set_bit(MMU_ACTIVATED, &data->state);
 525                         BUG_ON(!pservice->enabled);
 526                         if (pservice->enabled)
 527                                 rockchip_iovmm_activate(data->dev);
 528                 }
 529 #endif
 530                 return;
 531         }
 532
 533         if (pservice->curr_mode == data->mode)
 534                 return;
 535
 536         vpu_debug(3, "vcodec enter mode %d\n", data->mode);
 537 #if defined(CONFIG_VCODEC_MMU)
 538         list_for_each_entry_safe(subdata, n, &pservice->subdev_list, lnk_service) {
 539                 if (data != subdata && subdata->mmu_dev &&
 540                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 541                         clear_bit(MMU_ACTIVATED, &subdata->state);
 542                         rockchip_iovmm_deactivate(subdata->dev);
 543                 }
 544         }
 545 #endif
 546         bits = 1 << pservice->mode_bit;
 547 #ifdef CONFIG_MFD_SYSCON
 548         regmap_read(pservice->grf_base, pservice->mode_ctrl, &raw);
 549
 550         if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 551                 regmap_write(pservice->grf_base, pservice->mode_ctrl,
 552                         raw | bits | (bits << 16));
 553         else
 554                 regmap_write(pservice->grf_base, pservice->mode_ctrl,
 555                         (raw & (~bits)) | (bits << 16));
 556 #else
 557         raw = readl_relaxed(pservice->grf_base + pservice->mode_ctrl / 4);
 558         if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 559                 writel_relaxed(raw | bits | (bits << 16),
 560                         pservice->grf_base + pservice->mode_ctrl / 4);
 561         else
 562                 writel_relaxed((raw & (~bits)) | (bits << 16),
 563                         pservice->grf_base + pservice->mode_ctrl / 4);
 564 #endif
 565 #if defined(CONFIG_VCODEC_MMU)
 566         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 567                 set_bit(MMU_ACTIVATED, &data->state);
 568                 BUG_ON(!pservice->enabled);
 569                 if (pservice->enabled)
 570                         rockchip_iovmm_activate(data->dev);
 571         }
 572 #endif
 573         pservice->prev_mode = pservice->curr_mode;
 574         pservice->curr_mode = data->mode;
 575 }
 576
 577 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 578 {
 579         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 580                 clear_bit(MMU_ACTIVATED, &data->state);
 581                 rockchip_iovmm_deactivate(data->dev);
 582                 data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 583         }
 584 }
 585
 586 static int vpu_get_clk(struct vpu_service_info *pservice)
 587 {
 588 #if VCODEC_CLOCK_ENABLE
 589         switch (pservice->dev_id) {
 590         case VCODEC_DEVICE_ID_HEVC:
 591                 pservice->pd_video = devm_clk_get(pservice->dev, "pd_hevc");
 592                 if (IS_ERR(pservice->pd_video)) {
 593                         dev_err(pservice->dev, "failed on clk_get pd_hevc\n");
 594                         return -1;
 595                 }
 596         case VCODEC_DEVICE_ID_COMBO:
 597                 pservice->clk_cabac = devm_clk_get(pservice->dev, "clk_cabac");
 598                 if (IS_ERR(pservice->clk_cabac)) {
 599                         dev_err(pservice->dev, "failed on clk_get clk_cabac\n");
 600                         pservice->clk_cabac = NULL;
 601                 }
 602                 pservice->clk_core = devm_clk_get(pservice->dev, "clk_core");
 603                 if (IS_ERR(pservice->clk_core)) {
 604                         dev_err(pservice->dev, "failed on clk_get clk_core\n");
 605                         return -1;
 606                 }
 607         case VCODEC_DEVICE_ID_VPU:
 608                 pservice->aclk_vcodec = devm_clk_get(pservice->dev, "aclk_vcodec");
 609                 if (IS_ERR(pservice->aclk_vcodec)) {
 610                         dev_err(pservice->dev, "failed on clk_get aclk_vcodec\n");
 611                         return -1;
 612                 }
 613
 614                 pservice->hclk_vcodec = devm_clk_get(pservice->dev, "hclk_vcodec");
 615                 if (IS_ERR(pservice->hclk_vcodec)) {
 616                         dev_err(pservice->dev, "failed on clk_get hclk_vcodec\n");
 617                         return -1;
 618                 }
 619                 if (pservice->pd_video == NULL) {
 620                         pservice->pd_video = devm_clk_get(pservice->dev, "pd_video");
 621                         if (IS_ERR(pservice->pd_video))
 622                                 pservice->pd_video = NULL;
 623                 }
 624                 break;
 625         default:
 626                 ;
 627         }
 628
 629         return 0;
 630 #else
 631         return 0;
 632 #endif
 633 }
 634
 635 static void vpu_put_clk(struct vpu_service_info *pservice)
 636 {
 637 #if VCODEC_CLOCK_ENABLE
 638         if (pservice->pd_video)
 639                 devm_clk_put(pservice->dev, pservice->pd_video);
 640         if (pservice->aclk_vcodec)
 641                 devm_clk_put(pservice->dev, pservice->aclk_vcodec);
 642         if (pservice->hclk_vcodec)
 643                 devm_clk_put(pservice->dev, pservice->hclk_vcodec);
 644         if (pservice->clk_core)
 645                 devm_clk_put(pservice->dev, pservice->clk_core);
 646         if (pservice->clk_cabac)
 647                 devm_clk_put(pservice->dev, pservice->clk_cabac);
 648 #endif
 649 }
 650
 651 static void vpu_reset(struct vpu_subdev_data *data)
 652 {
 653         struct vpu_service_info *pservice = data->pservice;
 654 #if defined(CONFIG_ARCH_RK29)
 655         clk_disable(aclk_ddr_vepu);
 656         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, true);
 657         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, true);
 658         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, true);
 659         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, true);
 660         mdelay(10);
 661         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, false);
 662         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, false);
 663         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, false);
 664         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, false);
 665         clk_enable(aclk_ddr_vepu);
 666 #elif defined(CONFIG_ARCH_RK30)
 667         pmu_set_idle_request(IDLE_REQ_VIDEO, true);
 668         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, true);
 669         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, true);
 670         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, true);
 671         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, true);
 672         mdelay(1);
 673         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, false);
 674         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, false);
 675         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, false);
 676         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, false);
 677         pmu_set_idle_request(IDLE_REQ_VIDEO, false);
 678 #endif
 679         pservice->reg_codec = NULL;
 680         pservice->reg_pproc = NULL;
 681         pservice->reg_resev = NULL;
 682
 683 #if defined(CONFIG_VCODEC_MMU)
 684         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 685                 clear_bit(MMU_ACTIVATED, &data->state);
 686                 BUG_ON(!pservice->enabled);
 687                 if (pservice->enabled)
 688                         rockchip_iovmm_deactivate(data->dev);
 689         }
 690 #endif
 691 }
 692
 693 static void reg_deinit(struct vpu_subdev_data *data, vpu_reg *reg);
 694 static void vpu_service_session_clear(struct vpu_subdev_data *data, vpu_session *session)
 695 {
 696         vpu_reg *reg, *n;
 697         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 698                 reg_deinit(data, reg);
 699         }
 700         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 701                 reg_deinit(data, reg);
 702         }
 703         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 704                 reg_deinit(data, reg);
 705         }
 706 }
 707
 708 static void vpu_service_dump(struct vpu_service_info *pservice)
 709 {
 710 }
 711
 712 static void vpu_service_power_off(struct vpu_service_info *pservice)
 713 {
 714         int total_running;
 715         struct vpu_subdev_data *data = NULL, *n;
 716         if (!pservice->enabled)
 717                 return;
 718
 719         pservice->enabled = false;
 720         total_running = atomic_read(&pservice->total_running);
 721         if (total_running) {
 722                 pr_alert("alert: power off when %d task running!!\n", total_running);
 723                 mdelay(50);
 724                 pr_alert("alert: delay 50 ms for running task\n");
 725                 vpu_service_dump(pservice);
 726         }
 727
 728         pr_info("%s: power off...", dev_name(pservice->dev));
 729         udelay(10);
 730 #if defined(CONFIG_VCODEC_MMU)
 731         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 732                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 733                         clear_bit(MMU_ACTIVATED, &data->state);
 734                         rockchip_iovmm_deactivate(data->dev);
 735                 }
 736         }
 737         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 738 #endif
 739
 740 #if VCODEC_CLOCK_ENABLE
 741         if (pservice->pd_video)
 742                 clk_disable_unprepare(pservice->pd_video);
 743         if (pservice->hclk_vcodec)
 744                 clk_disable_unprepare(pservice->hclk_vcodec);
 745         if (pservice->aclk_vcodec)
 746                 clk_disable_unprepare(pservice->aclk_vcodec);
 747         if (pservice->clk_core)
 748                 clk_disable_unprepare(pservice->clk_core);
 749         if (pservice->clk_cabac)
 750                 clk_disable_unprepare(pservice->clk_cabac);
 751 #endif
 752
 753         wake_unlock(&pservice->wake_lock);
 754         pr_info("done\n");
 755 }
 756
 757 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 758 {
 759         queue_delayed_work(system_nrt_wq, &pservice->power_off_work, VPU_POWER_OFF_DELAY);
 760 }
 761
 762 static void vpu_power_off_work(struct work_struct *work_s)
 763 {
 764         struct delayed_work *dlwork = container_of(work_s, struct delayed_work, work);
 765         struct vpu_service_info *pservice = container_of(dlwork, struct vpu_service_info, power_off_work);
 766
 767         if (mutex_trylock(&pservice->lock)) {
 768                 vpu_service_power_off(pservice);
 769                 mutex_unlock(&pservice->lock);
 770         } else {
 771                 /* Come back later if the device is busy... */
 772                 vpu_queue_power_off_work(pservice);
 773         }
 774 }
 775
 776 static void vpu_service_power_on(struct vpu_service_info *pservice)
 777 {
 778         static ktime_t last;
 779         ktime_t now = ktime_get();
 780         if (ktime_to_ns(ktime_sub(now, last)) > NSEC_PER_SEC) {
 781                 cancel_delayed_work_sync(&pservice->power_off_work);
 782                 vpu_queue_power_off_work(pservice);
 783                 last = now;
 784         }
 785         if (pservice->enabled)
 786                 return ;
 787
 788         pr_info("%s: power on\n", dev_name(pservice->dev));
 789
 790 #define BIT_VCODEC_CLK_SEL      (1<<10)
 791         if (cpu_is_rk312x())
 792                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1) |
 793                         BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 794                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 795
 796 #if VCODEC_CLOCK_ENABLE
 797         if (pservice->aclk_vcodec)
 798                 clk_prepare_enable(pservice->aclk_vcodec);
 799         if (pservice->hclk_vcodec)
 800                 clk_prepare_enable(pservice->hclk_vcodec);
 801         if (pservice->clk_core)
 802                 clk_prepare_enable(pservice->clk_core);
 803         if (pservice->clk_cabac)
 804                 clk_prepare_enable(pservice->clk_cabac);
 805         if (pservice->pd_video)
 806                 clk_prepare_enable(pservice->pd_video);
 807 #endif
 808
 809         udelay(10);
 810         pservice->enabled = true;
 811         wake_lock(&pservice->wake_lock);
 812 }
 813
 814 static inline bool reg_check_rmvb_wmv(vpu_reg *reg)
 815 {
 816         u32 type = (reg->reg[3] & 0xF0000000) >> 28;
 817         return ((type == 8) || (type == 4));
 818 }
 819
 820 static inline bool reg_check_interlace(vpu_reg *reg)
 821 {
 822         u32 type = (reg->reg[3] & (1 << 23));
 823         return (type > 0);
 824 }
 825
 826 static inline enum VPU_DEC_FMT reg_check_fmt(vpu_reg *reg)
 827 {
 828         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] & 0xF0000000) >> 28);
 829         return type;
 830 }
 831
 832 static inline int reg_probe_width(vpu_reg *reg)
 833 {
 834         int width_in_mb = reg->reg[4] >> 23;
 835         return width_in_mb * 16;
 836 }
 837
 838 #if defined(CONFIG_VCODEC_MMU)
 839 static int vcodec_fd_to_iova(struct vpu_subdev_data *data, vpu_reg *reg,int fd)
 840 {
 841         struct vpu_service_info *pservice = data->pservice;
 842         struct ion_handle *hdl;
 843         int ret = 0;
 844         struct vcodec_mem_region *mem_region;
 845
 846         hdl = ion_import_dma_buf(pservice->ion_client, fd);
 847         if (IS_ERR(hdl)) {
 848                 vpu_err("import dma-buf from fd %d failed\n", fd);
 849                 return PTR_ERR(hdl);
 850         }
 851         mem_region = kzalloc(sizeof(struct vcodec_mem_region), GFP_KERNEL);
 852
 853         if (mem_region == NULL) {
 854                 vpu_err("allocate memory for iommu memory region failed\n");
 855                 ion_free(pservice->ion_client, hdl);
 856                 return -1;
 857         }
 858
 859         mem_region->hdl = hdl;
 860         ret = ion_map_iommu(data->dev, pservice->ion_client,
 861                 mem_region->hdl, &mem_region->iova, &mem_region->len);
 862
 863         if (ret < 0) {
 864                 vpu_err("ion map iommu failed\n");
 865                 kfree(mem_region);
 866                 ion_free(pservice->ion_client, hdl);
 867                 return ret;
 868         }
 869         INIT_LIST_HEAD(&mem_region->reg_lnk);
 870         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 871         return mem_region->iova;
 872 }
 873
 874 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data, u8 *tbl,
 875                                 int size, vpu_reg *reg,
 876                                 struct extra_info_for_iommu *ext_inf)
 877 {
 878         struct vpu_service_info *pservice = data->pservice;
 879         int i;
 880         int usr_fd = 0;
 881         int offset = 0;
 882
 883         if (tbl == NULL || size <= 0) {
 884                 dev_err(pservice->dev, "input arguments invalidate\n");
 885                 return -1;
 886         }
 887
 888         vpu_service_power_on(pservice);
 889
 890         for (i = 0; i < size; i++) {
 891                 usr_fd = reg->reg[tbl[i]] & 0x3FF;
 892
 893                 if (tbl[i] == 41 && data->hw_info->hw_id != HEVC_ID &&
 894                     (reg->type == VPU_DEC || reg->type == VPU_DEC_PP))
 895                         /* special for vpu dec num 41 regitster */
 896                         offset = reg->reg[tbl[i]] >> 10 << 4;
 897                 else
 898                         offset = reg->reg[tbl[i]] >> 10;
 899
 900                 if (usr_fd != 0) {
 901                         struct ion_handle *hdl;
 902                         int ret = 0;
 903                         struct vcodec_mem_region *mem_region;
 904
 905                         hdl = ion_import_dma_buf(pservice->ion_client, usr_fd);
 906                         if (IS_ERR(hdl)) {
 907                                 dev_err(pservice->dev, "import dma-buf from fd %d failed, reg[%d]\n", usr_fd, tbl[i]);
 908                                 return PTR_ERR(hdl);
 909                         }
 910
 911                         if (tbl[i] == 42 && data->hw_info->hw_id == HEVC_ID){
 912                                 int i = 0;
 913                                 char *pps;
 914                                 pps = (char *)ion_map_kernel(pservice->ion_client,hdl);
 915                                 for (i=0; i<64; i++) {
 916                                         u32 scaling_offset;
 917                                         u32 tmp;
 918                                         int scaling_fd= 0;
 919                                         scaling_offset = (u32)pps[i*80+74];
 920                                         scaling_offset += (u32)pps[i*80+75] << 8;
 921                                         scaling_offset += (u32)pps[i*80+76] << 16;
 922                                         scaling_offset += (u32)pps[i*80+77] << 24;
 923                                         scaling_fd = scaling_offset&0x3ff;
 924                                         scaling_offset = scaling_offset >> 10;
 925                                         if(scaling_fd > 0) {
 926                                                 tmp = vcodec_fd_to_iova(data, reg, scaling_fd);
 927                                                 tmp += scaling_offset;
 928                                                 pps[i*80+74] = tmp & 0xff;
 929                                                 pps[i*80+75] = (tmp >> 8) & 0xff;
 930                                                 pps[i*80+76] = (tmp >> 16) & 0xff;
 931                                                 pps[i*80+77] = (tmp >> 24) & 0xff;
 932                                         }
 933                                 }
 934                         }
 935
 936                         mem_region = kzalloc(sizeof(struct vcodec_mem_region), GFP_KERNEL);
 937
 938                         if (mem_region == NULL) {
 939                                 dev_err(pservice->dev, "allocate memory for iommu memory region failed\n");
 940                                 ion_free(pservice->ion_client, hdl);
 941                                 return -1;
 942                         }
 943
 944                         mem_region->hdl = hdl;
 945                         mem_region->reg_idx = tbl[i];
 946                         ret = ion_map_iommu(data->dev,
 947                                             pservice->ion_client,
 948                                             mem_region->hdl,
 949                                             &mem_region->iova,
 950                                             &mem_region->len);
 951
 952                         if (ret < 0) {
 953                                 dev_err(pservice->dev, "ion map iommu failed\n");
 954                                 kfree(mem_region);
 955                                 ion_free(pservice->ion_client, hdl);
 956                                 return ret;
 957                         }
 958                         reg->reg[tbl[i]] = mem_region->iova + offset;
 959                         INIT_LIST_HEAD(&mem_region->reg_lnk);
 960                         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 961                 }
 962         }
 963
 964         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
 965                 for (i=0; i<ext_inf->cnt; i++) {
 966                         vpu_debug(3, "reg[%d] + offset %d\n",
 967                                   ext_inf->elem[i].index,
 968                                   ext_inf->elem[i].offset);
 969                         reg->reg[ext_inf->elem[i].index] +=
 970                                 ext_inf->elem[i].offset;
 971                 }
 972         }
 973
 974         return 0;
 975 }
 976
 977 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
 978                                         vpu_reg *reg,
 979                                         struct extra_info_for_iommu *ext_inf)
 980 {
 981         VPU_HW_ID hw_id;
 982         u8 *tbl;
 983         int size = 0;
 984
 985         hw_id = data->hw_info->hw_id;
 986
 987         if (hw_id == HEVC_ID) {
 988                 tbl = addr_tbl_hevc_dec;
 989                 size = sizeof(addr_tbl_hevc_dec);
 990         } else {
 991                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
 992                         switch (reg_check_fmt(reg)) {
 993                         case VPU_DEC_FMT_H264:
 994                                 {
 995                                         tbl = addr_tbl_vpu_h264dec;
 996                                         size = sizeof(addr_tbl_vpu_h264dec);
 997                                         break;
 998                                 }
 999                         case VPU_DEC_FMT_VP8:
1000                         case VPU_DEC_FMT_VP7:
1001                                 {
1002                                         tbl = addr_tbl_vpu_vp8dec;
1003                                         size = sizeof(addr_tbl_vpu_vp8dec);
1004                                         break;
1005                                 }
1006
1007                         case VPU_DEC_FMT_VP6:
1008                                 {
1009                                         tbl = addr_tbl_vpu_vp6dec;
1010                                         size = sizeof(addr_tbl_vpu_vp6dec);
1011                                         break;
1012                                 }
1013                         case VPU_DEC_FMT_VC1:
1014                                 {
1015                                         tbl = addr_tbl_vpu_vc1dec;
1016                                         size = sizeof(addr_tbl_vpu_vc1dec);
1017                                         break;
1018                                 }
1019
1020                         case VPU_DEC_FMT_JPEG:
1021                                 {
1022                                         tbl = addr_tbl_vpu_jpegdec;
1023                                         size = sizeof(addr_tbl_vpu_jpegdec);
1024                                         break;
1025                                 }
1026                         default:
1027                                 tbl = addr_tbl_vpu_defaultdec;
1028                                 size = sizeof(addr_tbl_vpu_defaultdec);
1029                                 break;
1030                         }
1031                 } else if (reg->type == VPU_ENC) {
1032                         tbl = addr_tbl_vpu_enc;
1033                         size = sizeof(addr_tbl_vpu_enc);
1034                 }
1035         }
1036
1037         if (size != 0) {
1038                 return vcodec_bufid_to_iova(data, tbl, size, reg, ext_inf);
1039         } else {
1040                 return -1;
1041         }
1042 }
1043 #endif
1044
1045 static vpu_reg *reg_init(struct vpu_subdev_data *data,
1046         vpu_session *session, void __user *src, u32 size)
1047 {
1048         struct vpu_service_info *pservice = data->pservice;
1049         int extra_size = 0;
1050         struct extra_info_for_iommu extra_info;
1051         vpu_reg *reg = kmalloc(sizeof(vpu_reg) + data->reg_size, GFP_KERNEL);
1052
1053         vpu_debug_enter();
1054
1055         if (NULL == reg) {
1056                 vpu_err("error: kmalloc fail in reg_init\n");
1057                 return NULL;
1058         }
1059
1060         if (size > data->reg_size) {
1061                 /*printk("warning: vpu reg size %lu is larger than hw reg size %lu\n",
1062                   size, pservice->reg_size);
1063                 size = pservice->reg_size;*/
1064                 extra_size = size - data->reg_size;
1065                 size = data->reg_size;
1066         }
1067         reg->session = session;
1068         reg->data = data;
1069         reg->type = session->type;
1070         reg->size = size;
1071         reg->freq = VPU_FREQ_DEFAULT;
1072         reg->reg = (u32 *)&reg[1];
1073         INIT_LIST_HEAD(&reg->session_link);
1074         INIT_LIST_HEAD(&reg->status_link);
1075
1076 #if defined(CONFIG_VCODEC_MMU)
1077         if (data->mmu_dev)
1078                 INIT_LIST_HEAD(&reg->mem_region_list);
1079 #endif
1080
1081         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1082                 vpu_err("error: copy_from_user failed in reg_init\n");
1083                 kfree(reg);
1084                 return NULL;
1085         }
1086
1087         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1088                 vpu_err("error: copy_from_user failed in reg_init\n");
1089                 kfree(reg);
1090                 return NULL;
1091         }
1092
1093 #if defined(CONFIG_VCODEC_MMU)
1094         if (data->mmu_dev &&
1095             0 > vcodec_reg_address_translate(data, reg, &extra_info)) {
1096                 vpu_err("error: translate reg address failed\n");
1097                 kfree(reg);
1098                 return NULL;
1099         }
1100 #endif
1101
1102         mutex_lock(&pservice->lock);
1103         list_add_tail(&reg->status_link, &pservice->waiting);
1104         list_add_tail(&reg->session_link, &session->waiting);
1105         mutex_unlock(&pservice->lock);
1106
1107         if (pservice->auto_freq) {
1108                 if (!soc_is_rk2928g()) {
1109                         if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1110                                 if (reg_check_rmvb_wmv(reg)) {
1111                                         reg->freq = VPU_FREQ_200M;
1112                                 } else if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1113                                         if (reg_probe_width(reg) > 3200) {
1114                                                 /*raise frequency for 4k avc.*/
1115                                                 reg->freq = VPU_FREQ_500M;
1116                                         }
1117                                 } else {
1118                                         if (reg_check_interlace(reg)) {
1119                                                 reg->freq = VPU_FREQ_400M;
1120                                         }
1121                                 }
1122                         }
1123                         if (reg->type == VPU_PP) {
1124                                 reg->freq = VPU_FREQ_400M;
1125                         }
1126                 }
1127         }
1128         vpu_debug_leave();
1129         return reg;
1130 }
1131
1132 static void reg_deinit(struct vpu_subdev_data *data, vpu_reg *reg)
1133 {
1134         struct vpu_service_info *pservice = data->pservice;
1135 #if defined(CONFIG_VCODEC_MMU)
1136         struct vcodec_mem_region *mem_region = NULL, *n;
1137 #endif
1138
1139         list_del_init(&reg->session_link);
1140         list_del_init(&reg->status_link);
1141         if (reg == pservice->reg_codec)
1142                 pservice->reg_codec = NULL;
1143         if (reg == pservice->reg_pproc)
1144                 pservice->reg_pproc = NULL;
1145
1146 #if defined(CONFIG_VCODEC_MMU)
1147         /* release memory region attach to this registers table. */
1148         if (data->mmu_dev) {
1149                 list_for_each_entry_safe(mem_region, n,
1150                         &reg->mem_region_list, reg_lnk) {
1151                         /* do not unmap iommu manually,
1152                            unmap will proccess when memory release */
1153                         /*vcodec_enter_mode(data);
1154                         ion_unmap_iommu(data->dev,
1155                                         pservice->ion_client,
1156                                         mem_region->hdl);
1157                         vcodec_exit_mode();*/
1158                         ion_free(pservice->ion_client, mem_region->hdl);
1159                         list_del_init(&mem_region->reg_lnk);
1160                         kfree(mem_region);
1161                 }
1162         }
1163 #endif
1164
1165         kfree(reg);
1166 }
1167
1168 static void reg_from_wait_to_run(struct vpu_service_info *pservice, vpu_reg *reg)
1169 {
1170         vpu_debug_enter();
1171         list_del_init(&reg->status_link);
1172         list_add_tail(&reg->status_link, &pservice->running);
1173
1174         list_del_init(&reg->session_link);
1175         list_add_tail(&reg->session_link, &reg->session->running);
1176         vpu_debug_leave();
1177 }
1178
1179 static void reg_copy_from_hw(vpu_reg *reg, volatile u32 *src, u32 count)
1180 {
1181         int i;
1182         u32 *dst = (u32 *)&reg->reg[0];
1183         vpu_debug_enter();
1184         for (i = 0; i < count; i++)
1185                 *dst++ = *src++;
1186         vpu_debug_leave();
1187 }
1188
1189 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1190         vpu_reg *reg)
1191 {
1192         struct vpu_service_info *pservice = data->pservice;
1193         int irq_reg = -1;
1194
1195         vpu_debug_enter();
1196
1197         list_del_init(&reg->status_link);
1198         list_add_tail(&reg->status_link, &pservice->done);
1199
1200         list_del_init(&reg->session_link);
1201         list_add_tail(&reg->session_link, &reg->session->done);
1202
1203         /*vcodec_enter_mode(data);*/
1204         switch (reg->type) {
1205         case VPU_ENC : {
1206                 pservice->reg_codec = NULL;
1207                 reg_copy_from_hw(reg, data->enc_dev.hwregs, data->hw_info->enc_reg_num);
1208                 irq_reg = ENC_INTERRUPT_REGISTER;
1209                 break;
1210         }
1211         case VPU_DEC : {
1212                 int reg_len = REG_NUM_9190_DEC;
1213                 pservice->reg_codec = NULL;
1214                 reg_copy_from_hw(reg, data->dec_dev.hwregs, reg_len);
1215                 irq_reg = DEC_INTERRUPT_REGISTER;
1216                 break;
1217         }
1218         case VPU_PP : {
1219                 pservice->reg_pproc = NULL;
1220                 reg_copy_from_hw(reg, data->dec_dev.hwregs + PP_INTERRUPT_REGISTER, REG_NUM_9190_PP);
1221                 data->dec_dev.hwregs[PP_INTERRUPT_REGISTER] = 0;
1222                 break;
1223         }
1224         case VPU_DEC_PP : {
1225                 pservice->reg_codec = NULL;
1226                 pservice->reg_pproc = NULL;
1227                 reg_copy_from_hw(reg, data->dec_dev.hwregs, REG_NUM_9190_DEC_PP);
1228                 data->dec_dev.hwregs[PP_INTERRUPT_REGISTER] = 0;
1229                 break;
1230         }
1231         default : {
1232                 vpu_err("error: copy reg from hw with unknown type %d\n", reg->type);
1233                 break;
1234         }
1235         }
1236         vcodec_exit_mode(data);
1237
1238         if (irq_reg != -1)
1239                 reg->reg[irq_reg] = pservice->irq_status;
1240
1241         atomic_sub(1, &reg->session->task_running);
1242         atomic_sub(1, &pservice->total_running);
1243         wake_up(&reg->session->wait);
1244
1245         vpu_debug_leave();
1246 }
1247
1248 static void vpu_service_set_freq(struct vpu_service_info *pservice, vpu_reg *reg)
1249 {
1250         VPU_FREQ curr = atomic_read(&pservice->freq_status);
1251         if (curr == reg->freq)
1252                 return;
1253         atomic_set(&pservice->freq_status, reg->freq);
1254         switch (reg->freq) {
1255         case VPU_FREQ_200M : {
1256                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1257         } break;
1258         case VPU_FREQ_266M : {
1259                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1260         } break;
1261         case VPU_FREQ_300M : {
1262                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1263         } break;
1264         case VPU_FREQ_400M : {
1265                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1266         } break;
1267         case VPU_FREQ_500M : {
1268                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1269         } break;
1270         case VPU_FREQ_600M : {
1271                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1272         } break;
1273         default : {
1274                 if (soc_is_rk2928g())
1275                         clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1276                 else
1277                         clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1278         } break;
1279         }
1280 }
1281
1282 static void reg_copy_to_hw(struct vpu_subdev_data *data, vpu_reg *reg)
1283 {
1284         struct vpu_service_info *pservice = data->pservice;
1285         int i;
1286         u32 *src = (u32 *)&reg->reg[0];
1287         vpu_debug_enter();
1288
1289         atomic_add(1, &pservice->total_running);
1290         atomic_add(1, &reg->session->task_running);
1291         if (pservice->auto_freq)
1292                 vpu_service_set_freq(pservice, reg);
1293
1294         vcodec_enter_mode(data);
1295
1296         switch (reg->type) {
1297         case VPU_ENC : {
1298                 int enc_count = data->hw_info->enc_reg_num;
1299                 u32 *dst = (u32 *)data->enc_dev.hwregs;
1300
1301                 pservice->reg_codec = reg;
1302
1303                 dst[VPU_REG_EN_ENC] = src[VPU_REG_EN_ENC] & 0x6;
1304
1305                 for (i = 0; i < VPU_REG_EN_ENC; i++)
1306                         dst[i] = src[i];
1307
1308                 for (i = VPU_REG_EN_ENC + 1; i < enc_count; i++)
1309                         dst[i] = src[i];
1310
1311                 VEPU_CLEAN_CACHE(dst);
1312
1313                 dsb(sy);
1314
1315                 dst[VPU_REG_ENC_GATE] = src[VPU_REG_ENC_GATE] | VPU_REG_ENC_GATE_BIT;
1316                 dst[VPU_REG_EN_ENC]   = src[VPU_REG_EN_ENC];
1317
1318 #if VPU_SERVICE_SHOW_TIME
1319                 do_gettimeofday(&enc_start);
1320 #endif
1321
1322         } break;
1323         case VPU_DEC : {
1324                 u32 *dst = (u32 *)data->dec_dev.hwregs;
1325
1326                 pservice->reg_codec = reg;
1327
1328                 if (data->hw_info->hw_id != HEVC_ID) {
1329                         for (i = REG_NUM_9190_DEC - 1; i > VPU_REG_DEC_GATE; i--)
1330                                 dst[i] = src[i];
1331                         VDPU_CLEAN_CACHE(dst);
1332                 } else {
1333                         for (i = REG_NUM_HEVC_DEC - 1; i > VPU_REG_EN_DEC; i--)
1334                                 dst[i] = src[i];
1335                         HEVC_CLEAN_CACHE(dst);
1336                 }
1337
1338                 dsb(sy);
1339
1340                 if (data->hw_info->hw_id != HEVC_ID) {
1341                         dst[VPU_REG_DEC_GATE] = src[VPU_REG_DEC_GATE] | VPU_REG_DEC_GATE_BIT;
1342                         dst[VPU_REG_EN_DEC] = src[VPU_REG_EN_DEC];
1343                 } else {
1344                         dst[VPU_REG_EN_DEC] = src[VPU_REG_EN_DEC];
1345                 }
1346                 dsb(sy);
1347                 dmb(sy);
1348 #if VPU_SERVICE_SHOW_TIME
1349                 do_gettimeofday(&dec_start);
1350 #endif
1351         } break;
1352         case VPU_PP : {
1353                 u32 *dst = (u32 *)data->dec_dev.hwregs + PP_INTERRUPT_REGISTER;
1354                 pservice->reg_pproc = reg;
1355
1356                 dst[VPU_REG_PP_GATE] = src[VPU_REG_PP_GATE] | VPU_REG_PP_GATE_BIT;
1357
1358                 for (i = VPU_REG_PP_GATE + 1; i < REG_NUM_9190_PP; i++)
1359                         dst[i] = src[i];
1360
1361                 dsb(sy);
1362
1363                 dst[VPU_REG_EN_PP] = src[VPU_REG_EN_PP];
1364 #if VPU_SERVICE_SHOW_TIME
1365                 do_gettimeofday(&pp_start);
1366 #endif
1367
1368         } break;
1369         case VPU_DEC_PP : {
1370                 u32 *dst = (u32 *)data->dec_dev.hwregs;
1371                 pservice->reg_codec = reg;
1372                 pservice->reg_pproc = reg;
1373
1374                 VDPU_SOFT_RESET(dst);
1375                 VDPU_CLEAN_CACHE(dst);
1376
1377                 for (i = VPU_REG_EN_DEC_PP + 1; i < REG_NUM_9190_DEC_PP; i++)
1378                         dst[i] = src[i];
1379
1380                 dst[VPU_REG_EN_DEC_PP]   = src[VPU_REG_EN_DEC_PP] | 0x2;
1381                 dsb(sy);
1382
1383                 dst[VPU_REG_DEC_PP_GATE] = src[VPU_REG_DEC_PP_GATE] | VPU_REG_PP_GATE_BIT;
1384                 dst[VPU_REG_DEC_GATE]    = src[VPU_REG_DEC_GATE]    | VPU_REG_DEC_GATE_BIT;
1385                 dst[VPU_REG_EN_DEC]      = src[VPU_REG_EN_DEC];
1386 #if VPU_SERVICE_SHOW_TIME
1387                 do_gettimeofday(&dec_start);
1388 #endif
1389         } break;
1390         default : {
1391                 vpu_err("error: unsupport session type %d", reg->type);
1392                 atomic_sub(1, &pservice->total_running);
1393                 atomic_sub(1, &reg->session->task_running);
1394                 break;
1395         }
1396         }
1397
1398         /*vcodec_exit_mode(data);*/
1399         vpu_debug_leave();
1400 }
1401
1402 static void try_set_reg(struct vpu_subdev_data *data)
1403 {
1404         struct vpu_service_info *pservice = data->pservice;
1405         vpu_debug_enter();
1406         if (!list_empty(&pservice->waiting)) {
1407                 int can_set = 0;
1408                 vpu_reg *reg = list_entry(pservice->waiting.next, vpu_reg, status_link);
1409
1410                 vpu_service_power_on(pservice);
1411
1412                 switch (reg->type) {
1413                 case VPU_ENC : {
1414                         if ((NULL == pservice->reg_codec) &&  (NULL == pservice->reg_pproc))
1415                                 can_set = 1;
1416                 } break;
1417                 case VPU_DEC : {
1418                         if (NULL == pservice->reg_codec)
1419                                 can_set = 1;
1420                         if (pservice->auto_freq && (NULL != pservice->reg_pproc))
1421                                 can_set = 0;
1422                 } break;
1423                 case VPU_PP : {
1424                         if (NULL == pservice->reg_codec) {
1425                                 if (NULL == pservice->reg_pproc)
1426                                         can_set = 1;
1427                         } else {
1428                                 if ((VPU_DEC == pservice->reg_codec->type) && (NULL == pservice->reg_pproc))
1429                                         can_set = 1;
1430                                 /* can not charge frequency when vpu is working */
1431                                 if (pservice->auto_freq)
1432                                         can_set = 0;
1433                         }
1434                 } break;
1435                 case VPU_DEC_PP : {
1436                         if ((NULL == pservice->reg_codec) && (NULL == pservice->reg_pproc))
1437                                 can_set = 1;
1438                         } break;
1439                 default : {
1440                         printk("undefined reg type %d\n", reg->type);
1441                 } break;
1442                 }
1443                 if (can_set) {
1444                         reg_from_wait_to_run(pservice, reg);
1445                         reg_copy_to_hw(reg->data, reg);
1446                 }
1447         }
1448         vpu_debug_leave();
1449 }
1450
1451 static int return_reg(struct vpu_subdev_data *data,
1452         vpu_reg *reg, u32 __user *dst)
1453 {
1454         int ret = 0;
1455         vpu_debug_enter();
1456         switch (reg->type) {
1457         case VPU_ENC : {
1458                 if (copy_to_user(dst, &reg->reg[0], data->hw_info->enc_io_size))
1459                         ret = -EFAULT;
1460                 break;
1461         }
1462         case VPU_DEC : {
1463                 int reg_len = data->hw_info->hw_id == HEVC_ID ? REG_NUM_HEVC_DEC : REG_NUM_9190_DEC;
1464                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(reg_len)))
1465                         ret = -EFAULT;
1466                 break;
1467         }
1468         case VPU_PP : {
1469                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(REG_NUM_9190_PP)))
1470                         ret = -EFAULT;
1471                 break;
1472         }
1473         case VPU_DEC_PP : {
1474                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(REG_NUM_9190_DEC_PP)))
1475                         ret = -EFAULT;
1476                 break;
1477         }
1478         default : {
1479                 ret = -EFAULT;
1480                 vpu_err("error: copy reg to user with unknown type %d\n", reg->type);
1481                 break;
1482         }
1483         }
1484         reg_deinit(data, reg);
1485         vpu_debug_leave();
1486         return ret;
1487 }
1488
1489 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1490         unsigned long arg)
1491 {
1492         struct vpu_subdev_data *data =
1493                 container_of(filp->f_dentry->d_inode->i_cdev,
1494                         struct vpu_subdev_data, cdev);
1495         struct vpu_service_info *pservice = data->pservice;
1496         vpu_session *session = (vpu_session *)filp->private_data;
1497         vpu_debug_enter();
1498         vpu_debug(3, "cmd %x, VPU_IOC_SET_CLIENT_TYPE %x\n", cmd, (u32)VPU_IOC_SET_CLIENT_TYPE);
1499         if (NULL == session)
1500                 return -EINVAL;
1501
1502         switch (cmd) {
1503         case VPU_IOC_SET_CLIENT_TYPE : {
1504                 session->type = (enum VPU_CLIENT_TYPE)arg;
1505                 break;
1506         }
1507         case VPU_IOC_GET_HW_FUSE_STATUS : {
1508                 vpu_request req;
1509                 if (copy_from_user(&req, (void __user *)arg, sizeof(vpu_request))) {
1510                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_from_user failed\n");
1511                         return -EFAULT;
1512                 } else {
1513                         if (VPU_ENC != session->type) {
1514                                 if (copy_to_user((void __user *)req.req,
1515                                         &pservice->dec_config,
1516                                         sizeof(struct vpu_dec_config))) {
1517                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_to_user failed type %d\n",
1518                                                 session->type);
1519                                         return -EFAULT;
1520                                 }
1521                         } else {
1522                                 if (copy_to_user((void __user *)req.req,
1523                                         &pservice->enc_config,
1524                                         sizeof(struct vpu_enc_config ))) {
1525                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_to_user failed type %d\n",
1526                                                 session->type);
1527                                         return -EFAULT;
1528                                 }
1529                         }
1530                 }
1531
1532                 break;
1533         }
1534         case VPU_IOC_SET_REG : {
1535                 vpu_request req;
1536                 vpu_reg *reg;
1537                 if (copy_from_user(&req, (void __user *)arg,
1538                         sizeof(vpu_request))) {
1539                         vpu_err("error: VPU_IOC_SET_REG copy_from_user failed\n");
1540                         return -EFAULT;
1541                 }
1542                 reg = reg_init(data, session,
1543                         (void __user *)req.req, req.size);
1544                 if (NULL == reg) {
1545                         return -EFAULT;
1546                 } else {
1547                         mutex_lock(&pservice->lock);
1548                         try_set_reg(data);
1549                         mutex_unlock(&pservice->lock);
1550                 }
1551
1552                 break;
1553         }
1554         case VPU_IOC_GET_REG : {
1555                 vpu_request req;
1556                 vpu_reg *reg;
1557                 if (copy_from_user(&req, (void __user *)arg,
1558                         sizeof(vpu_request))) {
1559                         vpu_err("error: VPU_IOC_GET_REG copy_from_user failed\n");
1560                         return -EFAULT;
1561                 } else {
1562                         int ret = wait_event_timeout(session->wait, !list_empty(&session->done), VPU_TIMEOUT_DELAY);
1563                         if (!list_empty(&session->done)) {
1564                                 if (ret < 0) {
1565                                         vpu_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session->pid, ret);
1566                                 }
1567                                 ret = 0;
1568                         } else {
1569                                 if (unlikely(ret < 0)) {
1570                                         vpu_err("error: pid %d wait task ret %d\n", session->pid, ret);
1571                                 } else if (0 == ret) {
1572                                         vpu_err("error: pid %d wait %d task done timeout\n", session->pid, atomic_read(&session->task_running));
1573                                         ret = -ETIMEDOUT;
1574                                 }
1575                         }
1576                         if (ret < 0) {
1577                                 int task_running = atomic_read(&session->task_running);
1578                                 mutex_lock(&pservice->lock);
1579                                 vpu_service_dump(pservice);
1580                                 if (task_running) {
1581                                         atomic_set(&session->task_running, 0);
1582                                         atomic_sub(task_running, &pservice->total_running);
1583                                         printk("%d task is running but not return, reset hardware...", task_running);
1584                                         vpu_reset(data);
1585                                         printk("done\n");
1586                                 }
1587                                 vpu_service_session_clear(data, session);
1588                                 mutex_unlock(&pservice->lock);
1589                                 return ret;
1590                         }
1591                 }
1592                 mutex_lock(&pservice->lock);
1593                 reg = list_entry(session->done.next, vpu_reg, session_link);
1594                 return_reg(data, reg, (u32 __user *)req.req);
1595                 mutex_unlock(&pservice->lock);
1596                 break;
1597         }
1598         case VPU_IOC_PROBE_IOMMU_STATUS: {
1599                 int iommu_enable = 0;
1600
1601 #if defined(CONFIG_VCODEC_MMU)
1602                 iommu_enable = data->mmu_dev ? 1 : 0;
1603 #endif
1604
1605                 if (copy_to_user((void __user *)arg, &iommu_enable, sizeof(int))) {
1606                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1607                         return -EFAULT;
1608                 }
1609                 break;
1610         }
1611         default : {
1612                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1613                 break;
1614         }
1615         }
1616         vpu_debug_leave();
1617         return 0;
1618 }
1619
1620 #ifdef CONFIG_COMPAT
1621 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1622         unsigned long arg)
1623 {
1624         struct vpu_subdev_data *data =
1625                 container_of(filp->f_dentry->d_inode->i_cdev,
1626                         struct vpu_subdev_data, cdev);
1627         struct vpu_service_info *pservice = data->pservice;
1628         vpu_session *session = (vpu_session *)filp->private_data;
1629         vpu_debug_enter();
1630         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1631                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1632         if (NULL == session)
1633                 return -EINVAL;
1634
1635         switch (cmd) {
1636         case COMPAT_VPU_IOC_SET_CLIENT_TYPE : {
1637                 session->type = (enum VPU_CLIENT_TYPE)arg;
1638                 break;
1639         }
1640         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS : {
1641                 vpu_request req;
1642                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1643                                    sizeof(vpu_request))) {
1644                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS"
1645                                 " copy_from_user failed\n");
1646                         return -EFAULT;
1647                 } else {
1648                         if (VPU_ENC != session->type) {
1649                                 if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1650                                                  &pservice->dec_config,
1651                                                  sizeof(struct vpu_dec_config))) {
1652                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS "
1653                                                 "copy_to_user failed type %d\n",
1654                                                 session->type);
1655                                         return -EFAULT;
1656                                 }
1657                         } else {
1658                                 if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1659                                                  &pservice->enc_config,
1660                                                  sizeof(struct vpu_enc_config ))) {
1661                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS"
1662                                                 " copy_to_user failed type %d\n",
1663                                                 session->type);
1664                                         return -EFAULT;
1665                                 }
1666                         }
1667                 }
1668
1669                 break;
1670         }
1671         case COMPAT_VPU_IOC_SET_REG : {
1672                 vpu_request req;
1673                 vpu_reg *reg;
1674                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1675                                    sizeof(vpu_request))) {
1676                         vpu_err("VPU_IOC_SET_REG copy_from_user failed\n");
1677                         return -EFAULT;
1678                 }
1679                 reg = reg_init(data, session,
1680                                compat_ptr((compat_uptr_t)req.req), req.size);
1681                 if (NULL == reg) {
1682                         return -EFAULT;
1683                 } else {
1684                         mutex_lock(&pservice->lock);
1685                         try_set_reg(data);
1686                         mutex_unlock(&pservice->lock);
1687                 }
1688
1689                 break;
1690         }
1691         case COMPAT_VPU_IOC_GET_REG : {
1692                 vpu_request req;
1693                 vpu_reg *reg;
1694                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1695                                    sizeof(vpu_request))) {
1696                         vpu_err("VPU_IOC_GET_REG copy_from_user failed\n");
1697                         return -EFAULT;
1698                 } else {
1699                         int ret = wait_event_timeout(session->wait, !list_empty(&session->done), VPU_TIMEOUT_DELAY);
1700                         if (!list_empty(&session->done)) {
1701                                 if (ret < 0) {
1702                                         vpu_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session->pid, ret);
1703                                 }
1704                                 ret = 0;
1705                         } else {
1706                                 if (unlikely(ret < 0)) {
1707                                         vpu_err("error: pid %d wait task ret %d\n", session->pid, ret);
1708                                 } else if (0 == ret) {
1709                                         vpu_err("error: pid %d wait %d task done timeout\n", session->pid, atomic_read(&session->task_running));
1710                                         ret = -ETIMEDOUT;
1711                                 }
1712                         }
1713                         if (ret < 0) {
1714                                 int task_running = atomic_read(&session->task_running);
1715                                 mutex_lock(&pservice->lock);
1716                                 vpu_service_dump(pservice);
1717                                 if (task_running) {
1718                                         atomic_set(&session->task_running, 0);
1719                                         atomic_sub(task_running, &pservice->total_running);
1720                                         printk("%d task is running but not return, reset hardware...", task_running);
1721                                         vpu_reset(data);
1722                                         printk("done\n");
1723                                 }
1724                                 vpu_service_session_clear(data, session);
1725                                 mutex_unlock(&pservice->lock);
1726                                 return ret;
1727                         }
1728                 }
1729                 mutex_lock(&pservice->lock);
1730                 reg = list_entry(session->done.next, vpu_reg, session_link);
1731                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
1732                 mutex_unlock(&pservice->lock);
1733                 break;
1734         }
1735         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS : {
1736                 int iommu_enable = 0;
1737
1738 #if defined(CONFIG_VCODEC_MMU)
1739                 iommu_enable = data->mmu_dev ? 1 : 0;
1740 #endif
1741
1742                 if (copy_to_user(compat_ptr((compat_uptr_t)arg), &iommu_enable, sizeof(int))) {
1743                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1744                         return -EFAULT;
1745                 }
1746                 break;
1747         }
1748         default : {
1749                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1750                 break;
1751         }
1752         }
1753         vpu_debug_leave();
1754         return 0;
1755 }
1756 #endif
1757
1758 static int vpu_service_check_hw(struct vpu_subdev_data *data, u32 hw_addr)
1759 {
1760         int ret = -EINVAL, i = 0;
1761         volatile u32 *tmp = (volatile u32 *)ioremap_nocache(hw_addr, 0x4);
1762         u32 enc_id = *tmp;
1763
1764         enc_id = (enc_id >> 16) & 0xFFFF;
1765         pr_info("checking hw id %x\n", enc_id);
1766         data->hw_info = NULL;
1767         for (i = 0; i < ARRAY_SIZE(vpu_hw_set); i++) {
1768                 if (enc_id == vpu_hw_set[i].hw_id) {
1769                         data->hw_info = &vpu_hw_set[i];
1770                         ret = 0;
1771                         break;
1772                 }
1773         }
1774         iounmap((void *)tmp);
1775         return ret;
1776 }
1777
1778 static int vpu_service_open(struct inode *inode, struct file *filp)
1779 {
1780         struct vpu_subdev_data *data = container_of(inode->i_cdev, struct vpu_subdev_data, cdev);
1781         struct vpu_service_info *pservice = data->pservice;
1782         vpu_session *session = (vpu_session *)kmalloc(sizeof(vpu_session), GFP_KERNEL);
1783
1784         vpu_debug_enter();
1785
1786         if (NULL == session) {
1787                 vpu_err("error: unable to allocate memory for vpu_session.");
1788                 return -ENOMEM;
1789         }
1790
1791         session->type   = VPU_TYPE_BUTT;
1792         session->pid    = current->pid;
1793         INIT_LIST_HEAD(&session->waiting);
1794         INIT_LIST_HEAD(&session->running);
1795         INIT_LIST_HEAD(&session->done);
1796         INIT_LIST_HEAD(&session->list_session);
1797         init_waitqueue_head(&session->wait);
1798         atomic_set(&session->task_running, 0);
1799         mutex_lock(&pservice->lock);
1800         list_add_tail(&session->list_session, &pservice->session);
1801         filp->private_data = (void *)session;
1802         mutex_unlock(&pservice->lock);
1803
1804         pr_debug("dev opened\n");
1805         vpu_debug_leave();
1806         return nonseekable_open(inode, filp);
1807 }
1808
1809 static int vpu_service_release(struct inode *inode, struct file *filp)
1810 {
1811         struct vpu_subdev_data *data = container_of(inode->i_cdev, struct vpu_subdev_data, cdev);
1812         struct vpu_service_info *pservice = data->pservice;
1813         int task_running;
1814         vpu_session *session = (vpu_session *)filp->private_data;
1815         vpu_debug_enter();
1816         if (NULL == session)
1817                 return -EINVAL;
1818
1819         task_running = atomic_read(&session->task_running);
1820         if (task_running) {
1821                 vpu_err("error: vpu_service session %d still has %d task running when closing\n", session->pid, task_running);
1822                 msleep(50);
1823         }
1824         wake_up(&session->wait);
1825
1826         mutex_lock(&pservice->lock);
1827         /* remove this filp from the asynchronusly notified filp's */
1828         list_del_init(&session->list_session);
1829         vpu_service_session_clear(data, session);
1830         kfree(session);
1831         filp->private_data = NULL;
1832         mutex_unlock(&pservice->lock);
1833
1834         pr_debug("dev closed\n");
1835         vpu_debug_leave();
1836         return 0;
1837 }
1838
1839 static const struct file_operations vpu_service_fops = {
1840         .unlocked_ioctl = vpu_service_ioctl,
1841         .open           = vpu_service_open,
1842         .release        = vpu_service_release,
1843 #ifdef CONFIG_COMPAT
1844         .compat_ioctl   = compat_vpu_service_ioctl,
1845 #endif
1846 };
1847
1848 static irqreturn_t vdpu_irq(int irq, void *dev_id);
1849 static irqreturn_t vdpu_isr(int irq, void *dev_id);
1850 static irqreturn_t vepu_irq(int irq, void *dev_id);
1851 static irqreturn_t vepu_isr(int irq, void *dev_id);
1852 static void get_hw_info(struct vpu_subdev_data *data);
1853
1854 #ifdef CONFIG_VCODEC_MMU
1855 static struct device *rockchip_get_sysmmu_dev(const char *compt)
1856 {
1857         struct device_node *dn = NULL;
1858         struct platform_device *pd = NULL;
1859         struct device *ret = NULL ;
1860
1861         dn = of_find_compatible_node(NULL,NULL,compt);
1862         if(!dn) {
1863                 printk("can't find device node %s \r\n",compt);
1864                 return NULL;
1865         }
1866
1867         pd = of_find_device_by_node(dn);
1868         if(!pd) {
1869                 printk("can't find platform device in device node %s\n",compt);
1870                 return  NULL;
1871         }
1872         ret = &pd->dev;
1873
1874         return ret;
1875
1876 }
1877 #ifdef CONFIG_IOMMU_API
1878 static inline void platform_set_sysmmu(struct device *iommu,
1879         struct device *dev)
1880 {
1881         dev->archdata.iommu = iommu;
1882 }
1883 #else
1884 static inline void platform_set_sysmmu(struct device *iommu,
1885         struct device *dev)
1886 {
1887 }
1888 #endif
1889
1890 int vcodec_sysmmu_fault_hdl(struct device *dev,
1891                                 enum rk_iommu_inttype itype,
1892                                 unsigned long pgtable_base,
1893                                 unsigned long fault_addr, unsigned int status)
1894 {
1895         struct platform_device *pdev;
1896         struct vpu_subdev_data *data;
1897         struct vpu_service_info *pservice;
1898
1899         vpu_debug_enter();
1900
1901         pdev = container_of(dev, struct platform_device, dev);
1902
1903         data = platform_get_drvdata(pdev);
1904         pservice = data->pservice;
1905
1906         if (pservice->reg_codec) {
1907                 struct vcodec_mem_region *mem, *n;
1908                 int i = 0;
1909                 vpu_debug(3, "vcodec, fault addr 0x%08x\n", (u32)fault_addr);
1910                 list_for_each_entry_safe(mem, n,
1911                                          &pservice->reg_codec->mem_region_list,
1912                                          reg_lnk) {
1913                         vpu_debug(3, "vcodec, reg[%02u] mem region [%02d] 0x%08x %ld\n",
1914                                 mem->reg_idx, i, (u32)mem->iova, mem->len);
1915                         i++;
1916                 }
1917
1918                 pr_alert("vcodec, page fault occur, reset hw\n");
1919                 pservice->reg_codec->reg[101] = 1;
1920                 vpu_reset(data);
1921         }
1922
1923         return 0;
1924 }
1925 #endif
1926
1927 #if HEVC_TEST_ENABLE
1928 static int hevc_test_case0(vpu_service_info *pservice);
1929 #endif
1930 #if defined(CONFIG_ION_ROCKCHIP)
1931 extern struct ion_client *rockchip_ion_client_create(const char * name);
1932 #endif
1933
1934 static int vcodec_subdev_probe(struct platform_device *pdev,
1935         struct vpu_service_info *pservice)
1936 {
1937         int ret = 0;
1938         struct resource *res = NULL;
1939         u32 ioaddr = 0;
1940         struct device *dev = &pdev->dev;
1941         char *name = (char*)dev_name(dev);
1942         struct device_node *np = pdev->dev.of_node;
1943         struct vpu_subdev_data *data =
1944                 devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
1945 #if defined(CONFIG_VCODEC_MMU)
1946         u32 iommu_en = 0;
1947         char mmu_dev_dts_name[40];
1948         of_property_read_u32(np, "iommu_enabled", &iommu_en);
1949 #endif
1950         pr_info("probe device %s\n", dev_name(dev));
1951
1952         data->pservice = pservice;
1953         data->dev = dev;
1954
1955         of_property_read_string(np, "name", (const char**)&name);
1956         of_property_read_u32(np, "dev_mode", (u32*)&data->mode);
1957         /*dev_set_name(dev, name);*/
1958
1959         if (pservice->reg_base == 0) {
1960                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1961                 data->regs = devm_ioremap_resource(dev, res);
1962                 if (IS_ERR(data->regs)) {
1963                         ret = PTR_ERR(data->regs);
1964                         goto err;
1965                 }
1966                 ioaddr = res->start;
1967         } else {
1968                 data->regs = pservice->reg_base;
1969                 ioaddr = pservice->ioaddr;
1970         }
1971
1972         clear_bit(MMU_ACTIVATED, &data->state);
1973         vcodec_enter_mode(data);
1974         ret = vpu_service_check_hw(data, ioaddr);
1975         if (ret < 0) {
1976                 vpu_err("error: hw info check faild\n");
1977                 goto err;
1978         }
1979
1980         data->dec_dev.iosize = data->hw_info->dec_io_size;
1981         data->dec_dev.hwregs = (volatile u32 *)((u8 *)data->regs + data->hw_info->dec_offset);
1982         data->reg_size = data->dec_dev.iosize;
1983
1984         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
1985                 data->enc_dev.iosize = data->hw_info->enc_io_size;
1986                 data->reg_size = data->reg_size > data->enc_dev.iosize ? data->reg_size : data->enc_dev.iosize;
1987                 data->enc_dev.hwregs = (volatile u32 *)((u8 *)data->regs + data->hw_info->enc_offset);
1988         }
1989
1990         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
1991         if (data->irq_enc > 0) {
1992                 ret = devm_request_threaded_irq(dev,
1993                         data->irq_enc, vepu_irq, vepu_isr,
1994                         IRQF_SHARED, dev_name(dev),
1995                         (void *)data);
1996                 if (ret) {
1997                         dev_err(dev,
1998                                 "error: can't request vepu irq %d\n",
1999                                 data->irq_enc);
2000                         goto err;
2001                 }
2002         }
2003         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2004         if (data->irq_dec > 0) {
2005                 ret = devm_request_threaded_irq(dev,
2006                         data->irq_dec, vdpu_irq, vdpu_isr,
2007                         IRQF_SHARED, dev_name(dev),
2008                         (void *)data);
2009                 if (ret) {
2010                         dev_err(dev,
2011                                 "error: can't request vdpu irq %d\n",
2012                                 data->irq_dec);
2013                         goto err;
2014                 }
2015         }
2016         atomic_set(&data->dec_dev.irq_count_codec, 0);
2017         atomic_set(&data->dec_dev.irq_count_pp, 0);
2018         atomic_set(&data->enc_dev.irq_count_codec, 0);
2019         atomic_set(&data->enc_dev.irq_count_pp, 0);
2020 #if defined(CONFIG_VCODEC_MMU)
2021         if (iommu_en) {
2022                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
2023                         sprintf(mmu_dev_dts_name,
2024                                 HEVC_IOMMU_COMPATIBLE_NAME);
2025                 else
2026                         sprintf(mmu_dev_dts_name,
2027                                 VPU_IOMMU_COMPATIBLE_NAME);
2028
2029                 data->mmu_dev =
2030                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2031
2032                 if (data->mmu_dev)
2033                         platform_set_sysmmu(data->mmu_dev, dev);
2034
2035                 rockchip_iovmm_set_fault_handler(dev, vcodec_sysmmu_fault_hdl);
2036         }
2037 #endif
2038         vcodec_exit_mode(data);
2039         /* create device node */
2040         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2041         if (ret) {
2042                 dev_err(dev, "alloc dev_t failed\n");
2043                 goto err;
2044         }
2045
2046         cdev_init(&data->cdev, &vpu_service_fops);
2047
2048         data->cdev.owner = THIS_MODULE;
2049         data->cdev.ops = &vpu_service_fops;
2050
2051         ret = cdev_add(&data->cdev, data->dev_t, 1);
2052
2053         if (ret) {
2054                 dev_err(dev, "add dev_t failed\n");
2055                 goto err;
2056         }
2057
2058         data->cls = class_create(THIS_MODULE, name);
2059
2060         if (IS_ERR(data->cls)) {
2061                 ret = PTR_ERR(data->cls);
2062                 dev_err(dev, "class_create err:%d\n", ret);
2063                 goto err;
2064         }
2065
2066         data->child_dev = device_create(data->cls, dev,
2067                 data->dev_t, NULL, name);
2068
2069         get_hw_info(data);
2070
2071         platform_set_drvdata(pdev, data);
2072
2073         INIT_LIST_HEAD(&data->lnk_service);
2074         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2075
2076 #ifdef CONFIG_DEBUG_FS
2077         data->debugfs_dir =
2078                 vcodec_debugfs_create_device_dir((char*)name, parent);
2079         if (data->debugfs_dir == NULL)
2080                 vpu_err("create debugfs dir %s failed\n", name);
2081
2082         data->debugfs_file_regs =
2083                 debugfs_create_file("regs", 0664,
2084                                     data->debugfs_dir, data,
2085                                     &debug_vcodec_fops);
2086 #endif
2087         return 0;
2088 err:
2089         if (data->irq_enc > 0)
2090                 free_irq(data->irq_enc, (void *)data);
2091         if (data->irq_dec > 0)
2092                 free_irq(data->irq_dec, (void *)data);
2093
2094         if (data->child_dev) {
2095                 device_destroy(data->cls, data->dev_t);
2096                 cdev_del(&data->cdev);
2097                 unregister_chrdev_region(data->dev_t, 1);
2098         }
2099
2100         if (data->cls)
2101                 class_destroy(data->cls);
2102         return -1;
2103 }
2104
2105 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2106 {
2107         device_destroy(data->cls, data->dev_t);
2108         class_destroy(data->cls);
2109         cdev_del(&data->cdev);
2110         unregister_chrdev_region(data->dev_t, 1);
2111
2112         free_irq(data->irq_enc, (void *)&data);
2113         free_irq(data->irq_dec, (void *)&data);
2114
2115 #ifdef CONFIG_DEBUG_FS
2116         debugfs_remove(data->debugfs_file_regs);
2117         debugfs_remove(data->debugfs_dir);
2118 #endif
2119 }
2120
2121 static void vcodec_read_property(struct device_node *np,
2122         struct vpu_service_info *pservice)
2123 {
2124         pservice->mode_bit = 0;
2125         pservice->mode_ctrl = 0;
2126         pservice->subcnt = 0;
2127
2128         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2129
2130         if (pservice->subcnt > 1) {
2131                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2132                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2133         }
2134 #ifdef CONFIG_MFD_SYSCON
2135         pservice->grf_base = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2136 #else
2137         pservice->grf_base = (u32*)RK_GRF_VIRT;
2138 #endif
2139         if (IS_ERR(pservice->grf_base)) {
2140                 vpu_err("can't find vpu grf property\n");
2141                 return;
2142         }
2143         of_property_read_string(np, "name", (const char**)&pservice->name);
2144 }
2145
2146 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2147 {
2148         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2149         pservice->curr_mode = -1;
2150
2151         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2152         INIT_LIST_HEAD(&pservice->waiting);
2153         INIT_LIST_HEAD(&pservice->running);
2154         mutex_init(&pservice->lock);
2155
2156         INIT_LIST_HEAD(&pservice->done);
2157         INIT_LIST_HEAD(&pservice->session);
2158         INIT_LIST_HEAD(&pservice->subdev_list);
2159
2160         pservice->reg_pproc     = NULL;
2161         atomic_set(&pservice->total_running, 0);
2162         pservice->enabled = false;
2163
2164         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2165
2166         pservice->ion_client = rockchip_ion_client_create("vpu");
2167         if (IS_ERR(pservice->ion_client)) {
2168                 vpu_err("failed to create ion client for vcodec ret %ld\n",
2169                         PTR_ERR(pservice->ion_client));
2170         } else {
2171                 vpu_debug(3, "vcodec ion client create success!\n");
2172         }
2173 }
2174
2175 static int vcodec_probe(struct platform_device *pdev)
2176 {
2177         int i;
2178         int ret = 0;
2179         struct resource *res = NULL;
2180         struct device *dev = &pdev->dev;
2181         struct device_node *np = pdev->dev.of_node;
2182         struct vpu_service_info *pservice =
2183                 devm_kzalloc(dev, sizeof(struct vpu_service_info), GFP_KERNEL);
2184
2185         pr_info("probe device %s\n", dev_name(dev));
2186
2187         vcodec_read_property(np, pservice);
2188         vcodec_init_drvdata(pservice);
2189
2190         if (strncmp(pservice->name, "hevc_service", 12) == 0)
2191                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2192         else if (strncmp(pservice->name, "vpu_service", 11) == 0)
2193                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2194         else
2195                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2196
2197         pservice->dev = dev;
2198
2199         if (0 > vpu_get_clk(pservice))
2200                 goto err;
2201
2202         vpu_service_power_on(pservice);
2203
2204         if (of_property_read_bool(np, "reg")) {
2205                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2206
2207                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2208                 if (IS_ERR(pservice->reg_base)) {
2209                         vpu_err("ioremap registers base failed\n");
2210                         ret = PTR_ERR(pservice->reg_base);
2211                         goto err;
2212                 }
2213                 pservice->ioaddr = res->start;
2214         } else {
2215                 pservice->reg_base = 0;
2216         }
2217
2218         if (of_property_read_bool(np, "subcnt")) {
2219                 for (i = 0; i<pservice->subcnt; i++) {
2220                         struct device_node *sub_np;
2221                         struct platform_device *sub_pdev;
2222                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2223                         sub_pdev = of_find_device_by_node(sub_np);
2224
2225                         vcodec_subdev_probe(sub_pdev, pservice);
2226                 }
2227         } else {
2228                 vcodec_subdev_probe(pdev, pservice);
2229         }
2230         platform_set_drvdata(pdev, pservice);
2231
2232         vpu_service_power_off(pservice);
2233
2234         pr_info("init success\n");
2235
2236         return 0;
2237
2238 err:
2239         pr_info("init failed\n");
2240         vpu_service_power_off(pservice);
2241         vpu_put_clk(pservice);
2242         wake_lock_destroy(&pservice->wake_lock);
2243
2244         if (res)
2245                 devm_release_mem_region(&pdev->dev, res->start, resource_size(res));
2246
2247         return ret;
2248 }
2249
2250 static int vcodec_remove(struct platform_device *pdev)
2251 {
2252         struct vpu_service_info *pservice = platform_get_drvdata(pdev);
2253         struct resource *res;
2254         struct vpu_subdev_data *data, *n;
2255
2256         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
2257                 vcodec_subdev_remove(data);
2258         }
2259
2260         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2261         devm_release_mem_region(&pdev->dev, res->start, resource_size(res));
2262         vpu_put_clk(pservice);
2263         wake_lock_destroy(&pservice->wake_lock);
2264
2265         return 0;
2266 }
2267
2268 #if defined(CONFIG_OF)
2269 static const struct of_device_id vcodec_service_dt_ids[] = {
2270         {.compatible = "vpu_service",},
2271         {.compatible = "rockchip,hevc_service",},
2272         {.compatible = "rockchip,vpu_combo",},
2273         {},
2274 };
2275 #endif
2276
2277 static struct platform_driver vcodec_driver = {
2278         .probe = vcodec_probe,
2279         .remove = vcodec_remove,
2280         .driver = {
2281                 .name = "vcodec",
2282                 .owner = THIS_MODULE,
2283 #if defined(CONFIG_OF)
2284                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2285 #endif
2286         },
2287 };
2288
2289 static void get_hw_info(struct vpu_subdev_data *data)
2290 {
2291         struct vpu_service_info *pservice = data->pservice;
2292         struct vpu_dec_config *dec = &pservice->dec_config;
2293         struct vpu_enc_config *enc = &pservice->enc_config;
2294         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2295                 u32 configReg   = data->dec_dev.hwregs[VPU_DEC_HWCFG0];
2296                 u32 asicID      = data->dec_dev.hwregs[0];
2297
2298                 dec->h264_support    = (configReg >> DWL_H264_E) & 0x3U;
2299                 dec->jpegSupport    = (configReg >> DWL_JPEG_E) & 0x01U;
2300                 if (dec->jpegSupport && ((configReg >> DWL_PJPEG_E) & 0x01U))
2301                         dec->jpegSupport = JPEG_PROGRESSIVE;
2302                 dec->mpeg4Support   = (configReg >> DWL_MPEG4_E) & 0x3U;
2303                 dec->vc1Support     = (configReg >> DWL_VC1_E) & 0x3U;
2304                 dec->mpeg2Support   = (configReg >> DWL_MPEG2_E) & 0x01U;
2305                 dec->sorensonSparkSupport = (configReg >> DWL_SORENSONSPARK_E) & 0x01U;
2306                 dec->refBufSupport  = (configReg >> DWL_REF_BUFF_E) & 0x01U;
2307                 dec->vp6Support     = (configReg >> DWL_VP6_E) & 0x01U;
2308
2309                 dec->maxDecPicWidth = 4096;
2310
2311                 /* 2nd Config register */
2312                 configReg   = data->dec_dev.hwregs[VPU_DEC_HWCFG1];
2313                 if (dec->refBufSupport) {
2314                         if ((configReg >> DWL_REF_BUFF_ILACE_E) & 0x01U)
2315                                 dec->refBufSupport |= 2;
2316                         if ((configReg >> DWL_REF_BUFF_DOUBLE_E) & 0x01U)
2317                                 dec->refBufSupport |= 4;
2318                 }
2319                 dec->customMpeg4Support = (configReg >> DWL_MPEG4_CUSTOM_E) & 0x01U;
2320                 dec->vp7Support     = (configReg >> DWL_VP7_E) & 0x01U;
2321                 dec->vp8Support     = (configReg >> DWL_VP8_E) & 0x01U;
2322                 dec->avsSupport     = (configReg >> DWL_AVS_E) & 0x01U;
2323
2324                 /* JPEG xtensions */
2325                 if (((asicID >> 16) >= 0x8190U) || ((asicID >> 16) == 0x6731U))
2326                         dec->jpegESupport = (configReg >> DWL_JPEG_EXT_E) & 0x01U;
2327                 else
2328                         dec->jpegESupport = JPEG_EXT_NOT_SUPPORTED;
2329
2330                 if (((asicID >> 16) >= 0x9170U) || ((asicID >> 16) == 0x6731U) )
2331                         dec->rvSupport = (configReg >> DWL_RV_E) & 0x03U;
2332                 else
2333                         dec->rvSupport = RV_NOT_SUPPORTED;
2334                 dec->mvcSupport = (configReg >> DWL_MVC_E) & 0x03U;
2335
2336                 if (dec->refBufSupport && (asicID >> 16) == 0x6731U )
2337                         dec->refBufSupport |= 8; /* enable HW support for offset */
2338
2339                 if (!cpu_is_rk3036()) {
2340                         configReg = data->enc_dev.hwregs[63];
2341                         enc->maxEncodedWidth = configReg & ((1 << 11) - 1);
2342                         enc->h264Enabled = (configReg >> 27) & 1;
2343                         enc->mpeg4Enabled = (configReg >> 26) & 1;
2344                         enc->jpegEnabled = (configReg >> 25) & 1;
2345                         enc->vsEnabled = (configReg >> 24) & 1;
2346                         enc->rgbEnabled = (configReg >> 28) & 1;
2347                         enc->reg_size = data->reg_size;
2348                         enc->reserv[0] = enc->reserv[1] = 0;
2349                 }
2350                 pservice->auto_freq = true;
2351                 vpu_debug(3, "vpu_service set to auto frequency mode\n");
2352                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2353
2354                 pservice->bug_dec_addr = cpu_is_rk30xx();
2355         } else {
2356                 if (cpu_is_rk3036()  || cpu_is_rk312x())
2357                         dec->maxDecPicWidth = 1920;
2358                 else
2359                         dec->maxDecPicWidth = 4096;
2360                 /* disable frequency switch in hevc.*/
2361                 pservice->auto_freq = false;
2362         }
2363 }
2364
2365 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2366 {
2367         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2368         struct vpu_service_info *pservice = data->pservice;
2369         vpu_device *dev = &data->dec_dev;
2370         u32 raw_status;
2371         u32 irq_status;
2372
2373         /*vcodec_enter_mode(data);*/
2374
2375         irq_status = raw_status = readl(dev->hwregs + DEC_INTERRUPT_REGISTER);
2376
2377         vpu_debug(3, "%s status %08x\n", __func__, raw_status);
2378
2379         if (irq_status & DEC_INTERRUPT_BIT) {
2380                 pr_debug("dec_isr dec %x\n", irq_status);
2381                 if ((irq_status & 0x40001) == 0x40001) {
2382                         do {
2383                                 irq_status =
2384                                         readl(dev->hwregs +
2385                                                 DEC_INTERRUPT_REGISTER);
2386                         } while ((irq_status & 0x40001) == 0x40001);
2387                 }
2388
2389                 writel(0, dev->hwregs + DEC_INTERRUPT_REGISTER);
2390                 atomic_add(1, &dev->irq_count_codec);
2391         }
2392
2393         if (data->hw_info->hw_id != HEVC_ID) {
2394                 irq_status = readl(dev->hwregs + PP_INTERRUPT_REGISTER);
2395                 if (irq_status & PP_INTERRUPT_BIT) {
2396                         pr_debug("vdpu_isr pp  %x\n", irq_status);
2397                         /* clear pp IRQ */
2398                         writel(irq_status & (~DEC_INTERRUPT_BIT), dev->hwregs + PP_INTERRUPT_REGISTER);
2399                         atomic_add(1, &dev->irq_count_pp);
2400                 }
2401         }
2402
2403         pservice->irq_status = raw_status;
2404
2405         /*vcodec_exit_mode(pservice);*/
2406
2407         if (atomic_read(&dev->irq_count_pp) ||
2408             atomic_read(&dev->irq_count_codec))
2409                 return IRQ_WAKE_THREAD;
2410         else
2411                 return IRQ_NONE;
2412 }
2413
2414 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2415 {
2416         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2417         struct vpu_service_info *pservice = data->pservice;
2418         vpu_device *dev = &data->dec_dev;
2419
2420         mutex_lock(&pservice->lock);
2421         if (atomic_read(&dev->irq_count_codec)) {
2422 #if VPU_SERVICE_SHOW_TIME
2423                 do_gettimeofday(&dec_end);
2424                 vpu_debug(3, "dec task: %ld ms\n",
2425                         (dec_end.tv_sec  - dec_start.tv_sec)  * 1000 +
2426                         (dec_end.tv_usec - dec_start.tv_usec) / 1000);
2427 #endif
2428                 atomic_sub(1, &dev->irq_count_codec);
2429                 if (NULL == pservice->reg_codec) {
2430                         vpu_err("error: dec isr with no task waiting\n");
2431                 } else {
2432                         reg_from_run_to_done(data, pservice->reg_codec);
2433                         writel(1, dev->hwregs + 101);
2434                 }
2435         }
2436
2437         if (atomic_read(&dev->irq_count_pp)) {
2438 #if VPU_SERVICE_SHOW_TIME
2439                 do_gettimeofday(&pp_end);
2440                 printk("pp  task: %ld ms\n",
2441                         (pp_end.tv_sec  - pp_start.tv_sec)  * 1000 +
2442                         (pp_end.tv_usec - pp_start.tv_usec) / 1000);
2443 #endif
2444                 atomic_sub(1, &dev->irq_count_pp);
2445                 if (NULL == pservice->reg_pproc) {
2446                         vpu_err("error: pp isr with no task waiting\n");
2447                 } else {
2448                         reg_from_run_to_done(data, pservice->reg_pproc);
2449                 }
2450         }
2451         try_set_reg(data);
2452         mutex_unlock(&pservice->lock);
2453         return IRQ_HANDLED;
2454 }
2455
2456 static irqreturn_t vepu_irq(int irq, void *dev_id)
2457 {
2458         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2459         struct vpu_service_info *pservice = data->pservice;
2460         vpu_device *dev = &data->enc_dev;
2461         u32 irq_status;
2462
2463         /*vcodec_enter_mode(data);*/
2464         irq_status= readl(dev->hwregs + ENC_INTERRUPT_REGISTER);
2465
2466         pr_debug("vepu_irq irq status %x\n", irq_status);
2467
2468 #if VPU_SERVICE_SHOW_TIME
2469         do_gettimeofday(&enc_end);
2470         vpu_debug(3, "enc task: %ld ms\n",
2471                 (enc_end.tv_sec  - enc_start.tv_sec)  * 1000 +
2472                 (enc_end.tv_usec - enc_start.tv_usec) / 1000);
2473 #endif
2474         if (likely(irq_status & ENC_INTERRUPT_BIT)) {
2475                 /* clear enc IRQ */
2476                 writel(irq_status & (~ENC_INTERRUPT_BIT), dev->hwregs + ENC_INTERRUPT_REGISTER);
2477                 atomic_add(1, &dev->irq_count_codec);
2478         }
2479
2480         pservice->irq_status = irq_status;
2481
2482         /*vcodec_exit_mode(pservice);*/
2483
2484         if (atomic_read(&dev->irq_count_codec))
2485                 return IRQ_WAKE_THREAD;
2486         else
2487                 return IRQ_NONE;
2488 }
2489
2490 static irqreturn_t vepu_isr(int irq, void *dev_id)
2491 {
2492         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2493         struct vpu_service_info *pservice = data->pservice;
2494         vpu_device *dev = &data->enc_dev;
2495
2496         mutex_lock(&pservice->lock);
2497         if (atomic_read(&dev->irq_count_codec)) {
2498                 atomic_sub(1, &dev->irq_count_codec);
2499                 if (NULL == pservice->reg_codec) {
2500                         vpu_err("error: enc isr with no task waiting\n");
2501                 } else {
2502                         reg_from_run_to_done(data, pservice->reg_codec);
2503                 }
2504         }
2505         try_set_reg(data);
2506         mutex_unlock(&pservice->lock);
2507         return IRQ_HANDLED;
2508 }
2509
2510 static int __init vcodec_service_init(void)
2511 {
2512         int ret;
2513
2514         if ((ret = platform_driver_register(&vcodec_driver)) != 0) {
2515                 vpu_err("Platform device register failed (%d).\n", ret);
2516                 return ret;
2517         }
2518
2519 #ifdef CONFIG_DEBUG_FS
2520         vcodec_debugfs_init();
2521 #endif
2522
2523         return ret;
2524 }
2525
2526 static void __exit vcodec_service_exit(void)
2527 {
2528 #ifdef CONFIG_DEBUG_FS
2529         vcodec_debugfs_exit();
2530 #endif
2531
2532         platform_driver_unregister(&vcodec_driver);
2533 }
2534
2535 module_init(vcodec_service_init);
2536 module_exit(vcodec_service_exit);
2537
2538 #ifdef CONFIG_DEBUG_FS
2539 #include <linux/seq_file.h>
2540
2541 static int vcodec_debugfs_init()
2542 {
2543         parent = debugfs_create_dir("vcodec", NULL);
2544         if (!parent)
2545                 return -1;
2546
2547         return 0;
2548 }
2549
2550 static void vcodec_debugfs_exit()
2551 {
2552         debugfs_remove(parent);
2553 }
2554
2555 static struct dentry* vcodec_debugfs_create_device_dir(char *dirname, struct dentry *parent)
2556 {
2557         return debugfs_create_dir(dirname, parent);
2558 }
2559
2560 static int debug_vcodec_show(struct seq_file *s, void *unused)
2561 {
2562         struct vpu_subdev_data *data = s->private;
2563         struct vpu_service_info *pservice = data->pservice;
2564         unsigned int i, n;
2565         vpu_reg *reg, *reg_tmp;
2566         vpu_session *session, *session_tmp;
2567
2568         mutex_lock(&pservice->lock);
2569         vpu_service_power_on(pservice);
2570         if (data->hw_info->hw_id != HEVC_ID) {
2571                 seq_printf(s, "\nENC Registers:\n");
2572                 n = data->enc_dev.iosize >> 2;
2573                 for (i = 0; i < n; i++)
2574                         seq_printf(s, "\tswreg%d = %08X\n", i, readl(data->enc_dev.hwregs + i));
2575         }
2576         seq_printf(s, "\nDEC Registers:\n");
2577         n = data->dec_dev.iosize >> 2;
2578         for (i = 0; i < n; i++)
2579                 seq_printf(s, "\tswreg%d = %08X\n", i, readl(data->dec_dev.hwregs + i));
2580
2581         seq_printf(s, "\nvpu service status:\n");
2582         list_for_each_entry_safe(session, session_tmp, &pservice->session, list_session) {
2583                 seq_printf(s, "session pid %d type %d:\n", session->pid, session->type);
2584                 /*seq_printf(s, "waiting reg set %d\n");*/
2585                 list_for_each_entry_safe(reg, reg_tmp, &session->waiting, session_link) {
2586                         seq_printf(s, "waiting register set\n");
2587                 }
2588                 list_for_each_entry_safe(reg, reg_tmp, &session->running, session_link) {
2589                         seq_printf(s, "running register set\n");
2590                 }
2591                 list_for_each_entry_safe(reg, reg_tmp, &session->done, session_link) {
2592                         seq_printf(s, "done    register set\n");
2593                 }
2594         }
2595         mutex_unlock(&pservice->lock);
2596
2597         return 0;
2598 }
2599
2600 static int debug_vcodec_open(struct inode *inode, struct file *file)
2601 {
2602         return single_open(file, debug_vcodec_show, inode->i_private);
2603 }
2604
2605 #endif
2606
2607 #if HEVC_TEST_ENABLE & defined(CONFIG_ION_ROCKCHIP)
2608 #include "hevc_test_inc/pps_00.h"
2609 #include "hevc_test_inc/register_00.h"
2610 #include "hevc_test_inc/rps_00.h"
2611 #include "hevc_test_inc/scaling_list_00.h"
2612 #include "hevc_test_inc/stream_00.h"
2613
2614 #include "hevc_test_inc/pps_01.h"
2615 #include "hevc_test_inc/register_01.h"
2616 #include "hevc_test_inc/rps_01.h"
2617 #include "hevc_test_inc/scaling_list_01.h"
2618 #include "hevc_test_inc/stream_01.h"
2619
2620 #include "hevc_test_inc/cabac.h"
2621
2622 extern struct ion_client *rockchip_ion_client_create(const char * name);
2623
2624 static struct ion_client *ion_client = NULL;
2625 u8* get_align_ptr(u8* tbl, int len, u32 *phy)
2626 {
2627         int size = (len+15) & (~15);
2628         struct ion_handle *handle;
2629         u8 *ptr;
2630
2631         if (ion_client == NULL)
2632                 ion_client = rockchip_ion_client_create("vcodec");
2633
2634         handle = ion_alloc(ion_client, (size_t)len, 16, ION_HEAP(ION_CMA_HEAP_ID), 0);
2635
2636         ptr = ion_map_kernel(ion_client, handle);
2637
2638         ion_phys(ion_client, handle, phy, &size);
2639
2640         memcpy(ptr, tbl, len);
2641
2642         return ptr;
2643 }
2644
2645 u8* get_align_ptr_no_copy(int len, u32 *phy)
2646 {
2647         int size = (len+15) & (~15);
2648         struct ion_handle *handle;
2649         u8 *ptr;
2650
2651         if (ion_client == NULL)
2652                 ion_client = rockchip_ion_client_create("vcodec");
2653
2654         handle = ion_alloc(ion_client, (size_t)len, 16, ION_HEAP(ION_CMA_HEAP_ID), 0);
2655
2656         ptr = ion_map_kernel(ion_client, handle);
2657
2658         ion_phys(ion_client, handle, phy, &size);
2659
2660         return ptr;
2661 }
2662
2663 #define TEST_CNT    2
2664 static int hevc_test_case0(vpu_service_info *pservice)
2665 {
2666         vpu_session session;
2667         vpu_reg *reg;
2668         unsigned long size = 272;
2669         int testidx = 0;
2670         int ret = 0;
2671         u8 *pps_tbl[TEST_CNT];
2672         u8 *register_tbl[TEST_CNT];
2673         u8 *rps_tbl[TEST_CNT];
2674         u8 *scaling_list_tbl[TEST_CNT];
2675         u8 *stream_tbl[TEST_CNT];
2676
2677         int stream_size[2];
2678         int pps_size[2];
2679         int rps_size[2];
2680         int scl_size[2];
2681         int cabac_size[2];
2682
2683         u32 phy_pps;
2684         u32 phy_rps;
2685         u32 phy_scl;
2686         u32 phy_str;
2687         u32 phy_yuv;
2688         u32 phy_ref;
2689         u32 phy_cabac;
2690
2691         volatile u8 *stream_buf;
2692         volatile u8 *pps_buf;
2693         volatile u8 *rps_buf;
2694         volatile u8 *scl_buf;
2695         volatile u8 *yuv_buf;
2696         volatile u8 *cabac_buf;
2697         volatile u8 *ref_buf;
2698
2699         u8 *pps;
2700         u8 *yuv[2];
2701         int i;
2702
2703         pps_tbl[0] = pps_00;
2704         pps_tbl[1] = pps_01;
2705
2706         register_tbl[0] = register_00;
2707         register_tbl[1] = register_01;
2708
2709         rps_tbl[0] = rps_00;
2710         rps_tbl[1] = rps_01;
2711
2712         scaling_list_tbl[0] = scaling_list_00;
2713         scaling_list_tbl[1] = scaling_list_01;
2714
2715         stream_tbl[0] = stream_00;
2716         stream_tbl[1] = stream_01;
2717
2718         stream_size[0] = sizeof(stream_00);
2719         stream_size[1] = sizeof(stream_01);
2720
2721         pps_size[0] = sizeof(pps_00);
2722         pps_size[1] = sizeof(pps_01);
2723
2724         rps_size[0] = sizeof(rps_00);
2725         rps_size[1] = sizeof(rps_01);
2726
2727         scl_size[0] = sizeof(scaling_list_00);
2728         scl_size[1] = sizeof(scaling_list_01);
2729
2730         cabac_size[0] = sizeof(Cabac_table);
2731         cabac_size[1] = sizeof(Cabac_table);
2732
2733         /* create session */
2734         session.pid = current->pid;
2735         session.type = VPU_DEC;
2736         INIT_LIST_HEAD(&session.waiting);
2737         INIT_LIST_HEAD(&session.running);
2738         INIT_LIST_HEAD(&session.done);
2739         INIT_LIST_HEAD(&session.list_session);
2740         init_waitqueue_head(&session.wait);
2741         atomic_set(&session.task_running, 0);
2742         list_add_tail(&session.list_session, &pservice->session);
2743
2744         yuv[0] = get_align_ptr_no_copy(256*256*2, &phy_yuv);
2745         yuv[1] = get_align_ptr_no_copy(256*256*2, &phy_ref);
2746
2747         while (testidx < TEST_CNT) {
2748                 /* create registers */
2749                 reg = kmalloc(sizeof(vpu_reg)+pservice->reg_size, GFP_KERNEL);
2750                 if (NULL == reg) {
2751                         vpu_err("error: kmalloc fail in reg_init\n");
2752                         return -1;
2753                 }
2754
2755                 if (size > pservice->reg_size) {
2756                         printk("warning: vpu reg size %lu is larger than hw reg size %lu\n", size, pservice->reg_size);
2757                         size = pservice->reg_size;
2758                 }
2759                 reg->session = &session;
2760                 reg->type = session.type;
2761                 reg->size = size;
2762                 reg->freq = VPU_FREQ_DEFAULT;
2763                 reg->reg = (unsigned long *)&reg[1];
2764                 INIT_LIST_HEAD(&reg->session_link);
2765                 INIT_LIST_HEAD(&reg->status_link);
2766
2767                 /* TODO: stuff registers */
2768                 memcpy(&reg->reg[0], register_tbl[testidx], /*sizeof(register_00)*/ 176);
2769
2770                 stream_buf = get_align_ptr(stream_tbl[testidx], stream_size[testidx], &phy_str);
2771                 pps_buf = get_align_ptr(pps_tbl[0], pps_size[0], &phy_pps);
2772                 rps_buf = get_align_ptr(rps_tbl[testidx], rps_size[testidx], &phy_rps);
2773                 scl_buf = get_align_ptr(scaling_list_tbl[testidx], scl_size[testidx], &phy_scl);
2774                 cabac_buf = get_align_ptr(Cabac_table, cabac_size[testidx], &phy_cabac);
2775
2776                 pps = pps_buf;
2777
2778                 /* TODO: replace reigster address */
2779                 for (i=0; i<64; i++) {
2780                         u32 scaling_offset;
2781                         u32 tmp;
2782
2783                         scaling_offset = (u32)pps[i*80+74];
2784                         scaling_offset += (u32)pps[i*80+75] << 8;
2785                         scaling_offset += (u32)pps[i*80+76] << 16;
2786                         scaling_offset += (u32)pps[i*80+77] << 24;
2787
2788                         tmp = phy_scl + scaling_offset;
2789
2790                         pps[i*80+74] = tmp & 0xff;
2791                         pps[i*80+75] = (tmp >> 8) & 0xff;
2792                         pps[i*80+76] = (tmp >> 16) & 0xff;
2793                         pps[i*80+77] = (tmp >> 24) & 0xff;
2794                 }
2795
2796                 printk("%s %d, phy stream %08x, phy pps %08x, phy rps %08x\n",
2797                         __func__, __LINE__, phy_str, phy_pps, phy_rps);
2798
2799                 reg->reg[1] = 0x21;
2800                 reg->reg[4] = phy_str;
2801                 reg->reg[5] = ((stream_size[testidx]+15)&(~15))+64;
2802                 reg->reg[6] = phy_cabac;
2803                 reg->reg[7] = testidx?phy_ref:phy_yuv;
2804                 reg->reg[42] = phy_pps;
2805                 reg->reg[43] = phy_rps;
2806                 for (i = 10; i <= 24; i++)
2807                         reg->reg[i] = phy_yuv;
2808
2809                 mutex_lock(pservice->lock);
2810                 list_add_tail(&reg->status_link, &pservice->waiting);
2811                 list_add_tail(&reg->session_link, &session.waiting);
2812                 mutex_unlock(pservice->lock);
2813
2814                 /* stuff hardware */
2815                 try_set_reg(data);
2816
2817                 /* wait for result */
2818                 ret = wait_event_timeout(session.wait, !list_empty(&session.done), VPU_TIMEOUT_DELAY);
2819                 if (!list_empty(&session.done)) {
2820                         if (ret < 0)
2821                                 vpu_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session.pid, ret);
2822                         ret = 0;
2823                 } else {
2824                         if (unlikely(ret < 0)) {
2825                                 vpu_err("error: pid %d wait task ret %d\n", session.pid, ret);
2826                         } else if (0 == ret) {
2827                                 vpu_err("error: pid %d wait %d task done timeout\n", session.pid, atomic_read(&session.task_running));
2828                                 ret = -ETIMEDOUT;
2829                         }
2830                 }
2831                 if (ret < 0) {
2832                         int task_running = atomic_read(&session.task_running);
2833                         int n;
2834                         mutex_lock(pservice->lock);
2835                         vpu_service_dump(pservice);
2836                         if (task_running) {
2837                                 atomic_set(&session.task_running, 0);
2838                                 atomic_sub(task_running, &pservice->total_running);
2839                                 printk("%d task is running but not return, reset hardware...", task_running);
2840                                 vpu_reset(data);
2841                                 printk("done\n");
2842                         }
2843                         vpu_service_session_clear(pservice, &session);
2844                         mutex_unlock(pservice->lock);
2845
2846                         printk("\nDEC Registers:\n");
2847                         n = data->dec_dev.iosize >> 2;
2848                         for (i=0; i<n; i++)
2849                                 printk("\tswreg%d = %08X\n", i, readl(data->dec_dev.hwregs + i));
2850
2851                         vpu_err("test index %d failed\n", testidx);
2852                         break;
2853                 } else {
2854                         vpu_debug(3, "test index %d success\n", testidx);
2855
2856                         vpu_reg *reg = list_entry(session.done.next, vpu_reg, session_link);
2857
2858                         for (i=0; i<68; i++) {
2859                                 if (i % 4 == 0)
2860                                         printk("%02d: ", i);
2861                                 printk("%08x ", reg->reg[i]);
2862                                 if ((i+1) % 4 == 0)
2863                                         printk("\n");
2864                         }
2865
2866                         testidx++;
2867                 }
2868
2869                 reg_deinit(data, reg);
2870         }
2871
2872         return 0;
2873 }
2874
2875 #endif
2876