arch/arm/mach-rockchip/vcodec_service.c

   1 /**
   2  * Copyright (C) 2014 ROCKCHIP, Inc.
   3  * author: chenhengming chm@rock-chips.com
   4  *         Alpha Lin, alpha.lin@rock-chips.com
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/clk.h>
  20 #include <linux/delay.h>
  21 #include <linux/init.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/io.h>
  24 #include <linux/kernel.h>
  25 #include <linux/module.h>
  26 #include <linux/fs.h>
  27 #include <linux/ioport.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/mm.h>
  30 #include <linux/poll.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/sched.h>
  33 #include <linux/slab.h>
  34 #include <linux/wakelock.h>
  35 #include <linux/cdev.h>
  36 #include <linux/of.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/of_irq.h>
  39 #include <linux/rockchip/cpu.h>
  40 #include <linux/rockchip/cru.h>
  41 #ifdef CONFIG_MFD_SYSCON
  42 #include <linux/regmap.h>
  43 #endif
  44 #include <linux/mfd/syscon.h>
  45
  46 #include <asm/cacheflush.h>
  47 #include <linux/uaccess.h>
  48 #include <linux/rockchip/grf.h>
  49
  50 #if defined(CONFIG_ION_ROCKCHIP)
  51 #include <linux/rockchip_ion.h>
  52 #endif
  53
  54 #if defined(CONFIG_ROCKCHIP_IOMMU) & defined(CONFIG_ION_ROCKCHIP)
  55 #define CONFIG_VCODEC_MMU
  56 #endif
  57
  58 #ifdef CONFIG_VCODEC_MMU
  59 #include <linux/rockchip-iovmm.h>
  60 #include <linux/dma-buf.h>
  61 #endif
  62
  63 #ifdef CONFIG_DEBUG_FS
  64 #include <linux/debugfs.h>
  65 #endif
  66
  67 #if defined(CONFIG_ARCH_RK319X)
  68 #include <mach/grf.h>
  69 #endif
  70
  71 #include "vcodec_service.h"
  72
  73 static int debug;
  74 module_param(debug, int, S_IRUGO | S_IWUSR);
  75 MODULE_PARM_DESC(debug,
  76                  "Debug level - higher value produces more verbose messages");
  77
  78 #define HEVC_TEST_ENABLE        0
  79 #define VCODEC_CLOCK_ENABLE     1
  80
  81 typedef enum {
  82         VPU_DEC_ID_9190         = 0x6731,
  83         VPU_ID_8270             = 0x8270,
  84         VPU_ID_4831             = 0x4831,
  85         HEVC_ID                 = 0x6867,
  86 } VPU_HW_ID;
  87
  88 enum VPU_HW_SPEC {
  89         VPU_TYPE_VPU,
  90         VPU_TYPE_HEVC,
  91         VPU_TYPE_COMBO_NOENC,
  92         VPU_TYPE_COMBO
  93 };
  94
  95 typedef enum {
  96         VPU_DEC_TYPE_9190       = 0,
  97         VPU_ENC_TYPE_8270       = 0x100,
  98         VPU_ENC_TYPE_4831       ,
  99 } VPU_HW_TYPE_E;
 100
 101 typedef enum VPU_FREQ {
 102         VPU_FREQ_200M,
 103         VPU_FREQ_266M,
 104         VPU_FREQ_300M,
 105         VPU_FREQ_400M,
 106         VPU_FREQ_500M,
 107         VPU_FREQ_600M,
 108         VPU_FREQ_DEFAULT,
 109         VPU_FREQ_BUT,
 110 } VPU_FREQ;
 111
 112 typedef struct {
 113         VPU_HW_ID               hw_id;
 114         unsigned long           hw_addr;
 115         unsigned long           enc_offset;
 116         unsigned long           enc_reg_num;
 117         unsigned long           enc_io_size;
 118         unsigned long           dec_offset;
 119         unsigned long           dec_reg_num;
 120         unsigned long           dec_io_size;
 121 } VPU_HW_INFO_E;
 122
 123 struct extra_info_elem {
 124         u32 index;
 125         u32 offset;
 126 };
 127
 128 #define EXTRA_INFO_MAGIC        0x4C4A46
 129
 130 struct extra_info_for_iommu {
 131         u32 magic;
 132         u32 cnt;
 133         struct extra_info_elem elem[20];
 134 };
 135
 136 #define VPU_SERVICE_SHOW_TIME                   0
 137
 138 #if VPU_SERVICE_SHOW_TIME
 139 static struct timeval enc_start, enc_end;
 140 static struct timeval dec_start, dec_end;
 141 static struct timeval pp_start,  pp_end;
 142 #endif
 143
 144 #define MHZ                                     (1000*1000)
 145
 146 #define REG_NUM_9190_DEC                        (60)
 147 #define REG_NUM_9190_PP                         (41)
 148 #define REG_NUM_9190_DEC_PP                     (REG_NUM_9190_DEC+REG_NUM_9190_PP)
 149
 150 #define REG_NUM_DEC_PP                          (REG_NUM_9190_DEC+REG_NUM_9190_PP)
 151
 152 #define REG_NUM_ENC_8270                        (96)
 153 #define REG_SIZE_ENC_8270                       (0x200)
 154 #define REG_NUM_ENC_4831                        (164)
 155 #define REG_SIZE_ENC_4831                       (0x400)
 156
 157 #define REG_NUM_HEVC_DEC                        (68)
 158
 159 #define SIZE_REG(reg)                           ((reg)*4)
 160
 161 static VPU_HW_INFO_E vpu_hw_set[] = {
 162         [0] = {
 163                 .hw_id          = VPU_ID_8270,
 164                 .hw_addr        = 0,
 165                 .enc_offset     = 0x0,
 166                 .enc_reg_num    = REG_NUM_ENC_8270,
 167                 .enc_io_size    = REG_NUM_ENC_8270 * 4,
 168                 .dec_offset     = REG_SIZE_ENC_8270,
 169                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 170                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 171         },
 172         [1] = {
 173                 .hw_id          = VPU_ID_4831,
 174                 .hw_addr        = 0,
 175                 .enc_offset     = 0x0,
 176                 .enc_reg_num    = REG_NUM_ENC_4831,
 177                 .enc_io_size    = REG_NUM_ENC_4831 * 4,
 178                 .dec_offset     = REG_SIZE_ENC_4831,
 179                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 180                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 181         },
 182         [2] = {
 183                 .hw_id          = HEVC_ID,
 184                 .hw_addr        = 0,
 185                 .dec_offset     = 0x0,
 186                 .dec_reg_num    = REG_NUM_HEVC_DEC,
 187                 .dec_io_size    = REG_NUM_HEVC_DEC * 4,
 188         },
 189         [3] = {
 190                 .hw_id          = VPU_DEC_ID_9190,
 191                 .hw_addr        = 0,
 192                 .enc_offset     = 0x0,
 193                 .enc_reg_num    = 0,
 194                 .enc_io_size    = 0,
 195                 .dec_offset     = 0,
 196                 .dec_reg_num    = REG_NUM_9190_DEC_PP,
 197                 .dec_io_size    = REG_NUM_9190_DEC_PP * 4,
 198         },
 199 };
 200
 201
 202 #define DEC_INTERRUPT_REGISTER                  1
 203 #define PP_INTERRUPT_REGISTER                   60
 204 #define ENC_INTERRUPT_REGISTER                  1
 205
 206 #define DEC_INTERRUPT_BIT                       0x100
 207 #define DEC_BUFFER_EMPTY_BIT                    0x4000
 208 #define PP_INTERRUPT_BIT                        0x100
 209 #define ENC_INTERRUPT_BIT                       0x1
 210
 211 #define HEVC_DEC_INT_RAW_BIT                    0x200
 212 #define HEVC_DEC_STR_ERROR_BIT                  0x4000
 213 #define HEVC_DEC_BUS_ERROR_BIT                  0x2000
 214 #define HEVC_DEC_BUFFER_EMPTY_BIT               0x10000
 215
 216 #define VPU_REG_EN_ENC                          14
 217 #define VPU_REG_ENC_GATE                        2
 218 #define VPU_REG_ENC_GATE_BIT                    (1<<4)
 219
 220 #define VPU_REG_EN_DEC                          1
 221 #define VPU_REG_DEC_GATE                        2
 222 #define VPU_REG_DEC_GATE_BIT                    (1<<10)
 223 #define VPU_REG_EN_PP                           0
 224 #define VPU_REG_PP_GATE                         1
 225 #define VPU_REG_PP_GATE_BIT                     (1<<8)
 226 #define VPU_REG_EN_DEC_PP                       1
 227 #define VPU_REG_DEC_PP_GATE                     61
 228 #define VPU_REG_DEC_PP_GATE_BIT                 (1<<8)
 229
 230 #define DEBUG
 231 #ifdef DEBUG
 232 #define vpu_debug(level, fmt, args...)                          \
 233         do {                                                    \
 234                 if (debug >= level)                             \
 235                         pr_info("%s:%d: " fmt,                  \
 236                                  __func__, __LINE__, ##args);   \
 237         } while (0)
 238 #else
 239 #define vpu_debug(level, fmt, args...)
 240 #endif
 241
 242 #define vpu_debug_enter() vpu_debug(4, "enter\n")
 243 #define vpu_debug_leave() vpu_debug(4, "leave\n")
 244
 245 #define vpu_err(fmt, args...)                           \
 246                 pr_err("%s:%d: " fmt, __func__, __LINE__, ##args)
 247
 248 #if defined(CONFIG_VCODEC_MMU)
 249 static u8 addr_tbl_vpu_h264dec[] = {
 250         12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
 251         25, 26, 27, 28, 29, 40, 41
 252 };
 253
 254 static u8 addr_tbl_vpu_vp8dec[] = {
 255         10, 12, 13, 14, 18, 19, 22, 23, 24, 25, 26, 27, 28, 29, 40
 256 };
 257
 258 static u8 addr_tbl_vpu_vp6dec[] = {
 259         12, 13, 14, 18, 27, 40
 260 };
 261
 262 static u8 addr_tbl_vpu_vc1dec[] = {
 263         12, 13, 14, 15, 16, 17, 27, 41
 264 };
 265
 266 static u8 addr_tbl_vpu_jpegdec[] = {
 267         12, 40, 66, 67
 268 };
 269
 270 static u8 addr_tbl_vpu_defaultdec[] = {
 271         12, 13, 14, 15, 16, 17, 40, 41
 272 };
 273
 274 static u8 addr_tbl_vpu_enc[] = {
 275         5, 6, 7, 8, 9, 10, 11, 12, 13, 51
 276 };
 277
 278 static u8 addr_tbl_hevc_dec[] = {
 279         4, 6, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
 280         21, 22, 23, 24, 42, 43
 281 };
 282 #endif
 283
 284 enum VPU_DEC_FMT {
 285         VPU_DEC_FMT_H264,
 286         VPU_DEC_FMT_MPEG4,
 287         VPU_DEC_FMT_H263,
 288         VPU_DEC_FMT_JPEG,
 289         VPU_DEC_FMT_VC1,
 290         VPU_DEC_FMT_MPEG2,
 291         VPU_DEC_FMT_MPEG1,
 292         VPU_DEC_FMT_VP6,
 293         VPU_DEC_FMT_RV,
 294         VPU_DEC_FMT_VP7,
 295         VPU_DEC_FMT_VP8,
 296         VPU_DEC_FMT_AVS,
 297         VPU_DEC_FMT_SVC,
 298         VPU_DEC_FMT_VC2,
 299         VPU_DEC_FMT_MVC,
 300         VPU_DEC_FMT_THEORA,
 301         VPU_DEC_FMT_RES
 302 };
 303
 304 /**
 305  * struct for process session which connect to vpu
 306  *
 307  * @author ChenHengming (2011-5-3)
 308  */
 309 typedef struct vpu_session {
 310         enum VPU_CLIENT_TYPE type;
 311         /* a linked list of data so we can access them for debugging */
 312         struct list_head list_session;
 313         /* a linked list of register data waiting for process */
 314         struct list_head waiting;
 315         /* a linked list of register data in processing */
 316         struct list_head running;
 317         /* a linked list of register data processed */
 318         struct list_head done;
 319         wait_queue_head_t wait;
 320         pid_t pid;
 321         atomic_t task_running;
 322 } vpu_session;
 323
 324 /**
 325  * struct for process register set
 326  *
 327  * @author ChenHengming (2011-5-4)
 328  */
 329 typedef struct vpu_reg {
 330         enum VPU_CLIENT_TYPE type;
 331         VPU_FREQ freq;
 332         vpu_session *session;
 333         struct vpu_subdev_data *data;
 334         struct list_head session_link;          /* link to vpu service session */
 335         struct list_head status_link;           /* link to register set list */
 336         unsigned long size;
 337 #if defined(CONFIG_VCODEC_MMU)
 338         struct list_head mem_region_list;
 339 #endif
 340         u32 *reg;
 341 } vpu_reg;
 342
 343 typedef struct vpu_device {
 344         atomic_t                irq_count_codec;
 345         atomic_t                irq_count_pp;
 346         unsigned long           iobaseaddr;
 347         unsigned int            iosize;
 348         volatile u32            *hwregs;
 349 } vpu_device;
 350
 351 enum vcodec_device_id {
 352         VCODEC_DEVICE_ID_VPU,
 353         VCODEC_DEVICE_ID_HEVC,
 354         VCODEC_DEVICE_ID_COMBO
 355 };
 356
 357 enum VCODEC_RUNNING_MODE {
 358         VCODEC_RUNNING_MODE_NONE = -1,
 359         VCODEC_RUNNING_MODE_VPU,
 360         VCODEC_RUNNING_MODE_HEVC,
 361 };
 362
 363 struct vcodec_mem_region {
 364         struct list_head srv_lnk;
 365         struct list_head reg_lnk;
 366         struct list_head session_lnk;
 367         unsigned long iova;     /* virtual address for iommu */
 368         unsigned long len;
 369         u32 reg_idx;
 370         struct ion_handle *hdl;
 371 };
 372
 373 enum vpu_ctx_state {
 374         MMU_ACTIVATED   = BIT(0)
 375 };
 376
 377 struct vpu_subdev_data {
 378         struct cdev cdev;
 379         dev_t dev_t;
 380         struct class *cls;
 381         struct device *child_dev;
 382
 383         int irq_enc;
 384         int irq_dec;
 385         struct vpu_service_info *pservice;
 386
 387         u32 *regs;
 388         enum VCODEC_RUNNING_MODE mode;
 389         struct list_head lnk_service;
 390
 391         struct device *dev;
 392
 393         vpu_device enc_dev;
 394         vpu_device dec_dev;
 395         VPU_HW_INFO_E *hw_info;
 396
 397         u32 reg_size;
 398         unsigned long state;
 399
 400 #ifdef CONFIG_DEBUG_FS
 401         struct dentry *debugfs_dir;
 402         struct dentry *debugfs_file_regs;
 403 #endif
 404
 405 #if defined(CONFIG_VCODEC_MMU)
 406         struct device *mmu_dev;
 407 #endif
 408 };
 409
 410 typedef struct vpu_service_info {
 411         struct wake_lock wake_lock;
 412         struct delayed_work power_off_work;
 413         struct mutex lock;
 414         struct list_head waiting;               /* link to link_reg in struct vpu_reg */
 415         struct list_head running;               /* link to link_reg in struct vpu_reg */
 416         struct list_head done;                  /* link to link_reg in struct vpu_reg */
 417         struct list_head session;               /* link to list_session in struct vpu_session */
 418         atomic_t total_running;
 419         atomic_t enabled;
 420         atomic_t power_on_cnt;
 421         atomic_t power_off_cnt;
 422         vpu_reg *reg_codec;
 423         vpu_reg *reg_pproc;
 424         vpu_reg *reg_resev;
 425         struct vpu_dec_config dec_config;
 426         struct vpu_enc_config enc_config;
 427
 428         bool auto_freq;
 429         bool bug_dec_addr;
 430         atomic_t freq_status;
 431
 432         struct clk *aclk_vcodec;
 433         struct clk *hclk_vcodec;
 434         struct clk *clk_core;
 435         struct clk *clk_cabac;
 436         struct clk *pd_video;
 437
 438         struct device *dev;
 439
 440         u32 irq_status;
 441 #if defined(CONFIG_VCODEC_MMU)
 442         struct ion_client *ion_client;
 443         struct list_head mem_region_list;
 444 #endif
 445
 446         enum vcodec_device_id dev_id;
 447
 448         enum VCODEC_RUNNING_MODE curr_mode;
 449         u32 prev_mode;
 450
 451         struct delayed_work simulate_work;
 452
 453         u32 mode_bit;
 454         u32 mode_ctrl;
 455         u32 *reg_base;
 456         u32 ioaddr;
 457 #ifdef CONFIG_MFD_SYSCON
 458         struct regmap *grf_base;
 459 #else
 460         u32 *grf_base;
 461 #endif
 462         char *name;
 463
 464         u32 subcnt;
 465         struct list_head subdev_list;
 466 } vpu_service_info;
 467
 468 struct vcodec_combo {
 469         struct vpu_service_info *vpu_srv;
 470         struct vpu_service_info *hevc_srv;
 471         struct list_head waiting;
 472         struct list_head running;
 473         struct mutex run_lock;
 474         vpu_reg *reg_codec;
 475         enum vcodec_device_id current_hw_mode;
 476 };
 477
 478 typedef struct vpu_request {
 479         u64 req;
 480         u32 size;
 481 } vpu_request;
 482
 483 /* debugfs root directory for all device (vpu, hevc).*/
 484 static struct dentry *parent;
 485
 486 #ifdef CONFIG_DEBUG_FS
 487 static int vcodec_debugfs_init(void);
 488 static void vcodec_debugfs_exit(void);
 489 static struct dentry* vcodec_debugfs_create_device_dir(char *dirname, struct dentry *parent);
 490 static int debug_vcodec_open(struct inode *inode, struct file *file);
 491
 492 static const struct file_operations debug_vcodec_fops = {
 493         .open = debug_vcodec_open,
 494         .read = seq_read,
 495         .llseek = seq_lseek,
 496         .release = single_release,
 497 };
 498 #endif
 499
 500 #define VDPU_SOFT_RESET_REG     101
 501 #define VDPU_CLEAN_CACHE_REG    516
 502 #define VEPU_CLEAN_CACHE_REG    772
 503 #define HEVC_CLEAN_CACHE_REG    260
 504
 505 #define VPU_REG_ENABLE(base, reg)       do { \
 506                                                 base[reg] = 1; \
 507                                         } while (0)
 508
 509 #define VDPU_SOFT_RESET(base)   VPU_REG_ENABLE(base, VDPU_SOFT_RESET_REG)
 510 #define VDPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VDPU_CLEAN_CACHE_REG)
 511 #define VEPU_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, VEPU_CLEAN_CACHE_REG)
 512 #define HEVC_CLEAN_CACHE(base)  VPU_REG_ENABLE(base, HEVC_CLEAN_CACHE_REG)
 513
 514 #define VPU_POWER_OFF_DELAY             4*HZ /* 4s */
 515 #define VPU_TIMEOUT_DELAY               2*HZ /* 2s */
 516
 517 static void vcodec_enter_mode(struct vpu_subdev_data *data)
 518 {
 519         int bits;
 520         u32 raw = 0;
 521         struct vpu_service_info *pservice = data->pservice;
 522         struct vpu_subdev_data *subdata, *n;
 523         if (pservice->subcnt < 2) {
 524 #if defined(CONFIG_VCODEC_MMU)
 525                 if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 526                         set_bit(MMU_ACTIVATED, &data->state);
 527                         if (atomic_read(&pservice->enabled))
 528                                 rockchip_iovmm_activate(data->dev);
 529                         else
 530                                 BUG_ON(!atomic_read(&pservice->enabled));
 531                 }
 532 #endif
 533                 return;
 534         }
 535
 536         if (pservice->curr_mode == data->mode)
 537                 return;
 538
 539         vpu_debug(3, "vcodec enter mode %d\n", data->mode);
 540 #if defined(CONFIG_VCODEC_MMU)
 541         list_for_each_entry_safe(subdata, n, &pservice->subdev_list, lnk_service) {
 542                 if (data != subdata && subdata->mmu_dev &&
 543                     test_bit(MMU_ACTIVATED, &subdata->state)) {
 544                         clear_bit(MMU_ACTIVATED, &subdata->state);
 545                         rockchip_iovmm_deactivate(subdata->dev);
 546                 }
 547         }
 548 #endif
 549         bits = 1 << pservice->mode_bit;
 550 #ifdef CONFIG_MFD_SYSCON
 551         regmap_read(pservice->grf_base, pservice->mode_ctrl, &raw);
 552
 553         if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 554                 regmap_write(pservice->grf_base, pservice->mode_ctrl,
 555                         raw | bits | (bits << 16));
 556         else
 557                 regmap_write(pservice->grf_base, pservice->mode_ctrl,
 558                         (raw & (~bits)) | (bits << 16));
 559 #else
 560         raw = readl_relaxed(pservice->grf_base + pservice->mode_ctrl / 4);
 561         if (data->mode == VCODEC_RUNNING_MODE_HEVC)
 562                 writel_relaxed(raw | bits | (bits << 16),
 563                         pservice->grf_base + pservice->mode_ctrl / 4);
 564         else
 565                 writel_relaxed((raw & (~bits)) | (bits << 16),
 566                         pservice->grf_base + pservice->mode_ctrl / 4);
 567 #endif
 568 #if defined(CONFIG_VCODEC_MMU)
 569         if (data->mmu_dev && !test_bit(MMU_ACTIVATED, &data->state)) {
 570                 set_bit(MMU_ACTIVATED, &data->state);
 571                 if (atomic_read(&pservice->enabled))
 572                         rockchip_iovmm_activate(data->dev);
 573                 else
 574                         BUG_ON(!atomic_read(&pservice->enabled));
 575         }
 576 #endif
 577         pservice->prev_mode = pservice->curr_mode;
 578         pservice->curr_mode = data->mode;
 579 }
 580
 581 static void vcodec_exit_mode(struct vpu_subdev_data *data)
 582 {
 583         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 584                 clear_bit(MMU_ACTIVATED, &data->state);
 585                 rockchip_iovmm_deactivate(data->dev);
 586                 data->pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 587         }
 588 }
 589
 590 static int vpu_get_clk(struct vpu_service_info *pservice)
 591 {
 592 #if VCODEC_CLOCK_ENABLE
 593         switch (pservice->dev_id) {
 594         case VCODEC_DEVICE_ID_HEVC:
 595                 pservice->pd_video = devm_clk_get(pservice->dev, "pd_hevc");
 596                 if (IS_ERR(pservice->pd_video)) {
 597                         dev_err(pservice->dev, "failed on clk_get pd_hevc\n");
 598                         return -1;
 599                 }
 600         case VCODEC_DEVICE_ID_COMBO:
 601                 pservice->clk_cabac = devm_clk_get(pservice->dev, "clk_cabac");
 602                 if (IS_ERR(pservice->clk_cabac)) {
 603                         dev_err(pservice->dev, "failed on clk_get clk_cabac\n");
 604                         pservice->clk_cabac = NULL;
 605                 }
 606                 pservice->clk_core = devm_clk_get(pservice->dev, "clk_core");
 607                 if (IS_ERR(pservice->clk_core)) {
 608                         dev_err(pservice->dev, "failed on clk_get clk_core\n");
 609                         return -1;
 610                 }
 611         case VCODEC_DEVICE_ID_VPU:
 612                 pservice->aclk_vcodec = devm_clk_get(pservice->dev, "aclk_vcodec");
 613                 if (IS_ERR(pservice->aclk_vcodec)) {
 614                         dev_err(pservice->dev, "failed on clk_get aclk_vcodec\n");
 615                         return -1;
 616                 }
 617
 618                 pservice->hclk_vcodec = devm_clk_get(pservice->dev, "hclk_vcodec");
 619                 if (IS_ERR(pservice->hclk_vcodec)) {
 620                         dev_err(pservice->dev, "failed on clk_get hclk_vcodec\n");
 621                         return -1;
 622                 }
 623                 if (pservice->pd_video == NULL) {
 624                         pservice->pd_video = devm_clk_get(pservice->dev, "pd_video");
 625                         if (IS_ERR(pservice->pd_video)) {
 626                                 pservice->pd_video = NULL;
 627                                 dev_info(pservice->dev, "do not have pd_video\n");
 628                         }
 629                 }
 630                 break;
 631         default:
 632                 ;
 633         }
 634
 635         return 0;
 636 #else
 637         return 0;
 638 #endif
 639 }
 640
 641 static void vpu_put_clk(struct vpu_service_info *pservice)
 642 {
 643 #if VCODEC_CLOCK_ENABLE
 644         if (pservice->pd_video)
 645                 devm_clk_put(pservice->dev, pservice->pd_video);
 646         if (pservice->aclk_vcodec)
 647                 devm_clk_put(pservice->dev, pservice->aclk_vcodec);
 648         if (pservice->hclk_vcodec)
 649                 devm_clk_put(pservice->dev, pservice->hclk_vcodec);
 650         if (pservice->clk_core)
 651                 devm_clk_put(pservice->dev, pservice->clk_core);
 652         if (pservice->clk_cabac)
 653                 devm_clk_put(pservice->dev, pservice->clk_cabac);
 654 #endif
 655 }
 656
 657 static void vpu_reset(struct vpu_subdev_data *data)
 658 {
 659         struct vpu_service_info *pservice = data->pservice;
 660 #if defined(CONFIG_ARCH_RK29)
 661         clk_disable(aclk_ddr_vepu);
 662         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, true);
 663         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, true);
 664         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, true);
 665         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, true);
 666         mdelay(10);
 667         cru_set_soft_reset(SOFT_RST_VCODEC_AXI_BUS, false);
 668         cru_set_soft_reset(SOFT_RST_VCODEC_AHB_BUS, false);
 669         cru_set_soft_reset(SOFT_RST_DDR_VCODEC_PORT, false);
 670         cru_set_soft_reset(SOFT_RST_CPU_VODEC_A2A_AHB, false);
 671         clk_enable(aclk_ddr_vepu);
 672 #elif defined(CONFIG_ARCH_RK30)
 673         pmu_set_idle_request(IDLE_REQ_VIDEO, true);
 674         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, true);
 675         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, true);
 676         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, true);
 677         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, true);
 678         mdelay(1);
 679         cru_set_soft_reset(SOFT_RST_VCODEC_AXI, false);
 680         cru_set_soft_reset(SOFT_RST_VCODEC_AHB, false);
 681         cru_set_soft_reset(SOFT_RST_VCODEC_NIU_AXI, false);
 682         cru_set_soft_reset(SOFT_RST_CPU_VCODEC, false);
 683         pmu_set_idle_request(IDLE_REQ_VIDEO, false);
 684 #endif
 685         pservice->reg_codec = NULL;
 686         pservice->reg_pproc = NULL;
 687         pservice->reg_resev = NULL;
 688
 689 #if defined(CONFIG_VCODEC_MMU)
 690         if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 691                 clear_bit(MMU_ACTIVATED, &data->state);
 692                 if (atomic_read(&pservice->enabled))
 693                         rockchip_iovmm_deactivate(data->dev);
 694                 else
 695                         BUG_ON(!atomic_read(&pservice->enabled));
 696         }
 697 #endif
 698 }
 699
 700 static void reg_deinit(struct vpu_subdev_data *data, vpu_reg *reg);
 701 static void vpu_service_session_clear(struct vpu_subdev_data *data, vpu_session *session)
 702 {
 703         vpu_reg *reg, *n;
 704         list_for_each_entry_safe(reg, n, &session->waiting, session_link) {
 705                 reg_deinit(data, reg);
 706         }
 707         list_for_each_entry_safe(reg, n, &session->running, session_link) {
 708                 reg_deinit(data, reg);
 709         }
 710         list_for_each_entry_safe(reg, n, &session->done, session_link) {
 711                 reg_deinit(data, reg);
 712         }
 713 }
 714
 715 static void vpu_service_dump(struct vpu_service_info *pservice)
 716 {
 717 }
 718
 719 static void vpu_service_power_off(struct vpu_service_info *pservice)
 720 {
 721         int total_running;
 722         struct vpu_subdev_data *data = NULL, *n;
 723         int ret = atomic_add_unless(&pservice->enabled, -1, 0);
 724         if (!ret)
 725                 return;
 726
 727         total_running = atomic_read(&pservice->total_running);
 728         if (total_running) {
 729                 pr_alert("alert: power off when %d task running!!\n", total_running);
 730                 mdelay(50);
 731                 pr_alert("alert: delay 50 ms for running task\n");
 732                 vpu_service_dump(pservice);
 733         }
 734
 735         pr_info("%s: power off...", dev_name(pservice->dev));
 736         udelay(10);
 737 #if defined(CONFIG_VCODEC_MMU)
 738         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
 739                 if (data->mmu_dev && test_bit(MMU_ACTIVATED, &data->state)) {
 740                         clear_bit(MMU_ACTIVATED, &data->state);
 741                         rockchip_iovmm_deactivate(data->dev);
 742                 }
 743         }
 744         pservice->curr_mode = VCODEC_RUNNING_MODE_NONE;
 745 #endif
 746
 747 #if VCODEC_CLOCK_ENABLE
 748         if (pservice->pd_video)
 749                 clk_disable_unprepare(pservice->pd_video);
 750         if (pservice->hclk_vcodec)
 751                 clk_disable_unprepare(pservice->hclk_vcodec);
 752         if (pservice->aclk_vcodec)
 753                 clk_disable_unprepare(pservice->aclk_vcodec);
 754         if (pservice->clk_core)
 755                 clk_disable_unprepare(pservice->clk_core);
 756         if (pservice->clk_cabac)
 757                 clk_disable_unprepare(pservice->clk_cabac);
 758 #endif
 759
 760         atomic_add(1, &pservice->power_off_cnt);
 761         wake_unlock(&pservice->wake_lock);
 762         pr_info("done\n");
 763 }
 764
 765 static inline void vpu_queue_power_off_work(struct vpu_service_info *pservice)
 766 {
 767         queue_delayed_work(system_nrt_wq, &pservice->power_off_work, VPU_POWER_OFF_DELAY);
 768 }
 769
 770 static void vpu_power_off_work(struct work_struct *work_s)
 771 {
 772         struct delayed_work *dlwork = container_of(work_s, struct delayed_work, work);
 773         struct vpu_service_info *pservice = container_of(dlwork, struct vpu_service_info, power_off_work);
 774
 775         if (mutex_trylock(&pservice->lock)) {
 776                 vpu_service_power_off(pservice);
 777                 mutex_unlock(&pservice->lock);
 778         } else {
 779                 /* Come back later if the device is busy... */
 780                 vpu_queue_power_off_work(pservice);
 781         }
 782 }
 783
 784 static void vpu_service_power_on(struct vpu_service_info *pservice)
 785 {
 786         int ret;
 787         static ktime_t last;
 788         ktime_t now = ktime_get();
 789         if (ktime_to_ns(ktime_sub(now, last)) > NSEC_PER_SEC) {
 790                 cancel_delayed_work_sync(&pservice->power_off_work);
 791                 vpu_queue_power_off_work(pservice);
 792                 last = now;
 793         }
 794         ret = atomic_add_unless(&pservice->enabled, 1, 1);
 795         if (!ret)
 796                 return ;
 797
 798         pr_info("%s: power on\n", dev_name(pservice->dev));
 799
 800 #define BIT_VCODEC_CLK_SEL      (1<<10)
 801         if (cpu_is_rk312x())
 802                 writel_relaxed(readl_relaxed(RK_GRF_VIRT + RK312X_GRF_SOC_CON1) |
 803                         BIT_VCODEC_CLK_SEL | (BIT_VCODEC_CLK_SEL << 16),
 804                         RK_GRF_VIRT + RK312X_GRF_SOC_CON1);
 805
 806 #if VCODEC_CLOCK_ENABLE
 807         if (pservice->aclk_vcodec)
 808                 clk_prepare_enable(pservice->aclk_vcodec);
 809         if (pservice->hclk_vcodec)
 810                 clk_prepare_enable(pservice->hclk_vcodec);
 811         if (pservice->clk_core)
 812                 clk_prepare_enable(pservice->clk_core);
 813         if (pservice->clk_cabac)
 814                 clk_prepare_enable(pservice->clk_cabac);
 815         if (pservice->pd_video)
 816                 clk_prepare_enable(pservice->pd_video);
 817 #endif
 818
 819         udelay(10);
 820         atomic_add(1, &pservice->power_on_cnt);
 821         wake_lock(&pservice->wake_lock);
 822 }
 823
 824 static inline bool reg_check_rmvb_wmv(vpu_reg *reg)
 825 {
 826         u32 type = (reg->reg[3] & 0xF0000000) >> 28;
 827         return ((type == 8) || (type == 4));
 828 }
 829
 830 static inline bool reg_check_interlace(vpu_reg *reg)
 831 {
 832         u32 type = (reg->reg[3] & (1 << 23));
 833         return (type > 0);
 834 }
 835
 836 static inline enum VPU_DEC_FMT reg_check_fmt(vpu_reg *reg)
 837 {
 838         enum VPU_DEC_FMT type = (enum VPU_DEC_FMT)((reg->reg[3] & 0xF0000000) >> 28);
 839         return type;
 840 }
 841
 842 static inline int reg_probe_width(vpu_reg *reg)
 843 {
 844         int width_in_mb = reg->reg[4] >> 23;
 845         return width_in_mb * 16;
 846 }
 847
 848 #if defined(CONFIG_VCODEC_MMU)
 849 static int vcodec_fd_to_iova(struct vpu_subdev_data *data, vpu_reg *reg,int fd)
 850 {
 851         struct vpu_service_info *pservice = data->pservice;
 852         struct ion_handle *hdl;
 853         int ret = 0;
 854         struct vcodec_mem_region *mem_region;
 855
 856         hdl = ion_import_dma_buf(pservice->ion_client, fd);
 857         if (IS_ERR(hdl)) {
 858                 vpu_err("import dma-buf from fd %d failed\n", fd);
 859                 return PTR_ERR(hdl);
 860         }
 861         mem_region = kzalloc(sizeof(struct vcodec_mem_region), GFP_KERNEL);
 862
 863         if (mem_region == NULL) {
 864                 vpu_err("allocate memory for iommu memory region failed\n");
 865                 ion_free(pservice->ion_client, hdl);
 866                 return -1;
 867         }
 868
 869         mem_region->hdl = hdl;
 870         ret = ion_map_iommu(data->dev, pservice->ion_client,
 871                 mem_region->hdl, &mem_region->iova, &mem_region->len);
 872
 873         if (ret < 0) {
 874                 vpu_err("ion map iommu failed\n");
 875                 kfree(mem_region);
 876                 ion_free(pservice->ion_client, hdl);
 877                 return ret;
 878         }
 879         INIT_LIST_HEAD(&mem_region->reg_lnk);
 880         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 881         return mem_region->iova;
 882 }
 883
 884 static int vcodec_bufid_to_iova(struct vpu_subdev_data *data, u8 *tbl,
 885                                 int size, vpu_reg *reg,
 886                                 struct extra_info_for_iommu *ext_inf)
 887 {
 888         struct vpu_service_info *pservice = data->pservice;
 889         int i;
 890         int usr_fd = 0;
 891         int offset = 0;
 892
 893         if (tbl == NULL || size <= 0) {
 894                 dev_err(pservice->dev, "input arguments invalidate\n");
 895                 return -1;
 896         }
 897
 898         vpu_service_power_on(pservice);
 899
 900         for (i = 0; i < size; i++) {
 901                 usr_fd = reg->reg[tbl[i]] & 0x3FF;
 902
 903                 if (tbl[i] == 41 && data->hw_info->hw_id != HEVC_ID &&
 904                     (reg->type == VPU_DEC || reg->type == VPU_DEC_PP))
 905                         /* special for vpu dec num 41 regitster */
 906                         offset = reg->reg[tbl[i]] >> 10 << 4;
 907                 else
 908                         offset = reg->reg[tbl[i]] >> 10;
 909
 910                 if (usr_fd != 0) {
 911                         struct ion_handle *hdl;
 912                         int ret = 0;
 913                         struct vcodec_mem_region *mem_region;
 914
 915                         hdl = ion_import_dma_buf(pservice->ion_client, usr_fd);
 916                         if (IS_ERR(hdl)) {
 917                                 dev_err(pservice->dev, "import dma-buf from fd %d failed, reg[%d]\n", usr_fd, tbl[i]);
 918                                 return PTR_ERR(hdl);
 919                         }
 920
 921                         if (tbl[i] == 42 && data->hw_info->hw_id == HEVC_ID){
 922                                 int i = 0;
 923                                 char *pps;
 924                                 pps = (char *)ion_map_kernel(pservice->ion_client,hdl);
 925                                 for (i=0; i<64; i++) {
 926                                         u32 scaling_offset;
 927                                         u32 tmp;
 928                                         int scaling_fd= 0;
 929                                         scaling_offset = (u32)pps[i*80+74];
 930                                         scaling_offset += (u32)pps[i*80+75] << 8;
 931                                         scaling_offset += (u32)pps[i*80+76] << 16;
 932                                         scaling_offset += (u32)pps[i*80+77] << 24;
 933                                         scaling_fd = scaling_offset&0x3ff;
 934                                         scaling_offset = scaling_offset >> 10;
 935                                         if(scaling_fd > 0) {
 936                                                 tmp = vcodec_fd_to_iova(data, reg, scaling_fd);
 937                                                 tmp += scaling_offset;
 938                                                 pps[i*80+74] = tmp & 0xff;
 939                                                 pps[i*80+75] = (tmp >> 8) & 0xff;
 940                                                 pps[i*80+76] = (tmp >> 16) & 0xff;
 941                                                 pps[i*80+77] = (tmp >> 24) & 0xff;
 942                                         }
 943                                 }
 944                         }
 945
 946                         mem_region = kzalloc(sizeof(struct vcodec_mem_region), GFP_KERNEL);
 947
 948                         if (mem_region == NULL) {
 949                                 dev_err(pservice->dev, "allocate memory for iommu memory region failed\n");
 950                                 ion_free(pservice->ion_client, hdl);
 951                                 return -1;
 952                         }
 953
 954                         mem_region->hdl = hdl;
 955                         mem_region->reg_idx = tbl[i];
 956                         ret = ion_map_iommu(data->dev,
 957                                             pservice->ion_client,
 958                                             mem_region->hdl,
 959                                             &mem_region->iova,
 960                                             &mem_region->len);
 961
 962                         if (ret < 0) {
 963                                 dev_err(pservice->dev, "ion map iommu failed\n");
 964                                 kfree(mem_region);
 965                                 ion_free(pservice->ion_client, hdl);
 966                                 return ret;
 967                         }
 968                         reg->reg[tbl[i]] = mem_region->iova + offset;
 969                         INIT_LIST_HEAD(&mem_region->reg_lnk);
 970                         list_add_tail(&mem_region->reg_lnk, &reg->mem_region_list);
 971                 }
 972         }
 973
 974         if (ext_inf != NULL && ext_inf->magic == EXTRA_INFO_MAGIC) {
 975                 for (i=0; i<ext_inf->cnt; i++) {
 976                         vpu_debug(3, "reg[%d] + offset %d\n",
 977                                   ext_inf->elem[i].index,
 978                                   ext_inf->elem[i].offset);
 979                         reg->reg[ext_inf->elem[i].index] +=
 980                                 ext_inf->elem[i].offset;
 981                 }
 982         }
 983
 984         return 0;
 985 }
 986
 987 static int vcodec_reg_address_translate(struct vpu_subdev_data *data,
 988                                         vpu_reg *reg,
 989                                         struct extra_info_for_iommu *ext_inf)
 990 {
 991         VPU_HW_ID hw_id;
 992         u8 *tbl;
 993         int size = 0;
 994
 995         hw_id = data->hw_info->hw_id;
 996
 997         if (hw_id == HEVC_ID) {
 998                 tbl = addr_tbl_hevc_dec;
 999                 size = sizeof(addr_tbl_hevc_dec);
1000         } else {
1001                 if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1002                         switch (reg_check_fmt(reg)) {
1003                         case VPU_DEC_FMT_H264:
1004                                 {
1005                                         tbl = addr_tbl_vpu_h264dec;
1006                                         size = sizeof(addr_tbl_vpu_h264dec);
1007                                         break;
1008                                 }
1009                         case VPU_DEC_FMT_VP8:
1010                         case VPU_DEC_FMT_VP7:
1011                                 {
1012                                         tbl = addr_tbl_vpu_vp8dec;
1013                                         size = sizeof(addr_tbl_vpu_vp8dec);
1014                                         break;
1015                                 }
1016
1017                         case VPU_DEC_FMT_VP6:
1018                                 {
1019                                         tbl = addr_tbl_vpu_vp6dec;
1020                                         size = sizeof(addr_tbl_vpu_vp6dec);
1021                                         break;
1022                                 }
1023                         case VPU_DEC_FMT_VC1:
1024                                 {
1025                                         tbl = addr_tbl_vpu_vc1dec;
1026                                         size = sizeof(addr_tbl_vpu_vc1dec);
1027                                         break;
1028                                 }
1029
1030                         case VPU_DEC_FMT_JPEG:
1031                                 {
1032                                         tbl = addr_tbl_vpu_jpegdec;
1033                                         size = sizeof(addr_tbl_vpu_jpegdec);
1034                                         break;
1035                                 }
1036                         default:
1037                                 tbl = addr_tbl_vpu_defaultdec;
1038                                 size = sizeof(addr_tbl_vpu_defaultdec);
1039                                 break;
1040                         }
1041                 } else if (reg->type == VPU_ENC) {
1042                         tbl = addr_tbl_vpu_enc;
1043                         size = sizeof(addr_tbl_vpu_enc);
1044                 }
1045         }
1046
1047         if (size != 0) {
1048                 return vcodec_bufid_to_iova(data, tbl, size, reg, ext_inf);
1049         } else {
1050                 return -1;
1051         }
1052 }
1053 #endif
1054
1055 static vpu_reg *reg_init(struct vpu_subdev_data *data,
1056         vpu_session *session, void __user *src, u32 size)
1057 {
1058         struct vpu_service_info *pservice = data->pservice;
1059         int extra_size = 0;
1060         struct extra_info_for_iommu extra_info;
1061         vpu_reg *reg = kmalloc(sizeof(vpu_reg) + data->reg_size, GFP_KERNEL);
1062
1063         vpu_debug_enter();
1064
1065         if (NULL == reg) {
1066                 vpu_err("error: kmalloc fail in reg_init\n");
1067                 return NULL;
1068         }
1069
1070         if (size > data->reg_size) {
1071                 /*printk("warning: vpu reg size %lu is larger than hw reg size %lu\n",
1072                   size, pservice->reg_size);
1073                 size = pservice->reg_size;*/
1074                 extra_size = size - data->reg_size;
1075                 size = data->reg_size;
1076         }
1077         reg->session = session;
1078         reg->data = data;
1079         reg->type = session->type;
1080         reg->size = size;
1081         reg->freq = VPU_FREQ_DEFAULT;
1082         reg->reg = (u32 *)&reg[1];
1083         INIT_LIST_HEAD(&reg->session_link);
1084         INIT_LIST_HEAD(&reg->status_link);
1085
1086 #if defined(CONFIG_VCODEC_MMU)
1087         if (data->mmu_dev)
1088                 INIT_LIST_HEAD(&reg->mem_region_list);
1089 #endif
1090
1091         if (copy_from_user(&reg->reg[0], (void __user *)src, size)) {
1092                 vpu_err("error: copy_from_user failed in reg_init\n");
1093                 kfree(reg);
1094                 return NULL;
1095         }
1096
1097         if (copy_from_user(&extra_info, (u8 *)src + size, extra_size)) {
1098                 vpu_err("error: copy_from_user failed in reg_init\n");
1099                 kfree(reg);
1100                 return NULL;
1101         }
1102
1103 #if defined(CONFIG_VCODEC_MMU)
1104         if (data->mmu_dev &&
1105             0 > vcodec_reg_address_translate(data, reg, &extra_info)) {
1106                 vpu_err("error: translate reg address failed\n");
1107                 kfree(reg);
1108                 return NULL;
1109         }
1110 #endif
1111
1112         mutex_lock(&pservice->lock);
1113         list_add_tail(&reg->status_link, &pservice->waiting);
1114         list_add_tail(&reg->session_link, &session->waiting);
1115         mutex_unlock(&pservice->lock);
1116
1117         if (pservice->auto_freq) {
1118                 if (!soc_is_rk2928g()) {
1119                         if (reg->type == VPU_DEC || reg->type == VPU_DEC_PP) {
1120                                 if (reg_check_rmvb_wmv(reg)) {
1121                                         reg->freq = VPU_FREQ_200M;
1122                                 } else if (reg_check_fmt(reg) == VPU_DEC_FMT_H264) {
1123                                         if (reg_probe_width(reg) > 3200) {
1124                                                 /*raise frequency for 4k avc.*/
1125                                                 reg->freq = VPU_FREQ_500M;
1126                                         }
1127                                 } else {
1128                                         if (reg_check_interlace(reg)) {
1129                                                 reg->freq = VPU_FREQ_400M;
1130                                         }
1131                                 }
1132                         }
1133                         if (reg->type == VPU_PP) {
1134                                 reg->freq = VPU_FREQ_400M;
1135                         }
1136                 }
1137         }
1138         vpu_debug_leave();
1139         return reg;
1140 }
1141
1142 static void reg_deinit(struct vpu_subdev_data *data, vpu_reg *reg)
1143 {
1144         struct vpu_service_info *pservice = data->pservice;
1145 #if defined(CONFIG_VCODEC_MMU)
1146         struct vcodec_mem_region *mem_region = NULL, *n;
1147 #endif
1148
1149         list_del_init(&reg->session_link);
1150         list_del_init(&reg->status_link);
1151         if (reg == pservice->reg_codec)
1152                 pservice->reg_codec = NULL;
1153         if (reg == pservice->reg_pproc)
1154                 pservice->reg_pproc = NULL;
1155
1156 #if defined(CONFIG_VCODEC_MMU)
1157         /* release memory region attach to this registers table. */
1158         if (data->mmu_dev) {
1159                 list_for_each_entry_safe(mem_region, n,
1160                         &reg->mem_region_list, reg_lnk) {
1161                         /* do not unmap iommu manually,
1162                            unmap will proccess when memory release */
1163                         /*vcodec_enter_mode(data);
1164                         ion_unmap_iommu(data->dev,
1165                                         pservice->ion_client,
1166                                         mem_region->hdl);
1167                         vcodec_exit_mode();*/
1168                         ion_free(pservice->ion_client, mem_region->hdl);
1169                         list_del_init(&mem_region->reg_lnk);
1170                         kfree(mem_region);
1171                 }
1172         }
1173 #endif
1174
1175         kfree(reg);
1176 }
1177
1178 static void reg_from_wait_to_run(struct vpu_service_info *pservice, vpu_reg *reg)
1179 {
1180         vpu_debug_enter();
1181         list_del_init(&reg->status_link);
1182         list_add_tail(&reg->status_link, &pservice->running);
1183
1184         list_del_init(&reg->session_link);
1185         list_add_tail(&reg->session_link, &reg->session->running);
1186         vpu_debug_leave();
1187 }
1188
1189 static void reg_copy_from_hw(vpu_reg *reg, volatile u32 *src, u32 count)
1190 {
1191         int i;
1192         u32 *dst = (u32 *)&reg->reg[0];
1193         vpu_debug_enter();
1194         for (i = 0; i < count; i++)
1195                 *dst++ = *src++;
1196         vpu_debug_leave();
1197 }
1198
1199 static void reg_from_run_to_done(struct vpu_subdev_data *data,
1200         vpu_reg *reg)
1201 {
1202         struct vpu_service_info *pservice = data->pservice;
1203         int irq_reg = -1;
1204
1205         vpu_debug_enter();
1206
1207         list_del_init(&reg->status_link);
1208         list_add_tail(&reg->status_link, &pservice->done);
1209
1210         list_del_init(&reg->session_link);
1211         list_add_tail(&reg->session_link, &reg->session->done);
1212
1213         /*vcodec_enter_mode(data);*/
1214         switch (reg->type) {
1215         case VPU_ENC : {
1216                 pservice->reg_codec = NULL;
1217                 reg_copy_from_hw(reg, data->enc_dev.hwregs, data->hw_info->enc_reg_num);
1218                 irq_reg = ENC_INTERRUPT_REGISTER;
1219                 break;
1220         }
1221         case VPU_DEC : {
1222                 int reg_len = REG_NUM_9190_DEC;
1223                 pservice->reg_codec = NULL;
1224                 reg_copy_from_hw(reg, data->dec_dev.hwregs, reg_len);
1225                 irq_reg = DEC_INTERRUPT_REGISTER;
1226                 break;
1227         }
1228         case VPU_PP : {
1229                 pservice->reg_pproc = NULL;
1230                 reg_copy_from_hw(reg, data->dec_dev.hwregs + PP_INTERRUPT_REGISTER, REG_NUM_9190_PP);
1231                 data->dec_dev.hwregs[PP_INTERRUPT_REGISTER] = 0;
1232                 break;
1233         }
1234         case VPU_DEC_PP : {
1235                 pservice->reg_codec = NULL;
1236                 pservice->reg_pproc = NULL;
1237                 reg_copy_from_hw(reg, data->dec_dev.hwregs, REG_NUM_9190_DEC_PP);
1238                 data->dec_dev.hwregs[PP_INTERRUPT_REGISTER] = 0;
1239                 break;
1240         }
1241         default : {
1242                 vpu_err("error: copy reg from hw with unknown type %d\n", reg->type);
1243                 break;
1244         }
1245         }
1246         vcodec_exit_mode(data);
1247
1248         if (irq_reg != -1)
1249                 reg->reg[irq_reg] = pservice->irq_status;
1250
1251         atomic_sub(1, &reg->session->task_running);
1252         atomic_sub(1, &pservice->total_running);
1253         wake_up(&reg->session->wait);
1254
1255         vpu_debug_leave();
1256 }
1257
1258 static void vpu_service_set_freq(struct vpu_service_info *pservice, vpu_reg *reg)
1259 {
1260         VPU_FREQ curr = atomic_read(&pservice->freq_status);
1261         if (curr == reg->freq)
1262                 return;
1263         atomic_set(&pservice->freq_status, reg->freq);
1264         switch (reg->freq) {
1265         case VPU_FREQ_200M : {
1266                 clk_set_rate(pservice->aclk_vcodec, 200*MHZ);
1267         } break;
1268         case VPU_FREQ_266M : {
1269                 clk_set_rate(pservice->aclk_vcodec, 266*MHZ);
1270         } break;
1271         case VPU_FREQ_300M : {
1272                 clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1273         } break;
1274         case VPU_FREQ_400M : {
1275                 clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1276         } break;
1277         case VPU_FREQ_500M : {
1278                 clk_set_rate(pservice->aclk_vcodec, 500*MHZ);
1279         } break;
1280         case VPU_FREQ_600M : {
1281                 clk_set_rate(pservice->aclk_vcodec, 600*MHZ);
1282         } break;
1283         default : {
1284                 if (soc_is_rk2928g())
1285                         clk_set_rate(pservice->aclk_vcodec, 400*MHZ);
1286                 else
1287                         clk_set_rate(pservice->aclk_vcodec, 300*MHZ);
1288         } break;
1289         }
1290 }
1291
1292 static void reg_copy_to_hw(struct vpu_subdev_data *data, vpu_reg *reg)
1293 {
1294         struct vpu_service_info *pservice = data->pservice;
1295         int i;
1296         u32 *src = (u32 *)&reg->reg[0];
1297         vpu_debug_enter();
1298
1299         atomic_add(1, &pservice->total_running);
1300         atomic_add(1, &reg->session->task_running);
1301         if (pservice->auto_freq)
1302                 vpu_service_set_freq(pservice, reg);
1303
1304         vcodec_enter_mode(data);
1305
1306         switch (reg->type) {
1307         case VPU_ENC : {
1308                 int enc_count = data->hw_info->enc_reg_num;
1309                 u32 *dst = (u32 *)data->enc_dev.hwregs;
1310
1311                 pservice->reg_codec = reg;
1312
1313                 dst[VPU_REG_EN_ENC] = src[VPU_REG_EN_ENC] & 0x6;
1314
1315                 for (i = 0; i < VPU_REG_EN_ENC; i++)
1316                         dst[i] = src[i];
1317
1318                 for (i = VPU_REG_EN_ENC + 1; i < enc_count; i++)
1319                         dst[i] = src[i];
1320
1321                 VEPU_CLEAN_CACHE(dst);
1322
1323                 dsb(sy);
1324
1325                 dst[VPU_REG_ENC_GATE] = src[VPU_REG_ENC_GATE] | VPU_REG_ENC_GATE_BIT;
1326                 dst[VPU_REG_EN_ENC]   = src[VPU_REG_EN_ENC];
1327
1328 #if VPU_SERVICE_SHOW_TIME
1329                 do_gettimeofday(&enc_start);
1330 #endif
1331
1332         } break;
1333         case VPU_DEC : {
1334                 u32 *dst = (u32 *)data->dec_dev.hwregs;
1335
1336                 pservice->reg_codec = reg;
1337
1338                 if (data->hw_info->hw_id != HEVC_ID) {
1339                         for (i = REG_NUM_9190_DEC - 1; i > VPU_REG_DEC_GATE; i--)
1340                                 dst[i] = src[i];
1341                         VDPU_CLEAN_CACHE(dst);
1342                 } else {
1343                         for (i = REG_NUM_HEVC_DEC - 1; i > VPU_REG_EN_DEC; i--)
1344                                 dst[i] = src[i];
1345                         HEVC_CLEAN_CACHE(dst);
1346                 }
1347
1348                 dsb(sy);
1349
1350                 if (data->hw_info->hw_id != HEVC_ID) {
1351                         dst[VPU_REG_DEC_GATE] = src[VPU_REG_DEC_GATE] | VPU_REG_DEC_GATE_BIT;
1352                         dst[VPU_REG_EN_DEC] = src[VPU_REG_EN_DEC];
1353                 } else {
1354                         dst[VPU_REG_EN_DEC] = src[VPU_REG_EN_DEC];
1355                 }
1356                 dsb(sy);
1357                 dmb(sy);
1358 #if VPU_SERVICE_SHOW_TIME
1359                 do_gettimeofday(&dec_start);
1360 #endif
1361         } break;
1362         case VPU_PP : {
1363                 u32 *dst = (u32 *)data->dec_dev.hwregs + PP_INTERRUPT_REGISTER;
1364                 pservice->reg_pproc = reg;
1365
1366                 dst[VPU_REG_PP_GATE] = src[VPU_REG_PP_GATE] | VPU_REG_PP_GATE_BIT;
1367
1368                 for (i = VPU_REG_PP_GATE + 1; i < REG_NUM_9190_PP; i++)
1369                         dst[i] = src[i];
1370
1371                 dsb(sy);
1372
1373                 dst[VPU_REG_EN_PP] = src[VPU_REG_EN_PP];
1374 #if VPU_SERVICE_SHOW_TIME
1375                 do_gettimeofday(&pp_start);
1376 #endif
1377
1378         } break;
1379         case VPU_DEC_PP : {
1380                 u32 *dst = (u32 *)data->dec_dev.hwregs;
1381                 pservice->reg_codec = reg;
1382                 pservice->reg_pproc = reg;
1383
1384                 VDPU_SOFT_RESET(dst);
1385                 VDPU_CLEAN_CACHE(dst);
1386
1387                 for (i = VPU_REG_EN_DEC_PP + 1; i < REG_NUM_9190_DEC_PP; i++)
1388                         dst[i] = src[i];
1389
1390                 dst[VPU_REG_EN_DEC_PP]   = src[VPU_REG_EN_DEC_PP] | 0x2;
1391                 dsb(sy);
1392
1393                 dst[VPU_REG_DEC_PP_GATE] = src[VPU_REG_DEC_PP_GATE] | VPU_REG_PP_GATE_BIT;
1394                 dst[VPU_REG_DEC_GATE]    = src[VPU_REG_DEC_GATE]    | VPU_REG_DEC_GATE_BIT;
1395                 dst[VPU_REG_EN_DEC]      = src[VPU_REG_EN_DEC];
1396 #if VPU_SERVICE_SHOW_TIME
1397                 do_gettimeofday(&dec_start);
1398 #endif
1399         } break;
1400         default : {
1401                 vpu_err("error: unsupport session type %d", reg->type);
1402                 atomic_sub(1, &pservice->total_running);
1403                 atomic_sub(1, &reg->session->task_running);
1404                 break;
1405         }
1406         }
1407
1408         /*vcodec_exit_mode(data);*/
1409         vpu_debug_leave();
1410 }
1411
1412 static void try_set_reg(struct vpu_subdev_data *data)
1413 {
1414         struct vpu_service_info *pservice = data->pservice;
1415         vpu_debug_enter();
1416         if (!list_empty(&pservice->waiting)) {
1417                 int can_set = 0;
1418                 vpu_reg *reg = list_entry(pservice->waiting.next, vpu_reg, status_link);
1419
1420                 vpu_service_power_on(pservice);
1421
1422                 switch (reg->type) {
1423                 case VPU_ENC : {
1424                         if ((NULL == pservice->reg_codec) &&  (NULL == pservice->reg_pproc))
1425                                 can_set = 1;
1426                 } break;
1427                 case VPU_DEC : {
1428                         if (NULL == pservice->reg_codec)
1429                                 can_set = 1;
1430                         if (pservice->auto_freq && (NULL != pservice->reg_pproc))
1431                                 can_set = 0;
1432                 } break;
1433                 case VPU_PP : {
1434                         if (NULL == pservice->reg_codec) {
1435                                 if (NULL == pservice->reg_pproc)
1436                                         can_set = 1;
1437                         } else {
1438                                 if ((VPU_DEC == pservice->reg_codec->type) && (NULL == pservice->reg_pproc))
1439                                         can_set = 1;
1440                                 /* can not charge frequency when vpu is working */
1441                                 if (pservice->auto_freq)
1442                                         can_set = 0;
1443                         }
1444                 } break;
1445                 case VPU_DEC_PP : {
1446                         if ((NULL == pservice->reg_codec) && (NULL == pservice->reg_pproc))
1447                                 can_set = 1;
1448                         } break;
1449                 default : {
1450                         printk("undefined reg type %d\n", reg->type);
1451                 } break;
1452                 }
1453                 if (can_set) {
1454                         reg_from_wait_to_run(pservice, reg);
1455                         reg_copy_to_hw(reg->data, reg);
1456                 }
1457         }
1458         vpu_debug_leave();
1459 }
1460
1461 static int return_reg(struct vpu_subdev_data *data,
1462         vpu_reg *reg, u32 __user *dst)
1463 {
1464         int ret = 0;
1465         vpu_debug_enter();
1466         switch (reg->type) {
1467         case VPU_ENC : {
1468                 if (copy_to_user(dst, &reg->reg[0], data->hw_info->enc_io_size))
1469                         ret = -EFAULT;
1470                 break;
1471         }
1472         case VPU_DEC : {
1473                 int reg_len = data->hw_info->hw_id == HEVC_ID ? REG_NUM_HEVC_DEC : REG_NUM_9190_DEC;
1474                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(reg_len)))
1475                         ret = -EFAULT;
1476                 break;
1477         }
1478         case VPU_PP : {
1479                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(REG_NUM_9190_PP)))
1480                         ret = -EFAULT;
1481                 break;
1482         }
1483         case VPU_DEC_PP : {
1484                 if (copy_to_user(dst, &reg->reg[0], SIZE_REG(REG_NUM_9190_DEC_PP)))
1485                         ret = -EFAULT;
1486                 break;
1487         }
1488         default : {
1489                 ret = -EFAULT;
1490                 vpu_err("error: copy reg to user with unknown type %d\n", reg->type);
1491                 break;
1492         }
1493         }
1494         reg_deinit(data, reg);
1495         vpu_debug_leave();
1496         return ret;
1497 }
1498
1499 static long vpu_service_ioctl(struct file *filp, unsigned int cmd,
1500         unsigned long arg)
1501 {
1502         struct vpu_subdev_data *data =
1503                 container_of(filp->f_dentry->d_inode->i_cdev,
1504                         struct vpu_subdev_data, cdev);
1505         struct vpu_service_info *pservice = data->pservice;
1506         vpu_session *session = (vpu_session *)filp->private_data;
1507         vpu_debug_enter();
1508         vpu_debug(3, "cmd %x, VPU_IOC_SET_CLIENT_TYPE %x\n", cmd, (u32)VPU_IOC_SET_CLIENT_TYPE);
1509         if (NULL == session)
1510                 return -EINVAL;
1511
1512         switch (cmd) {
1513         case VPU_IOC_SET_CLIENT_TYPE : {
1514                 session->type = (enum VPU_CLIENT_TYPE)arg;
1515                 break;
1516         }
1517         case VPU_IOC_GET_HW_FUSE_STATUS : {
1518                 vpu_request req;
1519                 if (copy_from_user(&req, (void __user *)arg, sizeof(vpu_request))) {
1520                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_from_user failed\n");
1521                         return -EFAULT;
1522                 } else {
1523                         if (VPU_ENC != session->type) {
1524                                 if (copy_to_user((void __user *)req.req,
1525                                         &pservice->dec_config,
1526                                         sizeof(struct vpu_dec_config))) {
1527                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_to_user failed type %d\n",
1528                                                 session->type);
1529                                         return -EFAULT;
1530                                 }
1531                         } else {
1532                                 if (copy_to_user((void __user *)req.req,
1533                                         &pservice->enc_config,
1534                                         sizeof(struct vpu_enc_config ))) {
1535                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS copy_to_user failed type %d\n",
1536                                                 session->type);
1537                                         return -EFAULT;
1538                                 }
1539                         }
1540                 }
1541
1542                 break;
1543         }
1544         case VPU_IOC_SET_REG : {
1545                 vpu_request req;
1546                 vpu_reg *reg;
1547                 if (copy_from_user(&req, (void __user *)arg,
1548                         sizeof(vpu_request))) {
1549                         vpu_err("error: VPU_IOC_SET_REG copy_from_user failed\n");
1550                         return -EFAULT;
1551                 }
1552                 reg = reg_init(data, session,
1553                         (void __user *)req.req, req.size);
1554                 if (NULL == reg) {
1555                         return -EFAULT;
1556                 } else {
1557                         mutex_lock(&pservice->lock);
1558                         try_set_reg(data);
1559                         mutex_unlock(&pservice->lock);
1560                 }
1561
1562                 break;
1563         }
1564         case VPU_IOC_GET_REG : {
1565                 vpu_request req;
1566                 vpu_reg *reg;
1567                 if (copy_from_user(&req, (void __user *)arg,
1568                         sizeof(vpu_request))) {
1569                         vpu_err("error: VPU_IOC_GET_REG copy_from_user failed\n");
1570                         return -EFAULT;
1571                 } else {
1572                         int ret = wait_event_timeout(session->wait, !list_empty(&session->done), VPU_TIMEOUT_DELAY);
1573                         if (!list_empty(&session->done)) {
1574                                 if (ret < 0) {
1575                                         vpu_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session->pid, ret);
1576                                 }
1577                                 ret = 0;
1578                         } else {
1579                                 if (unlikely(ret < 0)) {
1580                                         vpu_err("error: pid %d wait task ret %d\n", session->pid, ret);
1581                                 } else if (0 == ret) {
1582                                         vpu_err("error: pid %d wait %d task done timeout\n", session->pid, atomic_read(&session->task_running));
1583                                         ret = -ETIMEDOUT;
1584                                 }
1585                         }
1586                         if (ret < 0) {
1587                                 int task_running = atomic_read(&session->task_running);
1588                                 mutex_lock(&pservice->lock);
1589                                 vpu_service_dump(pservice);
1590                                 if (task_running) {
1591                                         atomic_set(&session->task_running, 0);
1592                                         atomic_sub(task_running, &pservice->total_running);
1593                                         printk("%d task is running but not return, reset hardware...", task_running);
1594                                         vpu_reset(data);
1595                                         printk("done\n");
1596                                 }
1597                                 vpu_service_session_clear(data, session);
1598                                 mutex_unlock(&pservice->lock);
1599                                 return ret;
1600                         }
1601                 }
1602                 mutex_lock(&pservice->lock);
1603                 reg = list_entry(session->done.next, vpu_reg, session_link);
1604                 return_reg(data, reg, (u32 __user *)req.req);
1605                 mutex_unlock(&pservice->lock);
1606                 break;
1607         }
1608         case VPU_IOC_PROBE_IOMMU_STATUS: {
1609                 int iommu_enable = 0;
1610
1611 #if defined(CONFIG_VCODEC_MMU)
1612                 iommu_enable = data->mmu_dev ? 1 : 0;
1613 #endif
1614
1615                 if (copy_to_user((void __user *)arg, &iommu_enable, sizeof(int))) {
1616                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1617                         return -EFAULT;
1618                 }
1619                 break;
1620         }
1621         default : {
1622                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1623                 break;
1624         }
1625         }
1626         vpu_debug_leave();
1627         return 0;
1628 }
1629
1630 #ifdef CONFIG_COMPAT
1631 static long compat_vpu_service_ioctl(struct file *filp, unsigned int cmd,
1632         unsigned long arg)
1633 {
1634         struct vpu_subdev_data *data =
1635                 container_of(filp->f_dentry->d_inode->i_cdev,
1636                         struct vpu_subdev_data, cdev);
1637         struct vpu_service_info *pservice = data->pservice;
1638         vpu_session *session = (vpu_session *)filp->private_data;
1639         vpu_debug_enter();
1640         vpu_debug(3, "cmd %x, COMPAT_VPU_IOC_SET_CLIENT_TYPE %x\n", cmd,
1641                   (u32)COMPAT_VPU_IOC_SET_CLIENT_TYPE);
1642         if (NULL == session)
1643                 return -EINVAL;
1644
1645         switch (cmd) {
1646         case COMPAT_VPU_IOC_SET_CLIENT_TYPE : {
1647                 session->type = (enum VPU_CLIENT_TYPE)arg;
1648                 break;
1649         }
1650         case COMPAT_VPU_IOC_GET_HW_FUSE_STATUS : {
1651                 vpu_request req;
1652                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1653                                    sizeof(vpu_request))) {
1654                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS"
1655                                 " copy_from_user failed\n");
1656                         return -EFAULT;
1657                 } else {
1658                         if (VPU_ENC != session->type) {
1659                                 if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1660                                                  &pservice->dec_config,
1661                                                  sizeof(struct vpu_dec_config))) {
1662                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS "
1663                                                 "copy_to_user failed type %d\n",
1664                                                 session->type);
1665                                         return -EFAULT;
1666                                 }
1667                         } else {
1668                                 if (copy_to_user(compat_ptr((compat_uptr_t)req.req),
1669                                                  &pservice->enc_config,
1670                                                  sizeof(struct vpu_enc_config ))) {
1671                                         vpu_err("error: VPU_IOC_GET_HW_FUSE_STATUS"
1672                                                 " copy_to_user failed type %d\n",
1673                                                 session->type);
1674                                         return -EFAULT;
1675                                 }
1676                         }
1677                 }
1678
1679                 break;
1680         }
1681         case COMPAT_VPU_IOC_SET_REG : {
1682                 vpu_request req;
1683                 vpu_reg *reg;
1684                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1685                                    sizeof(vpu_request))) {
1686                         vpu_err("VPU_IOC_SET_REG copy_from_user failed\n");
1687                         return -EFAULT;
1688                 }
1689                 reg = reg_init(data, session,
1690                                compat_ptr((compat_uptr_t)req.req), req.size);
1691                 if (NULL == reg) {
1692                         return -EFAULT;
1693                 } else {
1694                         mutex_lock(&pservice->lock);
1695                         try_set_reg(data);
1696                         mutex_unlock(&pservice->lock);
1697                 }
1698
1699                 break;
1700         }
1701         case COMPAT_VPU_IOC_GET_REG : {
1702                 vpu_request req;
1703                 vpu_reg *reg;
1704                 if (copy_from_user(&req, compat_ptr((compat_uptr_t)arg),
1705                                    sizeof(vpu_request))) {
1706                         vpu_err("VPU_IOC_GET_REG copy_from_user failed\n");
1707                         return -EFAULT;
1708                 } else {
1709                         int ret = wait_event_timeout(session->wait, !list_empty(&session->done), VPU_TIMEOUT_DELAY);
1710                         if (!list_empty(&session->done)) {
1711                                 if (ret < 0) {
1712                                         vpu_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session->pid, ret);
1713                                 }
1714                                 ret = 0;
1715                         } else {
1716                                 if (unlikely(ret < 0)) {
1717                                         vpu_err("error: pid %d wait task ret %d\n", session->pid, ret);
1718                                 } else if (0 == ret) {
1719                                         vpu_err("error: pid %d wait %d task done timeout\n", session->pid, atomic_read(&session->task_running));
1720                                         ret = -ETIMEDOUT;
1721                                 }
1722                         }
1723                         if (ret < 0) {
1724                                 int task_running = atomic_read(&session->task_running);
1725                                 mutex_lock(&pservice->lock);
1726                                 vpu_service_dump(pservice);
1727                                 if (task_running) {
1728                                         atomic_set(&session->task_running, 0);
1729                                         atomic_sub(task_running, &pservice->total_running);
1730                                         printk("%d task is running but not return, reset hardware...", task_running);
1731                                         vpu_reset(data);
1732                                         printk("done\n");
1733                                 }
1734                                 vpu_service_session_clear(data, session);
1735                                 mutex_unlock(&pservice->lock);
1736                                 return ret;
1737                         }
1738                 }
1739                 mutex_lock(&pservice->lock);
1740                 reg = list_entry(session->done.next, vpu_reg, session_link);
1741                 return_reg(data, reg, compat_ptr((compat_uptr_t)req.req));
1742                 mutex_unlock(&pservice->lock);
1743                 break;
1744         }
1745         case COMPAT_VPU_IOC_PROBE_IOMMU_STATUS : {
1746                 int iommu_enable = 0;
1747
1748 #if defined(CONFIG_VCODEC_MMU)
1749                 iommu_enable = data->mmu_dev ? 1 : 0;
1750 #endif
1751
1752                 if (copy_to_user(compat_ptr((compat_uptr_t)arg), &iommu_enable, sizeof(int))) {
1753                         vpu_err("error: VPU_IOC_PROBE_IOMMU_STATUS copy_to_user failed\n");
1754                         return -EFAULT;
1755                 }
1756                 break;
1757         }
1758         default : {
1759                 vpu_err("error: unknow vpu service ioctl cmd %x\n", cmd);
1760                 break;
1761         }
1762         }
1763         vpu_debug_leave();
1764         return 0;
1765 }
1766 #endif
1767
1768 static int vpu_service_check_hw(struct vpu_subdev_data *data, u32 hw_addr)
1769 {
1770         int ret = -EINVAL, i = 0;
1771         volatile u32 *tmp = (volatile u32 *)ioremap_nocache(hw_addr, 0x4);
1772         u32 enc_id = *tmp;
1773
1774         enc_id = (enc_id >> 16) & 0xFFFF;
1775         pr_info("checking hw id %x\n", enc_id);
1776         data->hw_info = NULL;
1777         for (i = 0; i < ARRAY_SIZE(vpu_hw_set); i++) {
1778                 if (enc_id == vpu_hw_set[i].hw_id) {
1779                         data->hw_info = &vpu_hw_set[i];
1780                         ret = 0;
1781                         break;
1782                 }
1783         }
1784         iounmap((void *)tmp);
1785         return ret;
1786 }
1787
1788 static int vpu_service_open(struct inode *inode, struct file *filp)
1789 {
1790         struct vpu_subdev_data *data = container_of(inode->i_cdev, struct vpu_subdev_data, cdev);
1791         struct vpu_service_info *pservice = data->pservice;
1792         vpu_session *session = (vpu_session *)kmalloc(sizeof(vpu_session), GFP_KERNEL);
1793
1794         vpu_debug_enter();
1795
1796         if (NULL == session) {
1797                 vpu_err("error: unable to allocate memory for vpu_session.");
1798                 return -ENOMEM;
1799         }
1800
1801         session->type   = VPU_TYPE_BUTT;
1802         session->pid    = current->pid;
1803         INIT_LIST_HEAD(&session->waiting);
1804         INIT_LIST_HEAD(&session->running);
1805         INIT_LIST_HEAD(&session->done);
1806         INIT_LIST_HEAD(&session->list_session);
1807         init_waitqueue_head(&session->wait);
1808         atomic_set(&session->task_running, 0);
1809         mutex_lock(&pservice->lock);
1810         list_add_tail(&session->list_session, &pservice->session);
1811         filp->private_data = (void *)session;
1812         mutex_unlock(&pservice->lock);
1813
1814         pr_debug("dev opened\n");
1815         vpu_debug_leave();
1816         return nonseekable_open(inode, filp);
1817 }
1818
1819 static int vpu_service_release(struct inode *inode, struct file *filp)
1820 {
1821         struct vpu_subdev_data *data = container_of(inode->i_cdev, struct vpu_subdev_data, cdev);
1822         struct vpu_service_info *pservice = data->pservice;
1823         int task_running;
1824         vpu_session *session = (vpu_session *)filp->private_data;
1825         vpu_debug_enter();
1826         if (NULL == session)
1827                 return -EINVAL;
1828
1829         task_running = atomic_read(&session->task_running);
1830         if (task_running) {
1831                 vpu_err("error: vpu_service session %d still has %d task running when closing\n", session->pid, task_running);
1832                 msleep(50);
1833         }
1834         wake_up(&session->wait);
1835
1836         mutex_lock(&pservice->lock);
1837         /* remove this filp from the asynchronusly notified filp's */
1838         list_del_init(&session->list_session);
1839         vpu_service_session_clear(data, session);
1840         kfree(session);
1841         filp->private_data = NULL;
1842         mutex_unlock(&pservice->lock);
1843
1844         pr_debug("dev closed\n");
1845         vpu_debug_leave();
1846         return 0;
1847 }
1848
1849 static const struct file_operations vpu_service_fops = {
1850         .unlocked_ioctl = vpu_service_ioctl,
1851         .open           = vpu_service_open,
1852         .release        = vpu_service_release,
1853 #ifdef CONFIG_COMPAT
1854         .compat_ioctl   = compat_vpu_service_ioctl,
1855 #endif
1856 };
1857
1858 static irqreturn_t vdpu_irq(int irq, void *dev_id);
1859 static irqreturn_t vdpu_isr(int irq, void *dev_id);
1860 static irqreturn_t vepu_irq(int irq, void *dev_id);
1861 static irqreturn_t vepu_isr(int irq, void *dev_id);
1862 static void get_hw_info(struct vpu_subdev_data *data);
1863
1864 #ifdef CONFIG_VCODEC_MMU
1865 static struct device *rockchip_get_sysmmu_dev(const char *compt)
1866 {
1867         struct device_node *dn = NULL;
1868         struct platform_device *pd = NULL;
1869         struct device *ret = NULL ;
1870
1871         dn = of_find_compatible_node(NULL,NULL,compt);
1872         if(!dn) {
1873                 printk("can't find device node %s \r\n",compt);
1874                 return NULL;
1875         }
1876
1877         pd = of_find_device_by_node(dn);
1878         if(!pd) {
1879                 printk("can't find platform device in device node %s\n",compt);
1880                 return  NULL;
1881         }
1882         ret = &pd->dev;
1883
1884         return ret;
1885
1886 }
1887 #ifdef CONFIG_IOMMU_API
1888 static inline void platform_set_sysmmu(struct device *iommu,
1889         struct device *dev)
1890 {
1891         dev->archdata.iommu = iommu;
1892 }
1893 #else
1894 static inline void platform_set_sysmmu(struct device *iommu,
1895         struct device *dev)
1896 {
1897 }
1898 #endif
1899
1900 int vcodec_sysmmu_fault_hdl(struct device *dev,
1901                                 enum rk_iommu_inttype itype,
1902                                 unsigned long pgtable_base,
1903                                 unsigned long fault_addr, unsigned int status)
1904 {
1905         struct platform_device *pdev;
1906         struct vpu_subdev_data *data;
1907         struct vpu_service_info *pservice;
1908
1909         vpu_debug_enter();
1910
1911         pdev = container_of(dev, struct platform_device, dev);
1912
1913         data = platform_get_drvdata(pdev);
1914         pservice = data->pservice;
1915
1916         if (pservice->reg_codec) {
1917                 struct vcodec_mem_region *mem, *n;
1918                 int i = 0;
1919                 vpu_debug(3, "vcodec, fault addr 0x%08x\n", (u32)fault_addr);
1920                 list_for_each_entry_safe(mem, n,
1921                                          &pservice->reg_codec->mem_region_list,
1922                                          reg_lnk) {
1923                         vpu_debug(3, "vcodec, reg[%02u] mem region [%02d] 0x%08x %ld\n",
1924                                 mem->reg_idx, i, (u32)mem->iova, mem->len);
1925                         i++;
1926                 }
1927
1928                 pr_alert("vcodec, page fault occur, reset hw\n");
1929                 pservice->reg_codec->reg[101] = 1;
1930                 vpu_reset(data);
1931         }
1932
1933         return 0;
1934 }
1935 #endif
1936
1937 #if HEVC_TEST_ENABLE
1938 static int hevc_test_case0(vpu_service_info *pservice);
1939 #endif
1940 #if defined(CONFIG_ION_ROCKCHIP)
1941 extern struct ion_client *rockchip_ion_client_create(const char * name);
1942 #endif
1943
1944 static int vcodec_subdev_probe(struct platform_device *pdev,
1945         struct vpu_service_info *pservice)
1946 {
1947         int ret = 0;
1948         struct resource *res = NULL;
1949         u32 ioaddr = 0;
1950         struct device *dev = &pdev->dev;
1951         char *name = (char*)dev_name(dev);
1952         struct device_node *np = pdev->dev.of_node;
1953         struct vpu_subdev_data *data =
1954                 devm_kzalloc(dev, sizeof(struct vpu_subdev_data), GFP_KERNEL);
1955 #if defined(CONFIG_VCODEC_MMU)
1956         u32 iommu_en = 0;
1957         char mmu_dev_dts_name[40];
1958         of_property_read_u32(np, "iommu_enabled", &iommu_en);
1959 #endif
1960         pr_info("probe device %s\n", dev_name(dev));
1961
1962         data->pservice = pservice;
1963         data->dev = dev;
1964
1965         of_property_read_string(np, "name", (const char**)&name);
1966         of_property_read_u32(np, "dev_mode", (u32*)&data->mode);
1967         /*dev_set_name(dev, name);*/
1968
1969         if (pservice->reg_base == 0) {
1970                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1971                 data->regs = devm_ioremap_resource(dev, res);
1972                 if (IS_ERR(data->regs)) {
1973                         ret = PTR_ERR(data->regs);
1974                         goto err;
1975                 }
1976                 ioaddr = res->start;
1977         } else {
1978                 data->regs = pservice->reg_base;
1979                 ioaddr = pservice->ioaddr;
1980         }
1981
1982         clear_bit(MMU_ACTIVATED, &data->state);
1983         vcodec_enter_mode(data);
1984         ret = vpu_service_check_hw(data, ioaddr);
1985         if (ret < 0) {
1986                 vpu_err("error: hw info check faild\n");
1987                 goto err;
1988         }
1989
1990         data->dec_dev.iosize = data->hw_info->dec_io_size;
1991         data->dec_dev.hwregs = (volatile u32 *)((u8 *)data->regs + data->hw_info->dec_offset);
1992         data->reg_size = data->dec_dev.iosize;
1993
1994         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
1995                 data->enc_dev.iosize = data->hw_info->enc_io_size;
1996                 data->reg_size = data->reg_size > data->enc_dev.iosize ? data->reg_size : data->enc_dev.iosize;
1997                 data->enc_dev.hwregs = (volatile u32 *)((u8 *)data->regs + data->hw_info->enc_offset);
1998         }
1999
2000         data->irq_enc = platform_get_irq_byname(pdev, "irq_enc");
2001         if (data->irq_enc > 0) {
2002                 ret = devm_request_threaded_irq(dev,
2003                         data->irq_enc, vepu_irq, vepu_isr,
2004                         IRQF_SHARED, dev_name(dev),
2005                         (void *)data);
2006                 if (ret) {
2007                         dev_err(dev,
2008                                 "error: can't request vepu irq %d\n",
2009                                 data->irq_enc);
2010                         goto err;
2011                 }
2012         }
2013         data->irq_dec = platform_get_irq_byname(pdev, "irq_dec");
2014         if (data->irq_dec > 0) {
2015                 ret = devm_request_threaded_irq(dev,
2016                         data->irq_dec, vdpu_irq, vdpu_isr,
2017                         IRQF_SHARED, dev_name(dev),
2018                         (void *)data);
2019                 if (ret) {
2020                         dev_err(dev,
2021                                 "error: can't request vdpu irq %d\n",
2022                                 data->irq_dec);
2023                         goto err;
2024                 }
2025         }
2026         atomic_set(&data->dec_dev.irq_count_codec, 0);
2027         atomic_set(&data->dec_dev.irq_count_pp, 0);
2028         atomic_set(&data->enc_dev.irq_count_codec, 0);
2029         atomic_set(&data->enc_dev.irq_count_pp, 0);
2030 #if defined(CONFIG_VCODEC_MMU)
2031         if (iommu_en) {
2032                 if (data->mode == VCODEC_RUNNING_MODE_HEVC)
2033                         sprintf(mmu_dev_dts_name,
2034                                 HEVC_IOMMU_COMPATIBLE_NAME);
2035                 else
2036                         sprintf(mmu_dev_dts_name,
2037                                 VPU_IOMMU_COMPATIBLE_NAME);
2038
2039                 data->mmu_dev =
2040                         rockchip_get_sysmmu_dev(mmu_dev_dts_name);
2041
2042                 if (data->mmu_dev)
2043                         platform_set_sysmmu(data->mmu_dev, dev);
2044
2045                 rockchip_iovmm_set_fault_handler(dev, vcodec_sysmmu_fault_hdl);
2046         }
2047 #endif
2048         vcodec_exit_mode(data);
2049         /* create device node */
2050         ret = alloc_chrdev_region(&data->dev_t, 0, 1, name);
2051         if (ret) {
2052                 dev_err(dev, "alloc dev_t failed\n");
2053                 goto err;
2054         }
2055
2056         cdev_init(&data->cdev, &vpu_service_fops);
2057
2058         data->cdev.owner = THIS_MODULE;
2059         data->cdev.ops = &vpu_service_fops;
2060
2061         ret = cdev_add(&data->cdev, data->dev_t, 1);
2062
2063         if (ret) {
2064                 dev_err(dev, "add dev_t failed\n");
2065                 goto err;
2066         }
2067
2068         data->cls = class_create(THIS_MODULE, name);
2069
2070         if (IS_ERR(data->cls)) {
2071                 ret = PTR_ERR(data->cls);
2072                 dev_err(dev, "class_create err:%d\n", ret);
2073                 goto err;
2074         }
2075
2076         data->child_dev = device_create(data->cls, dev,
2077                 data->dev_t, NULL, name);
2078
2079         get_hw_info(data);
2080
2081         platform_set_drvdata(pdev, data);
2082
2083         INIT_LIST_HEAD(&data->lnk_service);
2084         list_add_tail(&data->lnk_service, &pservice->subdev_list);
2085
2086 #ifdef CONFIG_DEBUG_FS
2087         data->debugfs_dir =
2088                 vcodec_debugfs_create_device_dir((char*)name, parent);
2089         if (data->debugfs_dir == NULL)
2090                 vpu_err("create debugfs dir %s failed\n", name);
2091
2092         data->debugfs_file_regs =
2093                 debugfs_create_file("regs", 0664,
2094                                     data->debugfs_dir, data,
2095                                     &debug_vcodec_fops);
2096 #endif
2097         return 0;
2098 err:
2099         if (data->irq_enc > 0)
2100                 free_irq(data->irq_enc, (void *)data);
2101         if (data->irq_dec > 0)
2102                 free_irq(data->irq_dec, (void *)data);
2103
2104         if (data->child_dev) {
2105                 device_destroy(data->cls, data->dev_t);
2106                 cdev_del(&data->cdev);
2107                 unregister_chrdev_region(data->dev_t, 1);
2108         }
2109
2110         if (data->cls)
2111                 class_destroy(data->cls);
2112         return -1;
2113 }
2114
2115 static void vcodec_subdev_remove(struct vpu_subdev_data *data)
2116 {
2117         device_destroy(data->cls, data->dev_t);
2118         class_destroy(data->cls);
2119         cdev_del(&data->cdev);
2120         unregister_chrdev_region(data->dev_t, 1);
2121
2122         free_irq(data->irq_enc, (void *)&data);
2123         free_irq(data->irq_dec, (void *)&data);
2124
2125 #ifdef CONFIG_DEBUG_FS
2126         debugfs_remove(data->debugfs_file_regs);
2127         debugfs_remove(data->debugfs_dir);
2128 #endif
2129 }
2130
2131 static void vcodec_read_property(struct device_node *np,
2132         struct vpu_service_info *pservice)
2133 {
2134         pservice->mode_bit = 0;
2135         pservice->mode_ctrl = 0;
2136         pservice->subcnt = 0;
2137
2138         of_property_read_u32(np, "subcnt", &pservice->subcnt);
2139
2140         if (pservice->subcnt > 1) {
2141                 of_property_read_u32(np, "mode_bit", &pservice->mode_bit);
2142                 of_property_read_u32(np, "mode_ctrl", &pservice->mode_ctrl);
2143         }
2144 #ifdef CONFIG_MFD_SYSCON
2145         pservice->grf_base = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
2146 #else
2147         pservice->grf_base = (u32*)RK_GRF_VIRT;
2148 #endif
2149         if (IS_ERR(pservice->grf_base)) {
2150 #ifdef CONFIG_ARM
2151                 pservice->grf_base = RK_GRF_VIRT;
2152 #else
2153                 vpu_err("can't find vpu grf property\n");
2154                 return;
2155 #endif
2156         }
2157         of_property_read_string(np, "name", (const char**)&pservice->name);
2158 }
2159
2160 static void vcodec_init_drvdata(struct vpu_service_info *pservice)
2161 {
2162         pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2163         pservice->curr_mode = -1;
2164
2165         wake_lock_init(&pservice->wake_lock, WAKE_LOCK_SUSPEND, "vpu");
2166         INIT_LIST_HEAD(&pservice->waiting);
2167         INIT_LIST_HEAD(&pservice->running);
2168         mutex_init(&pservice->lock);
2169
2170         INIT_LIST_HEAD(&pservice->done);
2171         INIT_LIST_HEAD(&pservice->session);
2172         INIT_LIST_HEAD(&pservice->subdev_list);
2173
2174         pservice->reg_pproc     = NULL;
2175         atomic_set(&pservice->total_running, 0);
2176         atomic_set(&pservice->enabled,       0);
2177         atomic_set(&pservice->power_on_cnt,  0);
2178         atomic_set(&pservice->power_off_cnt, 0);
2179
2180         INIT_DELAYED_WORK(&pservice->power_off_work, vpu_power_off_work);
2181
2182         pservice->ion_client = rockchip_ion_client_create("vpu");
2183         if (IS_ERR(pservice->ion_client)) {
2184                 vpu_err("failed to create ion client for vcodec ret %ld\n",
2185                         PTR_ERR(pservice->ion_client));
2186         } else {
2187                 vpu_debug(3, "vcodec ion client create success!\n");
2188         }
2189 }
2190
2191 static int vcodec_probe(struct platform_device *pdev)
2192 {
2193         int i;
2194         int ret = 0;
2195         struct resource *res = NULL;
2196         struct device *dev = &pdev->dev;
2197         struct device_node *np = pdev->dev.of_node;
2198         struct vpu_service_info *pservice =
2199                 devm_kzalloc(dev, sizeof(struct vpu_service_info), GFP_KERNEL);
2200
2201         pr_info("probe device %s\n", dev_name(dev));
2202
2203         vcodec_read_property(np, pservice);
2204         vcodec_init_drvdata(pservice);
2205
2206         if (strncmp(pservice->name, "hevc_service", 12) == 0)
2207                 pservice->dev_id = VCODEC_DEVICE_ID_HEVC;
2208         else if (strncmp(pservice->name, "vpu_service", 11) == 0)
2209                 pservice->dev_id = VCODEC_DEVICE_ID_VPU;
2210         else
2211                 pservice->dev_id = VCODEC_DEVICE_ID_COMBO;
2212
2213         pservice->dev = dev;
2214
2215         if (0 > vpu_get_clk(pservice))
2216                 goto err;
2217
2218         vpu_service_power_on(pservice);
2219
2220         if (of_property_read_bool(np, "reg")) {
2221                 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2222
2223                 pservice->reg_base = devm_ioremap_resource(pservice->dev, res);
2224                 if (IS_ERR(pservice->reg_base)) {
2225                         vpu_err("ioremap registers base failed\n");
2226                         ret = PTR_ERR(pservice->reg_base);
2227                         goto err;
2228                 }
2229                 pservice->ioaddr = res->start;
2230         } else {
2231                 pservice->reg_base = 0;
2232         }
2233
2234         if (of_property_read_bool(np, "subcnt")) {
2235                 for (i = 0; i<pservice->subcnt; i++) {
2236                         struct device_node *sub_np;
2237                         struct platform_device *sub_pdev;
2238                         sub_np = of_parse_phandle(np, "rockchip,sub", i);
2239                         sub_pdev = of_find_device_by_node(sub_np);
2240
2241                         vcodec_subdev_probe(sub_pdev, pservice);
2242                 }
2243         } else {
2244                 vcodec_subdev_probe(pdev, pservice);
2245         }
2246         platform_set_drvdata(pdev, pservice);
2247
2248         vpu_service_power_off(pservice);
2249
2250         pr_info("init success\n");
2251
2252         return 0;
2253
2254 err:
2255         pr_info("init failed\n");
2256         vpu_service_power_off(pservice);
2257         vpu_put_clk(pservice);
2258         wake_lock_destroy(&pservice->wake_lock);
2259
2260         if (res)
2261                 devm_release_mem_region(&pdev->dev, res->start, resource_size(res));
2262
2263         return ret;
2264 }
2265
2266 static int vcodec_remove(struct platform_device *pdev)
2267 {
2268         struct vpu_service_info *pservice = platform_get_drvdata(pdev);
2269         struct resource *res;
2270         struct vpu_subdev_data *data, *n;
2271
2272         list_for_each_entry_safe(data, n, &pservice->subdev_list, lnk_service) {
2273                 vcodec_subdev_remove(data);
2274         }
2275
2276         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2277         devm_release_mem_region(&pdev->dev, res->start, resource_size(res));
2278         vpu_put_clk(pservice);
2279         wake_lock_destroy(&pservice->wake_lock);
2280
2281         return 0;
2282 }
2283
2284 #if defined(CONFIG_OF)
2285 static const struct of_device_id vcodec_service_dt_ids[] = {
2286         {.compatible = "vpu_service",},
2287         {.compatible = "rockchip,hevc_service",},
2288         {.compatible = "rockchip,vpu_combo",},
2289         {},
2290 };
2291 #endif
2292
2293 static struct platform_driver vcodec_driver = {
2294         .probe = vcodec_probe,
2295         .remove = vcodec_remove,
2296         .driver = {
2297                 .name = "vcodec",
2298                 .owner = THIS_MODULE,
2299 #if defined(CONFIG_OF)
2300                 .of_match_table = of_match_ptr(vcodec_service_dt_ids),
2301 #endif
2302         },
2303 };
2304
2305 static void get_hw_info(struct vpu_subdev_data *data)
2306 {
2307         struct vpu_service_info *pservice = data->pservice;
2308         struct vpu_dec_config *dec = &pservice->dec_config;
2309         struct vpu_enc_config *enc = &pservice->enc_config;
2310         if (data->mode == VCODEC_RUNNING_MODE_VPU) {
2311                 u32 configReg   = data->dec_dev.hwregs[VPU_DEC_HWCFG0];
2312                 u32 asicID      = data->dec_dev.hwregs[0];
2313
2314                 dec->h264_support    = (configReg >> DWL_H264_E) & 0x3U;
2315                 dec->jpegSupport    = (configReg >> DWL_JPEG_E) & 0x01U;
2316                 if (dec->jpegSupport && ((configReg >> DWL_PJPEG_E) & 0x01U))
2317                         dec->jpegSupport = JPEG_PROGRESSIVE;
2318                 dec->mpeg4Support   = (configReg >> DWL_MPEG4_E) & 0x3U;
2319                 dec->vc1Support     = (configReg >> DWL_VC1_E) & 0x3U;
2320                 dec->mpeg2Support   = (configReg >> DWL_MPEG2_E) & 0x01U;
2321                 dec->sorensonSparkSupport = (configReg >> DWL_SORENSONSPARK_E) & 0x01U;
2322                 dec->refBufSupport  = (configReg >> DWL_REF_BUFF_E) & 0x01U;
2323                 dec->vp6Support     = (configReg >> DWL_VP6_E) & 0x01U;
2324
2325                 dec->maxDecPicWidth = 4096;
2326
2327                 /* 2nd Config register */
2328                 configReg   = data->dec_dev.hwregs[VPU_DEC_HWCFG1];
2329                 if (dec->refBufSupport) {
2330                         if ((configReg >> DWL_REF_BUFF_ILACE_E) & 0x01U)
2331                                 dec->refBufSupport |= 2;
2332                         if ((configReg >> DWL_REF_BUFF_DOUBLE_E) & 0x01U)
2333                                 dec->refBufSupport |= 4;
2334                 }
2335                 dec->customMpeg4Support = (configReg >> DWL_MPEG4_CUSTOM_E) & 0x01U;
2336                 dec->vp7Support     = (configReg >> DWL_VP7_E) & 0x01U;
2337                 dec->vp8Support     = (configReg >> DWL_VP8_E) & 0x01U;
2338                 dec->avsSupport     = (configReg >> DWL_AVS_E) & 0x01U;
2339
2340                 /* JPEG xtensions */
2341                 if (((asicID >> 16) >= 0x8190U) || ((asicID >> 16) == 0x6731U))
2342                         dec->jpegESupport = (configReg >> DWL_JPEG_EXT_E) & 0x01U;
2343                 else
2344                         dec->jpegESupport = JPEG_EXT_NOT_SUPPORTED;
2345
2346                 if (((asicID >> 16) >= 0x9170U) || ((asicID >> 16) == 0x6731U) )
2347                         dec->rvSupport = (configReg >> DWL_RV_E) & 0x03U;
2348                 else
2349                         dec->rvSupport = RV_NOT_SUPPORTED;
2350                 dec->mvcSupport = (configReg >> DWL_MVC_E) & 0x03U;
2351
2352                 if (dec->refBufSupport && (asicID >> 16) == 0x6731U )
2353                         dec->refBufSupport |= 8; /* enable HW support for offset */
2354
2355                 if (!cpu_is_rk3036()) {
2356                         configReg = data->enc_dev.hwregs[63];
2357                         enc->maxEncodedWidth = configReg & ((1 << 11) - 1);
2358                         enc->h264Enabled = (configReg >> 27) & 1;
2359                         enc->mpeg4Enabled = (configReg >> 26) & 1;
2360                         enc->jpegEnabled = (configReg >> 25) & 1;
2361                         enc->vsEnabled = (configReg >> 24) & 1;
2362                         enc->rgbEnabled = (configReg >> 28) & 1;
2363                         enc->reg_size = data->reg_size;
2364                         enc->reserv[0] = enc->reserv[1] = 0;
2365                 }
2366                 pservice->auto_freq = true;
2367                 vpu_debug(3, "vpu_service set to auto frequency mode\n");
2368                 atomic_set(&pservice->freq_status, VPU_FREQ_BUT);
2369
2370                 pservice->bug_dec_addr = cpu_is_rk30xx();
2371         } else {
2372                 if (cpu_is_rk3036()  || cpu_is_rk312x())
2373                         dec->maxDecPicWidth = 1920;
2374                 else
2375                         dec->maxDecPicWidth = 4096;
2376                 /* disable frequency switch in hevc.*/
2377                 pservice->auto_freq = false;
2378         }
2379 }
2380
2381 static irqreturn_t vdpu_irq(int irq, void *dev_id)
2382 {
2383         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2384         struct vpu_service_info *pservice = data->pservice;
2385         vpu_device *dev = &data->dec_dev;
2386         u32 raw_status;
2387         u32 irq_status;
2388
2389         /*vcodec_enter_mode(data);*/
2390
2391         irq_status = raw_status = readl(dev->hwregs + DEC_INTERRUPT_REGISTER);
2392
2393         vpu_debug(3, "%s status %08x\n", __func__, raw_status);
2394
2395         if (irq_status & DEC_INTERRUPT_BIT) {
2396                 pr_debug("dec_isr dec %x\n", irq_status);
2397                 if ((irq_status & 0x40001) == 0x40001) {
2398                         do {
2399                                 irq_status =
2400                                         readl(dev->hwregs +
2401                                                 DEC_INTERRUPT_REGISTER);
2402                         } while ((irq_status & 0x40001) == 0x40001);
2403                 }
2404
2405                 writel(0, dev->hwregs + DEC_INTERRUPT_REGISTER);
2406                 atomic_add(1, &dev->irq_count_codec);
2407         }
2408
2409         if (data->hw_info->hw_id != HEVC_ID) {
2410                 irq_status = readl(dev->hwregs + PP_INTERRUPT_REGISTER);
2411                 if (irq_status & PP_INTERRUPT_BIT) {
2412                         pr_debug("vdpu_isr pp  %x\n", irq_status);
2413                         /* clear pp IRQ */
2414                         writel(irq_status & (~DEC_INTERRUPT_BIT), dev->hwregs + PP_INTERRUPT_REGISTER);
2415                         atomic_add(1, &dev->irq_count_pp);
2416                 }
2417         }
2418
2419         pservice->irq_status = raw_status;
2420
2421         /*vcodec_exit_mode(pservice);*/
2422
2423         if (atomic_read(&dev->irq_count_pp) ||
2424             atomic_read(&dev->irq_count_codec))
2425                 return IRQ_WAKE_THREAD;
2426         else
2427                 return IRQ_NONE;
2428 }
2429
2430 static irqreturn_t vdpu_isr(int irq, void *dev_id)
2431 {
2432         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2433         struct vpu_service_info *pservice = data->pservice;
2434         vpu_device *dev = &data->dec_dev;
2435
2436         mutex_lock(&pservice->lock);
2437         if (atomic_read(&dev->irq_count_codec)) {
2438 #if VPU_SERVICE_SHOW_TIME
2439                 do_gettimeofday(&dec_end);
2440                 vpu_debug(3, "dec task: %ld ms\n",
2441                         (dec_end.tv_sec  - dec_start.tv_sec)  * 1000 +
2442                         (dec_end.tv_usec - dec_start.tv_usec) / 1000);
2443 #endif
2444                 atomic_sub(1, &dev->irq_count_codec);
2445                 if (NULL == pservice->reg_codec) {
2446                         vpu_err("error: dec isr with no task waiting\n");
2447                 } else {
2448                         reg_from_run_to_done(data, pservice->reg_codec);
2449                         writel(1, dev->hwregs + 101);
2450                 }
2451         }
2452
2453         if (atomic_read(&dev->irq_count_pp)) {
2454 #if VPU_SERVICE_SHOW_TIME
2455                 do_gettimeofday(&pp_end);
2456                 printk("pp  task: %ld ms\n",
2457                         (pp_end.tv_sec  - pp_start.tv_sec)  * 1000 +
2458                         (pp_end.tv_usec - pp_start.tv_usec) / 1000);
2459 #endif
2460                 atomic_sub(1, &dev->irq_count_pp);
2461                 if (NULL == pservice->reg_pproc) {
2462                         vpu_err("error: pp isr with no task waiting\n");
2463                 } else {
2464                         reg_from_run_to_done(data, pservice->reg_pproc);
2465                 }
2466         }
2467         try_set_reg(data);
2468         mutex_unlock(&pservice->lock);
2469         return IRQ_HANDLED;
2470 }
2471
2472 static irqreturn_t vepu_irq(int irq, void *dev_id)
2473 {
2474         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2475         struct vpu_service_info *pservice = data->pservice;
2476         vpu_device *dev = &data->enc_dev;
2477         u32 irq_status;
2478
2479         /*vcodec_enter_mode(data);*/
2480         irq_status= readl(dev->hwregs + ENC_INTERRUPT_REGISTER);
2481
2482         pr_debug("vepu_irq irq status %x\n", irq_status);
2483
2484 #if VPU_SERVICE_SHOW_TIME
2485         do_gettimeofday(&enc_end);
2486         vpu_debug(3, "enc task: %ld ms\n",
2487                 (enc_end.tv_sec  - enc_start.tv_sec)  * 1000 +
2488                 (enc_end.tv_usec - enc_start.tv_usec) / 1000);
2489 #endif
2490         if (likely(irq_status & ENC_INTERRUPT_BIT)) {
2491                 /* clear enc IRQ */
2492                 writel(irq_status & (~ENC_INTERRUPT_BIT), dev->hwregs + ENC_INTERRUPT_REGISTER);
2493                 atomic_add(1, &dev->irq_count_codec);
2494         }
2495
2496         pservice->irq_status = irq_status;
2497
2498         /*vcodec_exit_mode(pservice);*/
2499
2500         if (atomic_read(&dev->irq_count_codec))
2501                 return IRQ_WAKE_THREAD;
2502         else
2503                 return IRQ_NONE;
2504 }
2505
2506 static irqreturn_t vepu_isr(int irq, void *dev_id)
2507 {
2508         struct vpu_subdev_data *data = (struct vpu_subdev_data*)dev_id;
2509         struct vpu_service_info *pservice = data->pservice;
2510         vpu_device *dev = &data->enc_dev;
2511
2512         mutex_lock(&pservice->lock);
2513         if (atomic_read(&dev->irq_count_codec)) {
2514                 atomic_sub(1, &dev->irq_count_codec);
2515                 if (NULL == pservice->reg_codec) {
2516                         vpu_err("error: enc isr with no task waiting\n");
2517                 } else {
2518                         reg_from_run_to_done(data, pservice->reg_codec);
2519                 }
2520         }
2521         try_set_reg(data);
2522         mutex_unlock(&pservice->lock);
2523         return IRQ_HANDLED;
2524 }
2525
2526 static int __init vcodec_service_init(void)
2527 {
2528         int ret;
2529
2530         if ((ret = platform_driver_register(&vcodec_driver)) != 0) {
2531                 vpu_err("Platform device register failed (%d).\n", ret);
2532                 return ret;
2533         }
2534
2535 #ifdef CONFIG_DEBUG_FS
2536         vcodec_debugfs_init();
2537 #endif
2538
2539         return ret;
2540 }
2541
2542 static void __exit vcodec_service_exit(void)
2543 {
2544 #ifdef CONFIG_DEBUG_FS
2545         vcodec_debugfs_exit();
2546 #endif
2547
2548         platform_driver_unregister(&vcodec_driver);
2549 }
2550
2551 module_init(vcodec_service_init);
2552 module_exit(vcodec_service_exit);
2553
2554 #ifdef CONFIG_DEBUG_FS
2555 #include <linux/seq_file.h>
2556
2557 static int vcodec_debugfs_init()
2558 {
2559         parent = debugfs_create_dir("vcodec", NULL);
2560         if (!parent)
2561                 return -1;
2562
2563         return 0;
2564 }
2565
2566 static void vcodec_debugfs_exit()
2567 {
2568         debugfs_remove(parent);
2569 }
2570
2571 static struct dentry* vcodec_debugfs_create_device_dir(char *dirname, struct dentry *parent)
2572 {
2573         return debugfs_create_dir(dirname, parent);
2574 }
2575
2576 static int debug_vcodec_show(struct seq_file *s, void *unused)
2577 {
2578         struct vpu_subdev_data *data = s->private;
2579         struct vpu_service_info *pservice = data->pservice;
2580         unsigned int i, n;
2581         vpu_reg *reg, *reg_tmp;
2582         vpu_session *session, *session_tmp;
2583
2584         mutex_lock(&pservice->lock);
2585         vpu_service_power_on(pservice);
2586         if (data->hw_info->hw_id != HEVC_ID) {
2587                 seq_printf(s, "\nENC Registers:\n");
2588                 n = data->enc_dev.iosize >> 2;
2589                 for (i = 0; i < n; i++)
2590                         seq_printf(s, "\tswreg%d = %08X\n", i, readl(data->enc_dev.hwregs + i));
2591         }
2592         seq_printf(s, "\nDEC Registers:\n");
2593         n = data->dec_dev.iosize >> 2;
2594         for (i = 0; i < n; i++)
2595                 seq_printf(s, "\tswreg%d = %08X\n", i, readl(data->dec_dev.hwregs + i));
2596
2597         seq_printf(s, "\nvpu service status:\n");
2598         list_for_each_entry_safe(session, session_tmp, &pservice->session, list_session) {
2599                 seq_printf(s, "session pid %d type %d:\n", session->pid, session->type);
2600                 /*seq_printf(s, "waiting reg set %d\n");*/
2601                 list_for_each_entry_safe(reg, reg_tmp, &session->waiting, session_link) {
2602                         seq_printf(s, "waiting register set\n");
2603                 }
2604                 list_for_each_entry_safe(reg, reg_tmp, &session->running, session_link) {
2605                         seq_printf(s, "running register set\n");
2606                 }
2607                 list_for_each_entry_safe(reg, reg_tmp, &session->done, session_link) {
2608                         seq_printf(s, "done    register set\n");
2609                 }
2610         }
2611
2612         seq_printf(s, "\npower counter: on %d off %d\n",
2613                         atomic_read(&pservice->power_on_cnt),
2614                         atomic_read(&pservice->power_off_cnt));
2615         mutex_unlock(&pservice->lock);
2616         vpu_service_power_off(pservice);
2617
2618         return 0;
2619 }
2620
2621 static int debug_vcodec_open(struct inode *inode, struct file *file)
2622 {
2623         return single_open(file, debug_vcodec_show, inode->i_private);
2624 }
2625
2626 #endif
2627
2628 #if HEVC_TEST_ENABLE & defined(CONFIG_ION_ROCKCHIP)
2629 #include "hevc_test_inc/pps_00.h"
2630 #include "hevc_test_inc/register_00.h"
2631 #include "hevc_test_inc/rps_00.h"
2632 #include "hevc_test_inc/scaling_list_00.h"
2633 #include "hevc_test_inc/stream_00.h"
2634
2635 #include "hevc_test_inc/pps_01.h"
2636 #include "hevc_test_inc/register_01.h"
2637 #include "hevc_test_inc/rps_01.h"
2638 #include "hevc_test_inc/scaling_list_01.h"
2639 #include "hevc_test_inc/stream_01.h"
2640
2641 #include "hevc_test_inc/cabac.h"
2642
2643 extern struct ion_client *rockchip_ion_client_create(const char * name);
2644
2645 static struct ion_client *ion_client = NULL;
2646 u8* get_align_ptr(u8* tbl, int len, u32 *phy)
2647 {
2648         int size = (len+15) & (~15);
2649         struct ion_handle *handle;
2650         u8 *ptr;
2651
2652         if (ion_client == NULL)
2653                 ion_client = rockchip_ion_client_create("vcodec");
2654
2655         handle = ion_alloc(ion_client, (size_t)len, 16, ION_HEAP(ION_CMA_HEAP_ID), 0);
2656
2657         ptr = ion_map_kernel(ion_client, handle);
2658
2659         ion_phys(ion_client, handle, phy, &size);
2660
2661         memcpy(ptr, tbl, len);
2662
2663         return ptr;
2664 }
2665
2666 u8* get_align_ptr_no_copy(int len, u32 *phy)
2667 {
2668         int size = (len+15) & (~15);
2669         struct ion_handle *handle;
2670         u8 *ptr;
2671
2672         if (ion_client == NULL)
2673                 ion_client = rockchip_ion_client_create("vcodec");
2674
2675         handle = ion_alloc(ion_client, (size_t)len, 16, ION_HEAP(ION_CMA_HEAP_ID), 0);
2676
2677         ptr = ion_map_kernel(ion_client, handle);
2678
2679         ion_phys(ion_client, handle, phy, &size);
2680
2681         return ptr;
2682 }
2683
2684 #define TEST_CNT    2
2685 static int hevc_test_case0(vpu_service_info *pservice)
2686 {
2687         vpu_session session;
2688         vpu_reg *reg;
2689         unsigned long size = 272;
2690         int testidx = 0;
2691         int ret = 0;
2692         u8 *pps_tbl[TEST_CNT];
2693         u8 *register_tbl[TEST_CNT];
2694         u8 *rps_tbl[TEST_CNT];
2695         u8 *scaling_list_tbl[TEST_CNT];
2696         u8 *stream_tbl[TEST_CNT];
2697
2698         int stream_size[2];
2699         int pps_size[2];
2700         int rps_size[2];
2701         int scl_size[2];
2702         int cabac_size[2];
2703
2704         u32 phy_pps;
2705         u32 phy_rps;
2706         u32 phy_scl;
2707         u32 phy_str;
2708         u32 phy_yuv;
2709         u32 phy_ref;
2710         u32 phy_cabac;
2711
2712         volatile u8 *stream_buf;
2713         volatile u8 *pps_buf;
2714         volatile u8 *rps_buf;
2715         volatile u8 *scl_buf;
2716         volatile u8 *yuv_buf;
2717         volatile u8 *cabac_buf;
2718         volatile u8 *ref_buf;
2719
2720         u8 *pps;
2721         u8 *yuv[2];
2722         int i;
2723
2724         pps_tbl[0] = pps_00;
2725         pps_tbl[1] = pps_01;
2726
2727         register_tbl[0] = register_00;
2728         register_tbl[1] = register_01;
2729
2730         rps_tbl[0] = rps_00;
2731         rps_tbl[1] = rps_01;
2732
2733         scaling_list_tbl[0] = scaling_list_00;
2734         scaling_list_tbl[1] = scaling_list_01;
2735
2736         stream_tbl[0] = stream_00;
2737         stream_tbl[1] = stream_01;
2738
2739         stream_size[0] = sizeof(stream_00);
2740         stream_size[1] = sizeof(stream_01);
2741
2742         pps_size[0] = sizeof(pps_00);
2743         pps_size[1] = sizeof(pps_01);
2744
2745         rps_size[0] = sizeof(rps_00);
2746         rps_size[1] = sizeof(rps_01);
2747
2748         scl_size[0] = sizeof(scaling_list_00);
2749         scl_size[1] = sizeof(scaling_list_01);
2750
2751         cabac_size[0] = sizeof(Cabac_table);
2752         cabac_size[1] = sizeof(Cabac_table);
2753
2754         /* create session */
2755         session.pid = current->pid;
2756         session.type = VPU_DEC;
2757         INIT_LIST_HEAD(&session.waiting);
2758         INIT_LIST_HEAD(&session.running);
2759         INIT_LIST_HEAD(&session.done);
2760         INIT_LIST_HEAD(&session.list_session);
2761         init_waitqueue_head(&session.wait);
2762         atomic_set(&session.task_running, 0);
2763         list_add_tail(&session.list_session, &pservice->session);
2764
2765         yuv[0] = get_align_ptr_no_copy(256*256*2, &phy_yuv);
2766         yuv[1] = get_align_ptr_no_copy(256*256*2, &phy_ref);
2767
2768         while (testidx < TEST_CNT) {
2769                 /* create registers */
2770                 reg = kmalloc(sizeof(vpu_reg)+pservice->reg_size, GFP_KERNEL);
2771                 if (NULL == reg) {
2772                         vpu_err("error: kmalloc fail in reg_init\n");
2773                         return -1;
2774                 }
2775
2776                 if (size > pservice->reg_size) {
2777                         printk("warning: vpu reg size %lu is larger than hw reg size %lu\n", size, pservice->reg_size);
2778                         size = pservice->reg_size;
2779                 }
2780                 reg->session = &session;
2781                 reg->type = session.type;
2782                 reg->size = size;
2783                 reg->freq = VPU_FREQ_DEFAULT;
2784                 reg->reg = (unsigned long *)&reg[1];
2785                 INIT_LIST_HEAD(&reg->session_link);
2786                 INIT_LIST_HEAD(&reg->status_link);
2787
2788                 /* TODO: stuff registers */
2789                 memcpy(&reg->reg[0], register_tbl[testidx], /*sizeof(register_00)*/ 176);
2790
2791                 stream_buf = get_align_ptr(stream_tbl[testidx], stream_size[testidx], &phy_str);
2792                 pps_buf = get_align_ptr(pps_tbl[0], pps_size[0], &phy_pps);
2793                 rps_buf = get_align_ptr(rps_tbl[testidx], rps_size[testidx], &phy_rps);
2794                 scl_buf = get_align_ptr(scaling_list_tbl[testidx], scl_size[testidx], &phy_scl);
2795                 cabac_buf = get_align_ptr(Cabac_table, cabac_size[testidx], &phy_cabac);
2796
2797                 pps = pps_buf;
2798
2799                 /* TODO: replace reigster address */
2800                 for (i=0; i<64; i++) {
2801                         u32 scaling_offset;
2802                         u32 tmp;
2803
2804                         scaling_offset = (u32)pps[i*80+74];
2805                         scaling_offset += (u32)pps[i*80+75] << 8;
2806                         scaling_offset += (u32)pps[i*80+76] << 16;
2807                         scaling_offset += (u32)pps[i*80+77] << 24;
2808
2809                         tmp = phy_scl + scaling_offset;
2810
2811                         pps[i*80+74] = tmp & 0xff;
2812                         pps[i*80+75] = (tmp >> 8) & 0xff;
2813                         pps[i*80+76] = (tmp >> 16) & 0xff;
2814                         pps[i*80+77] = (tmp >> 24) & 0xff;
2815                 }
2816
2817                 printk("%s %d, phy stream %08x, phy pps %08x, phy rps %08x\n",
2818                         __func__, __LINE__, phy_str, phy_pps, phy_rps);
2819
2820                 reg->reg[1] = 0x21;
2821                 reg->reg[4] = phy_str;
2822                 reg->reg[5] = ((stream_size[testidx]+15)&(~15))+64;
2823                 reg->reg[6] = phy_cabac;
2824                 reg->reg[7] = testidx?phy_ref:phy_yuv;
2825                 reg->reg[42] = phy_pps;
2826                 reg->reg[43] = phy_rps;
2827                 for (i = 10; i <= 24; i++)
2828                         reg->reg[i] = phy_yuv;
2829
2830                 mutex_lock(pservice->lock);
2831                 list_add_tail(&reg->status_link, &pservice->waiting);
2832                 list_add_tail(&reg->session_link, &session.waiting);
2833                 mutex_unlock(pservice->lock);
2834
2835                 /* stuff hardware */
2836                 try_set_reg(data);
2837
2838                 /* wait for result */
2839                 ret = wait_event_timeout(session.wait, !list_empty(&session.done), VPU_TIMEOUT_DELAY);
2840                 if (!list_empty(&session.done)) {
2841                         if (ret < 0)
2842                                 vpu_err("warning: pid %d wait task sucess but wait_evernt ret %d\n", session.pid, ret);
2843                         ret = 0;
2844                 } else {
2845                         if (unlikely(ret < 0)) {
2846                                 vpu_err("error: pid %d wait task ret %d\n", session.pid, ret);
2847                         } else if (0 == ret) {
2848                                 vpu_err("error: pid %d wait %d task done timeout\n", session.pid, atomic_read(&session.task_running));
2849                                 ret = -ETIMEDOUT;
2850                         }
2851                 }
2852                 if (ret < 0) {
2853                         int task_running = atomic_read(&session.task_running);
2854                         int n;
2855                         mutex_lock(pservice->lock);
2856                         vpu_service_dump(pservice);
2857                         if (task_running) {
2858                                 atomic_set(&session.task_running, 0);
2859                                 atomic_sub(task_running, &pservice->total_running);
2860                                 printk("%d task is running but not return, reset hardware...", task_running);
2861                                 vpu_reset(data);
2862                                 printk("done\n");
2863                         }
2864                         vpu_service_session_clear(pservice, &session);
2865                         mutex_unlock(pservice->lock);
2866
2867                         printk("\nDEC Registers:\n");
2868                         n = data->dec_dev.iosize >> 2;
2869                         for (i=0; i<n; i++)
2870                                 printk("\tswreg%d = %08X\n", i, readl(data->dec_dev.hwregs + i));
2871
2872                         vpu_err("test index %d failed\n", testidx);
2873                         break;
2874                 } else {
2875                         vpu_debug(3, "test index %d success\n", testidx);
2876
2877                         vpu_reg *reg = list_entry(session.done.next, vpu_reg, session_link);
2878
2879                         for (i=0; i<68; i++) {
2880                                 if (i % 4 == 0)
2881                                         printk("%02d: ", i);
2882                                 printk("%08x ", reg->reg[i]);
2883                                 if ((i+1) % 4 == 0)
2884                                         printk("\n");
2885                         }
2886
2887                         testidx++;
2888                 }
2889
2890                 reg_deinit(data, reg);
2891         }
2892
2893         return 0;
2894 }
2895
2896 #endif
2897