include/libMinHook/src/HDE32/hde32.c

   1 /*
   2  * Hacker Disassembler Engine 32 C
   3  * Copyright (c) 2008-2009, Vyacheslav Patkov.
   4  * All rights reserved.
   5  *
   6  */
   7
   8 /* #include <stdint.h> */
   9 #include <string.h>
  10
  11 #include "hde32.h"
  12 #include "table32.h"
  13
  14 unsigned int hde32_disasm(const void *code, hde32s *hs)
  15 {
  16     uint8_t x, c, *p = (uint8_t *)code, cflags, opcode, pref = 0;
  17     uint8_t *ht = hde32_table, m_mod, m_reg, m_rm, disp_size = 0;
  18
  19     memset(hs,0,sizeof(hde32s));
  20
  21     for (x = 16; x; x--)
  22         switch (c = *p++) {
  23             case 0xf3:
  24                 hs->p_rep = c;
  25                 pref |= PRE_F3;
  26                 break;
  27             case 0xf2:
  28                 hs->p_rep = c;
  29                 pref |= PRE_F2;
  30                 break;
  31             case 0xf0:
  32                 hs->p_lock = c;
  33                 pref |= PRE_LOCK;
  34                 break;
  35             case 0x26: case 0x2e: case 0x36:
  36             case 0x3e: case 0x64: case 0x65:
  37                 hs->p_seg = c;
  38                 pref |= PRE_SEG;
  39                 break;
  40             case 0x66:
  41                 hs->p_66 = c;
  42                 pref |= PRE_66;
  43                 break;
  44             case 0x67:
  45                 hs->p_67 = c;
  46                 pref |= PRE_67;
  47                 break;
  48             default:
  49                 goto pref_done;
  50         }
  51   pref_done:
  52
  53     hs->flags = (uint32_t)pref << 23;
  54
  55     if (!pref)
  56         pref |= PRE_NONE;
  57
  58     if ((hs->opcode = c) == 0x0f) {
  59         hs->opcode2 = c = *p++;
  60         ht += DELTA_OPCODES;
  61     } else if (c >= 0xa0 && c <= 0xa3) {
  62         if (pref & PRE_67)
  63             pref |= PRE_66;
  64         else
  65             pref &= ~PRE_66;
  66     }
  67
  68     opcode = c;
  69     cflags = ht[ht[opcode / 4] + (opcode % 4)];
  70
  71     if (cflags == C_ERROR) {
  72         hs->flags |= F_ERROR | F_ERROR_OPCODE;
  73         cflags = 0;
  74         if ((opcode & -3) == 0x24)
  75             cflags++;
  76     }
  77
  78     x = 0;
  79     if (cflags & C_GROUP) {
  80         uint16_t t;
  81         t = *(uint16_t *)(ht + (cflags & 0x7f));
  82         cflags = (uint8_t)t;
  83         x = (uint8_t)(t >> 8);
  84     }
  85
  86     if (hs->opcode2) {
  87         ht = hde32_table + DELTA_PREFIXES;
  88         if (ht[ht[opcode / 4] + (opcode % 4)] & pref)
  89             hs->flags |= F_ERROR | F_ERROR_OPCODE;
  90     }
  91
  92     if (cflags & C_MODRM) {
  93         hs->flags |= F_MODRM;
  94         hs->modrm = c = *p++;
  95         hs->modrm_mod = m_mod = c >> 6;
  96         hs->modrm_rm = m_rm = c & 7;
  97         hs->modrm_reg = m_reg = (c & 0x3f) >> 3;
  98
  99         if (x && ((x << m_reg) & 0x80))
 100             hs->flags |= F_ERROR | F_ERROR_OPCODE;
 101
 102         if (!hs->opcode2 && opcode >= 0xd9 && opcode <= 0xdf) {
 103             uint8_t t = opcode - 0xd9;
 104             if (m_mod == 3) {
 105                 ht = hde32_table + DELTA_FPU_MODRM + t*8;
 106                 t = ht[m_reg] << m_rm;
 107             } else {
 108                 ht = hde32_table + DELTA_FPU_REG;
 109                 t = ht[t] << m_reg;
 110             }
 111             if (t & 0x80)
 112                 hs->flags |= F_ERROR | F_ERROR_OPCODE;
 113         }
 114
 115         if (pref & PRE_LOCK) {
 116             if (m_mod == 3) {
 117                 hs->flags |= F_ERROR | F_ERROR_LOCK;
 118             } else {
 119                 uint8_t *table_end, op = opcode;
 120                 if (hs->opcode2) {
 121                     ht = hde32_table + DELTA_OP2_LOCK_OK;
 122                     table_end = ht + DELTA_OP_ONLY_MEM - DELTA_OP2_LOCK_OK;
 123                 } else {
 124                     ht = hde32_table + DELTA_OP_LOCK_OK;
 125                     table_end = ht + DELTA_OP2_LOCK_OK - DELTA_OP_LOCK_OK;
 126                     op &= -2;
 127                 }
 128                 for (; ht != table_end; ht++)
 129                     if (*ht++ == op) {
 130                         if (!((*ht << m_reg) & 0x80))
 131                             goto no_lock_error;
 132                         else
 133                             break;
 134                     }
 135                 hs->flags |= F_ERROR | F_ERROR_LOCK;
 136               no_lock_error:
 137                 ;
 138             }
 139         }
 140
 141         if (hs->opcode2) {
 142             switch (opcode) {
 143                 case 0x20: case 0x22:
 144                     m_mod = 3;
 145                     if (m_reg > 4 || m_reg == 1)
 146                         goto error_operand;
 147                     else
 148                         goto no_error_operand;
 149                 case 0x21: case 0x23:
 150                     m_mod = 3;
 151                     if (m_reg == 4 || m_reg == 5)
 152                         goto error_operand;
 153                     else
 154                         goto no_error_operand;
 155             }
 156         } else {
 157             switch (opcode) {
 158                 case 0x8c:
 159                     if (m_reg > 5)
 160                         goto error_operand;
 161                     else
 162                         goto no_error_operand;
 163                 case 0x8e:
 164                     if (m_reg == 1 || m_reg > 5)
 165                         goto error_operand;
 166                     else
 167                         goto no_error_operand;
 168             }
 169         }
 170
 171         if (m_mod == 3) {
 172             uint8_t *table_end;
 173             if (hs->opcode2) {
 174                 ht = hde32_table + DELTA_OP2_ONLY_MEM;
 175                 table_end = ht + sizeof(hde32_table) - DELTA_OP2_ONLY_MEM;
 176             } else {
 177                 ht = hde32_table + DELTA_OP_ONLY_MEM;
 178                 table_end = ht + DELTA_OP2_ONLY_MEM - DELTA_OP_ONLY_MEM;
 179             }
 180             for (; ht != table_end; ht += 2)
 181                 if (*ht++ == opcode) {
 182                     if (*ht++ & pref && !((*ht << m_reg) & 0x80))
 183                         goto error_operand;
 184                     else
 185                         break;
 186                 }
 187             goto no_error_operand;
 188         } else if (hs->opcode2) {
 189             switch (opcode) {
 190                 case 0x50: case 0xd7: case 0xf7:
 191                     if (pref & (PRE_NONE | PRE_66))
 192                         goto error_operand;
 193                     break;
 194                 case 0xd6:
 195                     if (pref & (PRE_F2 | PRE_F3))
 196                         goto error_operand;
 197                     break;
 198                 case 0xc5:
 199                     goto error_operand;
 200             }
 201             goto no_error_operand;
 202         } else
 203             goto no_error_operand;
 204
 205       error_operand:
 206         hs->flags |= F_ERROR | F_ERROR_OPERAND;
 207       no_error_operand:
 208
 209         c = *p++;
 210         if (m_reg <= 1) {
 211             if (opcode == 0xf6)
 212                 cflags |= C_IMM8;
 213             else if (opcode == 0xf7)
 214                 cflags |= C_IMM_P66;
 215         }
 216
 217         switch (m_mod) {
 218             case 0:
 219                 if (pref & PRE_67) {
 220                     if (m_rm == 6)
 221                         disp_size = 2;
 222                 } else
 223                     if (m_rm == 5)
 224                         disp_size = 4;
 225                 break;
 226             case 1:
 227                 disp_size = 1;
 228                 break;
 229             case 2:
 230                 disp_size = 2;
 231                 if (!(pref & PRE_67))
 232                     disp_size <<= 1;
 233         }
 234
 235         if (m_mod != 3 && m_rm == 4 && !(pref & PRE_67)) {
 236             hs->flags |= F_SIB;
 237             p++;
 238             hs->sib = c;
 239             hs->sib_scale = c >> 6;
 240             hs->sib_index = (c & 0x3f) >> 3;
 241             if ((hs->sib_base = c & 7) == 5 && !(m_mod & 1))
 242                 disp_size = 4;
 243         }
 244
 245         p--;
 246         switch (disp_size) {
 247             case 1:
 248                 hs->flags |= F_DISP8;
 249                 hs->disp.disp8 = *p;
 250                 break;
 251             case 2:
 252                 hs->flags |= F_DISP16;
 253                 hs->disp.disp16 = *(uint16_t *)p;
 254                 break;
 255             case 4:
 256                 hs->flags |= F_DISP32;
 257                 hs->disp.disp32 = *(uint32_t *)p;
 258         }
 259         p += disp_size;
 260     } else if (pref & PRE_LOCK)
 261         hs->flags |= F_ERROR | F_ERROR_LOCK;
 262
 263     if (cflags & C_IMM_P66) {
 264         if (cflags & C_REL32) {
 265             if (pref & PRE_66) {
 266                 hs->flags |= F_IMM16 | F_RELATIVE;
 267                 hs->imm.imm16 = *(uint16_t *)p;
 268                 p += 2;
 269                 goto disasm_done;
 270             }
 271             goto rel32_ok;
 272         }
 273         if (pref & PRE_66) {
 274             hs->flags |= F_IMM16;
 275             hs->imm.imm16 = *(uint16_t *)p;
 276             p += 2;
 277         } else {
 278             hs->flags |= F_IMM32;
 279             hs->imm.imm32 = *(uint32_t *)p;
 280             p += 4;
 281         }
 282     }
 283
 284     if (cflags & C_IMM16) {
 285         if (hs->flags & F_IMM32) {
 286             hs->flags |= F_IMM16;
 287             hs->disp.disp16 = *(uint16_t *)p;
 288         } else if (hs->flags & F_IMM16) {
 289             hs->flags |= F_2IMM16;
 290             hs->disp.disp16 = *(uint16_t *)p;
 291         } else {
 292             hs->flags |= F_IMM16;
 293             hs->imm.imm16 = *(uint16_t *)p;
 294         }
 295         p += 2;
 296     }
 297     if (cflags & C_IMM8) {
 298         hs->flags |= F_IMM8;
 299         hs->imm.imm8 = *p++;
 300     }
 301
 302     if (cflags & C_REL32) {
 303       rel32_ok:
 304         hs->flags |= F_IMM32 | F_RELATIVE;
 305         hs->imm.imm32 = *(uint32_t *)p;
 306         p += 4;
 307     } else if (cflags & C_REL8) {
 308         hs->flags |= F_IMM8 | F_RELATIVE;
 309         hs->imm.imm8 = *p++;
 310     }
 311
 312   disasm_done:
 313
 314     if ((hs->len = (uint8_t)(p-(uint8_t *)code)) > 15) {
 315         hs->flags |= F_ERROR | F_ERROR_LENGTH;
 316         hs->len = 15;
 317     }
 318
 319     return (unsigned int)hs->len;
 320 }