1: /*
    2:  * internal execution defines for qemu
    3:  *
    4:  *  Copyright (c) 2003 Fabrice Bellard
    5:  *
    6:  * This library is free software; you can redistribute it and/or
    7:  * modify it under the terms of the GNU Lesser General Public
    8:  * License as published by the Free Software Foundation; either
    9:  * version 2 of the License, or (at your option) any later version.
   10:  *
   11:  * This library is distributed in the hope that it will be useful,
   12:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   13:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   14:  * Lesser General Public License for more details.
   15:  *
   16:  * You should have received a copy of the GNU Lesser General Public
   17:  * License along with this library; if not, write to the Free Software
   18:  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   19:  */
   20: 
   21: /* allow to see translation results - the slowdown should be negligible, so we leave it */
   22: #define DEBUG_DISAS
   23: 
   24: /* is_jmp field values */
   25: #define DISAS_NEXT    0 /* next instruction can be analyzed */
   26: #define DISAS_JUMP    1 /* only pc was modified dynamically */
   27: #define DISAS_UPDATE  2 /* cpu state was modified dynamically */
   28: #define DISAS_TB_JUMP 3 /* only pc was modified statically */
   29: 
   30: struct TranslationBlock;
   31: 
   32: /* XXX: make safe guess about sizes */
   33: #define MAX_OP_PER_INSTR 32
   34: #define OPC_BUF_SIZE 512
   35: #define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
   36: 
   37: #define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * 3)
   38: 
   39: extern uint16_t gen_opc_buf[OPC_BUF_SIZE];
   40: extern uint32_t gen_opparam_buf[OPPARAM_BUF_SIZE];
   41: extern long gen_labels[OPC_BUF_SIZE];
   42: extern int nb_gen_labels;
   43: extern target_ulong gen_opc_pc[OPC_BUF_SIZE];
   44: extern target_ulong gen_opc_npc[OPC_BUF_SIZE];
   45: extern uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
   46: extern uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
   47: extern target_ulong gen_opc_jump_pc[2];
   48: extern uint32_t gen_opc_hflags[OPC_BUF_SIZE];
   49: 
   50: typedef void (GenOpFunc)(void);
   51: typedef void (GenOpFunc1)(long);
   52: typedef void (GenOpFunc2)(long, long);
   53: typedef void (GenOpFunc3)(long, long, long);
   54: 
   55: #if defined(TARGET_I386)
   56: 
   57: void optimize_flags_init(void);
   58: 
   59: #endif
   60: 
   61: extern FILE *logfile;
   62: extern int loglevel;
   63: 
   64: int gen_intermediate_code(CPUState *env, struct TranslationBlock *tb);
   65: int gen_intermediate_code_pc(CPUState *env, struct TranslationBlock *tb);
   66: void dump_ops(const uint16_t *opc_buf, const uint32_t *opparam_buf);
   67: unsigned long code_gen_max_block_size(void);
   68: int cpu_gen_code(CPUState *env, struct TranslationBlock *tb,
   69:                  int *gen_code_size_ptr);
   70: int cpu_restore_state(struct TranslationBlock *tb,
   71:                       CPUState *env, unsigned long searched_pc,
   72:                       void *puc);
   73: int cpu_gen_code_copy(CPUState *env, struct TranslationBlock *tb,
   74:                       int max_code_size, int *gen_code_size_ptr);
   75: int cpu_restore_state_copy(struct TranslationBlock *tb,
   76:                            CPUState *env, unsigned long searched_pc,
   77:                            void *puc);
   78: void cpu_resume_from_signal(CPUState *env1, void *puc);
   79: void cpu_exec_init(CPUState *env);
   80: int page_unprotect(target_ulong address, unsigned long pc, void *puc);
   81: void tb_invalidate_phys_page_range(target_ulong start, target_ulong end,
   82:                                    int is_cpu_write_access);
   83: void tb_invalidate_page_range(target_ulong start, target_ulong end);
   84: void tlb_flush_page(CPUState *env, target_ulong addr);
   85: void tlb_flush(CPUState *env, int flush_global);
   86: int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
   87:                       target_phys_addr_t paddr, int prot,
   88:                       int mmu_idx, int is_softmmu);
   89: static inline int tlb_set_page(CPUState *env, target_ulong vaddr,
   90:                                target_phys_addr_t paddr, int prot,
   91:                                int mmu_idx, int is_softmmu)
   92: {
   93:     if (prot & PAGE_READ)
   94:         prot |= PAGE_EXEC;
   95:     return tlb_set_page_exec(env, vaddr, paddr, prot, mmu_idx, is_softmmu);
   96: }
   97: 
   98: #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
   99: 
  100: #define CODE_GEN_PHYS_HASH_BITS     15
  101: #define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
  102: 
  103: /* maximum total translate dcode allocated */
  104: 
  105: /* NOTE: the translated code area cannot be too big because on some
  106:    archs the range of "fast" function calls is limited. Here is a
  107:    summary of the ranges:
  108: 
  109:    i386  : signed 32 bits
  110:    arm   : signed 26 bits
  111:    ppc   : signed 24 bits
  112:    sparc : signed 32 bits
  113:    alpha : signed 23 bits
  114: */
  115: 
  116: #if defined(__alpha__)
  117: #define CODE_GEN_BUFFER_SIZE     (2 * 1024 * 1024)
  118: #elif defined(__ia64)
  119: #define CODE_GEN_BUFFER_SIZE     (4 * 1024 * 1024)      /* range of addl */
  120: #elif defined(__powerpc__)
  121: #define CODE_GEN_BUFFER_SIZE     (6 * 1024 * 1024)
  122: #else
  123: #define CODE_GEN_BUFFER_SIZE     (16 * 1024 * 1024)
  124: #endif
  125: 
  126: //#define CODE_GEN_BUFFER_SIZE     (128 * 1024)
  127: 
  128: /* estimated block size for TB allocation */
  129: /* XXX: use a per code average code fragment size and modulate it
  130:    according to the host CPU */
  131: #if defined(CONFIG_SOFTMMU)
  132: #define CODE_GEN_AVG_BLOCK_SIZE 128
  133: #else
  134: #define CODE_GEN_AVG_BLOCK_SIZE 64
  135: #endif
  136: 
  137: #define CODE_GEN_MAX_BLOCKS    (CODE_GEN_BUFFER_SIZE / CODE_GEN_AVG_BLOCK_SIZE)
  138: 
  139: #if defined(__powerpc__)
  140: #define USE_DIRECT_JUMP
  141: #endif
  142: #if defined(__i386__) && !defined(_WIN32)
  143: #define USE_DIRECT_JUMP
  144: #endif
  145: 
  146: typedef struct TranslationBlock {
  147:     target_ulong pc;   /* simulated PC corresponding to this block (EIP + CS base) */
  148:     target_ulong cs_base; /* CS base for this block */
  149:     uint64_t flags; /* flags defining in which context the code was generated */
  150:     uint16_t size;      /* size of target code for this block (1 <=
  151:                            size <= TARGET_PAGE_SIZE) */
  152:     uint16_t cflags;    /* compile flags */
  153: #define CF_CODE_COPY   0x0001 /* block was generated in code copy mode */
  154: #define CF_TB_FP_USED  0x0002 /* fp ops are used in the TB */
  155: #define CF_FP_USED     0x0004 /* fp ops are used in the TB or in a chained TB */
  156: #define CF_SINGLE_INSN 0x0008 /* compile only a single instruction */
  157: 
  158:     uint8_t *tc_ptr;    /* pointer to the translated code */
  159:     /* next matching tb for physical address. */
  160:     struct TranslationBlock *phys_hash_next;
  161:     /* first and second physical page containing code. The lower bit
  162:        of the pointer tells the index in page_next[] */
  163:     struct TranslationBlock *page_next[2];
  164:     target_ulong page_addr[2];
  165: 
  166:     /* the following data are used to directly call another TB from
  167:        the code of this one. */
  168:     uint16_t tb_next_offset[2]; /* offset of original jump target */
  169: #ifdef USE_DIRECT_JUMP
  170:     uint16_t tb_jmp_offset[4]; /* offset of jump instruction */
  171: #else
  172:     uint32_t tb_next[2]; /* address of jump generated code */
  173: #endif
  174:     /* list of TBs jumping to this one. This is a circular list using
  175:        the two least significant bits of the pointers to tell what is
  176:        the next pointer: 0 = jmp_next[0], 1 = jmp_next[1], 2 =
  177:        jmp_first */
  178:     struct TranslationBlock *jmp_next[2];
  179:     struct TranslationBlock *jmp_first;
  180: } TranslationBlock;
  181: 
  182: static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc)
  183: {
  184:     target_ulong tmp;
  185:     tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
  186:     return (tmp >> TB_JMP_PAGE_BITS) & TB_JMP_PAGE_MASK;
  187: }
  188: 
  189: static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
  190: {
  191:     target_ulong tmp;
  192:     tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
  193:     return (((tmp >> TB_JMP_PAGE_BITS) & TB_JMP_PAGE_MASK) |
  194:             (tmp & TB_JMP_ADDR_MASK));
  195: }
  196: 
  197: static inline unsigned int tb_phys_hash_func(unsigned long pc)
  198: {
  199:     return pc & (CODE_GEN_PHYS_HASH_SIZE - 1);
  200: }
  201: 
  202: TranslationBlock *tb_alloc(target_ulong pc);
  203: void tb_flush(CPUState *env);
  204: void tb_link_phys(TranslationBlock *tb,
  205:                   target_ulong phys_pc, target_ulong phys_page2);
  206: 
  207: extern TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
  208: 
  209: extern uint8_t code_gen_buffer[CODE_GEN_BUFFER_SIZE];
  210: extern uint8_t *code_gen_ptr;
  211: 
  212: #if defined(USE_DIRECT_JUMP)
  213: 
  214: #if defined(__powerpc__)
  215: static inline void tb_set_jmp_target1(unsigned long jmp_addr, unsigned long addr)
  216: {
  217:     uint32_t val, *ptr;
  218: 
  219:     /* patch the branch destination */
  220:     ptr = (uint32_t *)jmp_addr;
  221:     val = *ptr;
  222:     val = (val & ~0x03fffffc) | ((addr - jmp_addr) & 0x03fffffc);
  223:     *ptr = val;
  224:     /* flush icache */
  225:     asm volatile ("dcbst 0,%0" : : "r"(ptr) : "memory");
  226:     asm volatile ("sync" : : : "memory");
  227:     asm volatile ("icbi 0,%0" : : "r"(ptr) : "memory");
  228:     asm volatile ("sync" : : : "memory");
  229:     asm volatile ("isync" : : : "memory");
  230: }
  231: #elif defined(__i386__)
  232: static inline void tb_set_jmp_target1(unsigned long jmp_addr, unsigned long addr)
  233: {
  234:     /* patch the branch destination */
  235:     *(uint32_t *)jmp_addr = addr - (jmp_addr + 4);
  236:     /* no need to flush icache explicitely */
  237: }
  238: #endif
  239: 
  240: static inline void tb_set_jmp_target(TranslationBlock *tb,
  241:                                      int n, unsigned long addr)
  242: {
  243:     unsigned long offset;
  244: 
  245:     offset = tb->tb_jmp_offset[n];
  246:     tb_set_jmp_target1((unsigned long)(tb->tc_ptr + offset), addr);
  247:     offset = tb->tb_jmp_offset[n + 2];
  248:     if (offset != 0xffff)
  249:         tb_set_jmp_target1((unsigned long)(tb->tc_ptr + offset), addr);
  250: }
  251: 
  252: #else
  253: 
  254: /* set the jump target */
  255: static inline void tb_set_jmp_target(TranslationBlock *tb,
  256:                                      int n, unsigned long addr)
  257: {
  258:     tb->tb_next[n] = addr;
  259: }
  260: 
  261: #endif
  262: 
  263: static inline void tb_add_jump(TranslationBlock *tb, int n,
  264:                                TranslationBlock *tb_next)
  265: {
  266:     /* NOTE: this test is only needed for thread safety */
  267:     if (!tb->jmp_next[n]) {
  268:         /* patch the native jump address */
  269:         tb_set_jmp_target(tb, n, (unsigned long)tb_next->tc_ptr);
  270: 
  271:         /* add in TB jmp circular list */
  272:         tb->jmp_next[n] = tb_next->jmp_first;
  273:         tb_next->jmp_first = (TranslationBlock *)((long)(tb) | (n));
  274:     }
  275: }
  276: 
  277: TranslationBlock *tb_find_pc(unsigned long pc_ptr);
  278: 
  279: #ifndef offsetof
  280: #define offsetof(type, field) ((size_t) &((type *)0)->field)
  281: #endif
  282: 
  283: #if defined(_WIN32)
  284: #define ASM_DATA_SECTION ".section \".data\"\n"
  285: #define ASM_PREVIOUS_SECTION ".section .text\n"
  286: #elif defined(__APPLE__)
  287: #define ASM_DATA_SECTION ".data\n"
  288: #define ASM_PREVIOUS_SECTION ".text\n"
  289: #else
  290: #define ASM_DATA_SECTION ".section \".data\"\n"
  291: #define ASM_PREVIOUS_SECTION ".previous\n"
  292: #endif
  293: 
  294: #define ASM_OP_LABEL_NAME(n, opname) \
  295:     ASM_NAME(__op_label) #n "." ASM_NAME(opname)
  296: 
  297: #if defined(__powerpc__)
  298: 
  299: /* we patch the jump instruction directly */
  300: #define GOTO_TB(opname, tbparam, n)\
  301: do {\
  302:     asm volatile (ASM_DATA_SECTION\
  303:                   ASM_OP_LABEL_NAME(n, opname) ":\n"\
  304:                   ".long 1f\n"\
  305:                   ASM_PREVIOUS_SECTION \
  306:                   "b " ASM_NAME(__op_jmp) #n "\n"\
  307:                   "1:\n");\
  308: } while (0)
  309: 
  310: #elif defined(__i386__) && defined(USE_DIRECT_JUMP)
  311: 
  312: /* we patch the jump instruction directly */
  313: #define GOTO_TB(opname, tbparam, n)\
  314: do {\
  315:     asm volatile (".section .data\n"\
  316:                   ASM_OP_LABEL_NAME(n, opname) ":\n"\
  317:                   ".long 1f\n"\
  318:                   ASM_PREVIOUS_SECTION \
  319:                   "jmp " ASM_NAME(__op_jmp) #n "\n"\
  320:                   "1:\n");\
  321: } while (0)
  322: 
  323: #else
  324: 
  325: /* jump to next block operations (more portable code, does not need
  326:    cache flushing, but slower because of indirect jump) */
  327: #define GOTO_TB(opname, tbparam, n)\
  328: do {\
  329:     static void __attribute__((used)) *dummy ## n = &&dummy_label ## n;\
  330:     static void __attribute__((used)) *__op_label ## n \
  331:         __asm__(ASM_OP_LABEL_NAME(n, opname)) = &&label ## n;\
  332:     goto *(void *)(((TranslationBlock *)tbparam)->tb_next[n]);\
  333: label ## n: ;\
  334: dummy_label ## n: ;\
  335: } while (0)
  336: 
  337: #endif
  338: 
  339: extern CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
  340: extern CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
  341: extern void *io_mem_opaque[IO_MEM_NB_ENTRIES];
  342: 
  343: #if defined(__powerpc__)
  344: static inline int testandset (int *p)
  345: {
  346:     int ret;
  347:     __asm__ __volatile__ (
  348:                           "0:    lwarx %0,0,%1\n"
  349:                           "      xor. %0,%3,%0\n"
  350:                           "      bne 1f\n"
  351:                           "      stwcx. %2,0,%1\n"
  352:                           "      bne- 0b\n"
  353:                           "1:    "
  354:                           : "=&r" (ret)
  355:                           : "r" (p), "r" (1), "r" (0)
  356:                           : "cr0", "memory");
  357:     return ret;
  358: }
  359: #elif defined(__i386__)
  360: static inline int testandset (int *p)
  361: {
  362:     long int readval = 0;
  363: 
  364:     __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  365:                           : "+m" (*p), "+a" (readval)
  366:                           : "r" (1)
  367:                           : "cc");
  368:     return readval;
  369: }
  370: #elif defined(__x86_64__)
  371: static inline int testandset (int *p)
  372: {
  373:     long int readval = 0;
  374: 
  375:     __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  376:                           : "+m" (*p), "+a" (readval)
  377:                           : "r" (1)
  378:                           : "cc");
  379:     return readval;
  380: }
  381: #elif defined(__s390__)
  382: static inline int testandset (int *p)
  383: {
  384:     int ret;
  385: 
  386:     __asm__ __volatile__ ("0: cs    %0,%1,0(%2)\n"
  387:                           "   jl    0b"
  388:                           : "=&d" (ret)
  389:                           : "r" (1), "a" (p), "0" (*p)
  390:                           : "cc", "memory" );
  391:     return ret;
  392: }
  393: #elif defined(__alpha__)
  394: static inline int testandset (int *p)
  395: {
  396:     int ret;
  397:     unsigned long one;
  398: 
  399:     __asm__ __volatile__ ("0:   mov 1,%2\n"
  400:                           "  ldl_l %0,%1\n"
  401:                           "  stl_c %2,%1\n"
  402:                           "  beq %2,1f\n"
  403:                           ".subsection 2\n"
  404:                           "1:        br 0b\n"
  405:                           ".previous"
  406:                           : "=r" (ret), "=m" (*p), "=r" (one)
  407:                           : "m" (*p));
  408:     return ret;
  409: }
  410: #elif defined(__sparc__)
  411: static inline int testandset (int *p)
  412: {
  413:         int ret;
  414: 
  415:         __asm__ __volatile__("ldstub   [%1], %0"
  416:                              : "=r" (ret)
  417:                              : "r" (p)
  418:                              : "memory");
  419: 
  420:         return (ret ? 1 : 0);
  421: }
  422: #elif defined(__arm__)
  423: static inline int testandset (int *spinlock)
  424: {
  425:     register unsigned int ret;
  426:     __asm__ __volatile__("swp %0, %1, [%2]"
  427:                          : "=r"(ret)
  428:                          : "0"(1), "r"(spinlock));
  429: 
  430:     return ret;
  431: }
  432: #elif defined(__mc68000)
  433: static inline int testandset (int *p)
  434: {
  435:     char ret;
  436:     __asm__ __volatile__("tas %1; sne %0"
  437:                          : "=r" (ret)
  438:                          : "m" (p)
  439:                          : "cc","memory");
  440:     return ret;
  441: }
  442: #elif defined(__ia64)
  443: 
  444: #include <ia64intrin.h>
  445: 
  446: static inline int testandset (int *p)
  447: {
  448:     return __sync_lock_test_and_set (p, 1);
  449: }
  450: #elif defined(__mips__)
  451: static inline int testandset (int *p)
  452: {
  453:     int ret;
  454: 
  455:     __asm__ __volatile__ (
  456:         "      .set push            \n"
  457:         "      .set noat            \n"
  458:         "      .set mips2           \n"
  459:         "1:    li $1, 1           \n"
  460:         "      ll   %0, %1            \n"
  461:         "      sc   $1, %1            \n"
  462:         "      beqz $1, 1b          \n"
  463:         "      .set pop             "
  464:         : "=r" (ret), "+R" (*p)
  465:         :
  466:         : "memory");
  467: 
  468:     return ret;
  469: }
  470: #else
  471: #error unimplemented CPU support
  472: #endif
  473: 
  474: typedef int spinlock_t;
  475: 
  476: #define SPIN_LOCK_UNLOCKED 0
  477: 
  478: #if defined(CONFIG_USER_ONLY)
  479: static inline void spin_lock(spinlock_t *lock)
  480: {
  481:     while (testandset(lock));
  482: }
  483: 
  484: static inline void spin_unlock(spinlock_t *lock)
  485: {
  486:     *lock = 0;
  487: }
  488: 
  489: static inline int spin_trylock(spinlock_t *lock)
  490: {
  491:     return !testandset(lock);
  492: }
  493: #else
  494: static inline void spin_lock(spinlock_t *lock)
  495: {
  496: }
  497: 
  498: static inline void spin_unlock(spinlock_t *lock)
  499: {
  500: }
  501: 
  502: static inline int spin_trylock(spinlock_t *lock)
  503: {
  504:     return 1;
  505: }
  506: #endif
  507: 
  508: extern spinlock_t tb_lock;
  509: 
  510: extern int tb_invalidated_flag;
  511: 
  512: #if !defined(CONFIG_USER_ONLY)
  513: 
  514: void tlb_fill(target_ulong addr, int is_write, int mmu_idx,
  515:               void *retaddr);
  516: 
  517: #define ACCESS_TYPE (NB_MMU_MODES + 1)
  518: #define MEMSUFFIX _code
  519: #define env cpu_single_env
  520: 
  521: #define DATA_SIZE 1
  522: #include "softmmu_header.h"
  523: 
  524: #define DATA_SIZE 2
  525: #include "softmmu_header.h"
  526: 
  527: #define DATA_SIZE 4
  528: #include "softmmu_header.h"
  529: 
  530: #define DATA_SIZE 8
  531: #include "softmmu_header.h"
  532: 
  533: #undef ACCESS_TYPE
  534: #undef MEMSUFFIX
  535: #undef env
  536: 
  537: #endif
  538: 
  539: #if defined(CONFIG_USER_ONLY)
  540: static inline target_ulong get_phys_addr_code(CPUState *env, target_ulong addr)
  541: {
  542:     return addr;
  543: }
  544: #else
  545: /* NOTE: this function can trigger an exception */
  546: /* NOTE2: the returned address is not exactly the physical address: it
  547:    is the offset relative to phys_ram_base */
  548: static inline target_ulong get_phys_addr_code(CPUState *env, target_ulong addr)
  549: {
  550:     int mmu_idx, index, pd;
  551: 
  552:     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
  553:     mmu_idx = cpu_mmu_index(env);
  554:     if (__builtin_expect(env->tlb_table[mmu_idx][index].addr_code !=
  55