1: /*
    2:  *  virtual page mapping and translated block handling
    3:  *
    4:  *  Copyright (c) 2003 Fabrice Bellard
    5:  *
    6:  * This library is free software; you can redistribute it and/or
    7:  * modify it under the terms of the GNU Lesser General Public
    8:  * License as published by the Free Software Foundation; either
    9:  * version 2 of the License, or (at your option) any later version.
   10:  *
   11:  * This library is distributed in the hope that it will be useful,
   12:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   13:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   14:  * Lesser General Public License for more details.
   15:  *
   16:  * You should have received a copy of the GNU Lesser General Public
   17:  * License along with this library; if not, write to the Free Software
   18:  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   19:  */
   20: #include "config.h"
   21: #ifdef _WIN32
   22: #define WIN32_LEAN_AND_MEAN
   23: #include <windows.h>
   24: #else
   25: #include <sys/types.h>
   26: #include <sys/mman.h>
   27: #endif
   28: #include <stdlib.h>
   29: #include <stdio.h>
   30: #include <stdarg.h>
   31: #include <string.h>
   32: #include <errno.h>
   33: #include <unistd.h>
   34: #include <inttypes.h>
   35: 
   36: #include "cpu.h"
   37: #include "exec-all.h"
   38: #if defined(CONFIG_USER_ONLY)
   39: #include <qemu.h>
   40: #endif
   41: 
   42: //#define DEBUG_TB_INVALIDATE
   43: //#define DEBUG_FLUSH
   44: //#define DEBUG_TLB
   45: //#define DEBUG_UNASSIGNED
   46: 
   47: /* make various TB consistency checks */
   48: //#define DEBUG_TB_CHECK
   49: //#define DEBUG_TLB_CHECK
   50: 
   51: //#define DEBUG_IOPORT
   52: //#define DEBUG_SUBPAGE
   53: 
   54: #if !defined(CONFIG_USER_ONLY)
   55: /* TB consistency checks only implemented for usermode emulation.  */
   56: #undef DEBUG_TB_CHECK
   57: #endif
   58: 
   59: /* threshold to flush the translated code buffer */
   60: #define CODE_GEN_BUFFER_MAX_SIZE (CODE_GEN_BUFFER_SIZE - code_gen_max_block_size())
   61: 
   62: #define SMC_BITMAP_USE_THRESHOLD 10
   63: 
   64: #define MMAP_AREA_START        0x00000000
   65: #define MMAP_AREA_END          0xa8000000
   66: 
   67: #if defined(TARGET_SPARC64)
   68: #define TARGET_PHYS_ADDR_SPACE_BITS 41
   69: #elif defined(TARGET_SPARC)
   70: #define TARGET_PHYS_ADDR_SPACE_BITS 36
   71: #elif defined(TARGET_ALPHA)
   72: #define TARGET_PHYS_ADDR_SPACE_BITS 42
   73: #define TARGET_VIRT_ADDR_SPACE_BITS 42
   74: #elif defined(TARGET_PPC64)
   75: #define TARGET_PHYS_ADDR_SPACE_BITS 42
   76: #else
   77: /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
   78: #define TARGET_PHYS_ADDR_SPACE_BITS 32
   79: #endif
   80: 
   81: TranslationBlock tbs[CODE_GEN_MAX_BLOCKS];
   82: TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
   83: int nb_tbs;
   84: /* any access to the tbs or the page table must use this lock */
   85: spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
   86: 
   87: uint8_t code_gen_buffer[CODE_GEN_BUFFER_SIZE] __attribute__((aligned (32)));
   88: uint8_t *code_gen_ptr;
   89: 
   90: int phys_ram_size;
   91: int phys_ram_fd;
   92: uint8_t *phys_ram_base;
   93: uint8_t *phys_ram_dirty;
   94: static ram_addr_t phys_ram_alloc_offset = 0;
   95: 
   96: CPUState *first_cpu;
   97: /* current CPU in the current thread. It is only valid inside
   98:    cpu_exec() */
   99: CPUState *cpu_single_env;
  100: 
  101: typedef struct PageDesc {
  102:     /* list of TBs intersecting this ram page */
  103:     TranslationBlock *first_tb;
  104:     /* in order to optimize self modifying code, we count the number
  105:        of lookups we do to a given page to use a bitmap */
  106:     unsigned int code_write_count;
  107:     uint8_t *code_bitmap;
  108: #if defined(CONFIG_USER_ONLY)
  109:     unsigned long flags;
  110: #endif
  111: } PageDesc;
  112: 
  113: typedef struct PhysPageDesc {
  114:     /* offset in host memory of the page + io_index in the low 12 bits */
  115:     uint32_t phys_offset;
  116: } PhysPageDesc;
  117: 
  118: #define L2_BITS 10
  119: #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
  120: /* XXX: this is a temporary hack for alpha target.
  121:  *      In the future, this is to be replaced by a multi-level table
  122:  *      to actually be able to handle the complete 64 bits address space.
  123:  */
  124: #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
  125: #else
  126: #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
  127: #endif
  128: 
  129: #define L1_SIZE (1 << L1_BITS)
  130: #define L2_SIZE (1 << L2_BITS)
  131: 
  132: static void io_mem_init(void);
  133: 
  134: unsigned long qemu_real_host_page_size;
  135: unsigned long qemu_host_page_bits;
  136: unsigned long qemu_host_page_size;
  137: unsigned long qemu_host_page_mask;
  138: 
  139: /* XXX: for system emulation, it could just be an array */
  140: static PageDesc *l1_map[L1_SIZE];
  141: PhysPageDesc **l1_phys_map;
  142: 
  143: /* io memory support */
  144: CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
  145: CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
  146: void *io_mem_opaque[IO_MEM_NB_ENTRIES];
  147: static int io_mem_nb;
  148: #if defined(CONFIG_SOFTMMU)
  149: static int io_mem_watch;
  150: #endif
  151: 
  152: /* log support */
  153: char *logfilename = "/tmp/qemu.log";
  154: FILE *logfile;
  155: int loglevel;
  156: static int log_append = 0;
  157: 
  158: /* statistics */
  159: static int tlb_flush_count;
  160: static int tb_flush_count;
  161: static int tb_phys_invalidate_count;
  162: 
  163: #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
  164: typedef struct subpage_t {
  165:     target_phys_addr_t base;
  166:     CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
  167:     CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
  168:     void *opaque[TARGET_PAGE_SIZE][2][4];
  169: } subpage_t;
  170: 
  171: static void page_init(void)
  172: {
  173:     /* NOTE: we can always suppose that qemu_host_page_size >=
  174:        TARGET_PAGE_SIZE */
  175: #ifdef _WIN32
  176:     {
  177:         SYSTEM_INFO system_info;
  178:         DWORD old_protect;
  179: 
  180:         GetSystemInfo(&system_info);
  181:         qemu_real_host_page_size = system_info.dwPageSize;
  182: 
  183:         VirtualProtect(code_gen_buffer, sizeof(code_gen_buffer),
  184:                        PAGE_EXECUTE_READWRITE, &old_protect);
  185:     }
  186: #else
  187:     qemu_real_host_page_size = getpagesize();
  188:     {
  189:         unsigned long start, end;
  190: 
  191:         start = (unsigned long)code_gen_buffer;
  192:         start &= ~(qemu_real_host_page_size - 1);
  193: 
  194:         end = (unsigned long)code_gen_buffer + sizeof(code_gen_buffer);
  195:         end += qemu_real_host_page_size - 1;
  196:         end &= ~(qemu_real_host_page_size - 1);
  197: 
  198:         mprotect((void *)start, end - start,
  199:                  PROT_READ | PROT_WRITE | PROT_EXEC);
  200:     }
  201: #endif
  202: 
  203:     if (qemu_host_page_size == 0)
  204:         qemu_host_page_size = qemu_real_host_page_size;
  205:     if (qemu_host_page_size < TARGET_PAGE_SIZE)
  206:         qemu_host_page_size = TARGET_PAGE_SIZE;
  207:     qemu_host_page_bits = 0;
  208:     while ((1 << qemu_host_page_bits) < qemu_host_page_size)
  209:         qemu_host_page_bits++;
  210:     qemu_host_page_mask = ~(qemu_host_page_size - 1);
  211:     l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
  212:     memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
  213: 
  214: #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
  215:     {
  216:         long long startaddr, endaddr;
  217:         FILE *f;
  218:         int n;
  219: 
  220:         f = fopen("/proc/self/maps", "r");
  221:         if (f) {
  222:             do {
  223:                 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
  224:                 if (n == 2) {
  225:                     page_set_flags(TARGET_PAGE_ALIGN(startaddr),
  226:                                    TARGET_PAGE_ALIGN(endaddr),
  227:                                    PAGE_RESERVED); 
  228:                 }
  229:             } while (!feof(f));
  230:             fclose(f);
  231:         }
  232:     }
  233: #endif
  234: }
  235: 
  236: static inline PageDesc *page_find_alloc(unsigned int index)
  237: {
  238:     PageDesc **lp, *p;
  239: 
  240:     lp = &l1_map[index >> L2_BITS];
  241:     p = *lp;
  242:     if (!p) {
  243:         /* allocate if not found */
  244:         p = qemu_malloc(sizeof(PageDesc) * L2_SIZE);
  245:         memset(p, 0, sizeof(PageDesc) * L2_SIZE);
  246:         *lp = p;
  247:     }
  248:     return p + (index & (L2_SIZE - 1));
  249: }
  250: 
  251: static inline PageDesc *page_find(unsigned int index)
  252: {
  253:     PageDesc *p;
  254: 
  255:     p = l1_map[index >> L2_BITS];
  256:     if (!p)
  257:         return 0;
  258:     return p + (index & (L2_SIZE - 1));
  259: }
  260: 
  261: static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
  262: {
  263:     void **lp, **p;
  264:     PhysPageDesc *pd;
  265: 
  266:     p = (void **)l1_phys_map;
  267: #if TARGET_PHYS_ADDR_SPACE_BITS > 32
  268: 
  269: #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
  270: #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
  271: #endif
  272:     lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
  273:     p = *lp;
  274:     if (!p) {
  275:         /* allocate if not found */
  276:         if (!alloc)
  277:             return NULL;
  278:         p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
  279:         memset(p, 0, sizeof(void *) * L1_SIZE);
  280:         *lp = p;
  281:     }
  282: #endif
  283:     lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
  284:     pd = *lp;
  285:     if (!pd) {
  286:         int i;
  287:         /* allocate if not found */
  288:         if (!alloc)
  289:             return NULL;
  290:         pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
  291:         *lp = pd;
  292:         for (i = 0; i < L2_SIZE; i++)
  293:           pd[i].phys_offset = IO_MEM_UNASSIGNED;
  294:     }
  295:     return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
  296: }
  297: 
  298: static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
  299: {
  300:     return phys_page_find_alloc(index, 0);
  301: }
  302: 
  303: #if !defined(CONFIG_USER_ONLY)
  304: static void tlb_protect_code(ram_addr_t ram_addr);
  305: static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
  306:                                     target_ulong vaddr);
  307: #endif
  308: 
  309: void cpu_exec_init(CPUState *env)
  310: {
  311:     CPUState **penv;
  312:     int cpu_index;
  313: 
  314:     if (!code_gen_ptr) {
  315:         code_gen_ptr = code_gen_buffer;
  316:         page_init();
  317:         io_mem_init();
  318:     }
  319:     env->next_cpu = NULL;
  320:     penv = &first_cpu;
  321:     cpu_index = 0;
  322:     while (*penv != NULL) {
  323:         penv = (CPUState **)&(*penv)->next_cpu;
  324:         cpu_index++;
  325:     }
  326:     env->cpu_index = cpu_index;
  327:     env->nb_watchpoints = 0;
  328:     *penv = env;
  329: }
  330: 
  331: static inline void invalidate_page_bitmap(PageDesc *p)
  332: {
  333:     if (p->code_bitmap) {
  334:         qemu_free(p->code_bitmap);
  335:         p->code_bitmap = NULL;
  336:     }
  337:     p->code_write_count = 0;
  338: }
  339: 
  340: /* set to NULL all the 'first_tb' fields in all PageDescs */
  341: static void page_flush_tb(void)
  342: {
  343:     int i, j;
  344:     PageDesc *p;
  345: 
  346:     for(i = 0; i < L1_SIZE; i++) {
  347:         p = l1_map[i];
  348:         if (p) {
  349:             for(j = 0; j < L2_SIZE; j++) {
  350:                 p->first_tb = NULL;
  351:                 invalidate_page_bitmap(p);
  352:                 p++;
  353:             }
  354:         }
  355:     }
  356: }
  357: 
  358: /* flush all the translation blocks */
  359: /* XXX: tb_flush is currently not thread safe */
  360: void tb_flush(CPUState *env1)
  361: {
  362:     CPUState *env;
  363: #if defined(DEBUG_FLUSH)
  364:     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
  365:            (unsigned long)(code_gen_ptr - code_gen_buffer),
  366:            nb_tbs, nb_tbs > 0 ?
  367:            ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
  368: #endif
  369:     nb_tbs = 0;
  370: 
  371:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  372:         memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
  373:     }
  374: 
  375:     memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
  376:     page_flush_tb();
  377: 
  378:     code_gen_ptr = code_gen_buffer;
  379:     /* XXX: flush processor icache at this point if cache flush is
  380:        expensive */
  381:     tb_flush_count++;
  382: }
  383: 
  384: #ifdef DEBUG_TB_CHECK
  385: 
  386: static void tb_invalidate_check(target_ulong address)
  387: {
  388:     TranslationBlock *tb;
  389:     int i;
  390:     address &= TARGET_PAGE_MASK;
  391:     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
  392:         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
  393:             if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
  394:                   address >= tb->pc + tb->size)) {
  395:                 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
  396:                        address, (long)tb->pc, tb->size);
  397:             }
  398:         }
  399:     }
  400: }
  401: 
  402: /* verify that all the pages have correct rights for code */
  403: static void tb_page_check(void)
  404: {
  405:     TranslationBlock *tb;
  406:     int i, flags1, flags2;
  407: 
  408:     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
  409:         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
  410:             flags1 = page_get_flags(tb->pc);
  411:             flags2 = page_get_flags(tb->pc + tb->size - 1);
  412:             if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
  413:                 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
  414:                        (long)tb->pc, tb->size, flags1, flags2);
  415:             }
  416:         }
  417:     }
  418: }
  419: 
  420: void tb_jmp_check(TranslationBlock *tb)
  421: {
  422:     TranslationBlock *tb1;
  423:     unsigned int n1;
  424: 
  425:     /* suppress any remaining jumps to this TB */
  426:     tb1 = tb->jmp_first;
  427:     for(;;) {
  428:         n1 = (long)tb1 & 3;
  429:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
  430:         if (n1 == 2)
  431:             break;
  432:         tb1 = tb1->jmp_next[n1];
  433:     }
  434:     /* check end of list */
  435:     if (tb1 != tb) {
  436:         printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
  437:     }
  438: }
  439: 
  440: #endif
  441: 
  442: /* invalidate one TB */
  443: static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
  444:                              int next_offset)
  445: {
  446:     TranslationBlock *tb1;
  447:     for(;;) {
  448:         tb1 = *ptb;
  449:         if (tb1 == tb) {
  450:             *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
  451:             break;
  452:         }
  453:         ptb = (TranslationBlock **)((char *)tb1 + next_offset);
  454:     }
  455: }
  456: 
  457: static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
  458: {
  459:     TranslationBlock *tb1;
  460:     unsigned int n1;
  461: 
  462:     for(;;) {
  463:         tb1 = *ptb;
  464:         n1 = (long)tb1 & 3;
  465:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
  466:         if (tb1 == tb) {
  467:             *ptb = tb1->page_next[n1];
  468:             break;
  469:         }
  470:         ptb = &tb1->page_next[n1];
  471:     }
  472: }
  473: 
  474: static inline void tb_jmp_remove(TranslationBlock *tb, int n)
  475: {
  476:     TranslationBlock *tb1, **ptb;
  477:     unsigned int n1;
  478: 
  479:     ptb = &tb->jmp_next[n];
  480:     tb1 = *ptb;
  481:     if (tb1) {
  482:         /* find tb(n) in circular list */
  483:         for(;;) {
  484:             tb1 = *ptb;
  485:             n1 = (long)tb1 & 3;
  486:             tb1 = (TranslationBlock *)((long)tb1 & ~3);
  487:             if (n1 == n && tb1 == tb)
  488:                 break;
  489:             if (n1 == 2) {
  490:                 ptb = &tb1->jmp_first;
  491:             } else {
  492:                 ptb = &tb1->jmp_next[n1];
  493:             }
  494:         }
  495:         /* now we can suppress tb(n) from the list */
  496:         *ptb = tb->jmp_next[n];
  497: 
  498:         tb->jmp_next[n] = NULL;
  499:     }
  500: }
  501: 
  502: /* reset the jump entry 'n' of a TB so that it is not chained to
  503:    another TB */
  504: static inline void tb_reset_jump(TranslationBlock *tb, int n)
  505: {
  506:     tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
  507: }
  508: 
  509: static inline void tb_phys_invalidate(TranslationBlock *tb, unsigned int page_addr)
  510: {
  511:     CPUState *env;
  512:     PageDesc *p;
  513:     unsigned int h, n1;
  514:     target_ulong phys_pc;
  515:     TranslationBlock *tb1, *tb2;
  516: 
  517:     /* remove the TB from the hash list */
  518:     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
  519:     h = tb_phys_hash_func(phys_pc);
  520:     tb_remove(&tb_phys_hash[h], tb,
  521:               offsetof(TranslationBlock, phys_hash_next));
  522: 
  523:     /* remove the TB from the page list */
  524:     if (tb->page_addr[0] != page_addr) {
  525:         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
  526:         tb_page_remove(&p->first_tb, tb);
  527:         invalidate_page_bitmap(p);
  528:     }
  529:     if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
  530:         p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
  531:         tb_page_remove(&p->first_tb, tb);
  532:         invalidate_page_bitmap(p);
  533:     }
  534: 
  535:     tb_invalidated_flag = 1;
  536: 
  537:     /* remove the TB from the hash list */
  538:     h = tb_jmp_cache_hash_func(tb->pc);
  539:     for(env = first_cpu; env != NULL; env = env->next_cpu) {
  540:         if (env->tb_jmp_cache[h] == tb)
  541:             env->tb_jmp_cache[h] = NULL;
  542:     }
  543: 
  544:     /* suppress this TB from the two jump lists */
  545:     tb_jmp_remove(tb, 0);
  546:     tb_jmp_remove(tb, 1);
  547: 
  548:     /* suppress any remaining jumps to this TB */
  549:     tb1 = tb->jmp_first;
  550:     for(;;) {
  551:         n1 = (long)tb1 & 3;
  552:         if (n1 == 2)
  553:             break;
  554:         tb1 = (TranslationBlock *)((long)tb1 & ~3);
  555:         tb2 = tb1->jmp_next[n1];
  556:         tb_reset_jump(tb1, n1);
  557:         tb1->jmp_next[n1] = NULL;
  558:         tb1 = tb2;
  559:     }
  560:     tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
  561: 
  562:     tb_phys_invalidate_count++;
  563: }
  564: 
  565: static inline void set_bits(uint8_t *tab, int start, int len)
  566: {
  567:     int end, mask, end1;
  568: 
  569:     end = start + len;
  570:     tab += start >> 3;
  571:     mask = 0xff << (start & 7);
  572:     if ((start & ~7) == (end & ~7)) {
  573:         if (start < end) {
  574:             mask &= ~(0xff << (end & 7));
  575:             *tab |= mask;
  576:         }
  577:     } else {
  578:         *tab++ |= mask;
  579:         start = (start + 8) & ~7;
  580:         end1 = end & ~7;
  581:         while (start < end1) {
  582:             *tab++ = 0xff;
  583:             start += 8;
  584:         }
  585:         if (start < end) {
  586:             mask = ~(0xff << (end & 7));
  587:             *tab |= mask;
  588:         }
  589:     }
  590: }
  591: 
  592: static void build_page_bitmap(PageDesc *p)
  593: {
  594:     int n, tb_start, tb_end;
  595:     TranslationBlock *tb;
  596: