diff --git a/src/codegen.h b/src/codegen.h index e76391b..39c101f 100644 --- a/src/codegen.h +++ b/src/codegen.h @@ -29,12 +29,6 @@ same page). */ -/*Hack until better memory management written*/ -/*This is deliberately _not_ a power of two, to avoid cache aliasing problems. - Try changing this to a power of two, and watch the performance plummet :) - It's probably best for this to be a multiple of the cache line size though*/ -#define BLOCK_DATA_SIZE 0xff80 - typedef struct codeblock_t { uint32_t pc; @@ -59,7 +53,7 @@ typedef struct codeblock_t present in two pages.*/ uint16_t prev, next; uint16_t prev_2, next_2; - + /*First mem_block_t used by this block. Any subsequent mem_block_ts will be in the list starting at head_mem_block->next.*/ struct mem_block_t *head_mem_block; @@ -77,6 +71,8 @@ extern uint8_t *block_write_data; #define CODEBLOCK_STATIC_TOP 2 /*Code block has been compiled*/ #define CODEBLOCK_WAS_RECOMPILED 4 +/*Code block is in free list and is not valid*/ +#define CODEBLOCK_IN_FREE_LIST 8 #define BLOCK_PC_INVALID 0xffffffff @@ -304,9 +300,10 @@ x86seg *codegen_generate_ea(struct ir_data_t *ir, x86seg *op_ea_seg, uint32_t fe void codegen_check_seg_read(codeblock_t *block, struct ir_data_t *ir, x86seg *seg); void codegen_check_seg_write(codeblock_t *block, struct ir_data_t *ir, x86seg *seg); +int codegen_purge_purgable_list(); /*Delete a random code block to free memory. This is obviously quite expensive, and will only be called when the allocator is out of memory*/ -void codegen_delete_random_block(); +void codegen_delete_random_block(int required_mem_block); extern int cpu_block_end; extern uint32_t codegen_endpc; diff --git a/src/codegen_allocator.c b/src/codegen_allocator.c index 3dba230..b6b8273 100644 --- a/src/codegen_allocator.c +++ b/src/codegen_allocator.c @@ -50,7 +50,8 @@ mem_block_t *codegen_allocator_allocate(mem_block_t *parent) while (!mem_block_free_list) { - codegen_delete_random_block(); + if (!codegen_purge_purgable_list()) + codegen_delete_random_block(1); } // fatal("codegen_allocator_allocate: free list empty!\n"); diff --git a/src/codegen_allocator.h b/src/codegen_allocator.h index 22504bf..ffe633c 100644 --- a/src/codegen_allocator.h +++ b/src/codegen_allocator.h @@ -16,7 +16,7 @@ #ifdef __ARM_EABI__ #define MEM_BLOCK_NR 32768 #else -#define MEM_BLOCK_NR 65536 +#define MEM_BLOCK_NR 131072 #endif #define MEM_BLOCK_SIZE 0x3c0 diff --git a/src/codegen_backend.c b/src/codegen_backend.c index 8dca3b2..034ad61 100644 --- a/src/codegen_backend.c +++ b/src/codegen_backend.c @@ -53,10 +53,71 @@ static int last_ssegs; uint32_t instr_counts[256*256]; #endif +static uint16_t block_free_list; +static void delete_block(codeblock_t *block); + +static void block_free_list_add(codeblock_t *block) +{ + if (block_free_list) + block->next = block_free_list; + else + block->next = 0; + block_free_list = get_block_nr(block); + block->flags = CODEBLOCK_IN_FREE_LIST; +// pclog("block_free_list_add: %p %p\n", block, block->next); +} + +int codegen_purge_purgable_list() +{ + if (purgable_page_list_head) + { + page_t *page = &pages[purgable_page_list_head]; + int c; +// pclog("Purge page %08x\n", purgable_page_list_head); + for (c = 0; c < 4; c++) + { +// pclog(" Check %016llx %016llx\n", page->code_present_mask[c], page->dirty_mask[c]); + if (page->code_present_mask[c] & page->dirty_mask[c]) + { + codegen_check_flush(page, page->dirty_mask[c], (purgable_page_list_head << 12) + (c << 10)); +// pclog(" Check %08x %i\n", (purgable_page_list_head << 12) + (c << 10), block_free_list); + if (block_free_list) + return 1; + } + } + } + return 0; +} + +static codeblock_t *block_free_list_get() +{ + codeblock_t *block = NULL; + + while (!block_free_list) + { + /*Free list is empty - free up a block*/ + if (!codegen_purge_purgable_list()) + codegen_delete_random_block(0); + } + + block = &codeblock[block_free_list]; + block_free_list = block->next; + block->flags &= ~CODEBLOCK_IN_FREE_LIST; + block->next = 0; +// pclog("block_free_list_get: %p %p\n", block, block_free_list); + return block; +} + void codegen_init() { + int c; + codegen_allocator_init(); + codegen_backend_init(); + block_free_list = 0; + for (c = 0; c < BLOCK_SIZE; c++) + block_free_list_add(&codeblock[c]); #ifdef DEBUG_EXTRA memset(instr_counts, 0, sizeof(instr_counts)); #endif @@ -106,9 +167,13 @@ void codegen_reset() memset(codeblock, 0, BLOCK_SIZE * sizeof(codeblock_t)); memset(codeblock_hash, 0, HASH_SIZE * sizeof(uint16_t)); mem_reset_page_blocks(); - + + block_free_list = 0; for (c = 0; c < BLOCK_SIZE; c++) + { codeblock[c].pc = BLOCK_PC_INVALID; + block_free_list_add(&codeblock[c]); + } } void dump_block() @@ -232,24 +297,26 @@ static void delete_block(codeblock_t *block) if (block->head_mem_block) codegen_allocator_free(block->head_mem_block); block->head_mem_block = NULL; + block_free_list_add(block); } -void codegen_delete_random_block() +void codegen_delete_random_block(int required_mem_block) { + int block_nr = rand() & BLOCK_MASK; + while (1) { - int block_nr = rand() & BLOCK_MASK; - if (block_nr && block_nr != block_current) { codeblock_t *block = &codeblock[block_nr]; - - if (block->pc != BLOCK_PC_INVALID && block->head_mem_block) + + if (block->pc != BLOCK_PC_INVALID && (!required_mem_block || block->head_mem_block)) { delete_block(block); return; } } + block_nr = (block_nr + 1) & BLOCK_MASK; } } @@ -260,15 +327,16 @@ void codegen_check_flush(page_t *page, uint64_t mask, uint32_t phys_addr) while (block_nr) { codeblock_t *block = &codeblock[block_nr]; + uint16_t next_block = block->next; if (mask & block->page_mask) { delete_block(block); cpu_recomp_evicted++; } - if (block == &codeblock[block->next]) + if (block_nr == next_block) fatal("Broken 1\n"); - block_nr = block->next; + block_nr = next_block; } block_nr = page->block_2[(phys_addr >> 10) & 3]; @@ -276,16 +344,26 @@ void codegen_check_flush(page_t *page, uint64_t mask, uint32_t phys_addr) while (block_nr) { codeblock_t *block = &codeblock[block_nr]; + uint16_t next_block = block->next_2; if (mask & block->page_mask2) { delete_block(block); cpu_recomp_evicted++; } - if (block == &codeblock[block->next_2]) + if (block_nr == next_block) fatal("Broken 2\n"); - block_nr = block->next_2; + block_nr = next_block; } + + page->code_present_mask[(phys_addr >> 10) & 3] &= ~mask; + page->dirty_mask[(phys_addr >> 10) & 3] &= ~mask; + + if (!(page->code_present_mask[0] & page->dirty_mask[0]) && + !(page->code_present_mask[1] & page->dirty_mask[1]) && + !(page->code_present_mask[2] & page->dirty_mask[2]) && + !(page->code_present_mask[3] & page->dirty_mask[3])) + page_remove_from_evict_list(page); } void codegen_block_init(uint32_t phys_addr) @@ -296,19 +374,11 @@ void codegen_block_init(uint32_t phys_addr) if (!page->block[(phys_addr >> 10) & 3]) mem_flush_write_page(phys_addr, cs+cpu_state.pc); - block_current = (block_current + 1) & BLOCK_MASK; - if (!block_current) - block_current++; - block = &codeblock[block_current]; + block = block_free_list_get(); + if (!block) + fatal("codegen_block_init: block_free_list_get() returned NULL\n"); + block_current = get_block_nr(block); -// if (block->pc == 0xb00b4ff5) -// pclog("Init target block\n"); - if (block->pc != BLOCK_PC_INVALID) - { -// pclog("Reuse block : was %08x now %08x\n", block->pc, cs+pc); - delete_block(block); - cpu_recomp_reuse++; - } block_num = HASH(phys_addr); codeblock_hash[block_num] = block_current; @@ -407,6 +477,7 @@ void codegen_block_generate_end_mask() codeblock_t *block = &codeblock[block_current]; uint32_t start_pc; uint32_t end_pc; + page_t *p; block->page_mask = 0; start_pc = (block->pc & 0x3ff) & ~15; @@ -426,7 +497,10 @@ void codegen_block_generate_end_mask() // pclog(" %08x %llx\n", start_pc, block->page_mask); } - pages[block->phys >> 12].code_present_mask[(block->phys >> 10) & 3] |= block->page_mask; + p = &pages[block->phys >> 12]; + p->code_present_mask[(block->phys >> 10) & 3] |= block->page_mask; + if ((p->dirty_mask[(block->phys >> 10) & 3] & block->page_mask) && !page_in_evict_list(p)) + page_add_to_evict_list(p); block->phys_2 = -1; block->page_mask2 = 0; @@ -443,7 +517,9 @@ void codegen_block_generate_end_mask() for (; start_pc <= end_pc; start_pc++) block->page_mask2 |= ((uint64_t)1 << start_pc); page_2->code_present_mask[(block->phys_2 >> 10) & 3] |= block->page_mask2; - + if ((page_2->dirty_mask[(block->phys_2 >> 10) & 3] & block->page_mask2) && !page_in_evict_list(page_2)) + page_add_to_evict_list(page_2); + if (!pages[block->phys_2 >> 12].block_2[(block->phys_2 >> 10) & 3]) mem_flush_write_page(block->phys_2, codegen_endpc); diff --git a/src/codegen_backend_x86.h b/src/codegen_backend_x86.h index 7e32927..db521a7 100644 --- a/src/codegen_backend_x86.h +++ b/src/codegen_backend_x86.h @@ -1,7 +1,7 @@ #include "codegen_backend_x86_defs.h" -#define BLOCK_SIZE 0x4000 -#define BLOCK_MASK 0x3fff +#define BLOCK_SIZE 0x10000 +#define BLOCK_MASK 0xffff #define BLOCK_START 0 #define HASH_SIZE 0x20000 diff --git a/src/mem.c b/src/mem.c index ad095a3..cc253d8 100644 --- a/src/mem.c +++ b/src/mem.c @@ -924,14 +924,51 @@ uint32_t mem_read_raml(uint32_t addr, void *priv) return *(uint32_t *)&ram[addr]; } +uint32_t purgable_page_list_head = 0; +int purgeable_page_count = 0; + +static inline int page_index(page_t *p) +{ + return ((uintptr_t)p - (uintptr_t)pages) / sizeof(page_t); +} +void page_add_to_evict_list(page_t *p) +{ +// pclog("page_add_to_evict_list: %08x %i\n", page_index(p), purgeable_page_count); + pages[purgable_page_list_head].evict_prev = page_index(p); + p->evict_next = purgable_page_list_head; + p->evict_prev = 0; + purgable_page_list_head = pages[purgable_page_list_head].evict_prev; + purgeable_page_count++; +} +void page_remove_from_evict_list(page_t *p) +{ +// pclog("page_remove_from_evict_list: %08x %i\n", page_index(p), purgeable_page_count); + if (!page_in_evict_list(p)) + fatal("page_remove_from_evict_list: not in evict list!\n"); + if (p->evict_prev) + pages[p->evict_prev].evict_next = p->evict_next; + else + purgable_page_list_head = p->evict_next; + if (p->evict_next) + pages[p->evict_next].evict_prev = p->evict_prev; + p->evict_prev = EVICT_NOT_IN_LIST; + purgeable_page_count--; +} + void mem_write_ramb_page(uint32_t addr, uint8_t val, page_t *p) { if (val != p->mem[addr & 0xfff] || codegen_in_recompile) { uint64_t mask = (uint64_t)1 << ((addr >> PAGE_MASK_SHIFT) & PAGE_MASK_MASK); -// pclog("mem_write_ramb_page: %08x %02x %08x %llx %llx\n", addr, val, cs+pc, p->dirty_mask, mask); - p->dirty_mask[(addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK] |= mask; + int index = (addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK; +// pclog("mem_write_ramb_page: %08x %02x %08x %llx %llx\n", addr, val, cs+cpu_state.pc, p->dirty_mask, mask); p->mem[addr & 0xfff] = val; + p->dirty_mask[index] |= mask; + if ((p->code_present_mask[index] & mask) && !page_in_evict_list(p)) + { +// pclog("ramb add %08x %016llx %016llx\n", addr, p->code_present_mask[index], mask); + page_add_to_evict_list(p); + } } } void mem_write_ramw_page(uint32_t addr, uint16_t val, page_t *p) @@ -939,11 +976,17 @@ void mem_write_ramw_page(uint32_t addr, uint16_t val, page_t *p) if (val != *(uint16_t *)&p->mem[addr & 0xfff] || codegen_in_recompile) { uint64_t mask = (uint64_t)1 << ((addr >> PAGE_MASK_SHIFT) & PAGE_MASK_MASK); + int index = (addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK; if ((addr & 0xf) == 0xf) mask |= (mask << 1); -// pclog("mem_write_ramw_page: %08x %04x %08x\n", addr, val, cs+pc); - p->dirty_mask[(addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK] |= mask; +// pclog("mem_write_ramw_page: %08x %04x %08x %016llx %016llx %016llx %08x %08x %p\n", addr, val, cs+cpu_state.pc, p->dirty_mask[index], p->code_present_mask[index], mask, p->evict_prev, p->evict_next, p); *(uint16_t *)&p->mem[addr & 0xfff] = val; + p->dirty_mask[index] |= mask; + if ((p->code_present_mask[index] & mask) && !page_in_evict_list(p)) + { +// pclog("ramw add %08x %016llx %016llx\n", addr, p->code_present_mask[index], mask); + page_add_to_evict_list(p); + } } } void mem_write_raml_page(uint32_t addr, uint32_t val, page_t *p) @@ -951,11 +994,17 @@ void mem_write_raml_page(uint32_t addr, uint32_t val, page_t *p) if (val != *(uint32_t *)&p->mem[addr & 0xfff] || codegen_in_recompile) { uint64_t mask = (uint64_t)1 << ((addr >> PAGE_MASK_SHIFT) & PAGE_MASK_MASK); + int index = (addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK; if ((addr & 0xf) >= 0xd) mask |= (mask << 1); -// pclog("mem_write_raml_page: %08x %08x %08x\n", addr, val, cs+pc); - p->dirty_mask[(addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK] |= mask; +// pclog("mem_write_raml_page: %08x %08x %08x %016llx %016llx %016llx\n", addr, val, cs+cpu_state.pc, p->dirty_mask[index], p->code_present_mask[index], mask); *(uint32_t *)&p->mem[addr & 0xfff] = val; + p->dirty_mask[index] |= mask; + if ((p->code_present_mask[index] & mask) && !page_in_evict_list(p)) + { +// pclog("raml add %08x %016llx %016llx\n", addr, p->code_present_mask[index], mask); + page_add_to_evict_list(p); + } } } @@ -1066,8 +1115,15 @@ void mem_invalidate_range(uint32_t start_addr, uint32_t end_addr) for (; start_addr <= end_addr; start_addr += (1 << PAGE_MASK_SHIFT)) { uint64_t mask = (uint64_t)1 << ((start_addr >> PAGE_MASK_SHIFT) & PAGE_MASK_MASK); - - pages[start_addr >> 12].dirty_mask[(start_addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK] |= mask; + page_t *p = &pages[start_addr >> 12]; + int index = (start_addr >> PAGE_MASK_INDEX_SHIFT) & PAGE_MASK_INDEX_MASK; +//pclog("mem_invalidate: %08x\n", start_addr); + p->dirty_mask[index] |= mask; + if ((p->code_present_mask[index] & mask) && !page_in_evict_list(p)) + { +// pclog("invalidate add %08x %016llx %016llx\n", start_addr, p->code_present_mask[index], mask); + page_add_to_evict_list(p); + } } } @@ -1330,6 +1386,7 @@ void mem_init() pages[c].write_b = mem_write_ramb_page; pages[c].write_w = mem_write_ramw_page; pages[c].write_l = mem_write_raml_page; + pages[c].evict_prev = EVICT_NOT_IN_LIST; } memset(isram, 0, sizeof(isram)); @@ -1439,6 +1496,7 @@ void mem_resize() pages[c].write_b = mem_write_ramb_page; pages[c].write_w = mem_write_ramw_page; pages[c].write_l = mem_write_raml_page; + pages[c].evict_prev = EVICT_NOT_IN_LIST; } memset(isram, 0, sizeof(isram)); diff --git a/src/mem.h b/src/mem.h index a015acc..3a27a92 100644 --- a/src/mem.h +++ b/src/mem.h @@ -111,6 +111,7 @@ mem_mapping_t bios_high_mapping[8]; extern mem_mapping_t ram_high_mapping; extern mem_mapping_t ram_remapped_mapping; +#define EVICT_NOT_IN_LIST ((uint32_t)-1) typedef struct page_t { void (*write_b)(uint32_t addr, uint8_t val, struct page_t *p); @@ -125,12 +126,22 @@ typedef struct page_t uint16_t head; uint64_t code_present_mask[4], dirty_mask[4]; + + uint32_t evict_prev, evict_next; } page_t; extern page_t *pages; extern page_t **page_lookup; +extern uint32_t purgable_page_list_head; +static inline int page_in_evict_list(page_t *p) +{ + return (p->evict_prev != EVICT_NOT_IN_LIST); +} +void page_remove_from_evict_list(page_t *p); +void page_add_to_evict_list(page_t *p); + uint32_t mmutranslate_noabrt(uint32_t addr, int rw); extern uint32_t get_phys_virt,get_phys_phys; static inline uint32_t get_phys(uint32_t addr) @@ -195,4 +206,6 @@ void mmu_invalidate(uint32_t addr); int loadbios(); extern unsigned char isram[0x10000]; + +extern int purgeable_page_count; #endif