/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ #include #include #include #include #include #include #include .macro tlb_do_page_fault, write SYM_FUNC_START(tlb_do_page_fault_\write) SAVE_ALL csrrd a2, LOONGARCH_CSR_BADV move a0, sp REG_S a2, sp, PT_BVADDR li.w a1, \write la.abs t0, do_page_fault jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(tlb_do_page_fault_\write) .endm tlb_do_page_fault 0 tlb_do_page_fault 1 SYM_FUNC_START(handle_tlb_protect) BACKUP_T0T1 SAVE_ALL move a0, sp move a1, zero csrrd a2, LOONGARCH_CSR_BADV REG_S a2, sp, PT_BVADDR la.abs t0, do_page_fault jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_tlb_protect) SYM_FUNC_START(handle_tlb_load) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV blt t0, $r0, vmalloc_load csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_load: /* Get PGD offset in bytes */ srli.d t0, t0, PGDIR_SHIFT andi t0, t0, (PTRS_PER_PGD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #if CONFIG_PGTABLE_LEVELS > 3 csrrd t0, LOONGARCH_CSR_BADV ld.d t1, t1, 0 srli.d t0, t0, PUD_SHIFT andi t0, t0, (PTRS_PER_PUD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #endif #if CONFIG_PGTABLE_LEVELS > 2 csrrd t0, LOONGARCH_CSR_BADV ld.d t1, t1, 0 srli.d t0, t0, PMD_SHIFT andi t0, t0, (PTRS_PER_PMD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #endif ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ andi t0, ra, _PAGE_HUGE bne t0, $r0, tlb_huge_update_load csrrd t0, LOONGARCH_CSR_BADV srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) andi t0, t0, (PTRS_PER_PTE - 1) slli.d t0, t0, _PTE_T_LOG2 add.d t1, ra, t0 #ifdef CONFIG_SMP smp_pgtable_change_load: #endif #ifdef CONFIG_SMP ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif tlbsrch srli.d ra, t0, _PAGE_PRESENT_SHIFT andi ra, ra, 1 beq ra, $r0, nopage_tlb_load ori t0, t0, _PAGE_VALID #ifdef CONFIG_SMP sc.d t0, t1, 0 beq t0, $r0, smp_pgtable_change_load #else st.d t0, t1, 0 #endif ori t1, t1, 8 xori t1, t1, 8 ld.d t0, t1, 0 ld.d t1, t1, 8 csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr leave_load: csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn #ifdef CONFIG_64BIT vmalloc_load: la.abs t1, swapper_pg_dir b vmalloc_done_load #endif /* * This is the entry point when build_tlbchange_handler_head * spots a huge page. */ tlb_huge_update_load: #ifdef CONFIG_SMP ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif srli.d ra, t0, _PAGE_PRESENT_SHIFT andi ra, ra, 1 beq ra, $r0, nopage_tlb_load tlbsrch ori t0, t0, _PAGE_VALID #ifdef CONFIG_SMP sc.d t0, t1, 0 beq t0, $r0, tlb_huge_update_load ld.d t0, t1, 0 #else st.d t0, t1, 0 #endif addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) addi.d ra, t1, 0 csrxchg ra, t1, LOONGARCH_CSR_TLBIDX tlbwr csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 addi.d ra, t0, 0 csrwr t0, LOONGARCH_CSR_TLBELO0 addi.d t0, ra, 0 /* Convert to entrylo1 */ addi.d t1, $r0, 1 slli.d t1, t1, (HPAGE_SHIFT - 1) add.d t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX nopage_tlb_load: dbar 0 csrrd ra, EXCEPTION_KS2 la.abs t0, tlb_do_page_fault_0 jirl $r0, t0, 0 SYM_FUNC_END(handle_tlb_load) SYM_FUNC_START(handle_tlb_store) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV blt t0, $r0, vmalloc_store csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_store: /* Get PGD offset in bytes */ srli.d t0, t0, PGDIR_SHIFT andi t0, t0, (PTRS_PER_PGD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #if CONFIG_PGTABLE_LEVELS > 3 csrrd t0, LOONGARCH_CSR_BADV ld.d t1, t1, 0 srli.d t0, t0, PUD_SHIFT andi t0, t0, (PTRS_PER_PUD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #endif #if CONFIG_PGTABLE_LEVELS > 2 csrrd t0, LOONGARCH_CSR_BADV ld.d t1, t1, 0 srli.d t0, t0, PMD_SHIFT andi t0, t0, (PTRS_PER_PMD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #endif ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ andi t0, ra, _PAGE_HUGE bne t0, $r0, tlb_huge_update_store csrrd t0, LOONGARCH_CSR_BADV srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) andi t0, t0, (PTRS_PER_PTE - 1) slli.d t0, t0, _PTE_T_LOG2 add.d t1, ra, t0 #ifdef CONFIG_SMP smp_pgtable_change_store: #endif #ifdef CONFIG_SMP ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif tlbsrch srli.d ra, t0, _PAGE_PRESENT_SHIFT andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) bne ra, $r0, nopage_tlb_store ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP sc.d t0, t1, 0 beq t0, $r0, smp_pgtable_change_store #else st.d t0, t1, 0 #endif ori t1, t1, 8 xori t1, t1, 8 ld.d t0, t1, 0 ld.d t1, t1, 8 csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr leave_store: csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn #ifdef CONFIG_64BIT vmalloc_store: la.abs t1, swapper_pg_dir b vmalloc_done_store #endif /* * This is the entry point when build_tlbchange_handler_head * spots a huge page. */ tlb_huge_update_store: #ifdef CONFIG_SMP ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif srli.d ra, t0, _PAGE_PRESENT_SHIFT andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) bne ra, $r0, nopage_tlb_store tlbsrch ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP sc.d t0, t1, 0 beq t0, $r0, tlb_huge_update_store ld.d t0, t1, 0 #else st.d t0, t1, 0 #endif addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) addi.d ra, t1, 0 csrxchg ra, t1, LOONGARCH_CSR_TLBIDX tlbwr csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 addi.d ra, t0, 0 csrwr t0, LOONGARCH_CSR_TLBELO0 addi.d t0, ra, 0 /* Convert to entrylo1 */ addi.d t1, $r0, 1 slli.d t1, t1, (HPAGE_SHIFT - 1) add.d t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill /* Reset default page size */ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX nopage_tlb_store: dbar 0 csrrd ra, EXCEPTION_KS2 la.abs t0, tlb_do_page_fault_1 jirl $r0, t0, 0 SYM_FUNC_END(handle_tlb_store) SYM_FUNC_START(handle_tlb_modify) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV blt t0, $r0, vmalloc_modify csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_modify: /* Get PGD offset in bytes */ srli.d t0, t0, PGDIR_SHIFT andi t0, t0, (PTRS_PER_PGD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #if CONFIG_PGTABLE_LEVELS > 3 csrrd t0, LOONGARCH_CSR_BADV ld.d t1, t1, 0 srli.d t0, t0, PUD_SHIFT andi t0, t0, (PTRS_PER_PUD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #endif #if CONFIG_PGTABLE_LEVELS > 2 csrrd t0, LOONGARCH_CSR_BADV ld.d t1, t1, 0 srli.d t0, t0, PMD_SHIFT andi t0, t0, (PTRS_PER_PMD - 1) slli.d t0, t0, 3 add.d t1, t1, t0 #endif ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ andi t0, ra, _PAGE_HUGE bne t0, $r0, tlb_huge_update_modify csrrd t0, LOONGARCH_CSR_BADV srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) andi t0, t0, (PTRS_PER_PTE - 1) slli.d t0, t0, _PTE_T_LOG2 add.d t1, ra, t0 #ifdef CONFIG_SMP smp_pgtable_change_modify: #endif #ifdef CONFIG_SMP ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif tlbsrch srli.d ra, t0, _PAGE_WRITE_SHIFT andi ra, ra, 1 beq ra, $r0, nopage_tlb_modify ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP sc.d t0, t1, 0 beq t0, $r0, smp_pgtable_change_modify #else st.d t0, t1, 0 #endif ori t1, t1, 8 xori t1, t1, 8 ld.d t0, t1, 0 ld.d t1, t1, 8 csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr leave_modify: csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn #ifdef CONFIG_64BIT vmalloc_modify: la.abs t1, swapper_pg_dir b vmalloc_done_modify #endif /* * This is the entry point when * build_tlbchange_handler_head spots a huge page. */ tlb_huge_update_modify: #ifdef CONFIG_SMP ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif srli.d ra, t0, _PAGE_WRITE_SHIFT andi ra, ra, 1 beq ra, $r0, nopage_tlb_modify tlbsrch ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP sc.d t0, t1, 0 beq t0, $r0, tlb_huge_update_modify ld.d t0, t1, 0 #else st.d t0, t1, 0 #endif /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 addi.d ra, t0, 0 csrwr t0, LOONGARCH_CSR_TLBELO0 addi.d t0, ra, 0 /* Convert to entrylo1 */ addi.d t1, $r0, 1 slli.d t1, t1, (HPAGE_SHIFT - 1) add.d t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbwr /* Reset default page size */ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX nopage_tlb_modify: dbar 0 csrrd ra, EXCEPTION_KS2 la.abs t0, tlb_do_page_fault_1 jirl $r0, t0, 0 SYM_FUNC_END(handle_tlb_modify) SYM_FUNC_START(handle_tlb_refill) csrwr t0, LOONGARCH_CSR_TLBRSAVE csrrd t0, LOONGARCH_CSR_PGD lddir t0, t0, 3 #if CONFIG_PGTABLE_LEVELS > 3 lddir t0, t0, 2 #endif #if CONFIG_PGTABLE_LEVELS > 2 lddir t0, t0, 1 #endif ldpte t0, 0 ldpte t0, 1 tlbfill csrrd t0, LOONGARCH_CSR_TLBRSAVE ertn SYM_FUNC_END(handle_tlb_refill)