diff --git a/include/lib/aarch32/arch.h b/include/lib/aarch32/arch.h index e571ddc..6653cd1 100644 --- a/include/lib/aarch32/arch.h +++ b/include/lib/aarch32/arch.h @@ -254,6 +254,8 @@ #define TTBCR_T1SZ_SHIFT 16 #define TTBCR_T1SZ_MASK (0x7) +#define TTBCR_TxSZ_MIN 0 +#define TTBCR_TxSZ_MAX 7 #define TTBCR_SH0_NON_SHAREABLE (0x0 << 12) #define TTBCR_SH0_OUTER_SHAREABLE (0x2 << 12) diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h index fa5cb12..fb8cbc0 100644 --- a/include/lib/aarch64/arch.h +++ b/include/lib/aarch64/arch.h @@ -243,6 +243,9 @@ #define TCR_EL1_IPS_SHIFT 32 #define TCR_EL3_PS_SHIFT 16 +#define TCR_TxSZ_MIN 16 +#define TCR_TxSZ_MAX 39 + /* (internal) physical address size bits in EL3/EL1 */ #define TCR_PS_BITS_4GB (0x0) #define TCR_PS_BITS_64GB (0x1) diff --git a/include/lib/xlat_tables.h b/include/lib/xlat_tables.h index 3f35e45..d2ac6db 100644 --- a/include/lib/xlat_tables.h +++ b/include/lib/xlat_tables.h @@ -32,29 +32,12 @@ #define __XLAT_TABLES_H__ /* Miscellaneous MMU related constants */ -#define NUM_2MB_IN_GB (1 << 9) -#define NUM_4K_IN_2MB (1 << 9) -#define NUM_GB_IN_4GB (1 << 2) - -#define TWO_MB_SHIFT 21 -#define ONE_GB_SHIFT 30 #define FOUR_KB_SHIFT 12 -#define ONE_GB_INDEX(x) ((x) >> ONE_GB_SHIFT) -#define TWO_MB_INDEX(x) ((x) >> TWO_MB_SHIFT) -#define FOUR_KB_INDEX(x) ((x) >> FOUR_KB_SHIFT) - #define INVALID_DESC 0x0 -#define BLOCK_DESC 0x1 -#define TABLE_DESC 0x3 - -#define FIRST_LEVEL_DESC_N ONE_GB_SHIFT -#define SECOND_LEVEL_DESC_N TWO_MB_SHIFT -#define THIRD_LEVEL_DESC_N FOUR_KB_SHIFT - -#define LEVEL1 1 -#define LEVEL2 2 -#define LEVEL3 3 +#define BLOCK_DESC 0x1 /* Table levels 0-2 */ +#define TABLE_DESC 0x3 /* Table levels 0-2 */ +#define PAGE_DESC 0x3 /* Table level 3 */ #define XN (1ull << 2) #define PXN (1ull << 1) @@ -78,6 +61,14 @@ #define XLAT_TABLE_SIZE_SHIFT PAGE_SIZE_SHIFT #define XLAT_TABLE_SIZE (1 << XLAT_TABLE_SIZE_SHIFT) +#ifdef AARCH32 +#define XLAT_TABLE_LEVEL_MIN 1 +#else +#define XLAT_TABLE_LEVEL_MIN 0 +#endif /* AARCH32 */ + +#define XLAT_TABLE_LEVEL_MAX 3 + /* Values for number of entries in each MMU translation table */ #define XLAT_TABLE_ENTRIES_SHIFT (XLAT_TABLE_SIZE_SHIFT - XLAT_ENTRY_SIZE_SHIFT) #define XLAT_TABLE_ENTRIES (1 << XLAT_TABLE_ENTRIES_SHIFT) @@ -87,6 +78,7 @@ #define L3_XLAT_ADDRESS_SHIFT PAGE_SIZE_SHIFT #define L2_XLAT_ADDRESS_SHIFT (L3_XLAT_ADDRESS_SHIFT + XLAT_TABLE_ENTRIES_SHIFT) #define L1_XLAT_ADDRESS_SHIFT (L2_XLAT_ADDRESS_SHIFT + XLAT_TABLE_ENTRIES_SHIFT) +#define L0_XLAT_ADDRESS_SHIFT (L1_XLAT_ADDRESS_SHIFT + XLAT_TABLE_ENTRIES_SHIFT) /* * AP[1] bit is ignored by hardware and is diff --git a/lib/xlat_tables/aarch32/xlat_tables.c b/lib/xlat_tables/aarch32/xlat_tables.c index a97cf31..d70a6ef 100644 --- a/lib/xlat_tables/aarch32/xlat_tables.c +++ b/lib/xlat_tables/aarch32/xlat_tables.c @@ -38,25 +38,57 @@ #include "../xlat_tables_private.h" /* - * The virtual address space size must be a power of two. As we start the initial - * lookup at level 1, it must also be between 2 GB and 4 GB. See section - * G4.6.5 in the ARMv8-A Architecture Reference Manual (DDI 0487A.j) for more - * information. + * Each platform can define the size of the virtual address space, which is + * defined in ADDR_SPACE_SIZE. TTBCR.TxSZ is calculated as 32 minus the width + * of said address space. The value of TTBCR.TxSZ must be in the range 0 to + * 7 [1], which means that the virtual address space width must be in the range + * 32 to 25 bits. + * + * Here we calculate the initial lookup level from the value of ADDR_SPACE_SIZE. + * For a 4 KB page size, level 1 supports virtual address spaces of widths 32 + * to 31 bits, and level 2 from 30 to 25. Wider or narrower address spaces are + * not supported. As a result, level 3 cannot be used as initial lookup level + * with 4 KB granularity [1]. + * + * For example, for a 31-bit address space (i.e. ADDR_SPACE_SIZE == 1 << 31), + * TTBCR.TxSZ will be programmed to (32 - 31) = 1. According to Table G4-5 in + * the ARM ARM, the initial lookup level for such an address space is 1. + * + * See the ARMv8-A Architecture Reference Manual (DDI 0487A.j) for more + * information: + * [1] Section G4.6.5 */ -CASSERT(ADDR_SPACE_SIZE >= (1ull << 31) && ADDR_SPACE_SIZE <= (1ull << 32) && - IS_POWER_OF_TWO(ADDR_SPACE_SIZE), assert_valid_addr_space_size); -#define NUM_L1_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT) +#if ADDR_SPACE_SIZE > (1ULL << (32 - TTBCR_TxSZ_MIN)) -static uint64_t l1_xlation_table[NUM_L1_ENTRIES] - __aligned(NUM_L1_ENTRIES * sizeof(uint64_t)); +# error "ADDR_SPACE_SIZE is too big." + +#elif ADDR_SPACE_SIZE > (1 << L1_XLAT_ADDRESS_SHIFT) + +# define XLAT_TABLE_LEVEL_BASE 1 +# define NUM_BASE_LEVEL_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT) + +#elif ADDR_SPACE_SIZE >= (1 << (32 - TTBCR_TxSZ_MAX)) + +# define XLAT_TABLE_LEVEL_BASE 2 +# define NUM_BASE_LEVEL_ENTRIES (ADDR_SPACE_SIZE >> L2_XLAT_ADDRESS_SHIFT) + +#else + +# error "ADDR_SPACE_SIZE is too small." + +#endif + +static uint64_t base_xlation_table[NUM_BASE_LEVEL_ENTRIES] + __aligned(NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t)); void init_xlat_tables(void) { unsigned long long max_pa; uintptr_t max_va; print_mmap(); - init_xlation_table(0, l1_xlation_table, 1, &max_va, &max_pa); + init_xlation_table(0, base_xlation_table, XLAT_TABLE_LEVEL_BASE, + &max_va, &max_pa); assert(max_va < ADDR_SPACE_SIZE); } @@ -95,7 +127,7 @@ write_ttbcr(ttbcr); /* Set TTBR0 bits as well */ - ttbr0 = (uintptr_t) l1_xlation_table; + ttbr0 = (uintptr_t) base_xlation_table; write64_ttbr0(ttbr0); write64_ttbr1(0); diff --git a/lib/xlat_tables/aarch64/xlat_tables.c b/lib/xlat_tables/aarch64/xlat_tables.c index f432237..5b639b7 100644 --- a/lib/xlat_tables/aarch64/xlat_tables.c +++ b/lib/xlat_tables/aarch64/xlat_tables.c @@ -38,19 +38,55 @@ #include "../xlat_tables_private.h" /* - * The virtual address space size must be a power of two (as set in TCR.T0SZ). - * As we start the initial lookup at level 1, it must also be between 2 GB and - * 512 GB (with the virtual address size therefore 31 to 39 bits). See section - * D4.2.5 in the ARMv8-A Architecture Reference Manual (DDI 0487A.i) for more - * information. + * Each platform can define the size of the virtual address space, which is + * defined in ADDR_SPACE_SIZE. TCR.TxSZ is calculated as 64 minus the width of + * said address space. The value of TCR.TxSZ must be in the range 16 to 39 [1], + * which means that the virtual address space width must be in the range 48 to + * 25 bits. + * + * Here we calculate the initial lookup level from the value of ADDR_SPACE_SIZE. + * For a 4 KB page size, level 0 supports virtual address spaces of widths 48 to + * 40 bits, level 1 from 39 to 31, and level 2 from 30 to 25. Wider or narrower + * address spaces are not supported. As a result, level 3 cannot be used as + * initial lookup level with 4 KB granularity. [2] + * + * For example, for a 35-bit address space (i.e. ADDR_SPACE_SIZE == 1 << 35), + * TCR.TxSZ will be programmed to (64 - 35) = 29. According to Table D4-11 in + * the ARM ARM, the initial lookup level for such an address space is 1. + * + * See the ARMv8-A Architecture Reference Manual (DDI 0487A.j) for more + * information: + * [1] Page 1730: 'Input address size', 'For all translation stages'. + * [2] Section D4.2.5 */ -CASSERT(ADDR_SPACE_SIZE >= (1ull << 31) && ADDR_SPACE_SIZE <= (1ull << 39) && - IS_POWER_OF_TWO(ADDR_SPACE_SIZE), assert_valid_addr_space_size); -#define NUM_L1_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT) +#if ADDR_SPACE_SIZE > (1ULL << (64 - TCR_TxSZ_MIN)) -static uint64_t l1_xlation_table[NUM_L1_ENTRIES] - __aligned(NUM_L1_ENTRIES * sizeof(uint64_t)); +# error "ADDR_SPACE_SIZE is too big." + +#elif ADDR_SPACE_SIZE > (1ULL << L0_XLAT_ADDRESS_SHIFT) + +# define XLAT_TABLE_LEVEL_BASE 0 +# define NUM_BASE_LEVEL_ENTRIES (ADDR_SPACE_SIZE >> L0_XLAT_ADDRESS_SHIFT) + +#elif ADDR_SPACE_SIZE > (1 << L1_XLAT_ADDRESS_SHIFT) + +# define XLAT_TABLE_LEVEL_BASE 1 +# define NUM_BASE_LEVEL_ENTRIES (ADDR_SPACE_SIZE >> L1_XLAT_ADDRESS_SHIFT) + +#elif ADDR_SPACE_SIZE >= (1 << (64 - TCR_TxSZ_MAX)) + +# define XLAT_TABLE_LEVEL_BASE 2 +# define NUM_BASE_LEVEL_ENTRIES (ADDR_SPACE_SIZE >> L2_XLAT_ADDRESS_SHIFT) + +#else + +# error "ADDR_SPACE_SIZE is too small." + +#endif + +static uint64_t base_xlation_table[NUM_BASE_LEVEL_ENTRIES] + __aligned(NUM_BASE_LEVEL_ENTRIES * sizeof(uint64_t)); static unsigned long long tcr_ps_bits; @@ -88,7 +124,8 @@ unsigned long long max_pa; uintptr_t max_va; print_mmap(); - init_xlation_table(0, l1_xlation_table, 1, &max_va, &max_pa); + init_xlation_table(0, base_xlation_table, XLAT_TABLE_LEVEL_BASE, + &max_va, &max_pa); tcr_ps_bits = calc_physical_addr_size_bits(max_pa); assert(max_va < ADDR_SPACE_SIZE); } @@ -124,7 +161,8 @@ _tlbi_fct(); \ \ /* Set TCR bits as well. */ \ - /* Inner & outer WBWA & shareable + T0SZ = 32 */ \ + /* Inner & outer WBWA & shareable. */ \ + /* Set T0SZ to (64 - width of virtual address space) */ \ tcr = TCR_SH_INNER_SHAREABLE | TCR_RGN_OUTER_WBA | \ TCR_RGN_INNER_WBA | \ (64 - __builtin_ctzl(ADDR_SPACE_SIZE)); \ @@ -132,7 +170,7 @@ write_tcr_el##_el(tcr); \ \ /* Set TTBR bits as well */ \ - ttbr = (uint64_t) l1_xlation_table; \ + ttbr = (uint64_t) base_xlation_table; \ write_ttbr0_el##_el(ttbr); \ \ /* Ensure all translation table writes have drained */ \ diff --git a/lib/xlat_tables/xlat_tables_common.c b/lib/xlat_tables/xlat_tables_common.c index bc7fed7..ebbc916 100644 --- a/lib/xlat_tables/xlat_tables_common.c +++ b/lib/xlat_tables/xlat_tables_common.c @@ -199,7 +199,11 @@ int mem_type; desc = addr_pa; - desc |= (level == 3) ? TABLE_DESC : BLOCK_DESC; + /* + * There are different translation table descriptors for level 3 and the + * rest. + */ + desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0; desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); desc |= LOWER_ATTRS(ACCESS_FLAG); @@ -311,14 +315,13 @@ uint64_t *table, int level) { - unsigned level_size_shift = L1_XLAT_ADDRESS_SHIFT - (level - 1) * - XLAT_TABLE_ENTRIES_SHIFT; - unsigned level_size = 1 << level_size_shift; - u_register_t level_index_mask = - (u_register_t)(((u_register_t) XLAT_TABLE_ENTRIES_MASK) - << level_size_shift); + assert(level >= XLAT_TABLE_LEVEL_MIN && level <= XLAT_TABLE_LEVEL_MAX); - assert(level > 0 && level <= 3); + unsigned int level_size_shift = + L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT; + u_register_t level_size = (u_register_t)1 << level_size_shift; + u_register_t level_index_mask = + ((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift; debug_print("New xlat table:\n"); @@ -334,8 +337,8 @@ continue; } - debug_print("%s VA:%p size:0x%x ", get_level_spacer(level), - (void *)base_va, level_size); + debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level), + (void *)base_va, (unsigned long long)level_size); if (mm->base_va > base_va + level_size - 1) { /* Next region is after this area. Nothing to map yet */ diff --git a/lib/xlat_tables/xlat_tables_private.h b/lib/xlat_tables/xlat_tables_private.h index 389bb0b..159d071 100644 --- a/lib/xlat_tables/xlat_tables_private.h +++ b/lib/xlat_tables/xlat_tables_private.h @@ -31,6 +31,12 @@ #ifndef __XLAT_TABLES_PRIVATE_H__ #define __XLAT_TABLES_PRIVATE_H__ +#include +#include + +/* The virtual address space size must be a power of two. */ +CASSERT(IS_POWER_OF_TWO(ADDR_SPACE_SIZE), assert_valid_addr_space_size); + void print_mmap(void); void init_xlation_table(uintptr_t base_va, uint64_t *table, int level, uintptr_t *max_va,