diff --git a/include/lib/xlat_tables/xlat_tables_v2.h b/include/lib/xlat_tables/xlat_tables_v2.h index 4dc2c5e..20a9ea1 100644 --- a/include/lib/xlat_tables/xlat_tables_v2.h +++ b/include/lib/xlat_tables/xlat_tables_v2.h @@ -123,10 +123,8 @@ /* * Translation regimes supported by this library. */ -typedef enum xlat_regime { - EL1_EL0_REGIME, - EL3_REGIME, -} xlat_regime_t; +#define EL1_EL0_REGIME 1 +#define EL3_REGIME 3 /* * Declare the translation context type. @@ -161,12 +159,12 @@ * (resp. PLAT_PHY_ADDR_SPACE_SIZE) for the translation context describing the * BL image currently executing. */ -#define REGISTER_XLAT_CONTEXT(_ctx_name, _mmap_count, _xlat_tables_count, \ - _virt_addr_space_size, _phy_addr_space_size) \ - _REGISTER_XLAT_CONTEXT_FULL_SPEC(_ctx_name, _mmap_count, \ - _xlat_tables_count, \ - _virt_addr_space_size, \ - _phy_addr_space_size, \ +#define REGISTER_XLAT_CONTEXT(_ctx_name, _mmap_count, _xlat_tables_count, \ + _virt_addr_space_size, _phy_addr_space_size) \ + _REGISTER_XLAT_CONTEXT_FULL_SPEC(_ctx_name, (_mmap_count), \ + (_xlat_tables_count), \ + (_virt_addr_space_size), \ + (_phy_addr_space_size), \ IMAGE_XLAT_DEFAULT_REGIME, \ "xlat_table") @@ -175,20 +173,20 @@ * * _xlat_regime: * Specify the translation regime managed by this xlat_ctx_t instance. The - * values are the one from xlat_regime_t enumeration. + * values are the one from the EL*_REGIME definitions. * * _section_name: * Specify the name of the section where the translation tables have to be * placed by the linker. */ -#define REGISTER_XLAT_CONTEXT2(_ctx_name, _mmap_count, _xlat_tables_count, \ - _virt_addr_space_size, _phy_addr_space_size, \ - _xlat_regime, _section_name) \ - _REGISTER_XLAT_CONTEXT_FULL_SPEC(_ctx_name, _mmap_count, \ - _xlat_tables_count, \ - _virt_addr_space_size, \ - _phy_addr_space_size, \ - _xlat_regime, _section_name) +#define REGISTER_XLAT_CONTEXT2(_ctx_name, _mmap_count, _xlat_tables_count, \ + _virt_addr_space_size, _phy_addr_space_size, \ + _xlat_regime, _section_name) \ + _REGISTER_XLAT_CONTEXT_FULL_SPEC(_ctx_name, (_mmap_count), \ + (_xlat_tables_count), \ + (_virt_addr_space_size), \ + (_phy_addr_space_size), \ + (_xlat_regime), (_section_name)) /****************************************************************************** * Generic translation table APIs. diff --git a/include/lib/xlat_tables/xlat_tables_v2_helpers.h b/include/lib/xlat_tables/xlat_tables_v2_helpers.h index e1ea2b6..56b9a93 100644 --- a/include/lib/xlat_tables/xlat_tables_v2_helpers.h +++ b/include/lib/xlat_tables/xlat_tables_v2_helpers.h @@ -109,10 +109,8 @@ unsigned int initialized; /* - * Translation regime managed by this xlat_ctx_t. It takes the values of - * the enumeration xlat_regime_t. The type is "int" to avoid a circular - * dependency on xlat_tables_v2.h, but this member must be treated as - * xlat_regime_t. + * Translation regime managed by this xlat_ctx_t. It should be one of + * the EL*_REGIME defines. */ int xlat_regime; }; @@ -157,7 +155,7 @@ .va_max_address = (_virt_addr_space_size) - 1, \ .pa_max_address = (_phy_addr_space_size) - 1, \ .mmap = _ctx_name##_mmap, \ - .mmap_num = _mmap_count, \ + .mmap_num = (_mmap_count), \ .base_level = GET_XLAT_TABLE_LEVEL_BASE(_virt_addr_space_size), \ .base_table = _ctx_name##_base_xlat_table, \ .base_table_entries = \ diff --git a/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c b/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c index 94dcf57..6e97192 100644 --- a/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c +++ b/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c @@ -61,7 +61,7 @@ tlbimvaais(TLBI_ADDR(va)); } -void xlat_arch_tlbi_va_regime(uintptr_t va, xlat_regime_t xlat_regime __unused) +void xlat_arch_tlbi_va_regime(uintptr_t va, int xlat_regime __unused) { /* * Ensure the translation table write has drained into memory before diff --git a/lib/xlat_tables_v2/aarch32/xlat_tables_arch_private.h b/lib/xlat_tables_v2/aarch32/xlat_tables_arch_private.h index 509395d..9b41f4d 100644 --- a/lib/xlat_tables_v2/aarch32/xlat_tables_arch_private.h +++ b/lib/xlat_tables_v2/aarch32/xlat_tables_arch_private.h @@ -14,7 +14,7 @@ * Return the execute-never mask that will prevent instruction fetch at the * given translation regime. */ -static inline uint64_t xlat_arch_regime_get_xn_desc(xlat_regime_t regime __unused) +static inline uint64_t xlat_arch_regime_get_xn_desc(int regime __unused) { return UPPER_ATTRS(XN); } diff --git a/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c b/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c index 71b9c8f..4bbbe54 100644 --- a/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c +++ b/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c @@ -7,11 +7,8 @@ #include #include #include -#include #include -#include #include -#include #include #include #include "../xlat_tables_private.h" @@ -128,7 +125,7 @@ #endif } -void xlat_arch_tlbi_va_regime(uintptr_t va, xlat_regime_t xlat_regime) +void xlat_arch_tlbi_va_regime(uintptr_t va, int xlat_regime) { /* * Ensure the translation table write has drained into memory before diff --git a/lib/xlat_tables_v2/aarch64/xlat_tables_arch_private.h b/lib/xlat_tables_v2/aarch64/xlat_tables_arch_private.h index d201590..39b0a65 100644 --- a/lib/xlat_tables_v2/aarch64/xlat_tables_arch_private.h +++ b/lib/xlat_tables_v2/aarch64/xlat_tables_arch_private.h @@ -15,7 +15,7 @@ * Return the execute-never mask that will prevent instruction fetch at all ELs * that are part of the given translation regime. */ -static inline uint64_t xlat_arch_regime_get_xn_desc(xlat_regime_t regime) +static inline uint64_t xlat_arch_regime_get_xn_desc(int regime) { if (regime == EL1_EL0_REGIME) { return UPPER_ATTRS(UXN) | UPPER_ATTRS(PXN); diff --git a/lib/xlat_tables_v2/xlat_tables.mk b/lib/xlat_tables_v2/xlat_tables.mk index 1e70f37..b25c805 100644 --- a/lib/xlat_tables_v2/xlat_tables.mk +++ b/lib/xlat_tables_v2/xlat_tables.mk @@ -7,6 +7,8 @@ XLAT_TABLES_LIB_SRCS := $(addprefix lib/xlat_tables_v2/, \ ${ARCH}/enable_mmu.S \ ${ARCH}/xlat_tables_arch.c \ - xlat_tables_internal.c) + xlat_tables_context.c \ + xlat_tables_core.c \ + xlat_tables_utils.c) INCLUDES += -Ilib/xlat_tables_v2/${ARCH} diff --git a/lib/xlat_tables_v2/xlat_tables_context.c b/lib/xlat_tables_v2/xlat_tables_context.c new file mode 100644 index 0000000..0964b49 --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_context.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include + +#include "xlat_tables_private.h" + +/* + * Each platform can define the size of its physical and virtual address spaces. + * If the platform hasn't defined one or both of them, default to + * ADDR_SPACE_SIZE. The latter is deprecated, though. + */ +#if ERROR_DEPRECATED +# ifdef ADDR_SPACE_SIZE +# error "ADDR_SPACE_SIZE is deprecated. Use PLAT_xxx_ADDR_SPACE_SIZE instead." +# endif +#elif defined(ADDR_SPACE_SIZE) +# ifndef PLAT_PHY_ADDR_SPACE_SIZE +# define PLAT_PHY_ADDR_SPACE_SIZE ADDR_SPACE_SIZE +# endif +# ifndef PLAT_VIRT_ADDR_SPACE_SIZE +# define PLAT_VIRT_ADDR_SPACE_SIZE ADDR_SPACE_SIZE +# endif +#endif + +/* + * Allocate and initialise the default translation context for the BL image + * currently executing. + */ +REGISTER_XLAT_CONTEXT(tf, MAX_MMAP_REGIONS, MAX_XLAT_TABLES, + PLAT_VIRT_ADDR_SPACE_SIZE, PLAT_PHY_ADDR_SPACE_SIZE); + +void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, size_t size, + unsigned int attr) +{ + mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); + + mmap_add_region_ctx(&tf_xlat_ctx, &mm); +} + +void mmap_add(const mmap_region_t *mm) +{ + mmap_add_ctx(&tf_xlat_ctx, mm); +} + +#if PLAT_XLAT_TABLES_DYNAMIC + +int mmap_add_dynamic_region(unsigned long long base_pa, uintptr_t base_va, + size_t size, unsigned int attr) +{ + mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); + + return mmap_add_dynamic_region_ctx(&tf_xlat_ctx, &mm); +} + +int mmap_remove_dynamic_region(uintptr_t base_va, size_t size) +{ + return mmap_remove_dynamic_region_ctx(&tf_xlat_ctx, + base_va, size); +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +void init_xlat_tables(void) +{ + init_xlat_tables_ctx(&tf_xlat_ctx); +} + +/* + * If dynamic allocation of new regions is disabled then by the time we call the + * function enabling the MMU, we'll have registered all the memory regions to + * map for the system's lifetime. Therefore, at this point we know the maximum + * physical address that will ever be mapped. + * + * If dynamic allocation is enabled then we can't make any such assumption + * because the maximum physical address could get pushed while adding a new + * region. Therefore, in this case we have to assume that the whole address + * space size might be mapped. + */ +#ifdef PLAT_XLAT_TABLES_DYNAMIC +#define MAX_PHYS_ADDR tf_xlat_ctx.pa_max_address +#else +#define MAX_PHYS_ADDR tf_xlat_ctx.max_pa +#endif + +#ifdef AARCH32 + +void enable_mmu_secure(unsigned int flags) +{ + setup_mmu_cfg(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address); + enable_mmu_direct(flags); +} + +#else + +void enable_mmu_el1(unsigned int flags) +{ + setup_mmu_cfg(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address); + enable_mmu_direct_el1(flags); +} + +void enable_mmu_el3(unsigned int flags) +{ + setup_mmu_cfg(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, + tf_xlat_ctx.va_max_address); + enable_mmu_direct_el3(flags); +} + +#endif /* AARCH32 */ diff --git a/lib/xlat_tables_v2/xlat_tables_core.c b/lib/xlat_tables_v2/xlat_tables_core.c new file mode 100644 index 0000000..f555524 --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_core.c @@ -0,0 +1,1013 @@ +/* + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xlat_tables_private.h" + +#if PLAT_XLAT_TABLES_DYNAMIC + +/* + * The following functions assume that they will be called using subtables only. + * The base table can't be unmapped, so it is not needed to do any special + * handling for it. + */ + +/* + * Returns the index of the array corresponding to the specified translation + * table. + */ +static int xlat_table_get_index(xlat_ctx_t *ctx, const uint64_t *table) +{ + for (unsigned int i = 0; i < ctx->tables_num; i++) + if (ctx->tables[i] == table) + return i; + + /* + * Maybe we were asked to get the index of the base level table, which + * should never happen. + */ + assert(0); + + return -1; +} + +/* Returns a pointer to an empty translation table. */ +static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) +{ + for (unsigned int i = 0; i < ctx->tables_num; i++) + if (ctx->tables_mapped_regions[i] == 0) + return ctx->tables[i]; + + return NULL; +} + +/* Increments region count for a given table. */ +static void xlat_table_inc_regions_count(xlat_ctx_t *ctx, const uint64_t *table) +{ + ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]++; +} + +/* Decrements region count for a given table. */ +static void xlat_table_dec_regions_count(xlat_ctx_t *ctx, const uint64_t *table) +{ + ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]--; +} + +/* Returns 0 if the specified table isn't empty, otherwise 1. */ +static int xlat_table_is_empty(xlat_ctx_t *ctx, const uint64_t *table) +{ + return !ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]; +} + +#else /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* Returns a pointer to the first empty translation table. */ +static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) +{ + assert(ctx->next_table < ctx->tables_num); + + return ctx->tables[ctx->next_table++]; +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* + * Returns a block/page table descriptor for the given level and attributes. + */ +uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, + unsigned long long addr_pa, int level) +{ + uint64_t desc; + int mem_type; + + /* Make sure that the granularity is fine enough to map this address. */ + assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0); + + desc = addr_pa; + /* + * There are different translation table descriptors for level 3 and the + * rest. + */ + desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; + /* + * Always set the access flag, as TF doesn't manage access flag faults. + * Deduce other fields of the descriptor based on the MT_NS and MT_RW + * memory region attributes. + */ + desc |= LOWER_ATTRS(ACCESS_FLAG); + + desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0; + desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); + + /* + * Do not allow unprivileged access when the mapping is for a privileged + * EL. For translation regimes that do not have mappings for access for + * lower exception levels, set AP[2] to AP_NO_ACCESS_UNPRIVILEGED. + */ + if (ctx->xlat_regime == EL1_EL0_REGIME) { + if (attr & MT_USER) { + /* EL0 mapping requested, so we give User access */ + desc |= LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED); + } else { + /* EL1 mapping requested, no User access granted */ + desc |= LOWER_ATTRS(AP_NO_ACCESS_UNPRIVILEGED); + } + } else { + assert(ctx->xlat_regime == EL3_REGIME); + desc |= LOWER_ATTRS(AP_ONE_VA_RANGE_RES1); + } + + /* + * Deduce shareability domain and executability of the memory region + * from the memory type of the attributes (MT_TYPE). + * + * Data accesses to device memory and non-cacheable normal memory are + * coherent for all observers in the system, and correspondingly are + * always treated as being Outer Shareable. Therefore, for these 2 types + * of memory, it is not strictly needed to set the shareability field + * in the translation tables. + */ + mem_type = MT_TYPE(attr); + if (mem_type == MT_DEVICE) { + desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); + /* + * Always map device memory as execute-never. + * This is to avoid the possibility of a speculative instruction + * fetch, which could be an issue if this memory region + * corresponds to a read-sensitive peripheral. + */ + desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); + + } else { /* Normal memory */ + /* + * Always map read-write normal memory as execute-never. + * (Trusted Firmware doesn't self-modify its code, therefore + * R/W memory is reserved for data storage, which must not be + * executable.) + * Note that setting the XN bit here is for consistency only. + * The function that enables the MMU sets the SCTLR_ELx.WXN bit, + * which makes any writable memory region to be treated as + * execute-never, regardless of the value of the XN bit in the + * translation table. + * + * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER + * attribute to figure out the value of the XN bit. The actual + * XN bit(s) to set in the descriptor depends on the context's + * translation regime and the policy applied in + * xlat_arch_regime_get_xn_desc(). + */ + if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER)) { + desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); + } + + if (mem_type == MT_MEMORY) { + desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); + } else { + assert(mem_type == MT_NON_CACHEABLE); + desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); + } + } + + return desc; +} + +/* + * Enumeration of actions that can be made when mapping table entries depending + * on the previous value in that entry and information about the region being + * mapped. + */ +typedef enum { + + /* Do nothing */ + ACTION_NONE, + + /* Write a block (or page, if in level 3) entry. */ + ACTION_WRITE_BLOCK_ENTRY, + + /* + * Create a new table and write a table entry pointing to it. Recurse + * into it for further processing. + */ + ACTION_CREATE_NEW_TABLE, + + /* + * There is a table descriptor in this entry, read it and recurse into + * that table for further processing. + */ + ACTION_RECURSE_INTO_TABLE, + +} action_t; + +#if PLAT_XLAT_TABLES_DYNAMIC + +/* + * Recursive function that writes to the translation tables and unmaps the + * specified region. + */ +static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm, + const uintptr_t table_base_va, + uint64_t *const table_base, + const int table_entries, + const unsigned int level) +{ + assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX); + + uint64_t *subtable; + uint64_t desc; + + uintptr_t table_idx_va; + uintptr_t table_idx_end_va; /* End VA of this entry */ + + uintptr_t region_end_va = mm->base_va + mm->size - 1; + + int table_idx; + + if (mm->base_va > table_base_va) { + /* Find the first index of the table affected by the region. */ + table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); + + table_idx = (table_idx_va - table_base_va) >> + XLAT_ADDR_SHIFT(level); + + assert(table_idx < table_entries); + } else { + /* Start from the beginning of the table. */ + table_idx_va = table_base_va; + table_idx = 0; + } + + while (table_idx < table_entries) { + + table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1; + + desc = table_base[table_idx]; + uint64_t desc_type = desc & DESC_MASK; + + action_t action = ACTION_NONE; + + if ((mm->base_va <= table_idx_va) && + (region_end_va >= table_idx_end_va)) { + + /* Region covers all block */ + + if (level == 3) { + /* + * Last level, only page descriptors allowed, + * erase it. + */ + assert(desc_type == PAGE_DESC); + + action = ACTION_WRITE_BLOCK_ENTRY; + } else { + /* + * Other levels can have table descriptors. If + * so, recurse into it and erase descriptors + * inside it as needed. If there is a block + * descriptor, just erase it. If an invalid + * descriptor is found, this table isn't + * actually mapped, which shouldn't happen. + */ + if (desc_type == TABLE_DESC) { + action = ACTION_RECURSE_INTO_TABLE; + } else { + assert(desc_type == BLOCK_DESC); + action = ACTION_WRITE_BLOCK_ENTRY; + } + } + + } else if ((mm->base_va <= table_idx_end_va) || + (region_end_va >= table_idx_va)) { + + /* + * Region partially covers block. + * + * It can't happen in level 3. + * + * There must be a table descriptor here, if not there + * was a problem when mapping the region. + */ + + assert(level < 3); + + assert(desc_type == TABLE_DESC); + + action = ACTION_RECURSE_INTO_TABLE; + } + + if (action == ACTION_WRITE_BLOCK_ENTRY) { + + table_base[table_idx] = INVALID_DESC; + xlat_arch_tlbi_va_regime(table_idx_va, ctx->xlat_regime); + + } else if (action == ACTION_RECURSE_INTO_TABLE) { + + subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); + + /* Recurse to write into subtable */ + xlat_tables_unmap_region(ctx, mm, table_idx_va, + subtable, XLAT_TABLE_ENTRIES, + level + 1); + + /* + * If the subtable is now empty, remove its reference. + */ + if (xlat_table_is_empty(ctx, subtable)) { + table_base[table_idx] = INVALID_DESC; + xlat_arch_tlbi_va_regime(table_idx_va, + ctx->xlat_regime); + } + + } else { + assert(action == ACTION_NONE); + } + + table_idx++; + table_idx_va += XLAT_BLOCK_SIZE(level); + + /* If reached the end of the region, exit */ + if (region_end_va <= table_idx_va) + break; + } + + if (level > ctx->base_level) + xlat_table_dec_regions_count(ctx, table_base); +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +/* + * From the given arguments, it decides which action to take when mapping the + * specified region. + */ +static action_t xlat_tables_map_region_action(const mmap_region_t *mm, + const int desc_type, const unsigned long long dest_pa, + const uintptr_t table_entry_base_va, const unsigned int level) +{ + uintptr_t mm_end_va = mm->base_va + mm->size - 1; + uintptr_t table_entry_end_va = + table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1; + + /* + * The descriptor types allowed depend on the current table level. + */ + + if ((mm->base_va <= table_entry_base_va) && + (mm_end_va >= table_entry_end_va)) { + + /* + * Table entry is covered by region + * -------------------------------- + * + * This means that this table entry can describe the whole + * translation with this granularity in principle. + */ + + if (level == 3) { + /* + * Last level, only page descriptors are allowed. + */ + if (desc_type == PAGE_DESC) { + /* + * There's another region mapped here, don't + * overwrite. + */ + return ACTION_NONE; + } else { + assert(desc_type == INVALID_DESC); + return ACTION_WRITE_BLOCK_ENTRY; + } + + } else { + + /* + * Other levels. Table descriptors are allowed. Block + * descriptors too, but they have some limitations. + */ + + if (desc_type == TABLE_DESC) { + /* There's already a table, recurse into it. */ + return ACTION_RECURSE_INTO_TABLE; + + } else if (desc_type == INVALID_DESC) { + /* + * There's nothing mapped here, create a new + * entry. + * + * Check if the destination granularity allows + * us to use a block descriptor or we need a + * finer table for it. + * + * Also, check if the current level allows block + * descriptors. If not, create a table instead. + */ + if ((dest_pa & XLAT_BLOCK_MASK(level)) || + (level < MIN_LVL_BLOCK_DESC) || + (mm->granularity < XLAT_BLOCK_SIZE(level))) + return ACTION_CREATE_NEW_TABLE; + else + return ACTION_WRITE_BLOCK_ENTRY; + + } else { + /* + * There's another region mapped here, don't + * overwrite. + */ + assert(desc_type == BLOCK_DESC); + + return ACTION_NONE; + } + } + + } else if ((mm->base_va <= table_entry_end_va) || + (mm_end_va >= table_entry_base_va)) { + + /* + * Region partially covers table entry + * ----------------------------------- + * + * This means that this table entry can't describe the whole + * translation, a finer table is needed. + + * There cannot be partial block overlaps in level 3. If that + * happens, some of the preliminary checks when adding the + * mmap region failed to detect that PA and VA must at least be + * aligned to PAGE_SIZE. + */ + assert(level < 3); + + if (desc_type == INVALID_DESC) { + /* + * The block is not fully covered by the region. Create + * a new table, recurse into it and try to map the + * region with finer granularity. + */ + return ACTION_CREATE_NEW_TABLE; + + } else { + assert(desc_type == TABLE_DESC); + /* + * The block is not fully covered by the region, but + * there is already a table here. Recurse into it and + * try to map with finer granularity. + * + * PAGE_DESC for level 3 has the same value as + * TABLE_DESC, but this code can't run on a level 3 + * table because there can't be overlaps in level 3. + */ + return ACTION_RECURSE_INTO_TABLE; + } + } + + /* + * This table entry is outside of the region specified in the arguments, + * don't write anything to it. + */ + return ACTION_NONE; +} + +/* + * Recursive function that writes to the translation tables and maps the + * specified region. On success, it returns the VA of the last byte that was + * successfully mapped. On error, it returns the VA of the next entry that + * should have been mapped. + */ +static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm, + const uintptr_t table_base_va, + uint64_t *const table_base, + const int table_entries, + const unsigned int level) +{ + assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX); + + uintptr_t mm_end_va = mm->base_va + mm->size - 1; + + uintptr_t table_idx_va; + unsigned long long table_idx_pa; + + uint64_t *subtable; + uint64_t desc; + + int table_idx; + + if (mm->base_va > table_base_va) { + /* Find the first index of the table affected by the region. */ + table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); + + table_idx = (table_idx_va - table_base_va) >> + XLAT_ADDR_SHIFT(level); + + assert(table_idx < table_entries); + } else { + /* Start from the beginning of the table. */ + table_idx_va = table_base_va; + table_idx = 0; + } + +#if PLAT_XLAT_TABLES_DYNAMIC + if (level > ctx->base_level) + xlat_table_inc_regions_count(ctx, table_base); +#endif + + while (table_idx < table_entries) { + + desc = table_base[table_idx]; + + table_idx_pa = mm->base_pa + table_idx_va - mm->base_va; + + action_t action = xlat_tables_map_region_action(mm, + desc & DESC_MASK, table_idx_pa, table_idx_va, level); + + if (action == ACTION_WRITE_BLOCK_ENTRY) { + + table_base[table_idx] = + xlat_desc(ctx, (uint32_t)mm->attr, table_idx_pa, + level); + + } else if (action == ACTION_CREATE_NEW_TABLE) { + + subtable = xlat_table_get_empty(ctx); + if (subtable == NULL) { + /* Not enough free tables to map this region */ + return table_idx_va; + } + + /* Point to new subtable from this one. */ + table_base[table_idx] = TABLE_DESC | (unsigned long)subtable; + + /* Recurse to write into subtable */ + uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va, + subtable, XLAT_TABLE_ENTRIES, + level + 1); + if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1) + return end_va; + + } else if (action == ACTION_RECURSE_INTO_TABLE) { + + subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); + /* Recurse to write into subtable */ + uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va, + subtable, XLAT_TABLE_ENTRIES, + level + 1); + if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1) + return end_va; + + } else { + + assert(action == ACTION_NONE); + + } + + table_idx++; + table_idx_va += XLAT_BLOCK_SIZE(level); + + /* If reached the end of the region, exit */ + if (mm_end_va <= table_idx_va) + break; + } + + return table_idx_va - 1; +} + +/* + * Function that verifies that a region can be mapped. + * Returns: + * 0: Success, the mapping is allowed. + * EINVAL: Invalid values were used as arguments. + * ERANGE: The memory limits were surpassed. + * ENOMEM: There is not enough memory in the mmap array. + * EPERM: Region overlaps another one in an invalid way. + */ +static int mmap_add_region_check(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + unsigned long long base_pa = mm->base_pa; + uintptr_t base_va = mm->base_va; + size_t size = mm->size; + size_t granularity = mm->granularity; + + unsigned long long end_pa = base_pa + size - 1; + uintptr_t end_va = base_va + size - 1; + + if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) || + !IS_PAGE_ALIGNED(size)) + return -EINVAL; + + if ((granularity != XLAT_BLOCK_SIZE(1)) && + (granularity != XLAT_BLOCK_SIZE(2)) && + (granularity != XLAT_BLOCK_SIZE(3))) { + return -EINVAL; + } + + /* Check for overflows */ + if ((base_pa > end_pa) || (base_va > end_va)) + return -ERANGE; + + if ((base_va + (uintptr_t)size - (uintptr_t)1) > ctx->va_max_address) + return -ERANGE; + + if ((base_pa + (unsigned long long)size - 1ULL) > ctx->pa_max_address) + return -ERANGE; + + /* Check that there is space in the ctx->mmap array */ + if (ctx->mmap[ctx->mmap_num - 1].size != 0) + return -ENOMEM; + + /* Check for PAs and VAs overlaps with all other regions */ + for (mmap_region_t *mm_cursor = ctx->mmap; + mm_cursor->size; ++mm_cursor) { + + uintptr_t mm_cursor_end_va = mm_cursor->base_va + + mm_cursor->size - 1; + + /* + * Check if one of the regions is completely inside the other + * one. + */ + int fully_overlapped_va = + ((base_va >= mm_cursor->base_va) && + (end_va <= mm_cursor_end_va)) || + + ((mm_cursor->base_va >= base_va) && + (mm_cursor_end_va <= end_va)); + + /* + * Full VA overlaps are only allowed if both regions are + * identity mapped (zero offset) or have the same VA to PA + * offset. Also, make sure that it's not the exact same area. + * This can only be done with static regions. + */ + if (fully_overlapped_va) { + +#if PLAT_XLAT_TABLES_DYNAMIC + if ((mm->attr & MT_DYNAMIC) || + (mm_cursor->attr & MT_DYNAMIC)) + return -EPERM; +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + if ((mm_cursor->base_va - mm_cursor->base_pa) != + (base_va - base_pa)) + return -EPERM; + + if ((base_va == mm_cursor->base_va) && + (size == mm_cursor->size)) + return -EPERM; + + } else { + /* + * If the regions do not have fully overlapping VAs, + * then they must have fully separated VAs and PAs. + * Partial overlaps are not allowed + */ + + unsigned long long mm_cursor_end_pa = + mm_cursor->base_pa + mm_cursor->size - 1; + + int separated_pa = + (end_pa < mm_cursor->base_pa) || + (base_pa > mm_cursor_end_pa); + int separated_va = + (end_va < mm_cursor->base_va) || + (base_va > mm_cursor_end_va); + + if (!(separated_va && separated_pa)) + return -EPERM; + } + } + + return 0; +} + +void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + mmap_region_t *mm_cursor = ctx->mmap, *mm_destination; + const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num; + mmap_region_t *mm_last; + unsigned long long end_pa = mm->base_pa + mm->size - 1; + uintptr_t end_va = mm->base_va + mm->size - 1; + int ret; + + /* Ignore empty regions */ + if (!mm->size) + return; + + /* Static regions must be added before initializing the xlat tables. */ + assert(!ctx->initialized); + + ret = mmap_add_region_check(ctx, mm); + if (ret != 0) { + ERROR("mmap_add_region_check() failed. error %d\n", ret); + assert(0); + return; + } + + /* + * Find correct place in mmap to insert new region. + * + * 1 - Lower region VA end first. + * 2 - Smaller region size first. + * + * VA 0 0xFF + * + * 1st |------| + * 2nd |------------| + * 3rd |------| + * 4th |---| + * 5th |---| + * 6th |----------| + * 7th |-------------------------------------| + * + * This is required for overlapping regions only. It simplifies adding + * regions with the loop in xlat_tables_init_internal because the outer + * ones won't overwrite block or page descriptors of regions added + * previously. + * + * Overlapping is only allowed for static regions. + */ + + while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va + && mm_cursor->size) + ++mm_cursor; + + while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) && + (mm_cursor->size != 0U) && (mm_cursor->size < mm->size)) + ++mm_cursor; + + /* + * Find the last entry marker in the mmap + */ + mm_last = ctx->mmap; + while ((mm_last->size != 0U) && (mm_last < mm_end)) { + ++mm_last; + } + + /* + * Check if we have enough space in the memory mapping table. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_last->size == 0U); + + /* Make room for new region by moving other regions up by one place */ + mm_destination = mm_cursor + 1; + memmove(mm_destination, mm_cursor, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check we haven't lost the empty sentinel from the end of the array. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_end->size == 0U); + + *mm_cursor = *mm; + + if (end_pa > ctx->max_pa) + ctx->max_pa = end_pa; + if (end_va > ctx->max_va) + ctx->max_va = end_va; +} + +void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) +{ + while (mm->size) { + mmap_add_region_ctx(ctx, mm); + mm++; + } +} + +#if PLAT_XLAT_TABLES_DYNAMIC + +int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) +{ + mmap_region_t *mm_cursor = ctx->mmap; + mmap_region_t *mm_last = mm_cursor + ctx->mmap_num; + unsigned long long end_pa = mm->base_pa + mm->size - 1; + uintptr_t end_va = mm->base_va + mm->size - 1; + int ret; + + /* Nothing to do */ + if (!mm->size) + return 0; + + /* Now this region is a dynamic one */ + mm->attr |= MT_DYNAMIC; + + ret = mmap_add_region_check(ctx, mm); + if (ret != 0) + return ret; + + /* + * Find the adequate entry in the mmap array in the same way done for + * static regions in mmap_add_region_ctx(). + */ + + while ((mm_cursor->base_va + mm_cursor->size - 1) + < end_va && mm_cursor->size) + ++mm_cursor; + + while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) + && (mm_cursor->size < mm->size)) + ++mm_cursor; + + /* Make room for new region by moving other regions up by one place */ + memmove(mm_cursor + 1, mm_cursor, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check we haven't lost the empty sentinal from the end of the array. + * This shouldn't happen as we have checked in mmap_add_region_check + * that there is free space. + */ + assert(mm_last->size == 0); + + *mm_cursor = *mm; + + /* + * Update the translation tables if the xlat tables are initialized. If + * not, this region will be mapped when they are initialized. + */ + if (ctx->initialized) { + uintptr_t end_va = xlat_tables_map_region(ctx, mm_cursor, + 0, ctx->base_table, ctx->base_table_entries, + ctx->base_level); + + /* Failed to map, remove mmap entry, unmap and return error. */ + if (end_va != mm_cursor->base_va + mm_cursor->size - 1) { + memmove(mm_cursor, mm_cursor + 1, + (uintptr_t)mm_last - (uintptr_t)mm_cursor); + + /* + * Check if the mapping function actually managed to map + * anything. If not, just return now. + */ + if (mm->base_va >= end_va) + return -ENOMEM; + + /* + * Something went wrong after mapping some table + * entries, undo every change done up to this point. + */ + mmap_region_t unmap_mm = { + .base_pa = 0, + .base_va = mm->base_va, + .size = end_va - mm->base_va, + .attr = 0 + }; + xlat_tables_unmap_region(ctx, &unmap_mm, 0, ctx->base_table, + ctx->base_table_entries, ctx->base_level); + + return -ENOMEM; + } + + /* + * Make sure that all entries are written to the memory. There + * is no need to invalidate entries when mapping dynamic regions + * because new table/block/page descriptors only replace old + * invalid descriptors, that aren't TLB cached. + */ + dsbishst(); + } + + if (end_pa > ctx->max_pa) + ctx->max_pa = end_pa; + if (end_va > ctx->max_va) + ctx->max_va = end_va; + + return 0; +} + +/* + * Removes the region with given base Virtual Address and size from the given + * context. + * + * Returns: + * 0: Success. + * EINVAL: Invalid values were used as arguments (region not found). + * EPERM: Tried to remove a static region. + */ +int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va, + size_t size) +{ + mmap_region_t *mm = ctx->mmap; + mmap_region_t *mm_last = mm + ctx->mmap_num; + int update_max_va_needed = 0; + int update_max_pa_needed = 0; + + /* Check sanity of mmap array. */ + assert(mm[ctx->mmap_num].size == 0); + + while (mm->size) { + if ((mm->base_va == base_va) && (mm->size == size)) + break; + ++mm; + } + + /* Check that the region was found */ + if (mm->size == 0) + return -EINVAL; + + /* If the region is static it can't be removed */ + if (!(mm->attr & MT_DYNAMIC)) + return -EPERM; + + /* Check if this region is using the top VAs or PAs. */ + if ((mm->base_va + mm->size - 1) == ctx->max_va) + update_max_va_needed = 1; + if ((mm->base_pa + mm->size - 1) == ctx->max_pa) + update_max_pa_needed = 1; + + /* Update the translation tables if needed */ + if (ctx->initialized) { + xlat_tables_unmap_region(ctx, mm, 0, ctx->base_table, + ctx->base_table_entries, + ctx->base_level); + xlat_arch_tlbi_va_sync(); + } + + /* Remove this region by moving the rest down by one place. */ + memmove(mm, mm + 1, (uintptr_t)mm_last - (uintptr_t)mm); + + /* Check if we need to update the max VAs and PAs */ + if (update_max_va_needed) { + ctx->max_va = 0; + mm = ctx->mmap; + while (mm->size) { + if ((mm->base_va + mm->size - 1) > ctx->max_va) + ctx->max_va = mm->base_va + mm->size - 1; + ++mm; + } + } + + if (update_max_pa_needed) { + ctx->max_pa = 0; + mm = ctx->mmap; + while (mm->size) { + if ((mm->base_pa + mm->size - 1) > ctx->max_pa) + ctx->max_pa = mm->base_pa + mm->size - 1; + ++mm; + } + } + + return 0; +} + +#endif /* PLAT_XLAT_TABLES_DYNAMIC */ + +void init_xlat_tables_ctx(xlat_ctx_t *ctx) +{ + assert(ctx != NULL); + assert(!ctx->initialized); + assert(ctx->xlat_regime == EL3_REGIME || ctx->xlat_regime == EL1_EL0_REGIME); + assert(!is_mmu_enabled_ctx(ctx)); + + mmap_region_t *mm = ctx->mmap; + + xlat_mmap_print(mm); + + /* All tables must be zeroed before mapping any region. */ + + for (unsigned int i = 0; i < ctx->base_table_entries; i++) + ctx->base_table[i] = INVALID_DESC; + + for (unsigned int j = 0; j < ctx->tables_num; j++) { +#if PLAT_XLAT_TABLES_DYNAMIC + ctx->tables_mapped_regions[j] = 0; +#endif + for (unsigned int i = 0; i < XLAT_TABLE_ENTRIES; i++) + ctx->tables[j][i] = INVALID_DESC; + } + + while (mm->size) { + uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0, ctx->base_table, + ctx->base_table_entries, ctx->base_level); + + if (end_va != mm->base_va + mm->size - 1) { + ERROR("Not enough memory to map region:\n" + " VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", + (void *)mm->base_va, mm->base_pa, mm->size, mm->attr); + panic(); + } + + mm++; + } + + assert(ctx->pa_max_address <= xlat_arch_get_max_supported_pa()); + assert(ctx->max_va <= ctx->va_max_address); + assert(ctx->max_pa <= ctx->pa_max_address); + + ctx->initialized = 1; + + xlat_tables_print(ctx); +} diff --git a/lib/xlat_tables_v2/xlat_tables_internal.c b/lib/xlat_tables_v2/xlat_tables_internal.c deleted file mode 100644 index 7f1d395..0000000 --- a/lib/xlat_tables_v2/xlat_tables_internal.c +++ /dev/null @@ -1,1682 +0,0 @@ -/* - * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "xlat_tables_private.h" - -/* - * Each platform can define the size of its physical and virtual address spaces. - * If the platform hasn't defined one or both of them, default to - * ADDR_SPACE_SIZE. The latter is deprecated, though. - */ -#if ERROR_DEPRECATED -# ifdef ADDR_SPACE_SIZE -# error "ADDR_SPACE_SIZE is deprecated. Use PLAT_xxx_ADDR_SPACE_SIZE instead." -# endif -#elif defined(ADDR_SPACE_SIZE) -# ifndef PLAT_PHY_ADDR_SPACE_SIZE -# define PLAT_PHY_ADDR_SPACE_SIZE ADDR_SPACE_SIZE -# endif -# ifndef PLAT_VIRT_ADDR_SPACE_SIZE -# define PLAT_VIRT_ADDR_SPACE_SIZE ADDR_SPACE_SIZE -# endif -#endif - -/* - * Allocate and initialise the default translation context for the BL image - * currently executing. - */ -REGISTER_XLAT_CONTEXT(tf, MAX_MMAP_REGIONS, MAX_XLAT_TABLES, - PLAT_VIRT_ADDR_SPACE_SIZE, PLAT_PHY_ADDR_SPACE_SIZE); - -#if PLAT_XLAT_TABLES_DYNAMIC - -/* - * The following functions assume that they will be called using subtables only. - * The base table can't be unmapped, so it is not needed to do any special - * handling for it. - */ - -/* - * Returns the index of the array corresponding to the specified translation - * table. - */ -static int xlat_table_get_index(xlat_ctx_t *ctx, const uint64_t *table) -{ - for (unsigned int i = 0; i < ctx->tables_num; i++) - if (ctx->tables[i] == table) - return i; - - /* - * Maybe we were asked to get the index of the base level table, which - * should never happen. - */ - assert(0); - - return -1; -} - -/* Returns a pointer to an empty translation table. */ -static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) -{ - for (unsigned int i = 0; i < ctx->tables_num; i++) - if (ctx->tables_mapped_regions[i] == 0) - return ctx->tables[i]; - - return NULL; -} - -/* Increments region count for a given table. */ -static void xlat_table_inc_regions_count(xlat_ctx_t *ctx, const uint64_t *table) -{ - ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]++; -} - -/* Decrements region count for a given table. */ -static void xlat_table_dec_regions_count(xlat_ctx_t *ctx, const uint64_t *table) -{ - ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]--; -} - -/* Returns 0 if the speficied table isn't empty, otherwise 1. */ -static int xlat_table_is_empty(xlat_ctx_t *ctx, const uint64_t *table) -{ - return !ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)]; -} - -#else /* PLAT_XLAT_TABLES_DYNAMIC */ - -/* Returns a pointer to the first empty translation table. */ -static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx) -{ - assert(ctx->next_table < ctx->tables_num); - - return ctx->tables[ctx->next_table++]; -} - -#endif /* PLAT_XLAT_TABLES_DYNAMIC */ - -/* - * Returns a block/page table descriptor for the given level and attributes. - */ -static uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, - unsigned long long addr_pa, int level) -{ - uint64_t desc; - int mem_type; - - /* Make sure that the granularity is fine enough to map this address. */ - assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0); - - desc = addr_pa; - /* - * There are different translation table descriptors for level 3 and the - * rest. - */ - desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; - /* - * Always set the access flag, as TF doesn't manage access flag faults. - * Deduce other fields of the descriptor based on the MT_NS and MT_RW - * memory region attributes. - */ - desc |= LOWER_ATTRS(ACCESS_FLAG); - - desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0; - desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); - - /* - * Do not allow unprivileged access when the mapping is for a privileged - * EL. For translation regimes that do not have mappings for access for - * lower exception levels, set AP[2] to AP_NO_ACCESS_UNPRIVILEGED. - */ - if (ctx->xlat_regime == EL1_EL0_REGIME) { - if (attr & MT_USER) { - /* EL0 mapping requested, so we give User access */ - desc |= LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED); - } else { - /* EL1 mapping requested, no User access granted */ - desc |= LOWER_ATTRS(AP_NO_ACCESS_UNPRIVILEGED); - } - } else { - assert(ctx->xlat_regime == EL3_REGIME); - desc |= LOWER_ATTRS(AP_ONE_VA_RANGE_RES1); - } - - /* - * Deduce shareability domain and executability of the memory region - * from the memory type of the attributes (MT_TYPE). - * - * Data accesses to device memory and non-cacheable normal memory are - * coherent for all observers in the system, and correspondingly are - * always treated as being Outer Shareable. Therefore, for these 2 types - * of memory, it is not strictly needed to set the shareability field - * in the translation tables. - */ - mem_type = MT_TYPE(attr); - if (mem_type == MT_DEVICE) { - desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); - /* - * Always map device memory as execute-never. - * This is to avoid the possibility of a speculative instruction - * fetch, which could be an issue if this memory region - * corresponds to a read-sensitive peripheral. - */ - desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); - - } else { /* Normal memory */ - /* - * Always map read-write normal memory as execute-never. - * (Trusted Firmware doesn't self-modify its code, therefore - * R/W memory is reserved for data storage, which must not be - * executable.) - * Note that setting the XN bit here is for consistency only. - * The function that enables the MMU sets the SCTLR_ELx.WXN bit, - * which makes any writable memory region to be treated as - * execute-never, regardless of the value of the XN bit in the - * translation table. - * - * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER - * attribute to figure out the value of the XN bit. The actual - * XN bit(s) to set in the descriptor depends on the context's - * translation regime and the policy applied in - * xlat_arch_regime_get_xn_desc(). - */ - if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER)) { - desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime); - } - - if (mem_type == MT_MEMORY) { - desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); - } else { - assert(mem_type == MT_NON_CACHEABLE); - desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); - } - } - - return desc; -} - -/* - * Enumeration of actions that can be made when mapping table entries depending - * on the previous value in that entry and information about the region being - * mapped. - */ -typedef enum { - - /* Do nothing */ - ACTION_NONE, - - /* Write a block (or page, if in level 3) entry. */ - ACTION_WRITE_BLOCK_ENTRY, - - /* - * Create a new table and write a table entry pointing to it. Recurse - * into it for further processing. - */ - ACTION_CREATE_NEW_TABLE, - - /* - * There is a table descriptor in this entry, read it and recurse into - * that table for further processing. - */ - ACTION_RECURSE_INTO_TABLE, - -} action_t; - -#if PLAT_XLAT_TABLES_DYNAMIC - -/* - * Recursive function that writes to the translation tables and unmaps the - * specified region. - */ -static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm, - const uintptr_t table_base_va, - uint64_t *const table_base, - const int table_entries, - const unsigned int level) -{ - assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX); - - uint64_t *subtable; - uint64_t desc; - - uintptr_t table_idx_va; - uintptr_t table_idx_end_va; /* End VA of this entry */ - - uintptr_t region_end_va = mm->base_va + mm->size - 1; - - int table_idx; - - if (mm->base_va > table_base_va) { - /* Find the first index of the table affected by the region. */ - table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); - - table_idx = (table_idx_va - table_base_va) >> - XLAT_ADDR_SHIFT(level); - - assert(table_idx < table_entries); - } else { - /* Start from the beginning of the table. */ - table_idx_va = table_base_va; - table_idx = 0; - } - - while (table_idx < table_entries) { - - table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1; - - desc = table_base[table_idx]; - uint64_t desc_type = desc & DESC_MASK; - - action_t action = ACTION_NONE; - - if ((mm->base_va <= table_idx_va) && - (region_end_va >= table_idx_end_va)) { - - /* Region covers all block */ - - if (level == 3) { - /* - * Last level, only page descriptors allowed, - * erase it. - */ - assert(desc_type == PAGE_DESC); - - action = ACTION_WRITE_BLOCK_ENTRY; - } else { - /* - * Other levels can have table descriptors. If - * so, recurse into it and erase descriptors - * inside it as needed. If there is a block - * descriptor, just erase it. If an invalid - * descriptor is found, this table isn't - * actually mapped, which shouldn't happen. - */ - if (desc_type == TABLE_DESC) { - action = ACTION_RECURSE_INTO_TABLE; - } else { - assert(desc_type == BLOCK_DESC); - action = ACTION_WRITE_BLOCK_ENTRY; - } - } - - } else if ((mm->base_va <= table_idx_end_va) || - (region_end_va >= table_idx_va)) { - - /* - * Region partially covers block. - * - * It can't happen in level 3. - * - * There must be a table descriptor here, if not there - * was a problem when mapping the region. - */ - - assert(level < 3); - - assert(desc_type == TABLE_DESC); - - action = ACTION_RECURSE_INTO_TABLE; - } - - if (action == ACTION_WRITE_BLOCK_ENTRY) { - - table_base[table_idx] = INVALID_DESC; - xlat_arch_tlbi_va_regime(table_idx_va, ctx->xlat_regime); - - } else if (action == ACTION_RECURSE_INTO_TABLE) { - - subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); - - /* Recurse to write into subtable */ - xlat_tables_unmap_region(ctx, mm, table_idx_va, - subtable, XLAT_TABLE_ENTRIES, - level + 1); - - /* - * If the subtable is now empty, remove its reference. - */ - if (xlat_table_is_empty(ctx, subtable)) { - table_base[table_idx] = INVALID_DESC; - xlat_arch_tlbi_va_regime(table_idx_va, - ctx->xlat_regime); - } - - } else { - assert(action == ACTION_NONE); - } - - table_idx++; - table_idx_va += XLAT_BLOCK_SIZE(level); - - /* If reached the end of the region, exit */ - if (region_end_va <= table_idx_va) - break; - } - - if (level > ctx->base_level) - xlat_table_dec_regions_count(ctx, table_base); -} - -#endif /* PLAT_XLAT_TABLES_DYNAMIC */ - -/* - * From the given arguments, it decides which action to take when mapping the - * specified region. - */ -static action_t xlat_tables_map_region_action(const mmap_region_t *mm, - const int desc_type, const unsigned long long dest_pa, - const uintptr_t table_entry_base_va, const unsigned int level) -{ - uintptr_t mm_end_va = mm->base_va + mm->size - 1; - uintptr_t table_entry_end_va = - table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1; - - /* - * The descriptor types allowed depend on the current table level. - */ - - if ((mm->base_va <= table_entry_base_va) && - (mm_end_va >= table_entry_end_va)) { - - /* - * Table entry is covered by region - * -------------------------------- - * - * This means that this table entry can describe the whole - * translation with this granularity in principle. - */ - - if (level == 3) { - /* - * Last level, only page descriptors are allowed. - */ - if (desc_type == PAGE_DESC) { - /* - * There's another region mapped here, don't - * overwrite. - */ - return ACTION_NONE; - } else { - assert(desc_type == INVALID_DESC); - return ACTION_WRITE_BLOCK_ENTRY; - } - - } else { - - /* - * Other levels. Table descriptors are allowed. Block - * descriptors too, but they have some limitations. - */ - - if (desc_type == TABLE_DESC) { - /* There's already a table, recurse into it. */ - return ACTION_RECURSE_INTO_TABLE; - - } else if (desc_type == INVALID_DESC) { - /* - * There's nothing mapped here, create a new - * entry. - * - * Check if the destination granularity allows - * us to use a block descriptor or we need a - * finer table for it. - * - * Also, check if the current level allows block - * descriptors. If not, create a table instead. - */ - if ((dest_pa & XLAT_BLOCK_MASK(level)) || - (level < MIN_LVL_BLOCK_DESC) || - (mm->granularity < XLAT_BLOCK_SIZE(level))) - return ACTION_CREATE_NEW_TABLE; - else - return ACTION_WRITE_BLOCK_ENTRY; - - } else { - /* - * There's another region mapped here, don't - * overwrite. - */ - assert(desc_type == BLOCK_DESC); - - return ACTION_NONE; - } - } - - } else if ((mm->base_va <= table_entry_end_va) || - (mm_end_va >= table_entry_base_va)) { - - /* - * Region partially covers table entry - * ----------------------------------- - * - * This means that this table entry can't describe the whole - * translation, a finer table is needed. - - * There cannot be partial block overlaps in level 3. If that - * happens, some of the preliminary checks when adding the - * mmap region failed to detect that PA and VA must at least be - * aligned to PAGE_SIZE. - */ - assert(level < 3); - - if (desc_type == INVALID_DESC) { - /* - * The block is not fully covered by the region. Create - * a new table, recurse into it and try to map the - * region with finer granularity. - */ - return ACTION_CREATE_NEW_TABLE; - - } else { - assert(desc_type == TABLE_DESC); - /* - * The block is not fully covered by the region, but - * there is already a table here. Recurse into it and - * try to map with finer granularity. - * - * PAGE_DESC for level 3 has the same value as - * TABLE_DESC, but this code can't run on a level 3 - * table because there can't be overlaps in level 3. - */ - return ACTION_RECURSE_INTO_TABLE; - } - } - - /* - * This table entry is outside of the region specified in the arguments, - * don't write anything to it. - */ - return ACTION_NONE; -} - -/* - * Recursive function that writes to the translation tables and maps the - * specified region. On success, it returns the VA of the last byte that was - * succesfully mapped. On error, it returns the VA of the next entry that - * should have been mapped. - */ -static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm, - const uintptr_t table_base_va, - uint64_t *const table_base, - const int table_entries, - const unsigned int level) -{ - assert(level >= ctx->base_level && level <= XLAT_TABLE_LEVEL_MAX); - - uintptr_t mm_end_va = mm->base_va + mm->size - 1; - - uintptr_t table_idx_va; - unsigned long long table_idx_pa; - - uint64_t *subtable; - uint64_t desc; - - int table_idx; - - if (mm->base_va > table_base_va) { - /* Find the first index of the table affected by the region. */ - table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level); - - table_idx = (table_idx_va - table_base_va) >> - XLAT_ADDR_SHIFT(level); - - assert(table_idx < table_entries); - } else { - /* Start from the beginning of the table. */ - table_idx_va = table_base_va; - table_idx = 0; - } - -#if PLAT_XLAT_TABLES_DYNAMIC - if (level > ctx->base_level) - xlat_table_inc_regions_count(ctx, table_base); -#endif - - while (table_idx < table_entries) { - - desc = table_base[table_idx]; - - table_idx_pa = mm->base_pa + table_idx_va - mm->base_va; - - action_t action = xlat_tables_map_region_action(mm, - desc & DESC_MASK, table_idx_pa, table_idx_va, level); - - if (action == ACTION_WRITE_BLOCK_ENTRY) { - - table_base[table_idx] = - xlat_desc(ctx, (uint32_t)mm->attr, table_idx_pa, - level); - - } else if (action == ACTION_CREATE_NEW_TABLE) { - - subtable = xlat_table_get_empty(ctx); - if (subtable == NULL) { - /* Not enough free tables to map this region */ - return table_idx_va; - } - - /* Point to new subtable from this one. */ - table_base[table_idx] = TABLE_DESC | (unsigned long)subtable; - - /* Recurse to write into subtable */ - uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va, - subtable, XLAT_TABLE_ENTRIES, - level + 1); - if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1) - return end_va; - - } else if (action == ACTION_RECURSE_INTO_TABLE) { - - subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); - /* Recurse to write into subtable */ - uintptr_t end_va = xlat_tables_map_region(ctx, mm, table_idx_va, - subtable, XLAT_TABLE_ENTRIES, - level + 1); - if (end_va != table_idx_va + XLAT_BLOCK_SIZE(level) - 1) - return end_va; - - } else { - - assert(action == ACTION_NONE); - - } - - table_idx++; - table_idx_va += XLAT_BLOCK_SIZE(level); - - /* If reached the end of the region, exit */ - if (mm_end_va <= table_idx_va) - break; - } - - return table_idx_va - 1; -} - -void print_mmap(mmap_region_t *const mmap) -{ -#if LOG_LEVEL >= LOG_LEVEL_VERBOSE - tf_printf("mmap:\n"); - mmap_region_t *mm = mmap; - - while (mm->size) { - tf_printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x", - (void *)mm->base_va, mm->base_pa, - mm->size, mm->attr); - tf_printf(" granularity:0x%zx\n", mm->granularity); - ++mm; - }; - tf_printf("\n"); -#endif -} - -/* - * Function that verifies that a region can be mapped. - * Returns: - * 0: Success, the mapping is allowed. - * EINVAL: Invalid values were used as arguments. - * ERANGE: The memory limits were surpassed. - * ENOMEM: There is not enough memory in the mmap array. - * EPERM: Region overlaps another one in an invalid way. - */ -static int mmap_add_region_check(xlat_ctx_t *ctx, const mmap_region_t *mm) -{ - unsigned long long base_pa = mm->base_pa; - uintptr_t base_va = mm->base_va; - size_t size = mm->size; - size_t granularity = mm->granularity; - - unsigned long long end_pa = base_pa + size - 1; - uintptr_t end_va = base_va + size - 1; - - if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) || - !IS_PAGE_ALIGNED(size)) - return -EINVAL; - - if ((granularity != XLAT_BLOCK_SIZE(1)) && - (granularity != XLAT_BLOCK_SIZE(2)) && - (granularity != XLAT_BLOCK_SIZE(3))) { - return -EINVAL; - } - - /* Check for overflows */ - if ((base_pa > end_pa) || (base_va > end_va)) - return -ERANGE; - - if ((base_va + (uintptr_t)size - (uintptr_t)1) > ctx->va_max_address) - return -ERANGE; - - if ((base_pa + (unsigned long long)size - 1ULL) > ctx->pa_max_address) - return -ERANGE; - - /* Check that there is space in the ctx->mmap array */ - if (ctx->mmap[ctx->mmap_num - 1].size != 0) - return -ENOMEM; - - /* Check for PAs and VAs overlaps with all other regions */ - for (mmap_region_t *mm_cursor = ctx->mmap; - mm_cursor->size; ++mm_cursor) { - - uintptr_t mm_cursor_end_va = mm_cursor->base_va - + mm_cursor->size - 1; - - /* - * Check if one of the regions is completely inside the other - * one. - */ - int fully_overlapped_va = - ((base_va >= mm_cursor->base_va) && - (end_va <= mm_cursor_end_va)) || - - ((mm_cursor->base_va >= base_va) && - (mm_cursor_end_va <= end_va)); - - /* - * Full VA overlaps are only allowed if both regions are - * identity mapped (zero offset) or have the same VA to PA - * offset. Also, make sure that it's not the exact same area. - * This can only be done with static regions. - */ - if (fully_overlapped_va) { - -#if PLAT_XLAT_TABLES_DYNAMIC - if ((mm->attr & MT_DYNAMIC) || - (mm_cursor->attr & MT_DYNAMIC)) - return -EPERM; -#endif /* PLAT_XLAT_TABLES_DYNAMIC */ - if ((mm_cursor->base_va - mm_cursor->base_pa) != - (base_va - base_pa)) - return -EPERM; - - if ((base_va == mm_cursor->base_va) && - (size == mm_cursor->size)) - return -EPERM; - - } else { - /* - * If the regions do not have fully overlapping VAs, - * then they must have fully separated VAs and PAs. - * Partial overlaps are not allowed - */ - - unsigned long long mm_cursor_end_pa = - mm_cursor->base_pa + mm_cursor->size - 1; - - int separated_pa = - (end_pa < mm_cursor->base_pa) || - (base_pa > mm_cursor_end_pa); - int separated_va = - (end_va < mm_cursor->base_va) || - (base_va > mm_cursor_end_va); - - if (!(separated_va && separated_pa)) - return -EPERM; - } - } - - return 0; -} - -void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) -{ - mmap_region_t *mm_cursor = ctx->mmap, *mm_destination; - const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num; - mmap_region_t *mm_last; - unsigned long long end_pa = mm->base_pa + mm->size - 1; - uintptr_t end_va = mm->base_va + mm->size - 1; - int ret; - - /* Ignore empty regions */ - if (!mm->size) - return; - - /* Static regions must be added before initializing the xlat tables. */ - assert(!ctx->initialized); - - ret = mmap_add_region_check(ctx, mm); - if (ret != 0) { - ERROR("mmap_add_region_check() failed. error %d\n", ret); - assert(0); - return; - } - - /* - * Find correct place in mmap to insert new region. - * - * 1 - Lower region VA end first. - * 2 - Smaller region size first. - * - * VA 0 0xFF - * - * 1st |------| - * 2nd |------------| - * 3rd |------| - * 4th |---| - * 5th |---| - * 6th |----------| - * 7th |-------------------------------------| - * - * This is required for overlapping regions only. It simplifies adding - * regions with the loop in xlat_tables_init_internal because the outer - * ones won't overwrite block or page descriptors of regions added - * previously. - * - * Overlapping is only allowed for static regions. - */ - - while ((mm_cursor->base_va + mm_cursor->size - 1) < end_va - && mm_cursor->size) - ++mm_cursor; - - while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) && - (mm_cursor->size != 0U) && (mm_cursor->size < mm->size)) - ++mm_cursor; - - /* - * Find the last entry marker in the mmap - */ - mm_last = ctx->mmap; - while ((mm_last->size != 0U) && (mm_last < mm_end)) { - ++mm_last; - } - - /* - * Check if we have enough space in the memory mapping table. - * This shouldn't happen as we have checked in mmap_add_region_check - * that there is free space. - */ - assert(mm_last->size == 0U); - - /* Make room for new region by moving other regions up by one place */ - mm_destination = mm_cursor + 1; - memmove(mm_destination, mm_cursor, - (uintptr_t)mm_last - (uintptr_t)mm_cursor); - - /* - * Check we haven't lost the empty sentinel from the end of the array. - * This shouldn't happen as we have checked in mmap_add_region_check - * that there is free space. - */ - assert(mm_end->size == 0U); - - *mm_cursor = *mm; - - if (end_pa > ctx->max_pa) - ctx->max_pa = end_pa; - if (end_va > ctx->max_va) - ctx->max_va = end_va; -} - -void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, size_t size, - unsigned int attr) -{ - mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); - mmap_add_region_ctx(&tf_xlat_ctx, &mm); -} - - -void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm) -{ - while (mm->size) { - mmap_add_region_ctx(ctx, mm); - mm++; - } -} - -void mmap_add(const mmap_region_t *mm) -{ - mmap_add_ctx(&tf_xlat_ctx, mm); -} - -#if PLAT_XLAT_TABLES_DYNAMIC - -int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm) -{ - mmap_region_t *mm_cursor = ctx->mmap; - mmap_region_t *mm_last = mm_cursor + ctx->mmap_num; - unsigned long long end_pa = mm->base_pa + mm->size - 1; - uintptr_t end_va = mm->base_va + mm->size - 1; - int ret; - - /* Nothing to do */ - if (!mm->size) - return 0; - - /* Now this region is a dynamic one */ - mm->attr |= MT_DYNAMIC; - - ret = mmap_add_region_check(ctx, mm); - if (ret != 0) - return ret; - - /* - * Find the adequate entry in the mmap array in the same way done for - * static regions in mmap_add_region_ctx(). - */ - - while ((mm_cursor->base_va + mm_cursor->size - 1) - < end_va && mm_cursor->size) - ++mm_cursor; - - while ((mm_cursor->base_va + mm_cursor->size - 1 == end_va) - && (mm_cursor->size < mm->size)) - ++mm_cursor; - - /* Make room for new region by moving other regions up by one place */ - memmove(mm_cursor + 1, mm_cursor, - (uintptr_t)mm_last - (uintptr_t)mm_cursor); - - /* - * Check we haven't lost the empty sentinal from the end of the array. - * This shouldn't happen as we have checked in mmap_add_region_check - * that there is free space. - */ - assert(mm_last->size == 0); - - *mm_cursor = *mm; - - /* - * Update the translation tables if the xlat tables are initialized. If - * not, this region will be mapped when they are initialized. - */ - if (ctx->initialized) { - uintptr_t end_va = xlat_tables_map_region(ctx, mm_cursor, - 0, ctx->base_table, ctx->base_table_entries, - ctx->base_level); - - /* Failed to map, remove mmap entry, unmap and return error. */ - if (end_va != mm_cursor->base_va + mm_cursor->size - 1) { - memmove(mm_cursor, mm_cursor + 1, - (uintptr_t)mm_last - (uintptr_t)mm_cursor); - - /* - * Check if the mapping function actually managed to map - * anything. If not, just return now. - */ - if (mm->base_va >= end_va) - return -ENOMEM; - - /* - * Something went wrong after mapping some table - * entries, undo every change done up to this point. - */ - mmap_region_t unmap_mm = { - .base_pa = 0, - .base_va = mm->base_va, - .size = end_va - mm->base_va, - .attr = 0 - }; - xlat_tables_unmap_region(ctx, &unmap_mm, 0, ctx->base_table, - ctx->base_table_entries, ctx->base_level); - - return -ENOMEM; - } - - /* - * Make sure that all entries are written to the memory. There - * is no need to invalidate entries when mapping dynamic regions - * because new table/block/page descriptors only replace old - * invalid descriptors, that aren't TLB cached. - */ - dsbishst(); - } - - if (end_pa > ctx->max_pa) - ctx->max_pa = end_pa; - if (end_va > ctx->max_va) - ctx->max_va = end_va; - - return 0; -} - -int mmap_add_dynamic_region(unsigned long long base_pa, uintptr_t base_va, - size_t size, unsigned int attr) -{ - mmap_region_t mm = MAP_REGION(base_pa, base_va, size, attr); - return mmap_add_dynamic_region_ctx(&tf_xlat_ctx, &mm); -} - -/* - * Removes the region with given base Virtual Address and size from the given - * context. - * - * Returns: - * 0: Success. - * EINVAL: Invalid values were used as arguments (region not found). - * EPERM: Tried to remove a static region. - */ -int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va, - size_t size) -{ - mmap_region_t *mm = ctx->mmap; - mmap_region_t *mm_last = mm + ctx->mmap_num; - int update_max_va_needed = 0; - int update_max_pa_needed = 0; - - /* Check sanity of mmap array. */ - assert(mm[ctx->mmap_num].size == 0); - - while (mm->size) { - if ((mm->base_va == base_va) && (mm->size == size)) - break; - ++mm; - } - - /* Check that the region was found */ - if (mm->size == 0) - return -EINVAL; - - /* If the region is static it can't be removed */ - if (!(mm->attr & MT_DYNAMIC)) - return -EPERM; - - /* Check if this region is using the top VAs or PAs. */ - if ((mm->base_va + mm->size - 1) == ctx->max_va) - update_max_va_needed = 1; - if ((mm->base_pa + mm->size - 1) == ctx->max_pa) - update_max_pa_needed = 1; - - /* Update the translation tables if needed */ - if (ctx->initialized) { - xlat_tables_unmap_region(ctx, mm, 0, ctx->base_table, - ctx->base_table_entries, - ctx->base_level); - xlat_arch_tlbi_va_sync(); - } - - /* Remove this region by moving the rest down by one place. */ - memmove(mm, mm + 1, (uintptr_t)mm_last - (uintptr_t)mm); - - /* Check if we need to update the max VAs and PAs */ - if (update_max_va_needed) { - ctx->max_va = 0; - mm = ctx->mmap; - while (mm->size) { - if ((mm->base_va + mm->size - 1) > ctx->max_va) - ctx->max_va = mm->base_va + mm->size - 1; - ++mm; - } - } - - if (update_max_pa_needed) { - ctx->max_pa = 0; - mm = ctx->mmap; - while (mm->size) { - if ((mm->base_pa + mm->size - 1) > ctx->max_pa) - ctx->max_pa = mm->base_pa + mm->size - 1; - ++mm; - } - } - - return 0; -} - -int mmap_remove_dynamic_region(uintptr_t base_va, size_t size) -{ - return mmap_remove_dynamic_region_ctx(&tf_xlat_ctx, - base_va, size); -} - -#endif /* PLAT_XLAT_TABLES_DYNAMIC */ - -#if LOG_LEVEL >= LOG_LEVEL_VERBOSE - -/* Print the attributes of the specified block descriptor. */ -static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc) -{ - int mem_type_index = ATTR_INDEX_GET(desc); - xlat_regime_t xlat_regime = ctx->xlat_regime; - - if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { - tf_printf("MEM"); - } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) { - tf_printf("NC"); - } else { - assert(mem_type_index == ATTR_DEVICE_INDEX); - tf_printf("DEV"); - } - - const char *priv_str = "(PRIV)"; - const char *user_str = "(USER)"; - - /* - * Showing Privileged vs Unprivileged only makes sense for EL1&0 - * mappings - */ - const char *ro_str = "-RO"; - const char *rw_str = "-RW"; - const char *no_access_str = "-NOACCESS"; - - if (xlat_regime == EL3_REGIME) { - /* For EL3, the AP[2] bit is all what matters */ - tf_printf("%s", (desc & LOWER_ATTRS(AP_RO)) ? ro_str : rw_str); - } else { - const char *ap_str = (desc & LOWER_ATTRS(AP_RO)) ? ro_str : rw_str; - tf_printf("%s", ap_str); - tf_printf("%s", priv_str); - /* - * EL0 can only have the same permissions as EL1 or no - * permissions at all. - */ - tf_printf("%s", - (desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) - ? ap_str : no_access_str); - tf_printf("%s", user_str); - } - - const char *xn_str = "-XN"; - const char *exec_str = "-EXEC"; - - if (xlat_regime == EL3_REGIME) { - /* For EL3, the XN bit is all what matters */ - tf_printf("%s", (UPPER_ATTRS(XN) & desc) ? xn_str : exec_str); - } else { - /* For EL0 and EL1, we need to know who has which rights */ - tf_printf("%s", (UPPER_ATTRS(PXN) & desc) ? xn_str : exec_str); - tf_printf("%s", priv_str); - - tf_printf("%s", (UPPER_ATTRS(UXN) & desc) ? xn_str : exec_str); - tf_printf("%s", user_str); - } - - tf_printf(LOWER_ATTRS(NS) & desc ? "-NS" : "-S"); -} - -static const char * const level_spacers[] = { - "[LV0] ", - " [LV1] ", - " [LV2] ", - " [LV3] " -}; - -static const char *invalid_descriptors_ommited = - "%s(%d invalid descriptors omitted)\n"; - -/* - * Recursive function that reads the translation tables passed as an argument - * and prints their status. - */ -static void xlat_tables_print_internal(xlat_ctx_t *ctx, - const uintptr_t table_base_va, - uint64_t *const table_base, const int table_entries, - const unsigned int level) -{ - assert(level <= XLAT_TABLE_LEVEL_MAX); - - uint64_t desc; - uintptr_t table_idx_va = table_base_va; - int table_idx = 0; - - size_t level_size = XLAT_BLOCK_SIZE(level); - - /* - * Keep track of how many invalid descriptors are counted in a row. - * Whenever multiple invalid descriptors are found, only the first one - * is printed, and a line is added to inform about how many descriptors - * have been omitted. - */ - int invalid_row_count = 0; - - while (table_idx < table_entries) { - - desc = table_base[table_idx]; - - if ((desc & DESC_MASK) == INVALID_DESC) { - - if (invalid_row_count == 0) { - tf_printf("%sVA:%p size:0x%zx\n", - level_spacers[level], - (void *)table_idx_va, level_size); - } - invalid_row_count++; - - } else { - - if (invalid_row_count > 1) { - tf_printf(invalid_descriptors_ommited, - level_spacers[level], - invalid_row_count - 1); - } - invalid_row_count = 0; - - /* - * Check if this is a table or a block. Tables are only - * allowed in levels other than 3, but DESC_PAGE has the - * same value as DESC_TABLE, so we need to check. - */ - if (((desc & DESC_MASK) == TABLE_DESC) && - (level < XLAT_TABLE_LEVEL_MAX)) { - /* - * Do not print any PA for a table descriptor, - * as it doesn't directly map physical memory - * but instead points to the next translation - * table in the translation table walk. - */ - tf_printf("%sVA:%p size:0x%zx\n", - level_spacers[level], - (void *)table_idx_va, level_size); - - uintptr_t addr_inner = desc & TABLE_ADDR_MASK; - - xlat_tables_print_internal(ctx, table_idx_va, - (uint64_t *)addr_inner, - XLAT_TABLE_ENTRIES, level + 1); - } else { - tf_printf("%sVA:%p PA:0x%llx size:0x%zx ", - level_spacers[level], - (void *)table_idx_va, - (unsigned long long)(desc & TABLE_ADDR_MASK), - level_size); - xlat_desc_print(ctx, desc); - tf_printf("\n"); - } - } - - table_idx++; - table_idx_va += level_size; - } - - if (invalid_row_count > 1) { - tf_printf(invalid_descriptors_ommited, - level_spacers[level], invalid_row_count - 1); - } -} - -#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ - -void xlat_tables_print(xlat_ctx_t *ctx) -{ -#if LOG_LEVEL >= LOG_LEVEL_VERBOSE - const char *xlat_regime_str; - if (ctx->xlat_regime == EL1_EL0_REGIME) { - xlat_regime_str = "1&0"; - } else { - assert(ctx->xlat_regime == EL3_REGIME); - xlat_regime_str = "3"; - } - VERBOSE("Translation tables state:\n"); - VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str); - VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address); - VERBOSE(" Max allowed VA: %p\n", (void *) ctx->va_max_address); - VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa); - VERBOSE(" Max mapped VA: %p\n", (void *) ctx->max_va); - - VERBOSE(" Initial lookup level: %i\n", ctx->base_level); - VERBOSE(" Entries @initial lookup level: %i\n", - ctx->base_table_entries); - - int used_page_tables; -#if PLAT_XLAT_TABLES_DYNAMIC - used_page_tables = 0; - for (unsigned int i = 0; i < ctx->tables_num; ++i) { - if (ctx->tables_mapped_regions[i] != 0) - ++used_page_tables; - } -#else - used_page_tables = ctx->next_table; -#endif - VERBOSE(" Used %i sub-tables out of %i (spare: %i)\n", - used_page_tables, ctx->tables_num, - ctx->tables_num - used_page_tables); - - xlat_tables_print_internal(ctx, 0, ctx->base_table, - ctx->base_table_entries, ctx->base_level); -#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ -} - -void init_xlat_tables_ctx(xlat_ctx_t *ctx) -{ - assert(ctx != NULL); - assert(!ctx->initialized); - assert(ctx->xlat_regime == EL3_REGIME || ctx->xlat_regime == EL1_EL0_REGIME); - assert(!is_mmu_enabled_ctx(ctx)); - - mmap_region_t *mm = ctx->mmap; - - print_mmap(mm); - - /* All tables must be zeroed before mapping any region. */ - - for (unsigned int i = 0; i < ctx->base_table_entries; i++) - ctx->base_table[i] = INVALID_DESC; - - for (unsigned int j = 0; j < ctx->tables_num; j++) { -#if PLAT_XLAT_TABLES_DYNAMIC - ctx->tables_mapped_regions[j] = 0; -#endif - for (unsigned int i = 0; i < XLAT_TABLE_ENTRIES; i++) - ctx->tables[j][i] = INVALID_DESC; - } - - while (mm->size) { - uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0, ctx->base_table, - ctx->base_table_entries, ctx->base_level); - - if (end_va != mm->base_va + mm->size - 1) { - ERROR("Not enough memory to map region:\n" - " VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", - (void *)mm->base_va, mm->base_pa, mm->size, mm->attr); - panic(); - } - - mm++; - } - - assert(ctx->pa_max_address <= xlat_arch_get_max_supported_pa()); - assert(ctx->max_va <= ctx->va_max_address); - assert(ctx->max_pa <= ctx->pa_max_address); - - ctx->initialized = 1; - - xlat_tables_print(ctx); -} - -void init_xlat_tables(void) -{ - init_xlat_tables_ctx(&tf_xlat_ctx); -} - -/* - * If dynamic allocation of new regions is disabled then by the time we call the - * function enabling the MMU, we'll have registered all the memory regions to - * map for the system's lifetime. Therefore, at this point we know the maximum - * physical address that will ever be mapped. - * - * If dynamic allocation is enabled then we can't make any such assumption - * because the maximum physical address could get pushed while adding a new - * region. Therefore, in this case we have to assume that the whole address - * space size might be mapped. - */ -#ifdef PLAT_XLAT_TABLES_DYNAMIC -#define MAX_PHYS_ADDR tf_xlat_ctx.pa_max_address -#else -#define MAX_PHYS_ADDR tf_xlat_ctx.max_pa -#endif - -#ifdef AARCH32 - -void enable_mmu_secure(unsigned int flags) -{ - setup_mmu_cfg(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, - tf_xlat_ctx.va_max_address); - enable_mmu_direct(flags); -} - -#else - -void enable_mmu_el1(unsigned int flags) -{ - setup_mmu_cfg(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, - tf_xlat_ctx.va_max_address); - enable_mmu_direct_el1(flags); -} - -void enable_mmu_el3(unsigned int flags) -{ - setup_mmu_cfg(flags, tf_xlat_ctx.base_table, MAX_PHYS_ADDR, - tf_xlat_ctx.va_max_address); - enable_mmu_direct_el3(flags); -} - -#endif /* AARCH32 */ - -/* - * Do a translation table walk to find the block or page descriptor that maps - * virtual_addr. - * - * On success, return the address of the descriptor within the translation - * table. Its lookup level is stored in '*out_level'. - * On error, return NULL. - * - * xlat_table_base - * Base address for the initial lookup level. - * xlat_table_base_entries - * Number of entries in the translation table for the initial lookup level. - * virt_addr_space_size - * Size in bytes of the virtual address space. - */ -static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr, - void *xlat_table_base, - int xlat_table_base_entries, - unsigned long long virt_addr_space_size, - int *out_level) -{ - unsigned int start_level; - uint64_t *table; - int entries; - - VERBOSE("%s(%p)\n", __func__, (void *)virtual_addr); - - start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size); - VERBOSE("Starting translation table walk from level %i\n", start_level); - - table = xlat_table_base; - entries = xlat_table_base_entries; - - for (unsigned int level = start_level; - level <= XLAT_TABLE_LEVEL_MAX; - ++level) { - int idx; - uint64_t desc; - uint64_t desc_type; - - VERBOSE("Table address: %p\n", (void *)table); - - idx = XLAT_TABLE_IDX(virtual_addr, level); - VERBOSE("Index into level %i table: %i\n", level, idx); - if (idx >= entries) { - VERBOSE("Invalid address\n"); - return NULL; - } - - desc = table[idx]; - desc_type = desc & DESC_MASK; - VERBOSE("Descriptor at level %i: 0x%llx\n", level, - (unsigned long long)desc); - - if (desc_type == INVALID_DESC) { - VERBOSE("Invalid entry (memory not mapped)\n"); - return NULL; - } - - if (level == XLAT_TABLE_LEVEL_MAX) { - /* - * There can't be table entries at the final lookup - * level. - */ - assert(desc_type == PAGE_DESC); - VERBOSE("Descriptor mapping a memory page (size: 0x%llx)\n", - (unsigned long long)XLAT_BLOCK_SIZE(XLAT_TABLE_LEVEL_MAX)); - *out_level = level; - return &table[idx]; - } - - if (desc_type == BLOCK_DESC) { - VERBOSE("Descriptor mapping a memory block (size: 0x%llx)\n", - (unsigned long long)XLAT_BLOCK_SIZE(level)); - *out_level = level; - return &table[idx]; - } - - assert(desc_type == TABLE_DESC); - VERBOSE("Table descriptor, continuing xlat table walk...\n"); - table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); - entries = XLAT_TABLE_ENTRIES; - } - - /* - * This shouldn't be reached, the translation table walk should end at - * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop. - */ - assert(0); - - return NULL; -} - - -static int get_mem_attributes_internal(const xlat_ctx_t *ctx, uintptr_t base_va, - uint32_t *attributes, uint64_t **table_entry, - unsigned long long *addr_pa, int *table_level) -{ - uint64_t *entry; - uint64_t desc; - int level; - unsigned long long virt_addr_space_size; - - /* - * Sanity-check arguments. - */ - assert(ctx != NULL); - assert(ctx->initialized); - assert(ctx->xlat_regime == EL1_EL0_REGIME || ctx->xlat_regime == EL3_REGIME); - - virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1; - assert(virt_addr_space_size > 0); - - entry = find_xlat_table_entry(base_va, - ctx->base_table, - ctx->base_table_entries, - virt_addr_space_size, - &level); - if (entry == NULL) { - WARN("Address %p is not mapped.\n", (void *)base_va); - return -EINVAL; - } - - if (addr_pa != NULL) { - *addr_pa = *entry & TABLE_ADDR_MASK; - } - - if (table_entry != NULL) { - *table_entry = entry; - } - - if (table_level != NULL) { - *table_level = level; - } - - desc = *entry; - -#if LOG_LEVEL >= LOG_LEVEL_VERBOSE - VERBOSE("Attributes: "); - xlat_desc_print(ctx, desc); - tf_printf("\n"); -#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ - - assert(attributes != NULL); - *attributes = 0; - - int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; - - if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { - *attributes |= MT_MEMORY; - } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) { - *attributes |= MT_NON_CACHEABLE; - } else { - assert(attr_index == ATTR_DEVICE_INDEX); - *attributes |= MT_DEVICE; - } - - int ap2_bit = (desc >> AP2_SHIFT) & 1; - - if (ap2_bit == AP2_RW) - *attributes |= MT_RW; - - if (ctx->xlat_regime == EL1_EL0_REGIME) { - int ap1_bit = (desc >> AP1_SHIFT) & 1; - if (ap1_bit == AP1_ACCESS_UNPRIVILEGED) - *attributes |= MT_USER; - } - - int ns_bit = (desc >> NS_SHIFT) & 1; - - if (ns_bit == 1) - *attributes |= MT_NS; - - uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime); - - if ((desc & xn_mask) == xn_mask) { - *attributes |= MT_EXECUTE_NEVER; - } else { - assert((desc & xn_mask) == 0); - } - - return 0; -} - - -int get_mem_attributes(const xlat_ctx_t *ctx, uintptr_t base_va, - uint32_t *attributes) -{ - return get_mem_attributes_internal(ctx, base_va, attributes, - NULL, NULL, NULL); -} - - -int change_mem_attributes(xlat_ctx_t *ctx, - uintptr_t base_va, - size_t size, - uint32_t attr) -{ - /* Note: This implementation isn't optimized. */ - - assert(ctx != NULL); - assert(ctx->initialized); - - unsigned long long virt_addr_space_size = - (unsigned long long)ctx->va_max_address + 1; - assert(virt_addr_space_size > 0); - - if (!IS_PAGE_ALIGNED(base_va)) { - WARN("%s: Address %p is not aligned on a page boundary.\n", - __func__, (void *)base_va); - return -EINVAL; - } - - if (size == 0) { - WARN("%s: Size is 0.\n", __func__); - return -EINVAL; - } - - if ((size % PAGE_SIZE) != 0) { - WARN("%s: Size 0x%zx is not a multiple of a page size.\n", - __func__, size); - return -EINVAL; - } - - if (((attr & MT_EXECUTE_NEVER) == 0) && ((attr & MT_RW) != 0)) { - WARN("%s() doesn't allow to remap memory as read-write and executable.\n", - __func__); - return -EINVAL; - } - - int pages_count = size / PAGE_SIZE; - - VERBOSE("Changing memory attributes of %i pages starting from address %p...\n", - pages_count, (void *)base_va); - - uintptr_t base_va_original = base_va; - - /* - * Sanity checks. - */ - for (int i = 0; i < pages_count; ++i) { - uint64_t *entry; - uint64_t desc; - int level; - - entry = find_xlat_table_entry(base_va, - ctx->base_table, - ctx->base_table_entries, - virt_addr_space_size, - &level); - if (entry == NULL) { - WARN("Address %p is not mapped.\n", (void *)base_va); - return -EINVAL; - } - - desc = *entry; - - /* - * Check that all the required pages are mapped at page - * granularity. - */ - if (((desc & DESC_MASK) != PAGE_DESC) || - (level != XLAT_TABLE_LEVEL_MAX)) { - WARN("Address %p is not mapped at the right granularity.\n", - (void *)base_va); - WARN("Granularity is 0x%llx, should be 0x%x.\n", - (unsigned long long)XLAT_BLOCK_SIZE(level), PAGE_SIZE); - return -EINVAL; - } - - /* - * If the region type is device, it shouldn't be executable. - */ - int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; - if (attr_index == ATTR_DEVICE_INDEX) { - if ((attr & MT_EXECUTE_NEVER) == 0) { - WARN("Setting device memory as executable at address %p.", - (void *)base_va); - return -EINVAL; - } - } - - base_va += PAGE_SIZE; - } - - /* Restore original value. */ - base_va = base_va_original; - - VERBOSE("%s: All pages are mapped, now changing their attributes...\n", - __func__); - - for (int i = 0; i < pages_count; ++i) { - - uint32_t old_attr, new_attr; - uint64_t *entry; - int level; - unsigned long long addr_pa; - - get_mem_attributes_internal(ctx, base_va, &old_attr, - &entry, &addr_pa, &level); - - VERBOSE("Old attributes: 0x%x\n", old_attr); - - /* - * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and - * MT_USER/MT_PRIVILEGED are taken into account. Any other - * information is ignored. - */ - - /* Clean the old attributes so that they can be rebuilt. */ - new_attr = old_attr & ~(MT_RW|MT_EXECUTE_NEVER|MT_USER); - - /* - * Update attributes, but filter out the ones this function - * isn't allowed to change. - */ - new_attr |= attr & (MT_RW|MT_EXECUTE_NEVER|MT_USER); - - VERBOSE("New attributes: 0x%x\n", new_attr); - - /* - * The break-before-make sequence requires writing an invalid - * descriptor and making sure that the system sees the change - * before writing the new descriptor. - */ - *entry = INVALID_DESC; - - /* Invalidate any cached copy of this mapping in the TLBs. */ - xlat_arch_tlbi_va_regime(base_va, ctx->xlat_regime); - - /* Ensure completion of the invalidation. */ - xlat_arch_tlbi_va_sync(); - - /* Write new descriptor */ - *entry = xlat_desc(ctx, new_attr, addr_pa, level); - - base_va += PAGE_SIZE; - } - - /* Ensure that the last descriptor writen is seen by the system. */ - dsbish(); - - return 0; -} diff --git a/lib/xlat_tables_v2/xlat_tables_private.h b/lib/xlat_tables_v2/xlat_tables_private.h index 777189f..4a54ec5 100644 --- a/lib/xlat_tables_v2/xlat_tables_private.h +++ b/lib/xlat_tables_v2/xlat_tables_private.h @@ -50,7 +50,7 @@ * S-EL1. */ void xlat_arch_tlbi_va(uintptr_t va); -void xlat_arch_tlbi_va_regime(uintptr_t va, xlat_regime_t xlat_regime); +void xlat_arch_tlbi_va_regime(uintptr_t va, int xlat_regime); /* * This function has to be called at the end of any code that uses the function @@ -59,7 +59,7 @@ void xlat_arch_tlbi_va_sync(void); /* Print VA, PA, size and attributes of all regions in the mmap array. */ -void print_mmap(mmap_region_t *const mmap); +void xlat_mmap_print(mmap_region_t *const mmap); /* * Print the current state of the translation tables by reading them from @@ -68,6 +68,12 @@ void xlat_tables_print(xlat_ctx_t *ctx); /* + * Returns a block/page table descriptor for the given level and attributes. + */ +uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr, + unsigned long long addr_pa, int level); + +/* * Architecture-specific initialization code. */ diff --git a/lib/xlat_tables_v2/xlat_tables_utils.c b/lib/xlat_tables_v2/xlat_tables_utils.c new file mode 100644 index 0000000..5a78434 --- /dev/null +++ b/lib/xlat_tables_v2/xlat_tables_utils.c @@ -0,0 +1,562 @@ +/* + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xlat_tables_private.h" + +#if LOG_LEVEL < LOG_LEVEL_VERBOSE + +void xlat_mmap_print(__unused mmap_region_t *const mmap) +{ + /* Empty */ +} + +void xlat_tables_print(__unused xlat_ctx_t *ctx) +{ + /* Empty */ +} + +#else /* if LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + +void xlat_mmap_print(mmap_region_t *const mmap) +{ + tf_printf("mmap:\n"); + const mmap_region_t *mm = mmap; + + while (mm->size != 0U) { + tf_printf(" VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x " + "granularity:0x%zx\n", mm->base_va, mm->base_pa, + mm->size, mm->attr, mm->granularity); + ++mm; + }; + tf_printf("\n"); +} + +/* Print the attributes of the specified block descriptor. */ +static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc) +{ + int mem_type_index = ATTR_INDEX_GET(desc); + int xlat_regime = ctx->xlat_regime; + + if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { + tf_printf("MEM"); + } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) { + tf_printf("NC"); + } else { + assert(mem_type_index == ATTR_DEVICE_INDEX); + tf_printf("DEV"); + } + + if (xlat_regime == EL3_REGIME) { + /* For EL3 only check the AP[2] and XN bits. */ + tf_printf((desc & LOWER_ATTRS(AP_RO)) ? "-RO" : "-RW"); + tf_printf((desc & UPPER_ATTRS(XN)) ? "-XN" : "-EXEC"); + } else { + assert(xlat_regime == EL1_EL0_REGIME); + /* + * For EL0 and EL1: + * - In AArch64 PXN and UXN can be set independently but in + * AArch32 there is no UXN (XN affects both privilege levels). + * For consistency, we set them simultaneously in both cases. + * - RO and RW permissions must be the same in EL1 and EL0. If + * EL0 can access that memory region, so can EL1, with the + * same permissions. + */ +#if ENABLE_ASSERTIONS + uint64_t xn_mask = xlat_arch_regime_get_xn_desc(EL1_EL0_REGIME); + uint64_t xn_perm = desc & xn_mask; + + assert((xn_perm == xn_mask) || (xn_perm == 0ULL)); +#endif + tf_printf((desc & LOWER_ATTRS(AP_RO)) ? "-RO" : "-RW"); + /* Only check one of PXN and UXN, the other one is the same. */ + tf_printf((desc & UPPER_ATTRS(PXN)) ? "-XN" : "-EXEC"); + /* + * Privileged regions can only be accessed from EL1, user + * regions can be accessed from EL1 and EL0. + */ + tf_printf((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) + ? "-USER" : "-PRIV"); + } + + tf_printf(LOWER_ATTRS(NS) & desc ? "-NS" : "-S"); +} + +static const char * const level_spacers[] = { + "[LV0] ", + " [LV1] ", + " [LV2] ", + " [LV3] " +}; + +static const char *invalid_descriptors_ommited = + "%s(%d invalid descriptors omitted)\n"; + +/* + * Recursive function that reads the translation tables passed as an argument + * and prints their status. + */ +static void xlat_tables_print_internal(xlat_ctx_t *ctx, + const uintptr_t table_base_va, + uint64_t *const table_base, const int table_entries, + const unsigned int level) +{ + assert(level <= XLAT_TABLE_LEVEL_MAX); + + uint64_t desc; + uintptr_t table_idx_va = table_base_va; + int table_idx = 0; + + size_t level_size = XLAT_BLOCK_SIZE(level); + + /* + * Keep track of how many invalid descriptors are counted in a row. + * Whenever multiple invalid descriptors are found, only the first one + * is printed, and a line is added to inform about how many descriptors + * have been omitted. + */ + int invalid_row_count = 0; + + while (table_idx < table_entries) { + + desc = table_base[table_idx]; + + if ((desc & DESC_MASK) == INVALID_DESC) { + + if (invalid_row_count == 0) { + tf_printf("%sVA:%p size:0x%zx\n", + level_spacers[level], + (void *)table_idx_va, level_size); + } + invalid_row_count++; + + } else { + + if (invalid_row_count > 1) { + tf_printf(invalid_descriptors_ommited, + level_spacers[level], + invalid_row_count - 1); + } + invalid_row_count = 0; + + /* + * Check if this is a table or a block. Tables are only + * allowed in levels other than 3, but DESC_PAGE has the + * same value as DESC_TABLE, so we need to check. + */ + if (((desc & DESC_MASK) == TABLE_DESC) && + (level < XLAT_TABLE_LEVEL_MAX)) { + /* + * Do not print any PA for a table descriptor, + * as it doesn't directly map physical memory + * but instead points to the next translation + * table in the translation table walk. + */ + tf_printf("%sVA:%p size:0x%zx\n", + level_spacers[level], + (void *)table_idx_va, level_size); + + uintptr_t addr_inner = desc & TABLE_ADDR_MASK; + + xlat_tables_print_internal(ctx, table_idx_va, + (uint64_t *)addr_inner, + XLAT_TABLE_ENTRIES, level + 1); + } else { + tf_printf("%sVA:%p PA:0x%llx size:0x%zx ", + level_spacers[level], + (void *)table_idx_va, + (unsigned long long)(desc & TABLE_ADDR_MASK), + level_size); + xlat_desc_print(ctx, desc); + tf_printf("\n"); + } + } + + table_idx++; + table_idx_va += level_size; + } + + if (invalid_row_count > 1) { + tf_printf(invalid_descriptors_ommited, + level_spacers[level], invalid_row_count - 1); + } +} + +void xlat_tables_print(xlat_ctx_t *ctx) +{ + const char *xlat_regime_str; + if (ctx->xlat_regime == EL1_EL0_REGIME) { + xlat_regime_str = "1&0"; + } else { + assert(ctx->xlat_regime == EL3_REGIME); + xlat_regime_str = "3"; + } + VERBOSE("Translation tables state:\n"); + VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str); + VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address); + VERBOSE(" Max allowed VA: %p\n", (void *) ctx->va_max_address); + VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa); + VERBOSE(" Max mapped VA: %p\n", (void *) ctx->max_va); + + VERBOSE(" Initial lookup level: %i\n", ctx->base_level); + VERBOSE(" Entries @initial lookup level: %i\n", + ctx->base_table_entries); + + int used_page_tables; +#if PLAT_XLAT_TABLES_DYNAMIC + used_page_tables = 0; + for (unsigned int i = 0; i < ctx->tables_num; ++i) { + if (ctx->tables_mapped_regions[i] != 0) + ++used_page_tables; + } +#else + used_page_tables = ctx->next_table; +#endif + VERBOSE(" Used %i sub-tables out of %i (spare: %i)\n", + used_page_tables, ctx->tables_num, + ctx->tables_num - used_page_tables); + + xlat_tables_print_internal(ctx, 0, ctx->base_table, + ctx->base_table_entries, ctx->base_level); +} + +#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + +/* + * Do a translation table walk to find the block or page descriptor that maps + * virtual_addr. + * + * On success, return the address of the descriptor within the translation + * table. Its lookup level is stored in '*out_level'. + * On error, return NULL. + * + * xlat_table_base + * Base address for the initial lookup level. + * xlat_table_base_entries + * Number of entries in the translation table for the initial lookup level. + * virt_addr_space_size + * Size in bytes of the virtual address space. + */ +static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr, + void *xlat_table_base, + int xlat_table_base_entries, + unsigned long long virt_addr_space_size, + int *out_level) +{ + unsigned int start_level; + uint64_t *table; + int entries; + + start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size); + + table = xlat_table_base; + entries = xlat_table_base_entries; + + for (unsigned int level = start_level; + level <= XLAT_TABLE_LEVEL_MAX; + ++level) { + int idx; + uint64_t desc; + uint64_t desc_type; + + idx = XLAT_TABLE_IDX(virtual_addr, level); + if (idx >= entries) { + WARN("Missing xlat table entry at address 0x%lx\n", + virtual_addr); + return NULL; + } + + desc = table[idx]; + desc_type = desc & DESC_MASK; + + if (desc_type == INVALID_DESC) { + VERBOSE("Invalid entry (memory not mapped)\n"); + return NULL; + } + + if (level == XLAT_TABLE_LEVEL_MAX) { + /* + * Only page descriptors allowed at the final lookup + * level. + */ + assert(desc_type == PAGE_DESC); + *out_level = level; + return &table[idx]; + } + + if (desc_type == BLOCK_DESC) { + *out_level = level; + return &table[idx]; + } + + assert(desc_type == TABLE_DESC); + table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); + entries = XLAT_TABLE_ENTRIES; + } + + /* + * This shouldn't be reached, the translation table walk should end at + * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop. + */ + assert(0); + + return NULL; +} + + +static int get_mem_attributes_internal(const xlat_ctx_t *ctx, uintptr_t base_va, + uint32_t *attributes, uint64_t **table_entry, + unsigned long long *addr_pa, int *table_level) +{ + uint64_t *entry; + uint64_t desc; + int level; + unsigned long long virt_addr_space_size; + + /* + * Sanity-check arguments. + */ + assert(ctx != NULL); + assert(ctx->initialized); + assert(ctx->xlat_regime == EL1_EL0_REGIME || ctx->xlat_regime == EL3_REGIME); + + virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1; + assert(virt_addr_space_size > 0); + + entry = find_xlat_table_entry(base_va, + ctx->base_table, + ctx->base_table_entries, + virt_addr_space_size, + &level); + if (entry == NULL) { + WARN("Address %p is not mapped.\n", (void *)base_va); + return -EINVAL; + } + + if (addr_pa != NULL) { + *addr_pa = *entry & TABLE_ADDR_MASK; + } + + if (table_entry != NULL) { + *table_entry = entry; + } + + if (table_level != NULL) { + *table_level = level; + } + + desc = *entry; + +#if LOG_LEVEL >= LOG_LEVEL_VERBOSE + VERBOSE("Attributes: "); + xlat_desc_print(ctx, desc); + tf_printf("\n"); +#endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ + + assert(attributes != NULL); + *attributes = 0; + + int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; + + if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { + *attributes |= MT_MEMORY; + } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) { + *attributes |= MT_NON_CACHEABLE; + } else { + assert(attr_index == ATTR_DEVICE_INDEX); + *attributes |= MT_DEVICE; + } + + int ap2_bit = (desc >> AP2_SHIFT) & 1; + + if (ap2_bit == AP2_RW) + *attributes |= MT_RW; + + if (ctx->xlat_regime == EL1_EL0_REGIME) { + int ap1_bit = (desc >> AP1_SHIFT) & 1; + if (ap1_bit == AP1_ACCESS_UNPRIVILEGED) + *attributes |= MT_USER; + } + + int ns_bit = (desc >> NS_SHIFT) & 1; + + if (ns_bit == 1) + *attributes |= MT_NS; + + uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime); + + if ((desc & xn_mask) == xn_mask) { + *attributes |= MT_EXECUTE_NEVER; + } else { + assert((desc & xn_mask) == 0); + } + + return 0; +} + + +int get_mem_attributes(const xlat_ctx_t *ctx, uintptr_t base_va, + uint32_t *attributes) +{ + return get_mem_attributes_internal(ctx, base_va, attributes, + NULL, NULL, NULL); +} + + +int change_mem_attributes(xlat_ctx_t *ctx, + uintptr_t base_va, + size_t size, + uint32_t attr) +{ + /* Note: This implementation isn't optimized. */ + + assert(ctx != NULL); + assert(ctx->initialized); + + unsigned long long virt_addr_space_size = + (unsigned long long)ctx->va_max_address + 1; + assert(virt_addr_space_size > 0); + + if (!IS_PAGE_ALIGNED(base_va)) { + WARN("%s: Address %p is not aligned on a page boundary.\n", + __func__, (void *)base_va); + return -EINVAL; + } + + if (size == 0) { + WARN("%s: Size is 0.\n", __func__); + return -EINVAL; + } + + if ((size % PAGE_SIZE) != 0) { + WARN("%s: Size 0x%zx is not a multiple of a page size.\n", + __func__, size); + return -EINVAL; + } + + if (((attr & MT_EXECUTE_NEVER) == 0) && ((attr & MT_RW) != 0)) { + WARN("%s: Mapping memory as read-write and executable not allowed.\n", + __func__); + return -EINVAL; + } + + int pages_count = size / PAGE_SIZE; + + VERBOSE("Changing memory attributes of %i pages starting from address %p...\n", + pages_count, (void *)base_va); + + uintptr_t base_va_original = base_va; + + /* + * Sanity checks. + */ + for (int i = 0; i < pages_count; ++i) { + uint64_t *entry; + uint64_t desc; + int level; + + entry = find_xlat_table_entry(base_va, + ctx->base_table, + ctx->base_table_entries, + virt_addr_space_size, + &level); + if (entry == NULL) { + WARN("Address %p is not mapped.\n", (void *)base_va); + return -EINVAL; + } + + desc = *entry; + + /* + * Check that all the required pages are mapped at page + * granularity. + */ + if (((desc & DESC_MASK) != PAGE_DESC) || + (level != XLAT_TABLE_LEVEL_MAX)) { + WARN("Address %p is not mapped at the right granularity.\n", + (void *)base_va); + WARN("Granularity is 0x%llx, should be 0x%x.\n", + (unsigned long long)XLAT_BLOCK_SIZE(level), PAGE_SIZE); + return -EINVAL; + } + + /* + * If the region type is device, it shouldn't be executable. + */ + int attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; + if (attr_index == ATTR_DEVICE_INDEX) { + if ((attr & MT_EXECUTE_NEVER) == 0) { + WARN("Setting device memory as executable at address %p.", + (void *)base_va); + return -EINVAL; + } + } + + base_va += PAGE_SIZE; + } + + /* Restore original value. */ + base_va = base_va_original; + + for (int i = 0; i < pages_count; ++i) { + + uint32_t old_attr, new_attr; + uint64_t *entry; + int level; + unsigned long long addr_pa; + + get_mem_attributes_internal(ctx, base_va, &old_attr, + &entry, &addr_pa, &level); + + /* + * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and + * MT_USER/MT_PRIVILEGED are taken into account. Any other + * information is ignored. + */ + + /* Clean the old attributes so that they can be rebuilt. */ + new_attr = old_attr & ~(MT_RW | MT_EXECUTE_NEVER | MT_USER); + + /* + * Update attributes, but filter out the ones this function + * isn't allowed to change. + */ + new_attr |= attr & (MT_RW | MT_EXECUTE_NEVER | MT_USER); + + /* + * The break-before-make sequence requires writing an invalid + * descriptor and making sure that the system sees the change + * before writing the new descriptor. + */ + *entry = INVALID_DESC; + + /* Invalidate any cached copy of this mapping in the TLBs. */ + xlat_arch_tlbi_va_regime(base_va, ctx->xlat_regime); + + /* Ensure completion of the invalidation. */ + xlat_arch_tlbi_va_sync(); + + /* Write new descriptor */ + *entry = xlat_desc(ctx, new_attr, addr_pa, level); + + base_va += PAGE_SIZE; + } + + /* Ensure that the last descriptor writen is seen by the system. */ + dsbish(); + + return 0; +}