diff --git a/arch/arm/boards/karo-tx53/lowlevel.c b/arch/arm/boards/karo-tx53/lowlevel.c
index 32d2c8f..f47f260 100644
--- a/arch/arm/boards/karo-tx53/lowlevel.c
+++ b/arch/arm/boards/karo-tx53/lowlevel.c
@@ -14,7 +14,7 @@
 	 * so do the PLL setup here.
 	 */
 	if (IS_ENABLED(CONFIG_TX53_REV_XX30))
-		imx53_init_lowlevel(800);
+		imx53_init_lowlevel_early(800);
 
 	imx53_barebox_entry(0);
 }
diff --git a/arch/arm/mach-imx/imx53.c b/arch/arm/mach-imx/imx53.c
index b06a5d0..193406f 100644
--- a/arch/arm/mach-imx/imx53.c
+++ b/arch/arm/mach-imx/imx53.c
@@ -75,7 +75,7 @@
 }
 postcore_initcall(imx53_init);
 
-void imx53_init_lowlevel(unsigned int cpufreq_mhz)
+void imx53_init_lowlevel_early(unsigned int cpufreq_mhz)
 {
 	void __iomem *ccm = (void __iomem *)MX53_CCM_BASE_ADDR;
 	u32 r;
@@ -188,8 +188,12 @@
 	writel(0xffffffff, ccm + MX5_CCM_CCGR6);
 	writel(0xffffffff, ccm + MX53_CCM_CCGR7);
 
-	if (!IS_ENABLED(__PBL__))
-		clock_notifier_call_chain();
-
 	writel(0, ccm + MX5_CCM_CCDR);
 }
+
+void imx53_init_lowlevel(unsigned int cpufreq_mhz)
+{
+	imx53_init_lowlevel_early(cpufreq_mhz);
+
+	clock_notifier_call_chain();
+}
diff --git a/arch/arm/mach-imx/include/mach/imx5.h b/arch/arm/mach-imx/include/mach/imx5.h
index ff570c4..7f5c2ef 100644
--- a/arch/arm/mach-imx/include/mach/imx5.h
+++ b/arch/arm/mach-imx/include/mach/imx5.h
@@ -3,6 +3,7 @@
 
 void imx51_init_lowlevel(unsigned int cpufreq_mhz);
 void imx53_init_lowlevel(unsigned int cpufreq_mhz);
+void imx53_init_lowlevel_early(unsigned int cpufreq_mhz);
 void imx5_init_lowlevel(void);
 
 void imx5_setup_pll(void __iomem *base, int freq, u32 op, u32 mfd, u32 mfn);