&\,\alpha  nJ^*\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}+\alpha\mathrm{\bold{D}}_2(\gamma,\gamma^*) +  (1+\vartheta)\lambda\mtbb\big\|(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}\nonumber\\\leq &\, \bold{\Delta}_{l}(\mu,\mu^*)+(1-\vartheta)\lambda\mtbb\big\|(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\big\|_{2,1}+K\lambda^2J^*.
\end{align}

\begin{align}\label{l1infbound}
\|\hat{\bar{\beta}}-\bar{\beta}^*\|_{2,\infty}
\leq C\frac{\sqrt{K\alpha}\lor\vartheta}{\alpha\sqrt{n}}A\Big\{\omega_{\bar\eta}+\frac{1}{\sqrt{J^*}}(\omega_{\bar m}+\omega_{\varsigma})\Big\}(\log p)^{\frac{1}{q}}
\end{align}

with probability at least $1-Cp^{-c(A \vartheta)^q}-C/\varphi(cA \vartheta (\log p)^{\frac{1}{q}}) $, where  $C,c$ are positive constants.

The element-wise error bound (<ref>) together with a signal strength condition guarantees faithful variable
selection  with high probability; see  Remark <ref>.

By the  optimality of $\hat{\mu}$: $l(\hat{\mu})+\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\leq l(\mu^*)+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}$,  we obtain the basic inequality

\begin{align}\label{thmvarselbasicineq}
\bold{\Delta}_{l}(\hat{\mu},\mu^*)\leq  \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}.
\end{align}

We follow the same lines as   in Section <ref> to bound the  stochastic terms on the right-hand side of (<ref>).

\begin{align}
\lambda_0=A\omega_{\bar\eta}(\log p)^{\frac{1}{q}}
\end{align}

for some large enough $A>0$ and $\lambda=\lambda_0/b'$ for some $b'>0$. Similar to the previous analysis, we have  with probability at least $1-Cp^{-A^q}$,
\begin{align}\label{thmvarseleta}
&\,\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\nonumber\\\leq &\, \big( 1+c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(1- c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1},
\end{align}

where the constant $c_0\geq \sqrt{2}$. Moreover, for any $a>0$, we get

\begin{align}\label{thmvarselmsigma}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+  \langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle\leq  acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma}+\frac{1}{2a}\big\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\big\},
\end{align}

with probability at least $1-1/\psi_q(cL)-C/\varphi(cL')$, where $L,L'>0$ can be customized.

Plugging the bounds of (<ref>) and (<ref>) into  (<ref>),  we get

\begin{align*}\label{thmvarselmedineq}
&\, \bold{\Delta}_{l}(\hat{\mu},\mu^*)-\frac{1}{2a}\big\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\big\} \\
\leq &\, \big( 1+c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(1- c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1}+acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma},
\end{align*}

with probability at least
$1-Cp^{-A^q}-1/\psi_q(cL)-C/\varphi(cL')$, where  $C,c$ are  positive constants.

With  the regularity condition and and $b'=\vartheta/c_0$, we obtain

\begin{align}\label{prop2ineq}
&\, \alpha \big(nJ^*\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}+\mathrm{\bold{D}}_2(\gamma,\gamma^*)\big)-\frac{1}{2a}\big\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\big\}\nonumber\\\leq &\, K\lambda^2J^*+acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma}.
\end{align}

Setting  $a=1/\alpha$ and using $\lambda=\lambda_0/b'=(c_0/\vartheta)\lambda_0$, we obtain

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}\leq &\, \frac{Kc_0^2}{n\alpha\vartheta^2}A^2\omega^2_{\bar\eta}(\log p)^{\frac{2}{q}}+\frac{cL^2\omega^2_{\bar m}}{n\alpha^2J^*}+\frac{2cL^{'2}\omega^2_{\varsigma}}{n\alpha^2J^*}
\end{align}

with probability at least $1-Cp^{-A^q}-1/\psi_q(cL)-C/\varphi(cL')$.
With $L=L'=A (\log p)^{\frac{1}{q}}$,

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}\leq &\, \frac{Kc_0^2}{n\alpha\vartheta^2}A^2\omega^2_{\bar\eta}(\log p)^{\frac{2}{q}}+\frac{c}{n\alpha^2J^*}A^2(\log p)^{\frac{2}{q}}(\omega^2_{\bar m}+\omega^2_{\varsigma})\nonumber\\\leq&\, C\frac{A^2}{n\alpha^2\vartheta^2}(\log p)^{\frac{2}{q}}\big\{K\alpha\omega^2_{\bar\eta}+\frac{\vartheta^2}{J^*}(\omega^2_{\bar m}+\omega^2_\varsigma)\big\},
\end{align}

which implies

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|_{2,\infty}\leq C \frac{\sqrt{K\alpha}\lor\vartheta}{\sqrt{n}\alpha\vartheta}A(\log p)^{\frac{1}{q}}\big\{\omega_{\bar\eta}+\frac{1}{\sqrt{J^*}}(\omega_{\bar m}+\omega_{\varsigma})\big\},
\end{align}

with probability at least $1-Cp^{-cA^q}-C/\varphi(cA (\log p)^{ {1}/{q}})$.
Hence by taking $A'=A/\vartheta$ and $\lambda=A'\omega_{\bar \eta}(\log p)^{\frac{1}{q}}$,  we have

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|_{2,\infty}\leq C \frac{\sqrt{K\alpha}\lor\vartheta}{\sqrt{n}\alpha}A'(\log p)^{\frac{1}{q}}\big\{\omega_{\bar\eta}+\frac{1}{\sqrt{J^*}}(\omega_{\bar m}+\omega_{\varsigma})\big\},
\end{align}

with probability at least $1-Cp^{-c(A'\vartheta)^q}-C/\varphi(cA'\vartheta (\log p)^{\frac{1}{q}})$. The proof is complete.

§.§ Analysis of a General Penalty

Consider  the following problem associated with
a general sparsity-inducing penalty $P$

\begin{align}\label{quadraticpenaltylossap}
\end{align}

where   $\|\cdot\|_{2,P}$ is  short  for $\|\cdot\|_{2,P(\cdot;\lambda)}$  and $\|\bar{\beta}\|_{2,P(\cdot;\lambda)}:=\sum^p_{k=1}P(\|\bar{\beta}_k\|_2;\lambda)$.

Since a sparsity-inducing penalty necessarily possesses thresholding power, we assume,  without loss of generality,  that $P(\cdot;\lambda)\geq P_{H}(\cdot;\lambda)$ throughout the subsection, where the “hard penalty” $ P_{H}(t;\lambda):=(-t^2/2+\lambda |t|)1_{|t|<\lambda}+(\lambda^2/2)1_{|t|\geq\lambda}$ is induced by the hard-thresholding, and

$P_{2,H}(\bar{\beta};\lambda):=\sum^p_{k=1}P_{H}(\|\bar{\beta}_k\|_2;\lambda)$ (see [56] for more details).   Furthermore, a penalty is referred to as subadditive if it satisfies  $P(t+s;\lambda)\leq P(t;\lambda)+P(s;\lambda)$.
In fact, when $P(t)$ is concave on $\mathbb R_+$ and $P(0)=0$,  $P(|t|)$ is necessarily subadditive. Well-known examples   include the widely used $\ell_1$-penalty, $\ell_0$-penalty, SCAD, MCP, and bridge $\ell_r$ $(0< r<1)$  in the literature.

Assume that the effective noises $\epsilon_{\bar\eta},\epsilon_{\bar m}$, and $\epsilon_\varsigma$ satisfy    $\|\epsilon_{\bar\eta}\|_{\psi_2}\leq \omega_{\bar\eta},\|\epsilon_{\bar m}\|_{\psi_2}\leq \omega_{\bar m}$, and $\|\epsilon_\varsigma\|_{\varphi}\leq \omega_\varsigma$ where  $\{\varphi^{-1}(\cdot)\}^2$  concave or $\{\varphi^{-1}(t)\}^2\lesssim t$ on $\mathbb{R}_+$ (for example, $\varphi$ can be an $L_q$-norm  with $q\geq 2$ or a $\psi_q$-norm with $q>0$).
Let  $\hat\unv$ denote the  optimal solution  of (<ref>) with $\varrho\geq \rhotwo: = \|\bar{X}\|_2$.

(a) Let $\lambda=A\omega_{\bar\eta}\sqrt{\log(ep)}/\sqrt{\etaa\land 1}$ with A a sufficiently large constant.  Then the following bound always holds

\begin{align}
\mathbb{E}\bigg\{ \bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\hat{\mu},\mu^*)\bigg\}\lesssim&\,\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}+\frac{1}{\etaa\land 1}\omega^2_{\bar\eta}+\frac{1}{\etaa}\big(\omega^2_{\bar m}+\omega_\varsigma^2\big)+\etaa\|\mu^*\|_{2}^2.
\end{align}

(b)  Let $P$ be a  sub-additive penalty.
Assume that there exist some $\alpha\geq0$, $\vartheta>0$, and some large $K>0$  such that

\begin{align}\label{alterregugeneral}
\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+ (1+\vartheta)\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\|_{2,P}\leq \bold{\Delta}_{l}(\mu,\mu^*)+(1-\vartheta)\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\|_{2,P}+K\lambda^2J^*,
\end{align}

for all $\unv^T=[\bar\beta^T,\bar m^T,\varsigma^T]$,
where $\lambda=A\omega_{\bar\eta}\sqrt{\log(ep)}/\sqrt{\{(\etaa+\alpha)\land \vartheta\}\vartheta}$ with A a sufficiently large constant.

\begin{align}
\mathbb{E}\,\mathrm{\bold{ D}}_2(\hat\mu,\mu^*)\lesssim \ &\,\frac{\omega_{\bar\eta}^2}{(\etaa+\alpha)\{(\etaa+\alpha)\land \vartheta\}\vartheta} \, \big\{KA^2J^*\log(ep)+\vartheta\big\}\nonumber\\ &+\frac{\omega^2_{\bar m}+\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}}{(\etaa+\alpha)^2}  +\big(1\land \frac{\etaa}{\alpha}\big)^2\|\mu^*\|_{2}^2.
\end{align}

According to  the proof below, (<ref>) can be relaxed to
$\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*};\lambda\big)+\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\|_{2,P}\nonumber \leq    \bold{\Delta}_{l}(\mu,\mu^*)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\|_{2,P}-\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}};\lambda\big)+K\lambda^2J^*
$, or $
\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*);\lambda\big)+\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+\| \varrho\bar{\beta}^*\|_{2,P}\leq  \bold{\Delta}_{l}(\mu,\mu^*)+\|\varrho\bar{\beta}\|_{2,P}+K\lambda^2J^*
$ if $P$ is not subadditive.

By definition,   $l(\hat{\zeta})+\frac{\etaa}{2}\|\hat\mu\|^2_2+\big\|\mtbb\hat {\bar\beta}\big\|_{2,P}\leq l(\zeta^*)+\frac{\etaa}{2}\|\mu^*\|^2_2+\big\| \mtbb\bar{\beta}^*\big\|_{2,P}$, which means

\begin{align}\label{basiceq}
&\,\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\|\hat{\mu}-\mu^*\|^2_2\nonumber\\\leq &\, \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle+\langle-\etaa\mu^*,\hat{\mu}-\mu^*\rangle\nonumber\\&\,+\big\| \mtbb\bar{\beta}^*\big\|_{2,P}-\big\|\mtbb\hat {\bar\beta}\big\|_{2,P}\nonumber\\\leq &\,\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle+\frac{b}{2}\|\hat\mu-\mu^*\|^2_2+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2\nonumber\\&\,+\big\| \mtbb\bar{\beta}^*\big\|_{2,P}-\big\|\mtbb\hat {\bar\beta}\big\|_{2,P},
\end{align}

where $b$ can be any positive number.

To bound the stochastic term $\langle\epsilon_{\bar\eta},\bar{X}(\hat{\bar{\beta}}-\bar{\beta}^*)\rangle$,
define  $\lambda_0=\omega_{\bar\eta}\sqrt{\log(ep)}$  and

\begin{align}\label{Rdef}
R=&\,\sup_{\bar{\beta}\in \mathbb{R}^{2p}}\Big\{\langle\epsilon_{\bar\eta},\bar{
\end{align}

By Lemma <ref>, for any $a'\geq 2b'>0$ and a sufficiently large constant $A$,  $\mathbb{P}(R\geq a'\omega^2_{\bar\eta} t)\leq C\exp(-ct)p^{{-cA^2}}$. Therefore, $\mathbb{E}[R]\lesssim a' \omega^2_{\bar\eta}$.

Next, we bound      $\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle$ by
\begin{align}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle=\langle P_{1_n}\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle\leq | u^T\epsilon_{\bar m}|\cdot\|\hat {\bar m}-\bar m^*\|_2\leq a(u^T\epsilon_{\bar m})^2+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)
\end{align}

for any $a>0$,
where $P_{1_n}=uu^T$ with $u=(1/\sqrt{n})1_n$. By the assumption on $\epsilon_{\bar m}$, $\|(u^T\epsilon_{\bar m})^2\|_{\psi_1}\lesssim\|u^T\epsilon_{\bar m}\|^2_{\psi_2}\leq \omega^2_{\bar m}$, from which it follows that $ \mathbb{E}[ (u^T\epsilon_{\bar m})^2]\leq C\omega^2_{\bar m}$.
Similarly, for the stochastic term $\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle$ with $\epsilon_{\varsigma}=[\epsilon^T_{\frac{1}{\sigma}1_n},\epsilon^T_{\frac{1}{\nu}1_n}]^T$ and $\hat {\varsigma}-\varsigma^*=[(1/\hat\sigma-1/\sigma^*)1^T_n,(1/\hat\nu-1/\nu^*)1^T_n]^T$, we have

\begin{align}
&\,\big\langle\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\=&\,\big\langle P_{1_n}\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle P_{1_n}\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\
\leq &\, a(u^T\epsilon_{\frac{1}{\sigma}1_n})^2+a(u^T\epsilon_{\frac{1}{\nu}1_n})^2+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*).\nonumber
\end{align}

From Lemma <ref>, we get a bound in $L_2$-norm:

\begin{align}
\mathbb{E} \big[( u^T\epsilon_{\frac{1}{\sigma}1_n})^2\big]\leq \{\varphi^{-1}(1)\}^2\|u^T\epsilon_{\frac{1}{\sigma}1_n}\|^2_{\varphi}\leq  \{\varphi^{-1}(1)\}^2\omega^2_{\varsigma},\nonumber
\end{align}

and  $ \mathbb{E} [( u^T\epsilon_{\frac{1}{\nu}1_n})^2]\leq  \{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}$.
To sum up, for any $a>0$,

\begin{align}\label{gammaineq1}
\mathbb{E} \big\{   \langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle\big\}\leq aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}.
\end{align}

Now, plugging the bounds  (<ref>) and  (<ref>) into (<ref>) yields

\begin{align}\label{middlebound1}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa-b}{2}\|\hat{\mu}-\mu^*\|^2_2\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\} \nonumber\\\leq &\, \frac{1}{2b'}P_{2,H}\big(\varrho(\hat{\bar{\beta}}-\bar{\beta}^*);\sqrt{a'b'}A\lambda_0\big)+\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}-\big\|\varrho\hat {\bar\beta}\big\|_{2,P(\cdot;\lambda)}\nonumber\\&+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta}.
\end{align}

To prove part  (a), we use the subadditivity of $P_H$:

\begin{align}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa-b}{2}\|\hat{\mu}-\mu^*\|^2_2\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\\leq &\, \frac{1}{2b'}P_{2,H}\big(\varrho\hat{\bar{\beta}};\sqrt{a'b'}A\lambda_0\big)+\frac{1}{2b'}P_{2,H}\big(\bar{\beta}^*;\sqrt{a'b'}A\lambda_0\big)+\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}-\big\|\varrho\hat {\bar\beta}\big\|_{2,P(\cdot;\lambda)} \nonumber\\&+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta}.
\end{align}

Because $P(\cdot;\lambda)\geq P_{H}(\cdot;\lambda)$, taking $b=\etaa/4,b'=1/2,a=2/\etaa,a'=4/(\etaa\land 1)$ gives

\begin{align}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{ D}}_2(\hat{\mu},\mu^*)\Big\}\nonumber\\\leq &\,2\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}+2\etaa\|\mu^*\|_{2}^2+\frac{2C}{\etaa}\omega^2_{\bar m}+\frac{4}{\etaa}\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{4C}{\etaa\land 1}\omega^2_{\bar\eta}\nonumber\\\lesssim &\,\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}+\etaa\|\mu^*\|_{2}^2+\frac{1}{\etaa}(\omega^2_{\bar m}+\omega^2_{\varsigma})+\frac{1}{\etaa\land 1}\omega^2_{\bar\eta}.
\end{align}

To prove part (b),
we use  $\big\| \varrho\bar{\beta}^*_{\mathcal{J^{*C}}}\big\|_{2,P(\cdot;\lambda)}=\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^{*C}}}\big\|_{2,P(\cdot;\lambda)}$ and $\big\| \varrho\bar{\beta}^*_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}-\big\|\varrho \bar\beta_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}\leq \|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}$ and rewrite    (<ref>) as

\begin{align}\label{middlebound}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa-b}{2}\|\hat{\mu}-\mu^*\|^2_2\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\\leq &\, \frac{1}{2b'}P_{2,H}\big(\varrho(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*};\sqrt{a'b'}A\lambda_0\big)+\frac{1}{2b'}P_{2,H}\big(\varrho(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^{*C}};\sqrt{a'b'}A\lambda_0\big)\nonumber\\&\, +\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}-\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^{*C}}}\big\|_{2,P(\cdot;\lambda)}+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}\nonumber\\&+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta}.
\end{align}

The condition (<ref>) implies

\begin{align}\label{realtt1}
&\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*};\lambda\big)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\|_{2,P(\cdot;\lambda)}\nonumber\\\leq&\,  \bold{\Delta}_{l}(\mu,\mu^*)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\|_{2,P(\cdot;\lambda)}-\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}};\lambda\big)+K\lambda^2J^*.
\end{align}

With   $b'=1/(2\vartheta)$, we add
(<ref>) and  (<ref>) to get

\begin{align}
&\,\mathbb{E}\Big\{(\etaa+\alpha-b)\mathrm{\bold{D}}_2(\hat\mu,\mu^*)\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\\leq&\,  K\lambda^2J^*+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta},
\end{align}

where $\lambda=\sqrt{a'b'}A\lambda_0$ and $A\geq A_0$ with $A_0$ given in the Lemma <ref>. Now, setting $b=(\etaa+\alpha)/4, a'=4/\{(\etaa+\alpha)\land \vartheta\}, a=2/(\etaa+\alpha)$ gives

\begin{align}
&\,\mathbb{E}\big\{\frac{\etaa+\alpha}{2}\mathrm{\bold{ D}}_2(\hat\mu,\mu^*)\big\}\nonumber\\\leq&\,  \frac{2KA^2}{\{(\etaa+\alpha)\land \vartheta\}\vartheta} {\omega_{\bar\eta}}^2 J^*\log(ep)+\frac{2C\omega^2_{\bar m}}{(\etaa+\alpha)}+\frac{4\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}}{(\etaa+\alpha)}+\frac{2\etaa^2\|\mu^*\|_{2}^2}{\etaa+\alpha}+\frac{4C\omega^2_{\bar\eta}}{(\etaa+\alpha)\land \vartheta}.\nonumber
\end{align}

The proof is complete.

Let  $\epsilon=[\epsilon_i]$ be an $n$-dimensional random vector (not necessarily mean centered or having independent components) satisfying  $\|\epsilon\|_{\psi_2}\leq \omega$. Suppose that $\bar{X}\in \mathbb{R}^{n\times 2p}$ satisfies $\|\bar{X}\|_2\leq \varrho$. Let $\lambda_0=\omega\sqrt{\log(ep)}$. Then there exist universal constants $A_0,C,c>0$ such that for any $a\geq 2b>0$, $A_1\geq A_0$, the following event
\begin{align}\label{basicboundlemma}
\sup_{\bar{\beta} \in \mathbb{R}^{2p}} 2\langle \epsilon,\bar{X}\bar{\beta}\rangle -\frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{b}P_{2,H}(\varrho\bar{\beta};\sqrt{ab}A_1\lambda_0)\geq a\omega^2 t
\end{align}

occurs with probability at most $C\exp(-ct)p^{-cA^2_1}$.

Let $P_0 = (\lambda^2/2)1_{t\ne 0}$. Define $l_H(\bar{\beta})=2\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{b}P_{2,H}(\bar{\beta};\sqrt{ab}A_0\lambda_0),l_0(\bar{\beta})=2\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{b}P_{2,0}(\bar{\beta};\sqrt{ab}A_0\lambda_0)$. Introduce two events $\varepsilon_H=\{\sup_{\bar{\beta}\in \Gamma}l_H(\bar{\beta})\geq at\omega^2\}$, and $\varepsilon_0=\{\sup_{\bar{\beta}\in \Gamma}l_0(\bar{\beta})\geq at\omega^2\}$

First, we use an optimization technique to prove that $\varepsilon_H=\varepsilon_0$. Since $P_0\geq P_H$,  $P_{2,0}\geq P_{2,H}$ and thus $\varepsilon_0\subset \varepsilon_H$. The occurrence of $\varepsilon_H$ implies that $l_H(\bar{\beta}^o)\geq a t\omega^2$ for any $\bar{\beta}^o$ defined by

\begin{align}
\bar{\beta}^o\in \argmin_{\bar{\beta}\in \mathbb{R}^{2p}} \frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-2\langle\epsilon,\bar{X}\bar{\beta}\rangle+\frac{1}{b}P_{2,H}(\bar{\beta};\sqrt{ab}A_0\lambda_0).
\end{align}

From    Lemma 5 in [56], under
$\|\bar{X}\|_2\leq 1$,  there exists a globally optimal solution $\bar{\beta}^0$ to the problem

\begin{align}
\min_{\bar{\beta}\in \mathbb{R}^{2p}}\frac{1}{2}\|y-\bar{X}\bar{\beta}\|^2_2+P_{2,H}(\bar{\beta};\lambda),
\end{align}

such that for any $j=1,\ldots,p$, either $\bar{\beta}^0_j=0$ or $\|\bar{\beta}^0_j\|_2\geq\lambda$. Therefore, with $a\geq 2b>0$, there exists at least one global minimizer $\bar{\beta}^{oo}$ satisfying $P_{2,H}(\bar{\beta}^{oo};\sqrt{ab}A_1\lambda_0)=P_{2,0}(\bar{\beta}^{oo};\sqrt{ab}A_1\lambda_0)$ and thus $l_H(\bar{\beta}^{oo})=l_0(\bar{\beta}^{oo})$. This means $\sup_{\bar{\beta}\in \mathbb{R}^{2p}} l_0(\bar{\beta})\geq l_0(\bar{\beta}^{oo})=l_{H}(\bar{\beta}^{oo})\geq a t\omega^2$, and so  $\varepsilon_H\subset\varepsilon_0$. It suffices to prove $\mathbb{P}(\varepsilon_0)\leq C\exp(-ct)p^{-cA^2_1}$.

Next, we   use Lemma <ref> to bound the tail probability of $R$ defined by

\begin{align}\label{rdefappend}
R= \sup_{1\leq J\leq p} \sup_{\bar\beta\in \Gamma_J}\big\{\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{2b}P_{2,0}(\bar\beta;\sqrt{ab}A_1\lambda_0)\big\},
\end{align}

where $P_{2,0}(\bar\beta;\lambda_0)=P_{0}(J;\lambda_0)=(1/2)J\lambda^2_0$ for $\bar\beta \in \Gamma_J$ and $\Gamma_J=\{\bar{\beta}\in\mathbb{R}^{2p}:J(\bar{\beta})=J\}$  (in the trivial case of $J=0$, the quantity inside the braces is 0).

By  a scaling argument, for any $a>0$,

\begin{align}\label{ineqdr2}
\langle\epsilon,\bar{X}\bar{\beta}/\|\bar{X}\bar{\beta}\|_2\rangle\|\bar{X}\bar{\beta}\|_2\leq \frac{a}{2}\sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2+\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2.
\end{align}

Applying Lemma <ref> with $G=2,q=2$ results in

\begin{align}
\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{a}{2}LJ\log (ep)\omega^2\geq \frac{a}{2}\omega^2t\big)\leq C\exp(-ct),
\end{align}

\begin{align}
\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle- \frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2- aLP_{2,0}(\bar\beta;\lambda_0)\geq a\omega^2t\big)\leq C\exp(-ct).
\end{align}

Set $A_1\geq \sqrt{2L}$. Noticing   that (i) $(A^2_1/2)P_0(J;\lambda_0)\geq LP_0(J;\lambda_0)+cA^2_1  P_0(J;\lambda_0)$ for some $c>0$, and (ii) $J\log(ep)\geq \log p+J$ for any $J\geq 1$, we get

\begin{align}\label{unionboundJ}
&\, \mathbb{P}\big(R\geq a\omega^2t\big)\nonumber\\\leq &\,\sum^p_{J=1}\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{2b}P_{2,0}(\bar\beta;\sqrt{ab}A_1\lambda_0)\geq a\omega^2t\big)\nonumber\\= &\,\sum^p_{J=1}\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{4}aA_1^2J\log (ep)\omega^2_{\bar\eta}\geq a\omega^2t\big)\nonumber\\\leq &\,\sum^p_{J=1}\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{a}{2}LJ\log (ep)\omega^2\geq a\omega^2t+caA^2_1 J\log (ep)\omega^2\big)\nonumber\\\leq &\,\sum^p_{J=1}C\exp(-ct)\sum^p_{J=1}\exp\{-cA_1^2(J+\log p)\}\nonumber\\\leq &\,C\exp(-ct)p^{-cA^2_1},
\end{align}

where the last inequality is due to the sum of geometric sequence.

Given a  matrix $X\in \mathbb{R}^{n\times pG}$ with   a block form of $X=[X_1,\ldots,X_p]$ with $X_j\in\mathbb{R}^{n\times G},1\leq j\leq p$,
let $X_{\mathcal{J}}$ denote the submatrix formed by the column blocks of $X$ indexed by $\mathcal{J}$. Define $\Gamma_{\mathcal{J}}=\{\alpha\in \mathbb{R}^{pG}: \|\alpha\|_2\leq 1, \alpha\in \mathcal{R}(X_{\mathcal{J}})\}$ and  $\Gamma_{J}=\bigcup_{|\mathcal{J}|=J}\Gamma_{\mathcal{J}}$, where $1\leq J\leq p$.
Let $\epsilon=[\epsilon_i]$ be an $n$-dimensional random vector. (i) Assume $\epsilon$ satisfies $\|\epsilon\|_{\psi_q}\leq \omega$ for some $q\geq 1$. Then for any $t>0$,

\begin{align}
&  \mathbb{P}\big( \sup_{\alpha\in \Gamma_{\mathcal{J}}}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2-L(JG)^{\frac{2}{q}}\omega^2\geq \omega^2t\big)\leq C\exp\big(-ct^{\frac{q}{2}}\big), \label{lemmaass1medb}\\
&  \mathbb{P}\big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-L\big[JG+(JG)^{\frac{2}{q}}+(2J)^{\frac{2}{q}}\{\log (ep)\}^{\frac{2}{q}}\big]\omega^2\geq \omega^2t\big)\leq C\exp\big(-ct^{\frac{q}{2}}\big),\label{lemmaass1finalb}
\end{align}

where   $C$ is a universal constant  and  $c,L$ are constants   depending on $q$ only.
(ii) Assume that  $\epsilon_1,\ldots,\epsilon_n$  are independent, centered,  and  $\|\epsilon_i\|_{\psi_q}\leq \omega$ for some $q\in(0,2]$. Then (<ref>) and (<ref>) are replaced by
\begin{align}
&     \mathbb{P}\Big( \sup_{\alpha\in \Gamma_{\mathcal{J}}}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2\geq \omega^2t\Big)\leq C\exp\big(-ct^{\frac{q}{2}}\big), \label{lemmaunionbound1}
\\
&   \mathbb{P}\Big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-L\big[JG+(2J)^{\frac{2}{q}}\{\log(ep)\}^\frac{2}{q}\big]\omega^2\geq \omega^2t\Big)\leq C\exp\big(-ct^{\frac{q}{2}}\big).\label{lemmameanzerobound}
\end{align}

First, we show  (<ref>) under the assumption that $\|\epsilon\|_{\psi_q}\leq \omega$ for some $q\geq 1$.
Define a  centered random vector $\epsilon_c=\epsilon-M$ with  $M=\mathbb{E}[\epsilon]$, then

\begin{align}
\sup_{\alpha\in \Gamma_{\mathcal{J}}} \langle\epsilon,\alpha\rangle\leq\sup_{\alpha\in \Gamma_{\mathcal{J}}}\langle \epsilon_c,\alpha\rangle+\sup_{\alpha\in \Gamma_{\mathcal{J}}}\langle P_{ X_{\mathcal{J}}}M,\alpha\rangle\leq \sup_{\alpha\in \Gamma_\mathcal{J}}\langle\epsilon_c,\alpha\rangle+\|U^TM\|_2,
\end{align}

where  $P_{ X_{\mathcal{J}}}=UU^T$ is the orthogonal projection matrix onto $\mathcal{R}(X_{\mathcal{J}})$ with $\{U_1,\ldots,U_{JG}\}$ as  an orthonormal basis and $\|P_{ X_{\mathcal{J}}}\|_2=1$. We claim that $\|U^TM\|^2_2=\sum^{JG}_{i=1}(\mathbb{E}[U^T_i\epsilon])^2\leq CJG\omega^2$. In fact,
because $\|U^T_i\epsilon\|_{\psi_q}\leq \omega$,

\begin{align}\label{probtomean}
\mathbb{P}\big(|U^T_i\epsilon|>t\omega\big)\leq \int^{+\infty}_{0}\frac{1}{\psi_q(\frac{t\omega}{\omega})}\rd t=\int^{+\infty}_{0}\exp(-t^q)\rd t<\infty,
\end{align}

and so $\mathbb{E}|U^T_i\epsilon|\leq C\omega,\,1\leq i\leq J$.

By definition, $\{\langle\epsilon_c,\alpha\rangle:\alpha\in \Gamma_{\mathcal{J}}\}$ is a (centered)      $\psi_q$-process. The induced metric on $\Gamma_{\mathcal{J}}$ is: $d(\alpha,\alpha')=\omega\|\alpha-\alpha'\|_2$.
To bound the metric entropy $\log (\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d)))$, where the $\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d))$ is the smallest cardinality of an $\varepsilon$-net that covers $\Gamma_{\mathcal{J}}$ under the metric $d$, we apply a standard volume argument to get

\begin{align}
\log (\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d)))\leq \log  (\frac{C\omega}{\varepsilon})^{JG}=JG\log(C\omega/\varepsilon).
\end{align}

By Theorem 5.36 in [65],  we get

\begin{align}
\mathbb{P}\Big( \sup_{\alpha,\alpha'\in \Gamma_{\mathcal{J}}}|\langle\epsilon_c,\alpha-\alpha'\rangle|-L\int^{D}_{0}\psi^{-1}_q(\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d)) \rd \varepsilon \geq Lt\Big)\leq 2\exp\big(-\frac{t^{q}}{D^q}\big),
\end{align}

for any $t>0$, where $D=\sup_{\alpha,\alpha'\in\Gamma_{\mathcal{J}}}d(\alpha,\alpha')=2\omega$. Equivalently,

\begin{align}
\mathbb{P}\Big( \sup_{\alpha,\alpha'\in \Gamma_{\mathcal{J}}}|\langle\epsilon_c,\alpha-\alpha'\rangle|-L\int^{2\omega}_{0}\psi^{-1}_q(\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d)) \rd \varepsilon \geq \omega t\Big)\leq C\exp\big(-ct^{q}\big),
\end{align}

which implies

\begin{align}\label{lemma3meanzeromedinq}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_\mathcal{J}}\langle\epsilon_c,\alpha\rangle-L\int^{2\omega}_{0}\psi^{-1}_q(\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d)) \rd \varepsilon \geq \omega t\Big)\leq C\exp\big(-ct^{q}\big).
\end{align}

Based on (<ref>) and (<ref>), we obtain

\begin{align}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_\mathcal{J}}\langle\epsilon,\alpha\rangle-L\sqrt{JG}\omega-L\int^{2\omega}_{0}\psi^{-1}_q(\mathcal{N}(\varepsilon,\Gamma_{\mathcal{J}},d)) \rd \varepsilon\geq \omega t\big)\leq C\exp\big(-ct^{q}\big),
\end{align}

\begin{align}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_\mathcal{J}}\langle\epsilon,\alpha\rangle-L\sqrt{JG}\omega-L(JG)^{\frac{1}{q}}\omega\geq \omega t\big)\leq C\exp\big(-ct^{q}\big).
\end{align}

\begin{align}
\mathbb{P}\Big( \sup_{\alpha\in \Gamma_{\mathcal{J}}}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2-L(JG)^{\frac{2}{q}}\omega^2\geq \omega^2t^2\Big)\leq C\exp\big(-ct^{q}\big),
\end{align}

and letting $s=t^2$ gives   (<ref>). With a union bound, we also obtain

\begin{align}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2-L(JG)^{\frac{2}{q}}\omega^2\geq \omega^2t\big)\leq &\, {p\choose J}C\exp\big(-ct^{\frac{q}{2}}\big)\nonumber\\\leq &\, C\exp\big\{-ct^{\frac{q}{2}}+J\log(ep)\big\},
\end{align}

from which it follows

\begin{align}
\mathbb{P}\Big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2-L(JG)^{\frac{2}{q}}\omega^2-L2^{\frac{2}{q}-1}\{J\log (ep)\}^{\frac{2}{q}}\omega^2\geq \omega^2t\Big)\leq C\exp\big(-ct^{\frac{q}{2}}\big).
\end{align}

Next, we prove (<ref>) assuming  $\epsilon_1,\ldots,\epsilon_n$  are independent, centered,  and $\|\epsilon_i\|_{\psi_q}\leq \omega$ for some $0<q\le 2$.
By Hölder's  inequality,

\begin{align}
\sup_{\alpha\in \Gamma_{\mathcal{J}}} \langle\epsilon,\alpha\rangle=\sup_{\alpha\in \Gamma_{\mathcal{J}}}\langle P_{ X_{\mathcal{J}}}\epsilon,\alpha\rangle\leq \|U^T\epsilon\|_2,
\end{align}

where $P_{ X_{\mathcal{J}}}=UU^T$ is the orthogonal projection matrix onto $\mathcal{R}(X_{\mathcal{J}})$ with $\{U_1,\ldots,U_{JG}\}$ as  an orthonormal basis and $\|P_{ X_{\mathcal{J}}}\|_2=1$.
It remains to get a tail bound for $\|U^T\epsilon\|^2_2$.

Noticing that    (i) $\mathbb{E}[\epsilon]=0$, (ii)   $\|\epsilon_i\|_{\psi_q}\leq \omega$ implies    $\mathbb{E}[\epsilon^2_i]\leq C\omega^2$, (iii) $\|UU^T\|_2=1$, and (iv) $Tr(UU^T)=JG$, we apply a generalized Hanson-Wright inequality ([53], Theorem 2.1) to obtain
\begin{align}\label{lemma3mean0medianineq1}
\mathbb{P}\big( \|U^T\epsilon\|^2_2-LJG\omega^2\geq \omega^2t\big)\leq C\exp\big(-ct^{\frac{q}{2}}\big),
\end{align}

where $C$ can be $2$  and  $c,L$ are constants   depending on $q$ only. This results in
\begin{align}\label{lemmamidleboundresult}
\mathbb{P}\Big( \sup_{\alpha\in \Gamma_{\mathcal{J}}}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2\geq \omega^2t\Big)\leq C\exp\big(-ct^{\frac{q}{2}}\big).
\end{align}

Finally, we prove (<ref>). Applying the union bound on  (<ref>) yields

\begin{align}\label{unionbondfirst}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2\geq \omega^2t\big)\leq {p\choose J}C\exp\big(-ct^{\frac{q}{2}}\big)\leq C\exp\big\{-ct^{\frac{q}{2}}+J\log(ep)\big\}.
\end{align}

Let $t'=t^{\frac{q}{2}}-(1/c)J\log(ep)$. Then (<ref>) can be rewritten as

\begin{align}\label{supineq1}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2\geq \omega^2\{t'+\frac{1}{c}J\log (ep)\}^{\frac{2}{q}}\big)\leq C\exp(-ct').
\end{align}

Given $0<q\leq 2$, by the convexity of $x^\frac{2}{q}$ on $\mathbb{R}_+$, we have  $(a+b)^\frac{2}{q}\leq 2^{\frac{2}{q}-1}(a^\frac{2}{q}+b^\frac{2}{q})$ for any $a,b\geq0$, and thus
(<ref>) becomes

\begin{align}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2-L2^{\frac{2}{q}-1}\{J\log (ep)\}^{\frac{2}{q}}\omega^2\geq 2^{\frac{2}{q}-1}\omega^2(t')^\frac{2}{q}\big)\leq C\exp(-ct'),
\end{align}

or equivalently,

\begin{align}
\mathbb{P}\big( \sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2-LJG\omega^2-L2^{\frac{2}{q}-1}\{J\log (ep)\}^{\frac{2}{q}}\omega^2\geq \omega^2s\big)\leq C\exp\big(-cs^{\frac{q}{2}}\big).
\end{align}

The proof is now complete.

§ FURTHER EXTENSIONS

We showcase two nonparametric statistical applications of skewed pivot-blend, one based on data ranks and the other based on kernels.

Skewed rank-based estimation.
Recently, there has been a lot of interest in rank-based  nonparametric estimation methods that minimize    $\sqrt{12}\sum^n_{i=1}r_i \{\mbox{R}(r_i)/(n+1) -\frac{1}{2} \}$ [33],
where $\mbox{R}(r_i)$ denotes the rank of $r_i$ among $r_1,\ldots,r_n$, or equivalently, the $\ell_1$ loss on the spread of residuals: $%\begin{align}
\frac{1}{ n(n-1) }\sum_{i\neq j}|r_i-r_j|
$ [28, 29, 67].

Applying the technique of U-statistics (with kernel $h(r_i, r_j)= |r_i-r_j|$) can show that the criterion results in estimators with desired asymptotic properties.

The adoption of the symmetric (and relatively robust) $\ell_1$-loss function heavily depends on the assumption that $r_i$ follows an i.i.d. distribution (when assessed against the statistical truth). In turn, it implies that the distribution of  differences  $r_i - r_j$ is symmetrical, as in the case of a double-exponential.

Nonetheless,   the assumption may not  hold in real-world applications beyond the i.i.d. setting. Such deviations can  result in extra skewness of $|r_i - r_j|$ that cannot be captured by an exponential (or half-normal) distribution. We introduce a skewed criterion that operates on the absolute differences $|r_i - r_j|$ and $m, \sigma, \nu>0$:

\begin{align}\label{skewrloss}
&\,\sum_{i\neq j}\Big\{ \big(\frac{|r_i-r_j|-m}{\sigma} + m\big) \,1_{m(1-\sigma)\le |r_i-r_j|\leq m}+ \big(\frac{|r_i-r_j|-m}{\nu} +m\big)\,1_{|r_i-r_j|>m}\Big\}\nonumber\\&\,\quad\quad+ n(n-1)\log \{ \sigma [1-\exp(-m)  ]+\nu \exp(-m) \}.
\end{align}

(<ref>) results from applying  skewed pivot-blend  to the exponential density (instead of   the double-exponential density). A regularization term (such as an $\ell_1$ penalty) can be incorporated to capture structural parsimony.

Kernel-assisted nonparametric skew estimation.
Assume  $y-X\beta^*\sim \mbox{SP}^{(\phi^*)}(\sigma^{*},\nu^{*},m^{*})$, where the functional form of the density  $\phi^{*}$ is also unknown.    We can employ  a backward-forward scheme for    nonparametric estimation, along with  explicit capture of skewness that  aligns with the theme of this paper.

To estimate   $\sigma^{*},\nu^{*},m^{*}$,       kernel may be employed to approximate    $\phi^{*}$, but the data  follow  $\mbox{SP}^{(\phi^*)} $. We can address this by using   backward pivot-blend. Holding $\beta, \sigma, \nu, m$ constant for now, and referring to the definitions of $\tilde{r}_i$ ($1\le i \le n$), $L(m)$, and $R(m)$ in Remark <ref>,  introduce a kernel density estimator with appropriate weights to estimate $\phi^*$ based on the transformed residuals $\tilde r_i$:

\begin{align}\label{rform}
\begin{split}
\phi_K(t;h)=\,\frac{1}{h}\sum^{n}_{i=1}&\Big\{\frac{\nu}{L(m)\nu+R(m)\sigma}K_h\big(\frac{r_i-m}{\sigma}+m-t\big)1_{r_i\leq m}\\&+\frac{\sigma}{L(m)\nu+R(m)\sigma}K_h\big(\frac{r_i-m}{\nu}+m-t\big)1_{r_i>m}\Big\}.
\end{split}
\end{align}

where   $K_h(t)=K\big(t/h\big)$, the kernel function $K(t)$ can be  any continuous symmetric function with $\int^{\infty}_{-\infty} K(t),dt=1$, and $h>0$ represents the bandwidth parameter.
The kernel approach in conjunction with (forward) skewed pivot-blend gives rise to a nonparametric skew-estimation problem, which warrants further investigation.

§ MORE EXPERIMENTS

In this part, we  consider some  densities which may not be unimodal or symmetric (as shown in   Figure <ref>). The experimental setup is the same as in Section  <ref>.

$\phi$: half-normal,  $m^*$: mean

$\phi$: Maxwell,   $m^*$: median

$\phi$: double Rayleigh,   $m^*$:  right mode

Plots of  $\phi$ for Ex 3–Ex 5. The choices  of $m^*$ are indicated in red dashed lines.

Ex 3. (Half-normal with the   mean as the pivotal point): Let $\phi$ be the half-normal density (with scale 1).  We set $n=700 $,   $\kappa=0.1$, $m^*$ as the mean,   $\nu^*=0.1$ and  $\sigma^*=0.2,0.3,0.4$.

Ex 4. (Maxwell-Boltzmann with the  median as the pivotal point): Here, $\phi$ is the Maxwell-Boltzmann density which is widely used in statistical mechanics  [31]. We set   $n=500$,   $\kappa=0.2$, $m^*$ as the median,  $\nu^*=0.1$ and  $\sigma^*=0.4,0.5,0.6$.

Ex 5. (Double Rayleigh with $m^*$ as the  right mode): Let $\phi$ be the double Rayleigh density (with scale 1),    $n=500$, $\kappa=0.1$,   $m^*$ be the  right mode,     $\nu^*=0.1$ and  $\sigma^*=0.2,0.3,0.4$.

13cSkewed half-normal

3c$\sigma^*/\nu^*=2$                              3c$\sigma^*/\nu^*=3$                               3c$\sigma^*/\nu^*=4$

3-5 7-9 11-13

Err($\beta$)      Err($\sigma$)     Err($\nu$)           Err($\beta$)      Err($\sigma$)     Err($\nu$)            Err($\beta$)      Err($\sigma$)     Err($\nu$)

QR$^*$          0.80     —      —          0.80     —       —                   0.80       —         —

BQR$^*$         0.80      0.49      0.62         0.80     0.51       0.64                   0.80       0.53        0.65

AME          0.64      0.99      1.22          0.56     1.0       2.02                   0.48       1.0         2.82

ZQR           0.65     0.97      0.09          0.59     0.95       0.37                   0.52       0.95         0.67

SPEUS          0.14     0.11      0.1          0.08     0.08       0.13                   0.06      0.09         0.17

13cSkewed Maxwell-Boltzmann
3c$\sigma^*/\nu^*=4$                              3c$\sigma^*/\nu^*=5$                               3c$\sigma^*/\nu^*=6$

3-5 7-9 11-13

Err($\beta$)      Err($\sigma$)     Err($\nu$)           Err($\beta$)      Err($\sigma$)     Err($\nu$)            Err($\beta$)      Err($\sigma$)     Err($\nu$)

QR$^*$          1.59     —      —          1.59     —       —                   1.59       —         —

BQR$^*$          1.59     0.41     0.74          1.59     0.43       0.75                   1.59       0.44         0.75

AME           1.47     0.22      0.78          1.46     0.19       0.84                   1.46      0.17        0.91

ZQR          1.49     0.58      0.30         1.48     0.58       0.32                   1.48       0.57         0.34

SPEUS          0.18     0.14      0.20          0.17     0.13      0.21                   0.16       0.12        0.30

13cSkewed double Rayleigh
3c$\sigma^*/\nu^*=2$                              3c$\sigma^*/\nu^*=3$                               3c$\sigma^*/\nu^*=4$

3-5 7-9 11-13

Err($\beta$)      Err($\sigma$)     Err($\nu$)           Err($\beta$)      Err($\sigma$)     Err($\nu$)            Err($\beta$)      Err($\sigma$)     Err($\nu$)

QR$^*$          1.0     —      —          1.0     —       —                   1.0       —         —

BQR$^*$           1.0     0.68      0.27          1.0     0.70       0.26                   1.0      0.71        0.26

AME            0.87     1.78      1.76         0.88     2.02       1.9                   0.89       2.15        1.92

ZQR          0.67     0.50      1.67          0.50     0.48       2.69                   0.32       0.45        3.78

SPEUS          0.22     0.03      0.18          0.15     0.03       0.24                   0.10      0.03      0.26

Performance comparison for skewed half-normal  with pivotal point at the  mean,  skewed Maxwell with pivotal point at the median, and skewed double Rayleigh with pivotal point at the right mode (Ex 3–Ex 5)

Table <ref> illustrates significant issues with conventional methods in capturing skewness, even when asymmetry is centered around the median, mean, or mode. Specifically, QR$^*$ and BQR$^*$ failed to accurately recover the true $\beta^*$ despite using the true quantile as input.  Moreover, in all three cases, AME or AME provided at least one misleading estimate of the scales. In contrast, our proposed method demonstrated remarkable performance, with the associated $\beta$-error being at most $1/3$ of that of the other methods.

We also conducted  experiments  on the air pollution data of
Leeds from 1994 to 1998
[27, 59]. The dataset contain
578 measurements of the daily maximum levels of  ozone  (O3), nitrogen dioxide (NO2), nitrogen oxide (NO), sulfur dioxide (SO2) and particulate matter  (PM10). We forecasted PM10 levels for the summer months (April to July) using  other air pollutants. We considered the Gumbel model and its skewed pivot-blend enhancement SPEUS, in addition to ZQR, AME, and ESN.
The p-values from the Kolmogorov-Smirnov tests for these models were  8e-2, 0.71,  9e-2,  0.20, and 8e-8,  respectively.  The results may appear surprising, considering the popularity of the skewed Gumbel distribution  for modeling such extreme values [12]. Figure <ref> presents   the residual Q-Q plots, demonstrating  that our skew-reinforced method significantly improves data fit compared to the classical Gumbel model.

Q-Q plots of residuals for Gumbel (left) and SPEUS (right) on air pollution  data.

In real-world applications, data scientists frequently confront challenges related to skewness. However,   fitting a   common distribution that inadequately addresses these distortions can lead to subpar model fits and  misleading inferences. The skewed pivot-blend technique refines skewness management in these distributions, thus enhancing the accuracy and reliability of the models.

[1]
Agency for Healthcare Research and Quality (2015).
MEPS HC-181: 2015 Full Year Consolidated Data File.

[2]
Agency for Healthcare Research and Quality (2019).
MEPS HC-216: 2019 Full Year Consolidated Data File.

[3]
Arellano-Valle, R. B., Gómez, H. W., and Quintana, F. A. (2005).
Statistical inference for a general class of asymmetric
Journal of Statistical Planning and Inference, 128(2):427–443.

[4]
Axon, D. R. and Kamel, A. (2021).
Patterns of healthcare expenditures among older United States
adults with pain and different perceived health status.
Healthcare, 9(10):1327.

[5]
Azzalini, A. (1985).
A class of distributions which includes the normal ones.
Scandinavian Journal of Statistics, 171–178.

[6]
Azzalini, A. (2005).
The skew-normal distribution and related multivariate families.
Scandinavian Journal of Statistics, 32(2):159–188.

[7]
Bartlett, P. L., Harvey, N., Liaw, C., and Mehrabian, A. (2019).
Nearly-tight VC-dimension and pseudo-dimension bounds for piecewise
linear neural networks.
The Journal of Machine Learning Research, 20(1):2285–2301.

[8]
Benoit, D. F. and Van den Poel, D. (2017).
bayesQR: A Bayesian approach to quantile regression.
Journal of Statistical Software, 76(1):1–32.

[9]
Bera, A. K., Galvao, A. F., Montes-Rojas, G. V., and Park, S. Y. (2016).
Asymmetric Laplace regression: Maximum likelihood, maximum entropy
and quantile regression.
Journal of Econometric Methods, 5(1):79–101.

[10]
Bernardi, A. and Bernardi, M. (2018).
Two-sided skew and shape dynamic conditional score models.
Mathematical and Statistical Methods for Actuarial Sciences and
Finance, pages 121–124.

[11]
Bickel, P. J., Ritov, Y., and Tsybakov, A. B. (2009).
Simultaneous analysis of Lasso and Dantzig selector.
The Annals of Statistics, 37(4):1705–1732.

[12]
Boldi, M. O. and Davison, A. C. (2007).
A mixture model for multivariate extremes.
Journal of the Royal Statistical Society: Series B (Statistical
Methodology), 69(2):217–229.

[13]
Boos, D. D. (1987).
Detecting skewed errors from regression residuals.
Technometrics, 29(1):83–90.

[14]
Bregman, L. M. (1967).
The relaxation method of finding the common point of convex sets and
its application to the solution of problems in convex programming.
USSR Computational Mathematics and Mathematical Physics,

[15]
Chai, H. S. and Bailey, K. R. (2008).
Use of log-skew-normal distribution in analysis of continuous data
with a discrete component at zero.
Statistics in Medicine, 27(18):3643–3655.

[16]
Chang, B. and Joe, H. (2019).
Prediction based on conditional distributions of vine copulas.
Computational Statistics $\&$ Data Analysis, 139:45–63.

[17]
Cooray, K. and Ananda, M. M. (2005).
Modeling actuarial data with a composite lognormal-pareto model.
Scandinavian Actuarial Journal, 2005(5):321–334.

[18]
Cragg, J. G. (1971).
Some statistical models for limited dependent variables with
application to the demand for durable goods.
Econometrica: Journal of the Econometric Society, 829–844.

[19]
Devroye, L., Györfi, L., and Lugosi, G. (1996).
A Probabilistic Theory of Pattern Recognition.
New York: Springer.

[20]
Dominicy, Y. and Sinner, C. (2017).
Distributions and composite models for size-type data.
In Hokimoto, T., editor, Advances in Statistical Methodologies
and Their Application to Real Problems, chapter 8, pages 159–183.
IntechOpen, Rijeka, Croatia.

[21]
Fernández, C. and Steel, M. F. (1998).
On Bayesian modeling of fat tails and skewness.
Journal of the American Statistical Association,

[22]
Giné, E. and Nickl, R. (2015).
Mathematical Foundations of Infinite-Dimensional Statistical
Cambridge: Cambridge University Press.

[23]
Gitman, I., Chen, J., Lei, E., and Dubrawski, A. (2018).
Novel prediction techniques based on clusterwise linear regression.
arXiv preprint:1804.10742.

[24]
Goldberg, P. W. and Jerrum, M. R. (1995).
Bounding the Vapnik-Chervonenkis dimension of concept classes
parameterized by real numbers.
Machine Learning, 18(2-3):131–148.

[25]
Götze, F., Sambale, H., and Sinulis, A. (2021).
Concentration inequalities for polynomials in
$\alpha$-sub-exponential random variables.
Electronic Journal of Probability, 26:1–22.

[26]
Hampel, F. R., Ronchetti, E. M., Rousseeuw, P. J., and Stahel, W. A. (2011).
Robust Statistics: The Approach Based on Influence Functions.
New Jersey: John Wiley & Sons.

[27]
Heffernan, J. E. and Tawn, J. A. (2004).
A conditional approach for multivariate extreme values.
Journal of the Royal Statistical Society: Series B (Statistical
Methodology), 66(3):497–546.

[28]
Hettmansperger, T. P. and McKean, J. W. (1978).
Statistical inference based on ranks.
Psychometrika, 43(1):69–79.

[29]
Hettmansperger, T. P. and McKean, J. W. (2010).
Robust Nonparametric Statistical Methods.
Florida: CRC Press.

[30]
Holle, M., Wolff, T., and Herant, M. (2021).
Trends in the concentration and distribution of health care
expenditures in the US, 2001-2018.
JAMA Network Open, 4(9):e2125179–e2125179.

[31]
Huang, K. (2008).
Statistical Mechanics.
New Jersey: John Wiley & Sons.

[32]
Hyndman, R. J. and Grunwald, G. K. (2000).
Applications: Generalized additive modelling of mixed distribution
Markov models with application to Melbourne's rainfall.
Australian $\&$ New Zealand Journal of Statistics,

[33]
Jaeckel, L. A. (1972).
Estimating regression coefficients by minimizing the dispersion of
the residuals.
The Annals of Mathematical Statistics, 43(5):1449–1458.

[34]
Jones, M. (2014).
Generating distributions by transformation of scale.
Statistica Sinica, 749–771.

[35]
Klugman, S. A., Panjer, H. H., and Willmot, G. E. (2012).
Loss models: from data to decisions, volume 715.
John Wiley & Sons.

[36]
Koenker, R. (2009).
Package `quantreg'.

[37]
Koenker, R. and Bassett Jr, G. (1978).
Regression quantiles.
Econometrica: Journal of the Econometric Society, 33–50.

[38]
Kottas, A. and Gelfand, A. E. (2001).
Bayesian semiparametric median regression modeling.
Journal of the American Statistical Association,

[39]
Kuchibhotla, A. K. and Chakrabortty, A. (2022).
Moving beyond sub-Gaussianity in high-dimensional statistics:
Applications in covariance estimation and linear regression.
Information and Inference: A Journal of the IMA, 11:1389–1456.

[40]
Lambert, D. (1992).
Zero-inflated Poisson regression, with an application to defects in
Technometrics, 34(1):1–14.

[41]
Lee, M. J. (1989).
Mode regression.
Journal of Econometrics, 42(3):337–349.

[42]
Linero, A. R., Sinha, D., and Lipsitz, S. R. (2020).
Semiparametric mixed-scale models using shared Bayesian forests.
Biometrics, 76(1):131–144.

[43]
Liu, L., Ma, J. Z., and Johnson, B. A. (2008).
A multi-level two-part random effects model, with application to an
alcohol-dependence study.
Statistics in Medicine, 27(18):3528–3539.

[44]
Ma, Y. and Genton, M. G. (2004).
Flexible class of skew-symmetric distributions.
Scandinavian Journal of Statistics, 31(3):459–468.

[45]
Maronna, R. A., Martin, D. R., and Yohai, V. J. (2006).
Robust Statistics: Theory and Methods.
New Jersey: John Wiley & Sons.

[46]
Mudholkar, G. S. and Hutson, A. D. (2000).
The epsilon-skew-normal distribution for analyzing near-normal data.
Journal of Statistical Planning and Inference, 83(2):291–309.

[47]
Mullahy, J. (1998).
Much ado about two: Reconsidering retransformation and the two-part
model in health econometrics.
Journal of Health Economics, 17(3):247–281.

[48]
Nadarajah, S. and Kotz, S. (2003).
Skewed distributions generated by the normal kernel.
Statistics $\&$ Probability Letters, 65(3):269–277.

[49]
Nash, W. J., Sellers, T. L., Talbot, S. R., Cawthorn, A. J., and Ford, W. B.
The population biology of abalone (Haliotis species) in Tasmania.
i. blacklip abalone (H. rubra) from the North Coast and islands of
Bass Strait.
Sea Fisheries Division, Technical Report, 48:411.

[50]
Olsen, M. K. and Schafer, J. L. (2001).
A two-part random-effects model for semicontinuous longitudinal data.
Journal of the American Statistical Association,

[51]
Rubio, F. J. and Steel, M. F. (2020).
The family of two-piece distributions.
Significance, 17(1):12–13.

[52]
Rubio, F. J. and Steel, M. F. J. (2015).
Bayesian modelling of skewness and kurtosis with two-piece scale and
shape distributions.
Electronic Journal of Statistics, 9(2):1884–1912.

[53]
Sambale, H. (2020).
Some notes on concentration for $\alpha $-subexponential random
arXiv preprint:2002.10761.

[54]
Sarul, L. S. and Sahin, S. (2015).
An application of claim frequency data using zero inflated and hurdle
models in general insurance.
Journal of Business Economics and Finance, 4(4).

[55]
Scollnik, D. (2007).
On composite lognormal-Pareto models.
Scandinavian Actuarial Journal, 2007(1):20–33.

[56]
She, Y. (2016).
On the finite-sample analysis of $\Theta$-estimators.
Electronic Journal of Statistics, 10:1874–1895.

[57]
She, Y. and Tran, H. (2019).
On cross-validation for sparse reduced rank regression.
Journal of the Royal Statistical Society: Series B (Statistical
Methodology), 81(1):145–161.

[58]
She, Y., Wang, Z., and Jin, J. (2021).
Analysis of generalized bregman surrogate algorithms for nonsmooth
nonconvex statistical learning.
The Annals of Statistics, 49(6):3434–3459.

[59]
Southworth, H., Heffernan, J. E., and Metcalfe, P. (2020).
Package ‘texmex’.

[60]
Stacy, E. W. (1962).
A generalization of the Gamma distribution.
The Annals of Mathematical Statistics, 33(3):1187–1192.

[61]
Tarbath, D. (1999).
Estimates of Growth and Natural Mortality of the Blacklip
Abalone (Haliotis Rubra) in Tasmania.
Technical report series. Tasmanian Aquaculture and Fisheries

[62]
Tobin, J. (1958).
Estimation of relationships for limited dependent variables.
Econometrica: Journal of the Econometric Society, 24–36.

[63]
van de Geer, S. A. and Bühlmann, P. (2009).
On the conditions used to prove oracle results for the Lasso.
Electronic Journal of Statistics, 3:1360–1392.

[64]
van der Vaart, A. W. and Wellner, J. A. (2013).
Weak Convergence and Empirical Processes: With Applications to
Berlin: Springer Science & Business Media.

[65]
Wainwright, M. J. (2019).
High-dimensional Statistics: A Non-Asymptotic Viewpoint.
Cambridge: Cambridge University Press.

[66]
Wang, J., Boyer, J., and Genton, M. G. (2004).
A skew-symmetric representation of multivariate distributions.
Statistica Sinica, 1259–1270.

[67]
Wang, L., Peng, B., Bradic, J., Li, R., and Wu, Y. (2020).
A tuning-free robust and efficient approach to high-dimensional
Journal of the American Statistical Association,

[68]
Yang, T., Gallagher, C. M., and McMahan, C. S. (2019).
A robust regression methodology via M-estimation.
Communications in Statistics-Theory and Methods,

[69]
Yu, K. and Moyeed, R. A. (2001).
Bayesian quantile regression.
Statistics $\&$ Probability Letters, 54(4):437–447.

[70]
Zhu, D. and Zinde-Walsh, V. (2009).
Properties and estimation of asymmetric exponential power
Journal of Econometrics, 148(1):86–99.

§ TECHNICAL DETAILS

Throughout the proofs, we use $C, c$ to denote positive constants, and they are not
necessarily the same at each occurrence. Given a matrix $A$, we use $\mathcal{R}(A)$ to denote the column space (range) of $A$ and  $A^+$ to denote the Moore-Penrose inverse of $A$.  Denote by $P_A=A(A^TA)^+A^T$ the orthogonal projection on $\mathcal{R}(A)$.

§.§ Basics

The conventional   definition of  Orlicz $\psi$-norms goes as follows:

Given  a strictly increasing convex function   $\psi$  on $\mathbb R_+ := [0,+\infty)$ with $\psi(0)=0$, then the Orlicz $\psi$-norm of a random variable $Y$ is defined as  $  \|Y\|_{\psi}=\inf\{t>0: \mathbb{E}\psi(|Y|/t)\leq 1\}$.

Some well-known examples of Orlicz $\psi$-norms  are the $L_q$-norms: $\|Y\|_q=(\mathbb{E}|Y|^q)^{1/q}$ associated with
\psi(x)=x^q $ $(q\geq 1)$ (e.g.,  the Pareto distribution), and the $\psi_q$-norms ($q\geq1$) with

\begin{align}\label{psiqfun}
\psi_q(x)=\exp(x^q)-1.
\end{align}

(<ref>) encompasses both  sub-Gaussian   and sub-Exponential type  random variables for $q=2,1$,  without the requirement for the random variables to be centered.

Our upcoming theorems  often relax the strict requirements of strict monotonicity and convexity for $\psi$.
This flexibility allows us to handle random variables with much heavier tails.   For instance, we consider the extension of (<ref>), known as sub-Weibull random variables, which have finite $\psi_q$-norms for   $q>0$   (cf. [39]). As $0 < q < 1$, $\psi_q$ is nonconvex, and these random variables, including Weibull, exhibit heavier tails compared to sub-Exponential ones [25].

Throughout the paper, when  we refer to the Orlicz norm of a random variable, denoted as  $\| \cdot \|_\psi$ or sometimes  $\| \cdot \|_\varphi$, it is always understood  that $\psi(\cdot)$ (or $\varphi(\cdot)$) is  an  nondecreasing  function defined on $\mathbb{R}_+$ with $\psi(0)=0$.  Theorem <ref> also requires       $\psi$  to satisfy the regularity condition   $\allowbreak\limsup_{x,y\to \infty}\psi(x)\psi(y)/\psi(cxy)\allowbreak<\infty$ for some constant $c>0$ [64]. It is easy to verify that  all these conditions are met by $L_q$ with $q>0$  and $\psi_q$ with $0<q\le 2$. Orlicz norms provide a useful framework for analyzing skewed random variables, including those without zero mean.

In this paper, we define that a random vector  $\epsilon\in\mathbb R^n$ has its Orlicz $\psi$-norm $\|\epsilon\|_{\psi}$ bounded above by $\omega$ if

\begin{align}
\|\langle\epsilon,\alpha\rangle\|_{\psi}\leq \omega\|\alpha\|_2,\ \forall \alpha\in\mathbb R^n. \label{orliczvecdef}
\end{align}

Note that (<ref>) is defined using the Euclidean norm $\|\cdot \|_2$, and the function $\psi$ may not necessarily be convex.   Furthermore, the components of $\epsilon$ are not required to be independent or centered. However, if $\epsilon$ does have centered, independent components, its vector Orlicz $\psi$-norm is bounded by the largest Orlicz $\psi$-norm among its components (up to a multiplicative constant)..

Let    $\epsilon_1,\ldots,\epsilon_n$ be centered, independent random variables satisfying  $\|\epsilon_i\|_{\psi_q}\leq \omega$ for some $q\in(0,2]$. Given any $\alpha \in \mathbb{R}^n$, we have $\|\langle \alpha,\epsilon \rangle\|_{\psi_q}\leq C\|\alpha\|_2\omega$, where $C$ is a constant  depending on $q$ only.

To prove the lemma, we first  introduce two lemmas. The first is       Theorem 1.5 in [25].
Let $\epsilon_1,\ldots,\epsilon_n$ be independent random variables satisfying $\|\epsilon_i\|_{\psi_q}\leq \omega$ for some $q\in(0,2]$. Let $f(\epsilon):\mathbb{R}^n\to \mathbb{R}$ be a polynomial of  degree $D\in \mathbb{N}$ and denote by $f^{(d)}$ the $d$-tensor of its $d$-th order partial derivatives for $1\leq d\leq D$. Then for all $t>0$, we have

\begin{align}
\mathbb{P}(|f(\epsilon)-\mathbb{E}f(\epsilon)|\geq t)\leq 2\exp\big(-C\min_{1\leq d\leq D} (\frac{t}{\omega^d \|\mathbb{E}f^{(d)}(\epsilon)\|_{HS}} )^\frac{q}{d}\big),
\end{align}

where $C$ is a constant that depends on $D$ and $q$ and $\|\cdot\|_{HS}$ denotes the Hilbert-Schmidt norm (or the Frobenius norm in the case of a matrix).

The second fact is a slight modification of Lemma 2.2.1 in [64].

Let $X$ be a random variable such that for some $q>0$,

\begin{align}\label{tailfubinilem}
P(|X|>t)\leq c\exp\big(-\big(\frac{t}{\omega}\big)^q\big),\quad \forall t>0,
\end{align}

where $\omega>0$ and $c$ is constant,
then we have $\|X\|_{\psi_q}\lesssim \omega$.

The proof is straightforward:

\begin{align*}
\mathbb{E}\big(\exp(M|X|^q)-1\big)= \, & \mathbb{E}\int^{|X|^q}_{0}M\exp(Mu)\rd u \\ =\, & \int^{\infty}_{0}\mathbb{P}(|X|>u^{\frac{1}{q}})M\exp(Mu)\rd u \\
=\, & \int^{\infty}_{0}cM\exp(Mu-\frac{u}{\omega^q})\rd u\leq \frac{cM}{\omega^{-q}-M}.
\end{align*}

It suffices to take $M^{-1/q}\geq c'\omega$ to have $\mathbb{E} (\exp(M|X|^q)-1)\le 1$. Therefore, $\|X\|_{\psi_q}\lesssim \omega$.

Now, given any $\alpha \in \mathbb{R}^n$, let $f_\alpha(\epsilon)=\langle \alpha,\epsilon \rangle=\sum^n_{i=1}\alpha_i\epsilon_i$. Then (i) $f_\alpha(\epsilon)$  is an 1-degree polynomial of $\epsilon_1,\ldots,\epsilon_n$,  (ii) $\|\mathbb{E}[\nabla f_\alpha(\epsilon)]\|_2=\|\alpha\|_2$ and $\mathbb{E}[\sum^n_{i=1}\alpha_i\epsilon_i]=0$, and (iii) $\epsilon_1,\ldots,\epsilon_n$ are independent and satisfy  $\|\epsilon_i\|_{\psi_q}\leq \omega$. Applying Lemma <ref> with $D=1, d=1$ yields

\begin{align}\label{tailbound}
\mathbb{P}\big(|\sum^n_{i=1}\alpha_i\epsilon_i|>t\big)=\mathbb{P}\big(|f_\alpha(\epsilon)|>t\big)\leq 2 \exp\Big(-C\big(\frac{t}{\omega\|\alpha\|_2}\big)^q\Big),
\end{align}

where $C$ is a constant depending on $q$ only.
By Lemma <ref>,
(<ref>) implies $\|\langle \alpha,\epsilon \rangle\|_{\psi_q}\leq C\|\alpha\|_2\omega$, where $C$ is a constant depending on $q$ only. The proof of Lemma <ref> is complete.

The following lemma is useful for stating the assumptions on effective noises.

Let $\psi,\varphi$ be any two    nondecreasing nonzero functions  defined on $\mathbb{R}_+$ with $\psi(0)=\varphi(0)=0$ (not necessarily convex). Define
$\varphi^{-1}(t):=\sup\{x\in\mathbb{R}_+:\varphi(x)\leq t\}.
and $\psi^{-1}$   similarly.
(i) Suppose that $\psi(\varphi^{-1}(t)/c_{0})$ is concave in $t$ on $\mathbb{R}_+$ for some $c_{0}> \varphi^{-1}(1)/\psi^{-1}(1)$. Then for any random variable $X$,  we have $\|X\|_\psi\leq c_{0}\|X\|_\varphi$.
(ii) Suppose that $\psi(\varphi^{-1}(t)/c_{0})\leq t$ for some $c_{0}>0$, then  $\|X\|_\psi\leq c_{0}\|X\|_\varphi$.

We remark that the condition for $c_0$ in part (i) can be replaced by $c_{0}\ge  \varphi^{-1}(1)/\psi^{-1}(1)$ when $\psi$ is continuous at $1$.
For completeness, we provide the proof below.

First, by the definition of $\varphi^{-1}$, $u\leq \varphi^{-1}(\varphi(u))$
for any $u\geq0$.
Therefore, $X/\|X\|_\varphi\leq \varphi^{-1}\big\{\varphi(X/\|X\|_\varphi)\big\}$, from which it follows that

\begin{align}\label{lemmabasicineq1}
\psi\Big(\frac{X}{c\|X\|_{\varphi}}\Big)\leq  \psi\Big[\frac{1}{c}\varphi^{-1}\big\{\varphi(\frac{X}{\|X\|_\varphi})\big\}\Big].
\end{align}

To prove part (i),
let $f(u)=\psi\big(\varphi^{-1}(u)/c\big)$ with $c>0$ to be determined.
Then $f(u)$ is an increasing function on $\mathbb{R}_+$. (In fact, for any $u\geq u'\geq0$,  $ \varphi^{-1}(u')\leq \varphi^{-1}(u)$, and so
$\psi\{\varphi^{-1}(u')/c\}\leq \psi\{\varphi^{-1}(u)/c\}$.)
From (<ref>), picking $t=\varphi(X/\|X\|_{\varphi})$ gives

\begin{align}\label{lemmaftineq}
\psi\Big(\frac{X}{c\|X\|_{\varphi}}\Big)\leq  \psi\Big[\frac{1}{c}\varphi^{-1}\big\{\varphi(\frac{X}{\|X\|_\varphi})\big\}\Big]=f(t).
\end{align}

With $c>\varphi^{-1}(1)/\psi^{-1}(1)$ (or $c\ge \varphi^{-1}(1)/\psi^{-1}(1)$ when $\psi$ is continuous at $1$), we can use   Jensen's inequality to get

\begin{align}
\mathbb{E}\Big\{\psi\Big(\frac{X}{c\|X\|_{\varphi}}\Big)\Big\}\leq \mathbb{E}\Big(\psi\Big[\frac{1}{c}\varphi^{-1}\big\{\varphi(\frac{X}{\|X\|_\varphi})\big\}\Big]\Big)=\mathbb{E}\big(f(t)\big)\leq f\big(\mathbb{E} (t)\big)\leq f(1)\leq1.
\end{align}

To prove part (ii), we still set $f(u)=\psi\big(\varphi^{-1}(u)/c\big)$ with $c>0$ and $t=\varphi(X/\|X\|_{\varphi})$. Based on (<ref>),  we get

\begin{align}\label{lemmaineqpartb}
\psi\big(\frac{X}{c\|X\|_{\varphi}}\big)\leq  f(t)\leq t= \varphi\big(\frac{X}{\|X\|_\varphi}\big).
\end{align}

The proof of Lemma <ref> is complete.

§.§ An  Excess Risk Bound

This part establishes an excess risk bound for location estimation using pivot-blend, which is uniform in scale parameters, shedding light on the impact of asymmetrical scales (skewness) and the presence of an unknown pivotal point. This result is non-asymptotic and non-parametric, making it applicable to various scenarios.

Let  $y_i\in\mathbb{R}$ and $X_i\in \mathbb{R}^p$ which satisfy satisfy $(X_i,y_i)\overset{i.i.d.}{\sim} F^{*}$, where    $F^{*}$ is a  distribution  that  depends on  $\beta^*,m^*,\sigma^*,\nu^*$, where we use superscript $^*$ to denote the statistical truth.  As discussed in Section <ref>, we consider the practice of estimating the scales beforehand and then minimizing a  criterion over all location parameters $\beta_j (1\le j\le p), m$.  For ease of presentation, given $(\sigma,\nu)$, let $  l_{\sigma,\nu}(\cdot;X_i,y_i)$, imposed on $   \theta:=(\beta,m)$, denote a loss motivated by skewed pivot-blend:

\begin{align}\label{globallossdef}
\begin{split}
l_{\sigma,\nu}(\theta;X_i,y_i):=\,&\, \rho\big(\frac{r_i-m}{\sigma}+m\big)1_{r_i-m\leq 0}+\rho\big(\frac{r_i-m}{\nu}+m\big)1_{r_i-m>0}\\&+\cc\log\big[\sigma\Phi(m)+\nu\{1-\Phi(m)\}\big] \quad \mbox{with} \quad r_i=y_i-X^T_i\beta,
\end{split}
\end{align}

where the calibration parameter $\cc>0$  and $0\le \Phi(m)\le 1$. Note that  $\Phi$ is not necessarily directly associated with $\rho$.
$\hat\theta_{\sigma,\nu}$ by   empirical risk minimization

\begin{align}\label{erm}
\hat\theta_{\sigma,\nu}=\argmin_{\theta }\sum^{n}_{i=1}l_{\sigma,\nu}(\theta;X_i,y_i).
\end{align}

Certainly, opting for different values of $\sigma$ and $\nu$  produces diverse  asymmetric losses and influences the overall risk.  In our analysis, no restrictions will be placed on $\sigma$ and $\nu$. The values of $(\sigma,\nu)$ can be specified based on domain knowledge or determined in a data-dependent manner. For notational simplicity,  we sometimes  abbreviate $\hat\theta_{\sigma,\nu}$ as $\hat\theta$ when there is no ambiguity.

To evaluate the generalization performance of   $\hat\theta$,  let $(X_0,y_0)$ be a new observation that follows $F^{*}$ but is independent of the training data $(X_i,y_i),(1\leq i\leq n)$, and define the population risk of $\hat\theta$ by

\begin{align}\label{riskdef}
\end{align}

where the expectation is taken with respect to the new observation $(X_0,y_0)$ only.
Due to the finite number of observations  in estimation, $R_{\sigma,\nu}(\hat\theta_{\sigma,\nu})$ is always greater than the population risk of the ideal $\theta^*_{\sigma,\nu}=\argmin_{\theta} R_{\sigma,\nu}(\theta)$, or $ R_{\sigma,\nu}(\theta^*_{\sigma,\nu})=\inf_{\theta\in \Omega}R_{\sigma,\nu}(\theta)$.
In such a setup, the notion of  excess risk $\mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)$
is helpful [19]:

\begin{align}
\mbox{Excess Risk:}\quad   \mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)\,:=\,R_{\sigma,\nu}(\hat\theta_{\sigma,\nu})-R_{\sigma,\nu}(\theta^*_{\sigma,\nu}).
\end{align}

Our main objective is to establish a non-asymptotic bound for $\mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)$ regardless of the data distribution for a broad range of $\rho$  that satisfy the following assumption.

$\mathrm{ASSUMPTION}$  $\mathcal{A}$: Assume that the loss $\rho$ satisfies (i) $\rho$ is bounded: $\rho\in [0,\rhob]$ for some $\rhob>0$, and (ii) $\rho$ is regular in the sense that $\rho$ is  piecewise polynomial   on $\kk\geq1$ intervals   $\rho(t)=P_i(t), \, \forall t\in [u_{i-1},u_i),\, i=1,\ldots,\kk,$
where $u_0=-\infty,u_{\kk}=\infty$, and each polynomial function $P_i$ has degree at most $\dd\geq 0$.

Assumption $\mathcal{A}$ encompasses a wide range of practically used loss functions in robust regression and classification, specifically designed to handle extreme outliers.  For example, some loss functions like   $\rho(t)=\int_{0}^{|t|}\psi(s)\rd s$, with a redescending  $\psi$,  such as  Tukey's bisquare $\psi(t)=t\{1-(t/c)^2\}^2$ if $|t|\leq c$, and $0$ otherwise [26], fit in the category.   Some other $\rho$ functions can be effectively approximated by piecewise polynomial functions.

As long as   the loss $\rho$ satisfies Assumption $\mathcal{A}$,
the   estimator $\hat\theta_{\sigma,\nu}$ defined in (<ref>)
satisfies the following probabilistic bound for all  $\sigma,\nu>0$,

\begin{align*}
\mathbb{P}&\left\{  \sup_{\sigma,\nu>0}\, \mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)-C\Bigg[\right.\frac{\rhob\sqrt{p\log\big\{(\kk+1)(\dd+1)\big\}}}{\sqrt{n}}+\frac{(\rhob\lor \cc)\big\{\log\frac{\sigma\lor\nu}{\sigma\land\nu}+|\log(\sigma\land\nu)| \big\}}{\sqrt{n}}\times\nonumber
\\
&\quad\quad\,\,\,\,\,\bigg\{ \sqrt{\log\Big(\log\frac{\sigma\lor\nu}{\sigma\land\nu}\lor|\log(\sigma\land\nu)|\lor1\Big)} + \sqrt{\log\frac{1}{\epsilon}}\bigg\}\left.\Bigg]\leq 0\right\}\geq 1-\epsilon.%\label{uniformbound}
\end{align*}

The theorem provides a bound for  the excess risk $\mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)$ that holds   uniformly in $\sigma$ and $\nu$ with probability at least $1-\epsilon$, as characterized by the following rate

\begin{align}\label{riskboundorderterm}
&\frac{\rhob\sqrt{p\log\big\{(\kk+1)(\dd+1)\big\}}}{\sqrt{n}}+\frac{\rhob\lor\cc\big\{\log\frac{\sigma\lor\nu}{\sigma\land\nu}+|\log(\sigma\land\nu)| \big\}}{\sqrt{n}}\times\bigg\{\sqrt{\log\frac{1}{\epsilon}}+\nonumber\\&\quad\sqrt{\log\Big(\log\frac{\sigma\lor\nu}{\sigma\land\nu}\lor|\log(\sigma\land\nu)|\lor1\Big)}\bigg\}.
\end{align}

The first term in (<ref>) illustrates the influence of loss complexity   and problem dimensions on the excess risk. The second term, which incorporates both $\sigma$ and $\nu$, arises due to the  pivot estimation. Clearly, when there is no skewness, $
\sigma=\nu  \Rightarrow  \log\frac{\sigma\lor\nu}{\sigma\land\nu}=0
$ and the rate becomes

\frac{\rhob\sqrt{p\log\big\{(\kk+1)(\dd+1)\big\}}}{\sqrt{n}}+\frac{(\rhob+\cc) |\log(\sigma\land\nu)|  }{\sqrt{n}}\times\bigg\{  \sqrt{\log (1+|\log(\sigma\land\nu)| )}+\sqrt{\log\frac{1}{\epsilon}}\bigg\}.

In the more practical scenario of unequal scales, the risk for location estimation can significantly increase, and the provided bound quantitatively characterizes how skewness inflates the risk non-asymptotically.

To prove the theorem, we first introduce a basic excess-risk bound  for fixed $\sigma,\nu>0$.

Suppose that the loss $\rho$ satisfies Assumption $\mathcal{A}$.  Fixing the values of $\sigma,\nu>0$ in (<ref>),  the corresponding estimator $\hat\theta_{\sigma,\nu}$ from (<ref>) satisfies

\begin{align}\label{fixriskbound}
\mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)\lesssim  \rhob\sqrt{\frac{p\log\big\{(\kk+1)(\dd+1)\big\}}{n}}+\frac{\rhob\lor\cc\big(|\log\sigma\wedge \nu|+\log\frac{\nu\vee\sigma}{\sigma\wedge \nu}\big)}{\sqrt{n}}\sqrt{\log\frac{1}{\epsilon}},
\end{align}

with probability
at least $1-\epsilon$.

Let $\{X,y\}$ denote the training data, i.e., $\{X,y\}:=\big\{(X_i,y_i),1\leq i\leq n\big\}$ with $(X_i,y_i)\overset{i.i.d.}{\sim}F^{*}$.
Given $\sigma,\nu>0$, define a function class consisting of all $l_{\sigma,\nu}(\theta;\cdot)$   $\theta\in \Omega=\mathbb R^p\times \mathbb R$

\begin{align}
\mathcal{L}_{\sigma,\nu}(\Omega):=\big\{l_{\sigma,\nu}(\theta;\cdot):\theta\in \Omega\big\}.
\end{align}

For  simplicity, we often use the shorthand notations
$R(\cdot)$, $R^{(n)}(\cdot)$, and $l(\theta;\cdot)$ to denote
$R_{\sigma,\nu}(\cdot)$, $(1/n)\sum^{n}_{i=1}l_{\sigma,\nu}(\cdot;X_i,y_i)$, and $l_{\sigma,\nu}(\theta;\cdot)$, respectively, when there is no ambiguity.

First,  the standard bound for excess risk through uniform laws yields
\begin{align}\label{symmetrizationbound}
R(\hat\theta_{\sigma,\nu})-R(\theta^*_{\sigma,\nu})\leq &\, R(\hat\theta_{\sigma,\nu})-R^{(n)}(\hat\theta_{\sigma,\nu})+R^{(n)}(\hat\theta_{\sigma,\nu})-R^{(n)}(\theta^*_{\sigma,\nu})+R^{(n)}(\theta^*_{\sigma,\nu})-R(\theta^*_{\sigma,\nu})\nonumber\\
\leq &\,
2\sup_{\theta\in\Omega}\big|R(\theta)-R^{(n)}(\theta)\big|  =2\sup_{l\in\mathcal{L}_{\sigma,\nu}(\Omega)}\big|\mathbb{P}_nl-\mathbb{P}l\big|=2\big\|\mathbb{P}_n-\mathbb{P}\big\|_{\mathcal{L}_{\sigma,\nu}(\Omega)},
\end{align}

where $\mathbb{P}$ is the distribution $F^*$ and $\mathbb{P}_n$ is the empirical measure that places  probability mass $1/n$ on each $(X_i,y_i),1\leq i\leq n$.

\begin{align}
\end{align}

Without loss of generality, we assume that $\sigma\leq \nu$, that is, $\sigma\land\nu=\sigma,\, \sigma\lor\nu=\nu$, then

\begin{align}
\end{align}

Because $\log(1+(\frac{\nu}{\sigma}-1)t)$ is an increasing function for $t\geq0$, we know

\begin{align}
\cc\log\sigma\leq   g_{\sigma,\nu}(m)\leq \cc\log\sigma+\cc\log\frac{\nu}{\sigma},
\end{align}

from which it follows that

\begin{align}\label{gbound}
|g_{\sigma,\nu}(m)|\leq \big|\cc\log\sigma+\cc\log\frac{\nu}{\sigma}\big|\lor |\cc\log\sigma|\leq |\cc\log\sigma|+\cc\log\frac{\nu}{\sigma}.
\end{align}

By Assumption $\mathcal{A}$ and (<ref>),

\begin{align}\label{lbounded}
|l_{\sigma,\nu}|\leq|\rho|+|g_{\sigma,\nu}|\leq \rhob+\cc\big(|\log\sigma|+\log\frac{\nu}{\sigma}\big).
\end{align}

Due to (<ref>) and $(X_i,y_i)\overset{i.i.d.}{\sim}F^{*}$,
applying   McDiarmid's inequality and symmetrization in  empirical process theory  yields a data-dependent bound with probability at least $1-\epsilon$,

\begin{align}\label{boundeddifferencebound}
\big\|\mathbb{P}_n-\mathbb{P}\big\|_{\mathcal{L}_{\sigma,\nu}(\Omega)}\leq 2\mathcal{R}_{X,y}(\mathcal{L}_{\sigma,\nu}(\Omega))+6\Big\{\rhob+\cc\big(|\log\sigma|+\log\frac{\nu}{\sigma}\big)\Big\}\sqrt{\frac{\log(3/\epsilon)}{2n}},
\end{align}

where $\mathcal{R}_{X,y}(\mathcal{L}_{\sigma,\nu}(\Omega))$ is the empirical Rademacher complexity of $\mathcal{L}_{\sigma,\nu}(\Omega)$ with respect to the training data $\{X,y\}$:

\begin{align}\label{originalrademacher}
\mathcal{R}_{X,y}(\mathcal{L}_{\sigma,\nu}(\Omega)) :=\frac{1}{n}\mathbb{E}_\epsilon\sup_{l\in\mathcal{L}_{\sigma,\nu}(\Omega)}\sum^n_{i=1}\epsilon_i l(\theta;X_i,y_i),
\end{align}

and $\epsilon_i$'s are i.i.d. Rademacher random variables;
see, e.g., Theorem 3.4.5 in [22]. Note that the  expectation in (<ref>) is taken with respect to $\epsilon$ only, and (<ref>) depends on $\sigma,\nu$  through  the function class $\mathcal{L}_{\sigma,\nu}(\Omega)$.

It remains to bound the empirical Rademacher complexity.
Toward this, denote by  $\nd_\sigma,\nd_\nu$  two augmented design matrices

\begin{align}
\nd_\sigma=-\frac{1}{\sigma}[X, (1-\sigma)1_n],\quad \nd_\nu=-\frac{1}{\nu}[X, (1-\nu)1_n],
\end{align}

where $1_n$ is a column vector of $n$ ones.

\begin{align}
\alpha_\sigma=\frac{1}{\sigma}y,\quad  \alpha_\nu=\frac{1}{\nu}y.
\end{align}

Suppose that  $\mathrm{rank}(\nd_\sigma)\leq r$ and $\mathrm{rank}(\nd_\nu)\leq r$.  By the singular value decomposition,

\begin{align}
\nd_\sigma=U_\sigma D_{\sigma}V^T_\sigma,\quad\nd_\nu=U_{\nu}D_\nu V^T_\nu,
\end{align}

where $U_\sigma,U_\nu$ are orthogonal matrices with   $r$ columns: $U^T_{\sigma}U_{\sigma}=I_{r\times r},U^T_{\nu}U_{\nu}=I_{r\times r}$.

\begin{align}\label{Udef}
\bar{U}_{\sigma}=[U_{\sigma},\alpha_\sigma],\quad \bar{U}_{\nu}=[U_{\nu},\alpha_\nu].
\end{align}

Now, by the sub-additivity of sup and (<ref>),

\begin{align}\label{newradecomplex bound}
&\,\mathcal{R}_{X,y}(\mathcal{L}_{\sigma,\nu}(\Omega)) \nonumber\\\leq &\, \frac{1}{n}\mathbb{E}_\epsilon\sup_{\theta\in\Omega}\langle\epsilon, \rho(\nd_\sigma\theta+\alpha_\sigma)\rangle+\frac{1}{n}\mathbb{E}_\epsilon\sup_{\theta\in\Omega}\langle\epsilon, \rho(\nd_\nu\theta+\alpha_\nu)\rangle+ \frac{1}{n}\mathbb{E}_\epsilon\sup_{m\in\mathbb{R}}\langle\epsilon, g_{\sigma,\nu}(1m)\rangle\nonumber\\\leq &\, \frac{1}{n}\mathbb{E}_\epsilon\sup_{\xi\in\mathbb{R}^{r+1}}\langle\epsilon, \rho(\bar{U}_{\sigma}\xi)\rangle+\frac{1}{n}\mathbb{E}_\epsilon\sup_{\xi\in\mathbb{R}^{r+1}}\langle\epsilon, \rho(\bar{U}_\nu\xi)\rangle+ \frac{1}{n}\mathbb{E}_\epsilon\sup_{m\in\mathbb{R}}\langle\epsilon, g_{\sigma,\nu}(1m)\rangle.
\end{align}

To  bound the first term in (<ref>),

\begin{align}
\end{align}

and given $\rho$, define a class of functions

\begin{align}
\mathcal{F}:=\big\{\rho(\langle\xi,\cdot\rangle):\xi\in\mathbb{R}^{r+1}\big\},
\end{align}

which does not depend on  the two scale parameters.
Given $Z(X,y,\sigma)$,  let $\mathbb{Q}_n$ be the empirical measure determined by $Z_i$, $1\leq i\leq n$,

\begin{align}
\|f-\tilde{f}\|^2_{\mathbb{Q}_n}:=\frac{1}{n}\sum^n_{i=1}\big\{f(Z_i)-\tilde{f}(Z_{ i})\big\}^2,\quad \forall f,\tilde{f}\in \mathcal{F}.
\end{align}

\begin{align}\label{rademacherequivalence}
\frac{1}{n}\mathbb{E}_\epsilon\sup_{\xi\in\mathbb{R}^{r+1}}\langle\epsilon, \rho(\bar{U}_{\sigma}\xi)\rangle=\frac{1}{\sqrt{n}}\mathbb{E}_\epsilon\sup_{f\in\mathcal{F}}\frac{1}{\sqrt{n}}\sum^n_{i=1}\epsilon_i f(Z_i)\leq \frac{C}{\sqrt{n}}\int^{\rhob}_{0}\sqrt{\log  \mathcal{N}(\varepsilon,\mathcal{F},\|\cdot\|_{\mathbb{Q}_n})}\rd \varepsilon,
\end{align}

where the last inequality is due to Dudley's integral bound. Because $|f|\leq \rhob,\, \forall f\in \mathcal{F}$, by Theorem 2.6.7 in [64] we know

\begin{align}\label{coveringnumberf}
\mathcal{N}(\varepsilon,\mathcal{F},\|\cdot\|_{\mathbb{Q}_n})\leq C\vc(\mathcal{F})\big(\frac{c\rhob}{\varepsilon}\big)^{2\vc(\mathcal{F})},
\end{align}

where $\vc(\mathcal{F})$  denotes the VC-dimension  of  $\mathcal{F}$  defined through the notion of subgraph (cf. Definition 3.6.8 in [22]).
In more details,
$\vc(\mathcal{F})$ is the VC-dimension of the following $\{0,1\}$-valued function class defined on $\mathbb{R}^{r+1}\times \mathbb{R}$:

\begin{align}
\mathcal{H}:=\big\{h_{\xi}(z,t)=1_{\rho(\langle\xi,z\rangle)> t}=\sign(\rho(\langle\xi,z\rangle)- t):\xi\in\mathbb{R}^{r+1}\big\}.
\end{align}

Here,  the sign function is defined by $\sign(a)=1$ if $a>0$, and $0$ otherwise, and recall that
$1_{\rho(\langle\xi,z\rangle)> t}$ is the indicator function of the set $\big\{(z,t):\rho(\langle\xi,z\rangle)> t,z\in\mathbb{R}^{r+1},t\in\mathbb{R}\big\}$ or the subgraph of $\rho(\langle\xi,\cdot\rangle)$ for each given $\xi$.

To bound $\vc(\mathcal{H})$,
we introduce
a more general function class $\tilde{\mathcal{H}}$

\begin{align}\label{allsubdef}
\tilde{\mathcal{H}}:=\big\{\tilde{h}_{\xi,\omega_1,\omega_2}(z,t)=\sign\big(\omega_1\rho(\langle\xi,z\rangle)+\omega_2 t\big):\xi\in\mathbb{R}^{r+1},\omega_1,\omega_2\in\mathbb{R}\big\},
\end{align}

where $\tilde{h}_{\xi,\omega_1,\omega_2}(z,t)$  is defined on $(z,t)\in\mathbb{R}^{r+1}\times \mathbb{R}$ and has two additional parameters $\omega_1$ and $\omega_2$ than $h_{\xi}(z,t)$ in the definition of $\mathcal{H}$.
Since $\mathcal{H}\subset   \tilde{\mathcal{H}}$ (by fixing $\omega_1=1,\omega_2=-1$),

\begin{align}\label{networkvcdim}
\vc(\mathcal{F})= \vc(\mathcal{H})\leq \vc( \tilde{\mathcal{H}}).
\end{align}

$\tilde{\mathcal{H}}$ corresponds to the set of functions computed by a
neural network $\mathbf{N}$ shown in Figure <ref>.
It has two computation units, one in the   hidden layer with activation function $\rho$, and the other  in the output layer applying a sign operation on the combined inputs  $\omega_1\rho(\langle\xi,z\rangle)+\omega_2 t$.
Lemma <ref> is Theorem 10 in [7] and can be  proved based on Theorem 2.2 of  [24].

Architecture of the  network $\mathbf{N}$ that computes $\tilde{h}_{\xi,\omega_1,\omega_2}(z,t)\in \tilde{\mathcal{H}}$.

Suppose that a neural network $\mathbf{N}_0$  satisfies (i) $\mathbf{N}_0$ has a directed acyclic graph, that is, the connections from input or computation units to computation units  do not form any loops, (ii)
the unique output unit is the only computation unit in the output layer (layer $J$), where $J\geq 2$ denotes the length of the longest path in the graph of $\mathbf{N}_0$, and the activation function of the output unit is a sign function that  takes inputs from units in any layer $j<J$, including the input layer (layer $0$), and
(iii) within each computation unit except the output unit, $\Psi$ is the  activation function  and  is piecewise polynomial  on $I\geq 1$ intervals $\Psi(t)=P_i(t),\,\forall t\in[u_{i-1},u_{i}),i=1,\ldots,I$, where $u_0=-\infty,u_I=\infty$, and each polynomial function $P_i$ has degree at most $M\geq0$. Let  $W\geq 2$  be the number of parameters (weights and biases) and $U\geq 2$ be the number of computation units.
Let $\mathcal{G}$  denote  the set of $\{0,1\}$-valued functions computed by the  network $\mathbf{N}_0$, then $\vc(\mathcal{G})\leq 2W\log_2\big[16e\{M^{U-1}+\sum^{U-1}_{i=0}M^i\}(1+I)^{U}\big]$.

Our network $\mathbf{N}$ as shown in Figure <ref>  is a feed-forward neural network and has a directed acyclic graph structure. The  output unit of $\mathbf{N}$ in the output layer  takes the input $\rho(\langle\xi,z\rangle)$ from the computation unit  in the hidden layer  and the input unit  $t\in \mathbb{R}$ in the input layer. Under Assumption $\mathcal{A}$,
our network $\mathbf{N}$ satisfies the assumptions in  Lemma <ref> and has $r+3$ parameters (no bias parameters) and two computation units. Using Lemma <ref>, we get

\begin{align}\label{networkvcbound}
\vc(\tilde{\mathcal{H}})\leq C(r+3)\log\big\{e(2 D+1)(1+K)^2\big\}\lesssim r\log\big\{(\kk+1)(\dd+1)\big\}.
\end{align}

Now, based on (<ref>) and (<ref>),

\begin{align}\label{pseudobound}
\vc(\mathcal{F})\leq  Cr\log\big\{(\kk+1)(\dd+1)\big\},
\end{align}

and combining (<ref>), (<ref>), and (<ref>) results in

\begin{align}\label{raderho1}
\frac{1}{n}\mathbb{E}_\epsilon\sup_{\xi\in\mathbb{R}^{r+1}}\langle\epsilon, \rho(\bar{U}_{\sigma}\xi)\rangle\leq&\,\frac{C}{\sqrt{n}}\int^{\rhob}_{0}\sqrt{\vc(\mathcal{F})\log\frac{\rhob}{\varepsilon}}\rd \varepsilon\lesssim \rhob\sqrt{\frac{r\log\big\{(\kk+1)(\dd+1)\big\}}{n}}.
\end{align}

Similarly, the second term in (<ref>) satisfies

\begin{align}\label{raderho2}
\frac{1}{n}\mathbb{E}_\epsilon\sup_{\xi\in\mathbb{R}^{r+1}}\langle\epsilon, \rho(\bar{U}_{\nu}\xi)\rangle \lesssim \rhob\sqrt{\frac{r\log\big\{(\kk+1)(\dd+1)\big\}}{n}}.
\end{align}

Note that  (<ref>)  and  (<ref>) hold for all  $\sigma,\nu>0$.

Finally, we bound the third  term on the right-hand side of  (<ref>). Let

\begin{align}
\end{align}

and so $0\leq q_{\sigma,\nu}(m)\leq \cc\log\frac{\nu}{\sigma}$.  Again, by the sub-additivity of sup and Dudley's integral bound,

\begin{align}\label{gconversion}
\frac{1}{n}\mathbb{E}_\epsilon\sup_{m\in\mathbb{R}}\langle\epsilon, g_{\sigma,\nu}(1m)\rangle\leq&\,\frac{1}{n}\cc\log\sigma\mathbb{E}_\epsilon\sup_{m\in\mathbb{R}}\langle\epsilon, 1\rangle+\frac{1}{n}\mathbb{E}_\epsilon\sup_{m\in\mathbb{R}}\langle \epsilon, q_{\sigma,\nu}(1m)\rangle\nonumber\\\leq &\,\frac{C}{n} \int^{\frac{1}{2}\cc\log\frac{\nu}{\sigma}\sqrt{n}}_{0}\sqrt{\log\mathcal{N}\big(\varepsilon,\big\{[q_{\sigma,\nu}(1m)]: m\in \mathbb{R}\big\},\|\cdot\|_2\big)}\rd \varepsilon\nonumber\\\leq &\,\frac{C}{n} \int^{\frac{1}{2}\cc\log\frac{\nu}{\sigma}\sqrt{n}}_{0}\sqrt{\log\mathcal{N}\Big(\frac{\varepsilon}{\sqrt{n}},\big[0,\cc\log\frac{\nu}{\sigma}\big],|\cdot|\Big)}\rd \varepsilon\nonumber\\\lesssim &\,  \frac{\cc\log\frac{\nu}{\sigma}}{\sqrt{n}}.
\end{align}

Plugging (<ref>), (<ref>), and (<ref>) into (<ref>) yields

\begin{align}\label{rcomplexbound}
\mathcal{R}_{X,y}(\mathcal{L}_{\sigma,\nu}(\Omega)) \lesssim \rhob\sqrt{\frac{r\log\big\{(\kk+1)(\dd+1)\big\}}{n}}+\frac{\cc\log\frac{\nu}{\sigma}}{\sqrt{n}}.
\end{align}

Summarizing  (<ref>), (<ref>), and (<ref>), we have   the following  bound with probability at least $1-\epsilon$ for any $0<\epsilon<1$,

\begin{align}\label{partabound}
&\, \mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)\nonumber\\\leq &\,
C\Bigg[\rhob\sqrt{\frac{r\log\big\{(\kk+1)(\dd+1)\big\}}{n}}+\frac{\cc\log\frac{\nu}{\sigma}}{\sqrt{n}}+\frac{\rhob+\cc\big(\log\frac{\nu}{\sigma}+|\log\sigma|\big)}{\sqrt{n}}\sqrt{\log\frac{1}{\epsilon}}\,\Bigg]\nonumber\\\lesssim&\, \rhob\sqrt{\frac{p\log\big\{(\kk+1)(\dd+1)\big\}}{n}}+\frac{\rhob\lor\cc\big(\log\frac{\nu}{\sigma}+|\log\sigma|\big)}{\sqrt{n}}\sqrt{\log\frac{1}{\epsilon}},
\end{align}

where the last inequality is due to $r\leq p+1\lesssim p$. The proof of Lemma <ref> is complete.

Lemma <ref>  shows the effect of skewness for fixed values of  $\sigma,\nu$. For example, when $\sigma=\sigma^*,\nu=\nu^*$,  the excess risk $\mathcal{E}(\hat\theta_{\sigma^*,\nu^*};\sigma^*,\nu^*)$ satisfies

\begin{align}
\mathcal{E}(\hat\theta_{\sigma^*,\nu^*};\sigma^*,\nu^*)\lesssim \frac{\rhob\sqrt{p\log\big\{(\kk+1)(\dd+1)\big\}}}{\sqrt{n}}+\frac{\rhob\lor\cc\big\{\log\frac{\sigma^*\lor\nu^*}{\sigma^*\land\nu^*}+|\log(\sigma^*\land\nu^*)| \big\}}{\sqrt{n}}\sqrt{\log\frac{1}{\epsilon}}
\end{align}

with probability at least $1-\epsilon$. Our objective is to establish a uniform law that applies to all scale parameters   $\sigma,\nu>0$. Toward this,    let

\begin{align}\label{taudef}
\tau=|\log(\sigma\land\nu)|+\log\frac{\sigma\lor\nu}{\sigma\land\nu},
\end{align}

which is nonnegative.
By Lemma <ref>,  there exists a universal constant $C$ such that the  event $\ev(\tau,t)$

\begin{align}
\ev(\tau,t) := \left\{ \mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)- C\Bigg[\rhob\sqrt{\frac{p\log\big\{(\kk+1)(\dd+1)\big\}}{n}}+\big(\rhob+\cc\tau\big)t\Bigg]\geq0\right\} \end{align}

occurs with probability

\begin{align}\label{eventprobsimple}
\mathbb{P}\big[\ev(\tau,t)\big]\leq \exp(-2nt^2),
\end{align}

for any $t>0$.

Let $[0,+\infty)=\bigcup\limits_{l=0}^{\infty}[\tau_l,\tau_{l+1}]$ with $0=\tau_0<\tau_1<\ldots<+\infty$ to be determined. Let $Q(\tau)\geq 0$ ($\forall \tau\geq0$) be an increasing function.  Then, for all $l\geq0$ and $l\in\mathbb{Z}$, $\tau\in [\tau_l,\tau_{l+1}]$  implies  $\tau\geq \tau_{l}$, $Q(\tau)\geq Q(\tau_l)$, and $\tau Q(\tau)\geq\tau Q(\tau_l)\geq \tau_l Q(\tau_l)$.
Based on (<ref>) and  the union bound,

\begin{align}\label{discretebound}
&\mathbb{P}\left(\sup_{\sigma,\nu>0}\mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)- C\Bigg[\rhob\sqrt{\frac{p\log\big\{(\kk+1)(\dd+1)\big\}}{n}}+\big(\rhob+\cc\tau\big)\big\{t+Q(\tau)\big\}\Bigg]\geq 0\right)\nonumber\\\leq& \sum^{\infty}_{l=0}\mathbb{P}\Big[\ev\big(\tau_l,t+Q(\tau_l)\big)\Big].
\end{align}

Take $Q(\tau)=\sqrt{\log(1+\tau)/n}$ and $\tau_l=l$ for all $l\geq0$ and $l\in\mathbb{Z}$, then (<ref>) is bounded by

\begin{align}\label{discretizationbound}
\sum^{\infty}_{l=0}\mathbb{P}\Bigg[\ev\bigg(l,t+\sqrt{\frac{\log(1+l)}{n}}\bigg)\Bigg]\leq\sum^{\infty}_{l=0}\frac{1}{(l+1)^2}\exp(-2nt^2) \leq &C\exp(-2nt^2).
\end{align}

Plugging (<ref>) into (<ref>) gives the following uniform law   for all $\sigma,\nu>0$ with probability $1-\epsilon$

\begin{align}
\mathcal{E}(\hat\theta_{\sigma,\nu};\sigma,\nu)\leq &\,C\Bigg[\frac{\rhob\sqrt{p\log\big\{(\kk+1)(\dd+1)\big\}}}{\sqrt{n}}+\frac{\rhob\lor\cc\big\{\log\frac{\sigma\lor\nu}{\sigma\land\nu}+|\log(\sigma\land\nu)| \big\}}{\sqrt{n}}\nonumber\\&\quad\,\times\bigg\{\sqrt{\log\frac{1}{\epsilon}}+\sqrt{\log\big(\log\frac{\sigma\lor\nu}{\sigma\land\nu}\lor|\log(\sigma\land\nu)|\lor1)+\log 2}\bigg\}\Bigg]\nonumber\\\lesssim &\,\frac{\rhob\sqrt{p\log\big\{(\kk+1)(\dd+1)\big\}}}{\sqrt{n}}+\frac{\rhob\lor\cc\big\{\log\frac{\sigma\lor\nu}{\sigma\land\nu}+|\log(\sigma\land\nu)| \big\}}{\sqrt{n}}\nonumber\\&\quad\,\times\bigg\{\sqrt{\log\frac{1}{\epsilon}}+\sqrt{\log\big(\log\frac{\sigma\lor\nu}{\sigma\land\nu}\lor|\log(\sigma\land\nu)|\lor1)}\bigg\}, \quad\forall\sigma,\nu>0,
\end{align}

The proof of Theorem <ref> is complete.

§.§ Proof of Theorem <ref>

By the  optimality of $\hat{\mu}$: $l(\hat{\mu})+\frac{\etaa}{2}\|\hat\gamma\|^2_2+\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\leq l(\mu^*)+\frac{\etaa}{2}\|\gamma^*\|^2_2+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}$,  we obtain a basic inequality as follows

\begin{align}\label{basicineql2}
&\,\mathbf{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\|\hat{\gamma}-\gamma^*\|^2_2\nonumber\\\leq &\, \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle+\langle-\etaa\gamma^*,\hat{\gamma}-\gamma^*\rangle\nonumber\\&\,+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\nonumber\\\leq &\,\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle+\frac{b}{2}\|\hat\gamma-\gamma^*\|^2_2+\frac{\etaa^2}{2b}\|\gamma^*\|_{2}^2\nonumber\\&\,+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}
\end{align}

for any $b>0$.

First, we  bound $\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle$. Let $\bar{X}_k$ denote the $k$th column of $\bar{X}$    and  $\Xi_k:=[\bar{X}_k,\bar{X}_{p+k}]^T$ and  recall $ {\bar{\beta}}_k = [\beta_k, b_k]^T$,   $k=1,\ldots,p$.  By Hölder's inequality,

\begin{align}
\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\rangle=  \sum^p_{k=1}\langle
\Xi_k\epsilon_{\bar{\eta}},\hat{\bar{\beta}}_k-\bar{\beta}^*_k\rangle
\leq  \sum^p_{k=1} \|\Xi_k\epsilon_{\bar{\eta}}\|_2\|\hat{\bar{\beta}}_k-\bar{\beta}^*_k\|_2
\leq  \sqrt{2}\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty} \|\hat{\bar{\beta}}-\bar{\beta}^*\|_{2,1}.
\end{align}

Let $\lambda=A\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}/\mtbb$ for $ A>1$. Then we have

\begin{align}
&\,\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\big\|\hat{\bar{\beta}}-\bar{\beta}^*\big\|_{2,1}+\lambda\mtbb\big\|\bar\beta^*\big\|_{2,1}-\lambda\mtbb\big\|\hat{\bar\beta}\big\|_{2,1}\nonumber\\\leq &\,\big(\lambda\mtbb+\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\big)\big\|\bar\beta^*\big\|_{2,1}+\big(\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}-\lambda\mtbb\big)\big\|\hat{\bar\beta}\big\|_{2,1}\nonumber\\= &\,(1+A)\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\big\|\bar\beta^*\big\|_{2,1}+(1-A)\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\big\|\hat{\bar\beta}\big\|_{2,1}\nonumber\\\leq &\,(1+A)\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\big\|\bar\beta^*\big\|_{2,1}.
\end{align}

By the  assumption on $\epsilon_{\bar{\eta}}$, the   $\psi$-norm of $| \bar{X}^T_k\epsilon_{\bar{\eta}}|$ is bounded by

\begin{align}
\big\| \bar{X}^T_k\epsilon_{\bar{\eta}}\big\|_{\psi}=\|\langle\epsilon_{\bar{\eta}},\bar{X}_k\rangle\|_{\psi}\leq \|\epsilon_{\bar{\eta}}\|_{\psi}\|\bar{X}_k\|_2\leq\omega_{\bar\eta}\mtbb.
\end{align}

The following inequality is essentially  Massart's finite class lemma, adapted for our purpose

\begin{align}
\big\|\max_{1\leq k\leq 2p} |\bar{X}^T_k\epsilon_{\bar{\eta}}|\big\|_{\psi}\leq   2 (1\vee c_1   \vee 2 \psi(c_2))^2 c_0    \psi^{-1}(p)\omega_{\bar\eta}\mtbb .
\end{align}

It can  be obtained by modifying  the proof of  Lemma 2.2.2 in [64] (details omitted).  By Lemma <ref>,

\begin{align}
\mathbb{E}\big[ \|\bar X^T\epsilon_{\bar\eta} \|_{\infty}\big]\le \left\| \|\bar X^T\epsilon_{\bar\eta} \|_{\infty}\right\|_{L_2} \le \psi^{-1}(1) \Big\|\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\Big\|_{\psi}\leq C c_\psi\psi^{-1}(  p)\omega_{\bar\eta}\mtbb,
\end{align}

where $c_\psi =    (1\vee c_1   \vee 2 \psi(c_2))^2 c_0 \psi^{-1}(1)$.

\begin{align}\label{derive1}
\mathbb{E}\Big\{ \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\lambda\mtbb\big\|\bar\beta^*\big\|_{2,1}-\lambda\mtbb\big\|\hat{\bar\beta}\big\|_{2,1}\Big\}\nonumber\leq &\, (1+A)\mathbb{E}\Big[\big\|\bar X^T\epsilon_{\bar\eta}\big\|_{\infty}\Big]\big\|\bar\beta^*\big\|_{2,1}\nonumber\\\leq&\, C(1+A) c_\psi \psi^{-1}(  p)\omega_{\bar\eta}\mtbb\big\|\bar\beta^*\big\|_{2,1}.
\end{align}

Next, we bound the stochastic term $\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle$. For any $a>0$,

\begin{align}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle=\langle P_{1_n}\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle\leq | u^T\epsilon_{\bar m}|\cdot\|\hat {\bar m}-\bar m^*\|_2\leq a( u^T\epsilon_{\bar m})^2+\frac{1}{2a}\mathrm{\mathbf{D}}_2(\hat {\bar m},\bar m^*),
\end{align}

where $P_{1_n}$ is the orthogonal projection matrix onto the column space of   $1_n$,  or equivalently,  $uu^T$ with $u=(1/\sqrt{n})1_n$.
By   Lemma <ref>,

\begin{align}
\mathbb{E} \big[( u^T\epsilon_{\bar m})^2\big]\leq \{\psi^{-1}(1)\}^2\|u^T\epsilon_{\bar m}\|^2_{\varphi}\leq  \{\psi^{-1}(1)\}^2\omega^2_{\bar m}.
\end{align}

Likewise, for the stochastic term $\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle$  with $\epsilon_{\varsigma}=[\epsilon^T_{\frac{1}{\sigma}1_n},\epsilon^T_{\frac{1}{\nu}1_n}]^T$ and $\hat {\varsigma}-\varsigma^*=[(1/\hat\sigma-1/\sigma^*)1^T_n,(1/\hat\nu-1/\nu^*)1^T_n]^T$, we have

\begin{align}
&\,\big\langle\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\=&\,\big\langle P_{1_n}\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle P_{1_n}\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\
\leq &\, a(u^T\epsilon_{\frac{1}{\sigma}1_n})^2+a(u^T\epsilon_{\frac{1}{\nu}1_n})^2+\frac{1}{2a}\mathrm{\mathbf{D}}_2(\hat {\varsigma},\varsigma^*),\nonumber
\end{align}

\begin{align}
\mathbb{E} \big[( u^T\epsilon_{\frac{1}{\sigma}1_n})^2\big]\leq \{\varphi^{-1}(1)\}^2\|u^T\epsilon_{\frac{1}{\sigma}1_n}\|^2_{\varphi}\leq  \{\varphi^{-1}(1)\}^2\omega^2_{\varsigma},\nonumber
\end{align}

and  $ \mathbb{E} [( u^T\epsilon_{\frac{1}{\nu}1_n})^2]\leq  \{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}$.
To sum up, for any $a>0$,

\begin{align}\label{gammaineq}
&\, \mathbb{E} \big\{   \langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle\big\}\nonumber\\ \leq&\, a\{\psi^{-1}(1)\}^2\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{1}{2a}\{\mathrm{\mathbf{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\mathbf{D}}_2(\hat {\varsigma},\varsigma^*)\}.
\end{align}

Plugging  (<ref>) and (<ref>) into   (<ref>) to obtain

\begin{align}
&\,\mathbb{E}\Big\{ \bold{\Delta}_{l}(\hat{\mu},\mu^*)+(\etaa-b)\mathrm{\bold{D}}_2(\hat{\gamma},\gamma^*)\Big\}-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\ \leq &\,C(1+A) c_\psi \psi^{-1}(  p)\omega_{\bar\eta}\mtbb\big\|\bar\beta^*\big\|_{2,1}+a\{\psi^{-1}(1)\}^2\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\gamma^*\|_{2}^2.
\end{align}

Choosing $b=\etaa/4$, $a=2/\etaa$, we get

\begin{align}
&\,\mathbb{E}\Big\{ \bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\hat{\gamma},\gamma^*)\Big\}\nonumber\\ \leq&\, C(1+A)c_\psi \psi^{-1}(  p)\omega_{\bar\eta}\mtbb\big\|\bar\beta^*\big\|_{2,1}+\frac{2}{\etaa}\big[\{\psi^{-1}(1)\}^2\omega^2_{\bar m}+2\{\varphi^{-1}\}^2\omega^2_{\varsigma}\big]+2\etaa\|\gamma^*\|_{2}^2\nonumber\\\lesssim&\,c_\psi \psi^{-1}( p)\omega_{\bar\eta}\mtbb\big\|\bar\beta^*\big\|_{2,1}+\frac{1}{\etaa} (\omega^2_{\bar m}+\omega^2_{\varsigma})+\etaa\|\gamma^*\|_{2}^2.
\end{align}

The proof is complete.

§.§ Proof of Theorem <ref>

We prove a  more general result, which includes  Theorem <ref> as a special case.

Assume that the effective noises $\epsilon_{\bar\eta}, \epsilon_{\bar m}$, and $\epsilon_\varsigma$ satisfy  $\|\epsilon_{\bar\eta}\|_{\psi}\leq \omega_{\bar\eta},\|\epsilon_{\bar m}\|_{\psi}\leq \omega_{\bar m}$, and $\|\epsilon_\varsigma\|_{\varphi}\leq \omega_\varsigma$.
Let  $\hat\unv$ denote the  optimal solution  for (<ref>) with $\varrho\geq \rhotwoinf$.    Suppose that  there exist some $\vartheta>0$ and some large $K >0$ such that the following condition holds for any $\bar\beta,\gamma$ (recall  $\unv^T=[\bar\beta^T,\gamma^T]$, $\mu^T=[\bar\eta^T,\gamma^T]$ defined based on (<ref>))

\begin{align}\label{them3regu}
(1+\vartheta)\lambda\mtbb\big\|(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}\leq \bold{\Delta}_{l}(\mu,\mu^*)+\lambda\mtbb\big\|(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\big\|_{2,1}+K\lambda^2J^*,
\end{align}

\begin{align} \label{lambdachoice-gen}
\lambda=\big(1\lor\frac{1}{\vartheta}\big)\omega_{\bar\eta}\psi^{-1}\big[p\psi\big\{A\psi^{-1}(p)\big\}\big]
\end{align}

for some large enough $A>0$.
Then for any   $L,L'>0$,

\begin{align}\label{thm3textbound}
\begin{split}  \bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\gamma,\gamma^*)\, \lesssim \, &\, (\frac{1}{\vartheta}\lor\frac{1}{\vartheta^3})K\omega^2_{\bar\eta}\Big(\psi^{-1}\Big[p\psi\big\{A\psi^{-1}(p)\big\}\Big]\Big)^2J^* \\&\,+(1\lor\frac{1}{\vartheta}) \frac{1}{\etaa}\big(L^2\omega^2_{\bar m}+L^{'2}\omega^2_{\varsigma}\big)+(1\lor\frac{1}{\vartheta})\etaa\|\gamma^*\|_{2}^2\end{split}
\end{align}

holds with probability at least $1-C/\psi\{A\psi^{-1}(p)\}-1/\psi(cL)-C/\varphi(cL')$, where  $C,c$ are  positive constants.

Theorem <ref> implies   Theorem <ref>. In fact,    (<ref>),  assuming $\vartheta$ is a constant, is just  (<ref>), and   letting  $\psi=\psi_q$ for some $q> 0$, (<ref>) yields  $\lambda=A\omega_{\bar\eta}(\log p)^{\frac{1}{q}}$.
Then,     choosing $L=C\psi^{-1}(p^{A^q}), L'=C\varphi^{-1}(p^{A^q})$, we can obtain the result in Theorem <ref>:
\begin{align*}
&\, \bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\hat\gamma,\gamma^*)\nonumber\\\lesssim  &\,
KA^2\omega^2_{\bar\eta}J^*(\log p)^{\frac{2}{q}}+ \frac{1}{\etaa}\{\psi_q^{-1}(p^{A^q})\}^2\omega^2_{\bar m}+\frac{1}{\etaa}\{\varphi^{-1}(p^{A^q})\}^{2}\omega^2_{\varsigma}+\etaa\|\gamma^*\|_{2}^2
\end{align*}

with probability at least $1-Cp^{-cA^q}$.

From $l(\hat{\mu})+\etaa\|\hat\gamma\|^2_2+\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\leq l(\mu^*)+\etaa\|\gamma^*\|^2_2+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}$,  we obtain

\begin{align}\label{basicineqgoupl2}
&\,\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\|\hat{\gamma}-\gamma^*\|^2_2\nonumber\\\leq &\, \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle+\langle-\etaa\gamma^*,\hat{\gamma}-\gamma^*\rangle\nonumber\\&\,+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\nonumber\\\leq &\,\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle+\frac{b}{2}\|\hat\gamma-\gamma^*\|^2_2+\frac{\etaa^2}{2b}\|\gamma^*\|_{2}^2\nonumber\\&\,+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}
\end{align}

for any $b>0$.

To bound $\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle$,
we use the same notations in Appendix <ref> and recall

\begin{align}\label{holderineq}
\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\rangle\leq c_0\max_{1\leq k\leq 2p} |\bar{X}^T_k\epsilon_{\bar{\eta}}| \|\hat{\bar{\beta}}-\bar{\beta}^*\|_{2,1},
\end{align}

where the constant $c_0\geq \sqrt{2}$,
\begin{align}
\big\| \bar{X}^T_k\epsilon_{\bar{\eta}}\big\|_{\psi}\leq\omega_{\bar\eta}\mtbb, \  \forall k.
\end{align}

\begin{align}
\lambda_0=\omega_{\bar\eta}\psi^{-1}\Big[p\psi\big\{A\psi^{-1}(p)\big\}\Big]
\end{align}

for some large enough $A>0$. By the union bound and Markov's inequality, the event
$\underset{1\leq k\leq 2p}{\max}|\bar{X}^T_k\epsilon_{\bar{\eta}}|\geq \lambda_0\mtbb$  occurs with probability

\begin{align}\label{probboundl1}
\mathbb{P}(\max_{1\leq k\leq 2p} |\bar{X}^T_k\epsilon_{\bar{\eta}}|\geq \lambda_0\mtbb)\leq  & \frac{2p}{\psi\Big(\frac{\omega_{\bar\eta}\mtbb\psi^{-1}\big[p\psi\{A\psi^{-1}(p)\}\big]}{ \omega_{\bar\eta}\mtbb}\Big)}\le \frac{2p}{ {  p\psi\{A\psi^{-1}(p)\}\big]}  }\leq    \frac{2}{\psi\{A\psi^{-1}(p)\}}.
\end{align}

Let $\lambda= \lambda_0/ b' $ with $b'>0$. By the subadditivity of the $\ell_1$-penalty, we get

\begin{align}\label{split}
&\, c_0\lambda_0\mtbb\big\|\hat{\bar{\beta}}-\bar{\beta}^*\big\|_{2,1}+\lambda\mtbb\big\|\bar\beta^*\big\|_{2,1}-\lambda\mtbb\big\|\hat{\bar\beta}\big\|_{2,1}\nonumber\\\leq &\, c_0\lambda_0\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}+ c_0\lambda_0\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1}+\lambda\mtbb\big\|\bar\beta^*_{\mathcal{J}^*}\big\|_{2,1}\nonumber\\&\,-\lambda\mtbb\big\|\hat{\bar\beta}_{_{\mathcal{J}^{*}}}\big\|_{2,1}-\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1}\nonumber\\
\leq &\,\big( c_0\lambda_0+\lambda\big)\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(\lambda- c_0\lambda_0\big)\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1}\nonumber\\=&\,\big( 1+c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(1- c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1},
\end{align}

where $\mathcal J^{*C}$ is the complement of $\mathcal J^*$.

For  the stochastic term $\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle$,   for any $a>0$,

\begin{align}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle=\langle P_{1_n}\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle\leq | u^T\epsilon_{\bar m}|\cdot\|\hat {\bar m}-\bar m^*\|_2\leq a( u^T\epsilon_{\bar m})^2+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*),
\end{align}

where $P_{1_n} = uu^T$ with $u=(1/\sqrt{n})1_n$.
By the assumption on $\epsilon_{\bar m}$, $\|u^T\epsilon_{\bar m}\|_\psi\leq \omega_{\bar m}$. By Markov's inequality,

\begin{align}
\mathbb{P}\Big(( u^T\epsilon_{\bar m})^2\geq cL^2\omega^2_{\bar m}\Big)\leq \frac{1}{\psi(\frac{cL\omega_{\bar m}}{\omega_{\bar m}})}=\frac{1}{\psi(cL)},
\end{align}

from which it follow that

\begin{align}\label{thm2mbound}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle\leq  acL^2\omega^2_{\bar m}+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)
\end{align}

occurs with probability at least $1-1/\psi(cL)$.
For the stochastic term $\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle$ with  $\epsilon_{\varsigma}=[\epsilon^T_{\frac{1}{\sigma}1_n},\epsilon^T_{\frac{1}{\nu}1_n}]^T$ and $\hat {\varsigma}-\varsigma^*=[(1/\hat\sigma-1/\sigma^*)1^T_n,(1/\hat\nu-1/\nu^*)1^T_n]^T$, similarly,
\begin{align}
&\,\big\langle\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\=&\,\big\langle P_{1_n}\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle P_{1_n}\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\
\leq &\, a(u^T\epsilon_{\frac{1}{\sigma}1_n})^2+a(u^T\epsilon_{\frac{1}{\nu}1_n})^2+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*).\nonumber
\end{align}

By the assumption on $\epsilon_\varsigma$, $\|u^T\epsilon_{\frac{1}{\sigma}1_n}\|_\varphi\leq \omega_\varsigma, \|u^T\epsilon_{\frac{1}{\nu}1_n}\|_\varphi\leq \omega_\varsigma$. It follows from
\begin{align}
\mathbb{P}\Big(( u^T\epsilon_{\frac{1}{\sigma}1_n})^2\geq cL^{'2}\omega^2_{\varsigma}\Big)\leq \frac{1}{\psi(\frac{cL\omega_\varsigma}{\omega_\varsigma})}=\frac{1}{\psi(cL')}
\end{align}

and $ \mathbb{P}\Big(( u^T\epsilon_{\frac{1}{\nu}1_n})^2\geq cL^{'2}\omega^2_{\varsigma}\Big)\leq 1/\psi(cL')$ that
\begin{align}\label{thm2varsigmabound}
\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle\leq 2acL^{'2}\omega^2_{\varsigma} +\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*),
\end{align}

with probability at least $1-2/\varphi(cL')$.

Plugging (<ref>), (<ref>), (<ref>), and (<ref>)   into   (<ref>)   results in

\begin{align}\label{groupl1basicineq}
&\,\bold{\Delta}_{l}(\hat{\mu},\mu^*)+(\etaa-b)\mathrm{\bold{D}}_2(\hat\gamma,\gamma^*)-\frac{1}{2a}\mathrm{\bold{D}}_2(\hat{\bar m},\bar m^*)-\frac{1}{2a}\mathrm{\bold{D}}_2(\hat\varsigma,\varsigma^*)\nonumber\\ \leq  &\,\big( 1+c_0b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(1- c_0b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1}+\frac{\etaa^2}{2b}\|\gamma^*\|_{2}^2\nonumber\\&\,+ acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma}.
\end{align}

with probability at least $1-C/\psi\{A\psi^{-1}(p)\}-1/\psi(cL)-C/\varphi(cL')$, where  $C,c$ are  positive constants.

The regularity condition (<ref>) implies
\begin{align}\label{thm3reguused}
&\,(1+\frac{\vartheta}{2+\vartheta})\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}\nonumber\\\leq&\, \frac{2}{2+\vartheta}\bold{\Delta}_{l}(\hat{\mu},\mu^*)+(1-\frac{\vartheta}{2+\vartheta})\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\big\|_{2,1}+\frac{2}{2+\vartheta}K\lambda^2J^*.
\end{align}

Set $b'=\vartheta/\{(2+\vartheta)c_0\}$ and
add    (<ref>) and  (<ref>) to get
\begin{align}
&\,  \frac{\vartheta}{2+\vartheta}\bold{\Delta}_{l}(\hat{\mu},\mu^*)+ (\etaa-b)\mathrm{\bold{D}}_2(\hat\gamma,\gamma^*)-\frac{1}{2a}\mathrm{\bold{D}}_2(\hat{\bar m},\bar m^*)-\frac{1}{2a}\mathrm{\bold{D}}_2(\hat\varsigma,\varsigma^*)\nonumber\\\leq &\,\frac{2}{2+\vartheta}K\lambda^2J^*+ acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\gamma^*\|_{2}^2.
\end{align}

Taking $b=\etaa/4$ and $  a=2/\etaa$ leads to

\begin{align}
\frac{\vartheta}{2+\vartheta}\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\hat\gamma,\gamma^*)\leq \frac{2}{2+\vartheta}K\lambda^2J^*+ \frac{2}{\etaa}cL^2\omega^2_{\bar m}+\frac{4}{\etaa}cL^{'2}\omega^2_{\varsigma}+2\etaa\|\gamma^*\|_{2}^2,
\end{align}

or equivalently

\begin{align}\label{thmmresult}
\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{(2+\vartheta)\etaa}{2\vartheta}\mathrm{\bold{D}}_2(\hat\gamma,\gamma^*)\leq \frac{2}{\vartheta}K\lambda^2J^*+\frac{2+\vartheta}{\vartheta}\frac{2}{\etaa}\big(cL^2\omega^2_{\bar m}+2cL^{'2}\omega^2_{\varsigma}\big)+\frac{2+\vartheta}{\vartheta}2\etaa\|\gamma^*\|_{2}^2.
\end{align}

Note that $\{\etaa(2+\vartheta)\}/(2\vartheta)\geq \etaa/2$. With $\lambda= \lambda_0/b'=\{(2+\vartheta)/\vartheta\}c_0\lambda_0$, we can derive from
(<ref>) that

\begin{align}\label{thm3finalbound}
&\,\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\hat\gamma,\gamma^*)\nonumber\\\leq&\, Kc^2_0\frac{2(2+\vartheta)^2}{\vartheta^3}\lambda^2_0J^*+\frac{2+\vartheta}{\vartheta}\frac{2}{\etaa}\big(cL^2\omega^2_{\bar m}+2cL^{'2}\omega^2_{\varsigma}\big)+\frac{2+\vartheta}{\vartheta}2\etaa\|\gamma^*\|_{2}^2\nonumber\\\lesssim &\,
(\frac{1}{\vartheta}\lor\frac{1}{\vartheta^3})K\omega^2_{\bar\eta}\bigg(\psi^{-1}\Big[p\psi\big\{A\psi^{-1}(p)\big\}\Big]\bigg)^2J^*+(1\lor\frac{1}{\vartheta}) \frac{1}{\etaa}\big(L^2\omega^2_{\bar m}+L^{'2}\omega^2_{\varsigma}\big)\nonumber\\&\,+(1\lor\frac{1}{\vartheta})\etaa\|\gamma^*\|_{2}^2,
\end{align}

with probability at least $1-C/\psi\{A\psi^{-1}(p)\}-1/\psi(cL)-C/\varphi(cL')$, where  $C,c$ are  positive constants.
The proof is complete.

§.§ An Elementwise  Estimation Error Bound

Assume that the effective noises $\epsilon_{\bar\eta},\epsilon_{\bar m}$, and $\epsilon_\varsigma$ satisfy    $\|\epsilon_{\bar\eta}\|_{\psi_q}\leq \omega_{\bar\eta},\|\epsilon_{\bar m}\|_{\psi_q}\leq \omega_{\bar m}$, and $\|\epsilon_\varsigma\|_{\varphi}\leq \omega_\varsigma$  for some  $q>0$.    Consider  $\hat\unv$ as the  optimal solution to (<ref>) with $\tau=0$, $\varrho\geq \rhotwoinf$, and

\begin{align}
\lambda=A\omega_{\bar\eta}(\log p)^{\frac{1}{q}}
\end{align}

for some large enough $A>0$.
Suppose that  there exist some $\vartheta,\alpha>0$ and some large $K >0$ such that  for any $\unv^T=[\bar\beta^T,\gamma^T]$

\begin{align}\label{l1infre}
&\,\alpha  nJ^*\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}+\alpha\mathrm{\bold{D}}_2(\gamma,\gamma^*) +  (1+\vartheta)\lambda\mtbb\big\|(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}\nonumber\\\leq &\, \bold{\Delta}_{l}(\mu,\mu^*)+(1-\vartheta)\lambda\mtbb\big\|(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\big\|_{2,1}+K\lambda^2J^*.
\end{align}

\begin{align}\label{l1infbound}
\|\hat{\bar{\beta}}-\bar{\beta}^*\|_{2,\infty}
\leq C\frac{\sqrt{K\alpha}\lor\vartheta}{\alpha\sqrt{n}}A\Big\{\omega_{\bar\eta}+\frac{1}{\sqrt{J^*}}(\omega_{\bar m}+\omega_{\varsigma})\Big\}(\log p)^{\frac{1}{q}}
\end{align}

with probability at least $1-Cp^{-c(A \vartheta)^q}-C/\varphi(cA \vartheta (\log p)^{\frac{1}{q}}) $, where  $C,c$ are positive constants.

The element-wise error bound (<ref>) together with a signal strength condition guarantees faithful variable
selection  with high probability; see  Remark <ref>.

By the  optimality of $\hat{\mu}$: $l(\hat{\mu})+\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\leq l(\mu^*)+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}$,  we obtain the basic inequality

\begin{align}\label{thmvarselbasicineq}
\bold{\Delta}_{l}(\hat{\mu},\mu^*)\leq  \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}.
\end{align}

We follow the same lines as   in Section <ref> to bound the  stochastic terms on the right-hand side of (<ref>).

\begin{align}
\lambda_0=A\omega_{\bar\eta}(\log p)^{\frac{1}{q}}
\end{align}

for some large enough $A>0$ and $\lambda=\lambda_0/b'$ for some $b'>0$. Similar to the previous analysis, we have  with probability at least $1-Cp^{-A^q}$,
\begin{align}\label{thmvarseleta}
&\,\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\lambda\mtbb\|\bar{\beta}^*\|_{2,1}-\lambda\mtbb\|\hat{\bar{\beta}}\|_{2,1}\nonumber\\\leq &\, \big( 1+c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(1- c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1},
\end{align}

where the constant $c_0\geq \sqrt{2}$. Moreover, for any $a>0$, we get

\begin{align}\label{thmvarselmsigma}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+  \langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma\rangle\leq  acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma}+\frac{1}{2a}\big\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\big\},
\end{align}

with probability at least $1-1/\psi_q(cL)-C/\varphi(cL')$, where $L,L'>0$ can be customized.

Plugging the bounds of (<ref>) and (<ref>) into  (<ref>),  we get

\begin{align*}\label{thmvarselmedineq}
&\, \bold{\Delta}_{l}(\hat{\mu},\mu^*)-\frac{1}{2a}\big\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\big\} \\
\leq &\, \big( 1+c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*}\big\|_{2,1}-\big(1- c_0 b'\big)\lambda\mtbb\big\|(\hat{\bar{\beta}}-\bar{\beta}^*)_{_{\mathcal{J}^{*C}}}\big\|_{2,1}+acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma},
\end{align*}

with probability at least
$1-Cp^{-A^q}-1/\psi_q(cL)-C/\varphi(cL')$, where  $C,c$ are  positive constants.

With  the regularity condition and and $b'=\vartheta/c_0$, we obtain

\begin{align}\label{prop2ineq}
&\, \alpha \big(nJ^*\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}+\mathrm{\bold{D}}_2(\gamma,\gamma^*)\big)-\frac{1}{2a}\big\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\big\}\nonumber\\\leq &\, K\lambda^2J^*+acL^2\omega^2_{\bar m}+2acL^{'2}\omega^2_{\varsigma}.
\end{align}

Setting  $a=1/\alpha$ and using $\lambda=\lambda_0/b'=(c_0/\vartheta)\lambda_0$, we obtain

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}\leq &\, \frac{Kc_0^2}{n\alpha\vartheta^2}A^2\omega^2_{\bar\eta}(\log p)^{\frac{2}{q}}+\frac{cL^2\omega^2_{\bar m}}{n\alpha^2J^*}+\frac{2cL^{'2}\omega^2_{\varsigma}}{n\alpha^2J^*}
\end{align}

with probability at least $1-Cp^{-A^q}-1/\psi_q(cL)-C/\varphi(cL')$.
With $L=L'=A (\log p)^{\frac{1}{q}}$,

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|^2_{2,\infty}\leq &\, \frac{Kc_0^2}{n\alpha\vartheta^2}A^2\omega^2_{\bar\eta}(\log p)^{\frac{2}{q}}+\frac{c}{n\alpha^2J^*}A^2(\log p)^{\frac{2}{q}}(\omega^2_{\bar m}+\omega^2_{\varsigma})\nonumber\\\leq&\, C\frac{A^2}{n\alpha^2\vartheta^2}(\log p)^{\frac{2}{q}}\big\{K\alpha\omega^2_{\bar\eta}+\frac{\vartheta^2}{J^*}(\omega^2_{\bar m}+\omega^2_\varsigma)\big\},
\end{align}

which implies

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|_{2,\infty}\leq C \frac{\sqrt{K\alpha}\lor\vartheta}{\sqrt{n}\alpha\vartheta}A(\log p)^{\frac{1}{q}}\big\{\omega_{\bar\eta}+\frac{1}{\sqrt{J^*}}(\omega_{\bar m}+\omega_{\varsigma})\big\},
\end{align}

with probability at least $1-Cp^{-cA^q}-C/\varphi(cA (\log p)^{ {1}/{q}})$.
Hence by taking $A'=A/\vartheta$ and $\lambda=A'\omega_{\bar \eta}(\log p)^{\frac{1}{q}}$,  we have

\begin{align}
\|\bar{\beta}-\bar{\beta}^*\|_{2,\infty}\leq C \frac{\sqrt{K\alpha}\lor\vartheta}{\sqrt{n}\alpha}A'(\log p)^{\frac{1}{q}}\big\{\omega_{\bar\eta}+\frac{1}{\sqrt{J^*}}(\omega_{\bar m}+\omega_{\varsigma})\big\},
\end{align}

with probability at least $1-Cp^{-c(A'\vartheta)^q}-C/\varphi(cA'\vartheta (\log p)^{\frac{1}{q}})$. The proof is complete.

§.§ Analysis of a General Penalty

Consider  the following problem associated with
a general sparsity-inducing penalty $P$

\begin{align}\label{quadraticpenaltylossap}
\end{align}

where   $\|\cdot\|_{2,P}$ is  short  for $\|\cdot\|_{2,P(\cdot;\lambda)}$  and $\|\bar{\beta}\|_{2,P(\cdot;\lambda)}:=\sum^p_{k=1}P(\|\bar{\beta}_k\|_2;\lambda)$.

Since a sparsity-inducing penalty necessarily possesses thresholding power, we assume,  without loss of generality,  that $P(\cdot;\lambda)\geq P_{H}(\cdot;\lambda)$ throughout the subsection, where the “hard penalty” $ P_{H}(t;\lambda):=(-t^2/2+\lambda |t|)1_{|t|<\lambda}+(\lambda^2/2)1_{|t|\geq\lambda}$ is induced by the hard-thresholding, and

$P_{2,H}(\bar{\beta};\lambda):=\sum^p_{k=1}P_{H}(\|\bar{\beta}_k\|_2;\lambda)$ (see [56] for more details).   Furthermore, a penalty is referred to as subadditive if it satisfies  $P(t+s;\lambda)\leq P(t;\lambda)+P(s;\lambda)$.
In fact, when $P(t)$ is concave on $\mathbb R_+$ and $P(0)=0$,  $P(|t|)$ is necessarily subadditive. Well-known examples   include the widely used $\ell_1$-penalty, $\ell_0$-penalty, SCAD, MCP, and bridge $\ell_r$ $(0< r<1)$  in the literature.

Assume that the effective noises $\epsilon_{\bar\eta},\epsilon_{\bar m}$, and $\epsilon_\varsigma$ satisfy    $\|\epsilon_{\bar\eta}\|_{\psi_2}\leq \omega_{\bar\eta},\|\epsilon_{\bar m}\|_{\psi_2}\leq \omega_{\bar m}$, and $\|\epsilon_\varsigma\|_{\varphi}\leq \omega_\varsigma$ where  $\{\varphi^{-1}(\cdot)\}^2$  concave or $\{\varphi^{-1}(t)\}^2\lesssim t$ on $\mathbb{R}_+$ (for example, $\varphi$ can be an $L_q$-norm  with $q\geq 2$ or a $\psi_q$-norm with $q>0$).
Let  $\hat\unv$ denote the  optimal solution  of (<ref>) with $\varrho\geq \rhotwo: = \|\bar{X}\|_2$.

(a) Let $\lambda=A\omega_{\bar\eta}\sqrt{\log(ep)}/\sqrt{\etaa\land 1}$ with A a sufficiently large constant.  Then the following bound always holds

\begin{align}
\mathbb{E}\bigg\{ \bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{D}}_2(\hat{\mu},\mu^*)\bigg\}\lesssim&\,\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}+\frac{1}{\etaa\land 1}\omega^2_{\bar\eta}+\frac{1}{\etaa}\big(\omega^2_{\bar m}+\omega_\varsigma^2\big)+\etaa\|\mu^*\|_{2}^2.
\end{align}

(b)  Let $P$ be a  sub-additive penalty.
Assume that there exist some $\alpha\geq0$, $\vartheta>0$, and some large $K>0$  such that

\begin{align}\label{alterregugeneral}
\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+ (1+\vartheta)\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\|_{2,P}\leq \bold{\Delta}_{l}(\mu,\mu^*)+(1-\vartheta)\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\|_{2,P}+K\lambda^2J^*,
\end{align}

for all $\unv^T=[\bar\beta^T,\bar m^T,\varsigma^T]$,
where $\lambda=A\omega_{\bar\eta}\sqrt{\log(ep)}/\sqrt{\{(\etaa+\alpha)\land \vartheta\}\vartheta}$ with A a sufficiently large constant.

\begin{align}
\mathbb{E}\,\mathrm{\bold{ D}}_2(\hat\mu,\mu^*)\lesssim \ &\,\frac{\omega_{\bar\eta}^2}{(\etaa+\alpha)\{(\etaa+\alpha)\land \vartheta\}\vartheta} \, \big\{KA^2J^*\log(ep)+\vartheta\big\}\nonumber\\ &+\frac{\omega^2_{\bar m}+\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}}{(\etaa+\alpha)^2}  +\big(1\land \frac{\etaa}{\alpha}\big)^2\|\mu^*\|_{2}^2.
\end{align}

According to  the proof below, (<ref>) can be relaxed to
$\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*};\lambda\big)+\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\|_{2,P}\nonumber \leq    \bold{\Delta}_{l}(\mu,\mu^*)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\|_{2,P}-\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}};\lambda\big)+K\lambda^2J^*
$, or $
\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*);\lambda\big)+\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+\| \varrho\bar{\beta}^*\|_{2,P}\leq  \bold{\Delta}_{l}(\mu,\mu^*)+\|\varrho\bar{\beta}\|_{2,P}+K\lambda^2J^*
$ if $P$ is not subadditive.

By definition,   $l(\hat{\zeta})+\frac{\etaa}{2}\|\hat\mu\|^2_2+\big\|\mtbb\hat {\bar\beta}\big\|_{2,P}\leq l(\zeta^*)+\frac{\etaa}{2}\|\mu^*\|^2_2+\big\| \mtbb\bar{\beta}^*\big\|_{2,P}$, which means

\begin{align}\label{basiceq}
&\,\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\|\hat{\mu}-\mu^*\|^2_2\nonumber\\\leq &\, \big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle+\langle-\etaa\mu^*,\hat{\mu}-\mu^*\rangle\nonumber\\&\,+\big\| \mtbb\bar{\beta}^*\big\|_{2,P}-\big\|\mtbb\hat {\bar\beta}\big\|_{2,P}\nonumber\\\leq &\,\big\langle\epsilon_{\bar{\eta}},\bar{X}\hat{\bar{\beta}}-\bar{X}\bar{\beta}^*\big\rangle+\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle+\frac{b}{2}\|\hat\mu-\mu^*\|^2_2+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2\nonumber\\&\,+\big\| \mtbb\bar{\beta}^*\big\|_{2,P}-\big\|\mtbb\hat {\bar\beta}\big\|_{2,P},
\end{align}

where $b$ can be any positive number.

To bound the stochastic term $\langle\epsilon_{\bar\eta},\bar{X}(\hat{\bar{\beta}}-\bar{\beta}^*)\rangle$,
define  $\lambda_0=\omega_{\bar\eta}\sqrt{\log(ep)}$  and

\begin{align}\label{Rdef}
R=&\,\sup_{\bar{\beta}\in \mathbb{R}^{2p}}\Big\{\langle\epsilon_{\bar\eta},\bar{
\end{align}

By Lemma <ref>, for any $a'\geq 2b'>0$ and a sufficiently large constant $A$,  $\mathbb{P}(R\geq a'\omega^2_{\bar\eta} t)\leq C\exp(-ct)p^{{-cA^2}}$. Therefore, $\mathbb{E}[R]\lesssim a' \omega^2_{\bar\eta}$.

Next, we bound      $\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle$ by
\begin{align}
\langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle=\langle P_{1_n}\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle\leq | u^T\epsilon_{\bar m}|\cdot\|\hat {\bar m}-\bar m^*\|_2\leq a(u^T\epsilon_{\bar m})^2+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)
\end{align}

for any $a>0$,
where $P_{1_n}=uu^T$ with $u=(1/\sqrt{n})1_n$. By the assumption on $\epsilon_{\bar m}$, $\|(u^T\epsilon_{\bar m})^2\|_{\psi_1}\lesssim\|u^T\epsilon_{\bar m}\|^2_{\psi_2}\leq \omega^2_{\bar m}$, from which it follows that $ \mathbb{E}[ (u^T\epsilon_{\bar m})^2]\leq C\omega^2_{\bar m}$.
Similarly, for the stochastic term $\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle$ with $\epsilon_{\varsigma}=[\epsilon^T_{\frac{1}{\sigma}1_n},\epsilon^T_{\frac{1}{\nu}1_n}]^T$ and $\hat {\varsigma}-\varsigma^*=[(1/\hat\sigma-1/\sigma^*)1^T_n,(1/\hat\nu-1/\nu^*)1^T_n]^T$, we have

\begin{align}
&\,\big\langle\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\=&\,\big\langle P_{1_n}\epsilon_{\frac{1}{\sigma}1_n},(\frac{1}{\hat\sigma}-\frac{1}{\sigma^*})1_n\big\rangle+\big\langle P_{1_n}\epsilon_{\frac{1}{\nu}1_n},(\frac{1}{\hat\nu}-\frac{1}{\nu^*})1_n\big\rangle\nonumber\\
\leq &\, a(u^T\epsilon_{\frac{1}{\sigma}1_n})^2+a(u^T\epsilon_{\frac{1}{\nu}1_n})^2+\frac{1}{2a}\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*).\nonumber
\end{align}

From Lemma <ref>, we get a bound in $L_2$-norm:

\begin{align}
\mathbb{E} \big[( u^T\epsilon_{\frac{1}{\sigma}1_n})^2\big]\leq \{\varphi^{-1}(1)\}^2\|u^T\epsilon_{\frac{1}{\sigma}1_n}\|^2_{\varphi}\leq  \{\varphi^{-1}(1)\}^2\omega^2_{\varsigma},\nonumber
\end{align}

and  $ \mathbb{E} [( u^T\epsilon_{\frac{1}{\nu}1_n})^2]\leq  \{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}$.
To sum up, for any $a>0$,

\begin{align}\label{gammaineq1}
\mathbb{E} \big\{   \langle\epsilon_{\bar m},\hat {\bar m}-\bar m^*\rangle+\langle\epsilon_{\varsigma},\hat {\varsigma}-\varsigma^*\rangle\big\}\leq aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}.
\end{align}

Now, plugging the bounds  (<ref>) and  (<ref>) into (<ref>) yields

\begin{align}\label{middlebound1}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa-b}{2}\|\hat{\mu}-\mu^*\|^2_2\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\} \nonumber\\\leq &\, \frac{1}{2b'}P_{2,H}\big(\varrho(\hat{\bar{\beta}}-\bar{\beta}^*);\sqrt{a'b'}A\lambda_0\big)+\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}-\big\|\varrho\hat {\bar\beta}\big\|_{2,P(\cdot;\lambda)}\nonumber\\&+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta}.
\end{align}

To prove part  (a), we use the subadditivity of $P_H$:

\begin{align}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa-b}{2}\|\hat{\mu}-\mu^*\|^2_2\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\\leq &\, \frac{1}{2b'}P_{2,H}\big(\varrho\hat{\bar{\beta}};\sqrt{a'b'}A\lambda_0\big)+\frac{1}{2b'}P_{2,H}\big(\bar{\beta}^*;\sqrt{a'b'}A\lambda_0\big)+\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}-\big\|\varrho\hat {\bar\beta}\big\|_{2,P(\cdot;\lambda)} \nonumber\\&+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta}.
\end{align}

Because $P(\cdot;\lambda)\geq P_{H}(\cdot;\lambda)$, taking $b=\etaa/4,b'=1/2,a=2/\etaa,a'=4/(\etaa\land 1)$ gives

\begin{align}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa}{2}\mathrm{\bold{ D}}_2(\hat{\mu},\mu^*)\Big\}\nonumber\\\leq &\,2\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}+2\etaa\|\mu^*\|_{2}^2+\frac{2C}{\etaa}\omega^2_{\bar m}+\frac{4}{\etaa}\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{4C}{\etaa\land 1}\omega^2_{\bar\eta}\nonumber\\\lesssim &\,\big\| \varrho\bar{\beta}^*\big\|_{2,P(\cdot;\lambda)}+\etaa\|\mu^*\|_{2}^2+\frac{1}{\etaa}(\omega^2_{\bar m}+\omega^2_{\varsigma})+\frac{1}{\etaa\land 1}\omega^2_{\bar\eta}.
\end{align}

To prove part (b),
we use  $\big\| \varrho\bar{\beta}^*_{\mathcal{J^{*C}}}\big\|_{2,P(\cdot;\lambda)}=\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^{*C}}}\big\|_{2,P(\cdot;\lambda)}$ and $\big\| \varrho\bar{\beta}^*_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}-\big\|\varrho \bar\beta_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}\leq \|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}$ and rewrite    (<ref>) as

\begin{align}\label{middlebound}
&\,\mathbb{E}\Big\{\bold{\Delta}_{l}(\hat{\mu},\mu^*)+\frac{\etaa-b}{2}\|\hat{\mu}-\mu^*\|^2_2\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\\leq &\, \frac{1}{2b'}P_{2,H}\big(\varrho(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^*};\sqrt{a'b'}A\lambda_0\big)+\frac{1}{2b'}P_{2,H}\big(\varrho(\hat{\bar{\beta}}-\bar{\beta}^*)_{\mathcal{J}^{*C}};\sqrt{a'b'}A\lambda_0\big)\nonumber\\&\, +\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^*}}\big\|_{2,P(\cdot;\lambda)}-\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J^{*C}}}\big\|_{2,P(\cdot;\lambda)}+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}\nonumber\\&+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta}.
\end{align}

The condition (<ref>) implies

\begin{align}\label{realtt1}
&\alpha  \mathrm{\bold{D}}_2(\mu,\mu^*)+\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*};\lambda\big)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^*}\|_{2,P(\cdot;\lambda)}\nonumber\\\leq&\,  \bold{\Delta}_{l}(\mu,\mu^*)+\|\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}}\|_{2,P(\cdot;\lambda)}-\vartheta P_{2,H}\big(\varrho(\bar{\beta}-\bar{\beta}^*)_{\mathcal{J}^{*C}};\lambda\big)+K\lambda^2J^*.
\end{align}

With   $b'=1/(2\vartheta)$, we add
(<ref>) and  (<ref>) to get

\begin{align}
&\,\mathbb{E}\Big\{(\etaa+\alpha-b)\mathrm{\bold{D}}_2(\hat\mu,\mu^*)\Big\}-\frac{1}{a'}\mathrm{\bold{D}}_2(\hat{\bar\eta},\bar{\eta}^*)-\frac{1}{2a}\{\mathrm{\bold{D}}_2(\hat {\bar m},\bar m^*)+\mathrm{\bold{D}}_2(\hat {\varsigma},\varsigma^*)\}\nonumber\\\leq&\,  K\lambda^2J^*+aC\omega^2_{\bar m}+2a\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}+\frac{\etaa^2}{2b}\|\mu^*\|_{2}^2+Ca'\omega^2_{\bar\eta},
\end{align}

where $\lambda=\sqrt{a'b'}A\lambda_0$ and $A\geq A_0$ with $A_0$ given in the Lemma <ref>. Now, setting $b=(\etaa+\alpha)/4, a'=4/\{(\etaa+\alpha)\land \vartheta\}, a=2/(\etaa+\alpha)$ gives

\begin{align}
&\,\mathbb{E}\big\{\frac{\etaa+\alpha}{2}\mathrm{\bold{ D}}_2(\hat\mu,\mu^*)\big\}\nonumber\\\leq&\,  \frac{2KA^2}{\{(\etaa+\alpha)\land \vartheta\}\vartheta} {\omega_{\bar\eta}}^2 J^*\log(ep)+\frac{2C\omega^2_{\bar m}}{(\etaa+\alpha)}+\frac{4\{\varphi^{-1}(1)\}^2\omega^2_{\varsigma}}{(\etaa+\alpha)}+\frac{2\etaa^2\|\mu^*\|_{2}^2}{\etaa+\alpha}+\frac{4C\omega^2_{\bar\eta}}{(\etaa+\alpha)\land \vartheta}.\nonumber
\end{align}

The proof is complete.

Let  $\epsilon=[\epsilon_i]$ be an $n$-dimensional random vector (not necessarily mean centered or having independent components) satisfying  $\|\epsilon\|_{\psi_2}\leq \omega$. Suppose that $\bar{X}\in \mathbb{R}^{n\times 2p}$ satisfies $\|\bar{X}\|_2\leq \varrho$. Let $\lambda_0=\omega\sqrt{\log(ep)}$. Then there exist universal constants $A_0,C,c>0$ such that for any $a\geq 2b>0$, $A_1\geq A_0$, the following event
\begin{align}\label{basicboundlemma}
\sup_{\bar{\beta} \in \mathbb{R}^{2p}} 2\langle \epsilon,\bar{X}\bar{\beta}\rangle -\frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{b}P_{2,H}(\varrho\bar{\beta};\sqrt{ab}A_1\lambda_0)\geq a\omega^2 t
\end{align}

occurs with probability at most $C\exp(-ct)p^{-cA^2_1}$.

Let $P_0 = (\lambda^2/2)1_{t\ne 0}$. Define $l_H(\bar{\beta})=2\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{b}P_{2,H}(\bar{\beta};\sqrt{ab}A_0\lambda_0),l_0(\bar{\beta})=2\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{b}P_{2,0}(\bar{\beta};\sqrt{ab}A_0\lambda_0)$. Introduce two events $\varepsilon_H=\{\sup_{\bar{\beta}\in \Gamma}l_H(\bar{\beta})\geq at\omega^2\}$, and $\varepsilon_0=\{\sup_{\bar{\beta}\in \Gamma}l_0(\bar{\beta})\geq at\omega^2\}$

First, we use an optimization technique to prove that $\varepsilon_H=\varepsilon_0$. Since $P_0\geq P_H$,  $P_{2,0}\geq P_{2,H}$ and thus $\varepsilon_0\subset \varepsilon_H$. The occurrence of $\varepsilon_H$ implies that $l_H(\bar{\beta}^o)\geq a t\omega^2$ for any $\bar{\beta}^o$ defined by

\begin{align}
\bar{\beta}^o\in \argmin_{\bar{\beta}\in \mathbb{R}^{2p}} \frac{1}{a}\|\bar{X}\bar{\beta}\|^2_2-2\langle\epsilon,\bar{X}\bar{\beta}\rangle+\frac{1}{b}P_{2,H}(\bar{\beta};\sqrt{ab}A_0\lambda_0).
\end{align}

From    Lemma 5 in [56], under
$\|\bar{X}\|_2\leq 1$,  there exists a globally optimal solution $\bar{\beta}^0$ to the problem

\begin{align}
\min_{\bar{\beta}\in \mathbb{R}^{2p}}\frac{1}{2}\|y-\bar{X}\bar{\beta}\|^2_2+P_{2,H}(\bar{\beta};\lambda),
\end{align}

such that for any $j=1,\ldots,p$, either $\bar{\beta}^0_j=0$ or $\|\bar{\beta}^0_j\|_2\geq\lambda$. Therefore, with $a\geq 2b>0$, there exists at least one global minimizer $\bar{\beta}^{oo}$ satisfying $P_{2,H}(\bar{\beta}^{oo};\sqrt{ab}A_1\lambda_0)=P_{2,0}(\bar{\beta}^{oo};\sqrt{ab}A_1\lambda_0)$ and thus $l_H(\bar{\beta}^{oo})=l_0(\bar{\beta}^{oo})$. This means $\sup_{\bar{\beta}\in \mathbb{R}^{2p}} l_0(\bar{\beta})\geq l_0(\bar{\beta}^{oo})=l_{H}(\bar{\beta}^{oo})\geq a t\omega^2$, and so  $\varepsilon_H\subset\varepsilon_0$. It suffices to prove $\mathbb{P}(\varepsilon_0)\leq C\exp(-ct)p^{-cA^2_1}$.

Next, we   use Lemma <ref> to bound the tail probability of $R$ defined by

\begin{align}\label{rdefappend}
R= \sup_{1\leq J\leq p} \sup_{\bar\beta\in \Gamma_J}\big\{\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{1}{2b}P_{2,0}(\bar\beta;\sqrt{ab}A_1\lambda_0)\big\},
\end{align}

where $P_{2,0}(\bar\beta;\lambda_0)=P_{0}(J;\lambda_0)=(1/2)J\lambda^2_0$ for $\bar\beta \in \Gamma_J$ and $\Gamma_J=\{\bar{\beta}\in\mathbb{R}^{2p}:J(\bar{\beta})=J\}$  (in the trivial case of $J=0$, the quantity inside the braces is 0).

By  a scaling argument, for any $a>0$,

\begin{align}\label{ineqdr2}
\langle\epsilon,\bar{X}\bar{\beta}/\|\bar{X}\bar{\beta}\|_2\rangle\|\bar{X}\bar{\beta}\|_2\leq \frac{a}{2}\sup_{\alpha\in \Gamma_J}(\langle\epsilon,\alpha\rangle)^2+\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2.
\end{align}

Applying Lemma <ref> with $G=2,q=2$ results in

\begin{align}
\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle-\frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2-\frac{a}{2}LJ\log (ep)\omega^2\geq \frac{a}{2}\omega^2t\big)\leq C\exp(-ct),
\end{align}

\begin{align}
\mathbb{P}\big(  \sup_{\bar\beta\in \Gamma_J}\langle\epsilon,\bar{X}\bar{\beta}\rangle- \frac{1}{2a}\|\bar{X}\bar{\beta}\|^2_2- aLP_{2,0}(\bar\beta;\lambda_0)\geq a\omega^2t\big)\leq C\exp(-ct).
\end{align}

Set $A_1\geq \sqrt{2L}$. Noticing   that (i) $(A^2_1/2)P_0(J;\lambda_0)\geq LP_0(J;\lambda_0)+cA^2_1  P_0(J;\lambda_0)$ for some $c>0$, and (ii) $J\log(ep)\geq \log p+J$ for any $J\geq 1$, we get

\begin{align}\label{unionboundJ}