@Article{CSIAM-AM-2-3,
author = {Tao, Luo and Zheng, Ma and Zhi-Qin, Xu, John and Zhang, Yaoyu},
title = {Theory of the Frequency Principle for General Deep Neural Networks},
journal = {CSIAM Transactions on Applied Mathematics},
year = {2021},
volume = {2},
number = {3},
pages = {484--507},
abstract = {<p style="text-align: justify;">Along with fruitful applications of Deep Neural Networks (DNNs) to realistic problems, recently, empirical studies reported a universal phenomenon of Frequency Principle (F-Principle), that is, a DNN tends to learn a target function from
low to high frequencies during the training. The F-Principle has been very useful in
providing both qualitative and quantitative understandings of DNNs. In this paper,
we rigorously investigate the F-Principle for the training dynamics of a general DNN
at three stages: initial stage, intermediate stage, and final stage. For each stage, a theorem is provided in terms of proper quantities characterizing the F-Principle. Our
results are general in the sense that they work for multilayer networks with general
activation functions, population densities of data, and a large class of loss functions.
Our work lays a theoretical foundation of the F-Principle for a better understanding of
the training process of DNNs.</p>},
issn = {2708-0579},
doi = {https://doi.org/10.4208/csiam-am.SO-2020-0005},
url = {https://global-sci.com/article/82354/theory-of-the-frequency-principle-for-general-deep-neural-networks}
}