@Article{CSIAM-AM-2-484, author = {Luo , TaoMa , ZhengXu , Zhi-Qin John and Zhang , Yaoyu}, title = {Theory of the Frequency Principle for General Deep Neural Networks}, journal = {CSIAM Transactions on Applied Mathematics}, year = {2021}, volume = {2}, number = {3}, pages = {484--507}, abstract = {

Along with fruitful applications of Deep Neural Networks (DNNs) to realistic problems, recently, empirical studies reported a universal phenomenon of Frequency Principle (F-Principle), that is, a DNN tends to learn a target function from low to high frequencies during the training. The F-Principle has been very useful in providing both qualitative and quantitative understandings of DNNs. In this paper, we rigorously investigate the F-Principle for the training dynamics of a general DNN at three stages: initial stage, intermediate stage, and final stage. For each stage, a theorem is provided in terms of proper quantities characterizing the F-Principle. Our results are general in the sense that they work for multilayer networks with general activation functions, population densities of data, and a large class of loss functions. Our work lays a theoretical foundation of the F-Principle for a better understanding of the training process of DNNs.

}, issn = {2708-0579}, doi = {https://doi.org/10.4208/csiam-am.SO-2020-0005}, url = {http://global-sci.org/intro/article_detail/csiam-am/19447.html} }