-
Notifications
You must be signed in to change notification settings - Fork 13
/
Copy pathweight normalization 原理和实现 - 知乎.html
106 lines (88 loc) · 199 KB
/
weight normalization 原理和实现 - 知乎.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
<!DOCTYPE html>
<html data-theme="light" data-react-helmet="data-theme" lang="zh"><head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8"><meta charset="utf-8"><title>weight normalization 原理和实现 - 知乎</title><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1"><meta name="renderer" content="webkit"><meta name="force-rendering" content="webkit"><meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"><meta name="google-site-verification" content="FTeR0c8arOPKh8c5DYh_9uu98_zJbaWw53J-Sch9MTg"><meta data-react-helmet="true" name="keywords" content="深度学习（Deep Learning）,Torch (深度学习框架),TensorLayer（深度学习库）"><meta data-react-helmet="true" name="description" content="除了最近很火的 spectral norm 之外，pytorch 还实现了 2016 年 OpenAI 的文章： Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks。该模块用于归一化网络权重…"><meta data-react-helmet="true" property="og:title" content="weight normalization 原理和实现"><meta data-react-helmet="true" property="og:url" content="https://zhuanlan.zhihu.com/p/114314389"><meta data-react-helmet="true" property="og:description" content="除了最近很火的 spectral norm 之外，pytorch 还实现了 2016 年 OpenAI 的文章： Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks。该模块用于归一化网络权重…"><meta data-react-helmet="true" property="og:image" content=""><meta data-react-helmet="true" property="og:type" content="article"><meta data-react-helmet="true" property="og:site_name" content="知乎专栏"><link data-react-helmet="true" rel="apple-touch-icon" href="https://static.zhihu.com/heifetz/assets/apple-touch-icon-152.a53ae37b.png"><link data-react-helmet="true" rel="apple-touch-icon" href="https://static.zhihu.com/heifetz/assets/apple-touch-icon-152.a53ae37b.png" sizes="152x152"><link data-react-helmet="true" rel="apple-touch-icon" href="https://static.zhihu.com/heifetz/assets/apple-touch-icon-120.bbce8f18.png" sizes="120x120"><link data-react-helmet="true" rel="apple-touch-icon" href="https://static.zhihu.com/heifetz/assets/apple-touch-icon-76.cbade8f9.png" sizes="76x76"><link data-react-helmet="true" rel="apple-touch-icon" href="https://static.zhihu.com/heifetz/assets/apple-touch-icon-60.8f6c52aa.png" sizes="60x60"><link rel="shortcut icon" type="image/x-icon" href="https://static.zhihu.com/heifetz/favicon.ico"><link rel="search" type="application/opensearchdescription+xml" href="https://static.zhihu.com/heifetz/search.xml" title="知乎"><link rel="dns-prefetch" href="https://static.zhimg.com/"><link rel="dns-prefetch" href="https://pic1.zhimg.com/"><link rel="dns-prefetch" href="https://pic2.zhimg.com/"><link rel="dns-prefetch" href="https://pic3.zhimg.com/"><link rel="dns-prefetch" href="https://pic4.zhimg.com/"><style>
.u-safeAreaInset-top {
height: constant(safe-area-inset-top) !important;
height: env(safe-area-inset-top) !important;
}
.u-safeAreaInset-bottom {
height: constant(safe-area-inset-bottom) !important;
height: env(safe-area-inset-bottom) !important;
}
</style><link href="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_005.css" rel="stylesheet"><script defer="defer" crossorigin="anonymous" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/init.js" data-sentry-config="{"dsn":"https://[email protected]/1224","sampleRate":0.1,"release":"1710-1a380e91","ignoreErrorNames":["NetworkError","SecurityError"],"ignoreErrors":["origin message","Network request failed","Loading chunk","这个系统不支持该功能。","Can't find variable: webkit","Can't find variable: $","内存不足","out of memory","DOM Exception 18","The operation is insecure","[object Event]","[object FileError]","[object DOMError]","[object Object]","拒绝访问。","Maximum call stack size exceeded","UploadError","无法 fetch","draft-js","缺少 JavaScript 对象","componentWillEnter","componentWillLeave","componentWillAppear","getInlineStyleAt","getCharacterList"],"whitelistUrls":["static.zhihu.com"]}"></script><style data-emotion-css="1cd9gw4">.css-1cd9gw4{margin-left:.3em;}</style><style data-emotion-css="qbubgm">.css-qbubgm{margin-left:0;}</style><script charset="utf-8" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column.js"></script><link rel="stylesheet" type="text/css" href="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_006.css"><script charset="utf-8" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_006.js"></script><link rel="stylesheet" type="text/css" href="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_003.css"><script charset="utf-8" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_007.js"></script><link rel="stylesheet" type="text/css" href="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_002.css"><script charset="utf-8" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_002.js"></script><link rel="stylesheet" type="text/css" href="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_004.css"><script charset="utf-8" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_005.js"></script><style data-emotion="css"></style><link rel="stylesheet" type="text/css" href="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column.css"><script charset="utf-8" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_003.js"></script></head><body class="WhiteBg-body" data-react-helmet="class" data-new-gr-c-s-check-loaded="8.869.0" data-gr-ext-installed="" data-gr-ext-disabled="forever"><div id="root"><div class="App"><div class="LoadingBar"></div><main role="main" class="App-main"><div class="Post-content" data-zop-usertoken="{"userToken":"yin-xiang-nan"}" data-zop="{"authorName":"杨指北","itemId":114314389,"title":"weight normalization 原理和实现","type":"article"}" data-za-detail-view-path-module="PostItem" data-za-extra-module="{"card":{"content":{"type":"Post","token":"114314389"}}}"><div class="ColumnPageHeader-Wrapper"><div><div class="Sticky ColumnPageHeader is-fixed" style="width: 1068px; top: 0px; left: 0px;"><div class="ColumnPageHeader-content"><a class="ZhihuLogoLink" href="https://www.zhihu.com/" aria-label="知乎"><svg viewBox="0 0 64 30" fill="#0066FF" width="64" height="30"><path d="M29.05 4.582H16.733V25.94h3.018l.403 2.572 4.081-2.572h4.815V4.582zm-5.207 18.69l-2.396 1.509-.235-1.508h-1.724V7.233h6.78v16.04h-2.425zM14.46 14.191H9.982c0-.471.033-.954.039-1.458v-5.5h5.106V5.935a1.352 1.352 0 0 0-.404-.957 1.378 1.378 0 0 0-.968-.396H5.783c.028-.088.056-.177.084-.255.274-.82 1.153-3.326 1.153-3.326a4.262 4.262 0 0 0-2.413.698c-.57.4-.912.682-1.371 1.946-.532 1.453-.997 2.856-1.31 3.693C1.444 8.674.28 11.025.28 11.025a5.85 5.85 0 0 0 2.52-.61c1.119-.593 1.679-1.502 2.054-2.883l.09-.3h2.334v5.5c0 .5-.045.982-.073 1.46h-4.12c-.71 0-1.39.278-1.893.775a2.638 2.638 0 0 0-.783 1.874h6.527a17.717 17.717 0 0 1-.778 3.649 16.796 16.796 0 0 1-3.012 5.273A33.104 33.104 0 0 1 0 28.74s3.13 1.175 5.425-.954c1.388-1.292 2.631-3.814 3.23-5.727a28.09 28.09 0 0 0 1.12-5.229h5.967v-1.37a1.254 1.254 0 0 0-.373-.899 1.279 1.279 0 0 0-.909-.37z"></path><path d="M11.27 19.675l-2.312 1.491 5.038 7.458a6.905 6.905 0 0 0 .672-2.218 3.15 3.15 0 0 0-.28-2.168l-3.118-4.563zM51.449 15.195V5.842c4.181-.205 7.988-.405 9.438-.483l.851-.05c.387-.399.885-2.395.689-3.021-.073-.25-.213-.666-.638-.555a33.279 33.279 0 0 1-4.277.727c-2.766.321-3.97.404-7.804.682-6.718.487-12.709.72-12.709.72a2.518 2.518 0 0 0 .788 1.834 2.567 2.567 0 0 0 1.883.706c2.278-.095 5.598-.25 8.996-.41v9.203h-12.78c0 .703.281 1.377.783 1.874a2.69 2.69 0 0 0 1.892.777h10.105v7.075c0 .887-.464 1.192-1.231 1.214h-3.92a4.15 4.15 0 0 0 .837 1.544 4.2 4.2 0 0 0 1.403 1.067 6.215 6.215 0 0 0 2.71.277c1.36-.066 2.967-.826 2.967-3.57v-7.607h11.28c.342 0 .67-.135.91-.374.242-.239.378-.563.378-.902v-1.375H51.449z"></path><path d="M42.614 8.873a2.304 2.304 0 0 0-1.508-.926 2.334 2.334 0 0 0-1.727.405l-.376.272 4.255 5.85 2.24-1.62-2.884-3.98zM57.35 8.68l-3.125 4.097 2.24 1.663 4.517-5.927-.375-.277a2.32 2.32 0 0 0-1.722-.452 2.327 2.327 0 0 0-1.536.896z"></path></svg></a><div class="ColumnPageHeader-Button"><button type="button" class="Button ColumnPageHeader-WriteButton Button--blue"><svg class="Zi Zi--EditSurround" fill="currentColor" viewBox="0 0 24 24" width="24" height="24"><path d="M18.453 7.992l-1.833-1.65.964-.978a1.223 1.223 0 0 1 1.73-.012l.005.006a1.24 1.24 0 0 1 .007 1.748l-.873.886zm-1.178 1.194l-5.578 5.66-1.935.697a.393.393 0 0 1-.504-.504l.697-1.935 5.488-5.567 1.832 1.65zM7.58 5.848l5.654.006-1.539 1.991-3.666.012A1.02 1.02 0 0 0 7 8.868v7.993c0 .558.46 1.01 1.029 1.01l7.941-.01c.568 0 1.03-.453 1.03-1.012v-4.061l2-1.442v6.002c0 1.397-1.2 2.501-2.62 2.501H7.574C6.153 19.85 5 18.717 5 17.32V8.35c0-1.397 1.16-2.502 2.58-2.502z"></path></svg>写文章</button><div class="Popover"><button title="更多" id="Popover1-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover1-content" type="button" class="Button ColumnPageHeader-MenuToggler Button--plain"><svg class="Zi Zi--Dots" fill="currentColor" viewBox="0 0 24 24" width="24" height="24"><path d="M5 14a2 2 0 1 1 0-4 2 2 0 0 1 0 4zm7 0a2 2 0 1 1 0-4 2 2 0 0 1 0 4zm7 0a2 2 0 1 1 0-4 2 2 0 0 1 0 4z" fill-rule="evenodd"></path></svg></button></div></div></div></div><div class="Sticky--holder" style="position: relative; inset: 0px; display: block; float: none; margin: 0px; height: 52px;"></div></div></div><article class="Post-Main Post-NormalMain" tabindex="-1"><header class="Post-Header"><h1 class="Post-Title">weight normalization 原理和实现</h1><div class="Post-Author"><div class="AuthorInfo" itemprop="author" itemscope="" itemtype="http://schema.org/Person"><meta itemprop="name" content="杨指北"><meta itemprop="image" content="https://pic3.zhimg.com/bf28449bb_l.jpg"><meta itemprop="url" content="https://www.zhihu.com/people/yin-xiang-nan"><meta itemprop="zhihu:followerCount"><span class="UserLink AuthorInfo-avatarWrapper"><div class="Popover"><div id="Popover9-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover9-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan"><img class="Avatar Avatar--round AuthorInfo-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/bf28449bb_xs.jpg" srcset="https://pic3.zhimg.com/bf28449bb_l.jpg 2x" alt="杨指北" width="38" height="38"></a></div></div></span><div class="AuthorInfo-content"><div class="AuthorInfo-head"><span class="UserLink AuthorInfo-name"><div class="Popover"><div id="Popover10-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover10-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan">杨指北</a></div></div></span></div><div class="AuthorInfo-detail"><div class="AuthorInfo-badge"><div class="ztext AuthorInfo-badgeText">无业游民</div></div></div></div></div></div><div class="LabelContainer-wrapper"></div><div><span class="Voters"><span><span class="UserLink"><div class="Popover"><div id="Popover14-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover14-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/li-lu-jun-39">李路军</a></div></div></span></span><span>、<span class="UserLink"><div class="Popover"><div id="Popover15-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover15-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/serena222">nana2</a></div></div></span></span><span>、<span class="UserLink"><div class="Popover"><div id="Popover16-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover16-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/huang-zhong-zhan">dede</a></div></div></span></span> 等 <button type="button" class="Button Button--plain">41 人赞同了该文章</button></span></div></header><div class="Post-RichTextContainer"><div class="RichText ztext Post-RichText"><p>除了最近很火的 spectral norm 之外，pytorch 还实现了 2016 年 OpenAI 的文章：<b>Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks</b>。该模块用于归一化网络权重，从而加速训练。本文介绍该模块的原理和 pytorch 的实现细节。</p><h2>Weight Normalization</h2><p>通常的梯度下降法直接将损失函数对 weight 求导得到梯度，然后以一定学习率，沿着梯度下降的方向更新权重。而 weight normalization 将每个神经元 weight 的<b>方向</b>和<b>长度</b>解耦，即把参数 weight 分为两部分：<b>v</b> 和 g，其中 <b>v</b> 代表方向，而 g 代表长度。然后将损失函数分别对这两部分求导，并更新它们的值。文章通过实验和理论分析，证明这种方式可以加速网络收敛。</p><figure data-size="normal"><noscript><img src="https://pic3.zhimg.com/v2-985fd9f93fbbb02d524022b07b1f4b62_b.jpg" data-size="normal" data-rawwidth="256" data-rawheight="349" class="content_image" width="256"/></noscript><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-985fd9f93fbbb02d524022b07b1f4b62_720w.jpg" data-size="normal" data-rawwidth="256" data-rawheight="349" class="content_image lazy" data-actualsrc="https://pic3.zhimg.com/v2-985fd9f93fbbb02d524022b07b1f4b62_b.jpg" data-lazy-status="ok" width="256"><figcaption>a simple neural network</figcaption></figure><p>用 <b>w</b>，<b>b</b> 分别表示某神经元（上图红色部分）的 weight 和 bias，其输入为 <b>x</b>。其输出通常按照如下公式计算：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_007.svg" alt="[公式]" eeimg="1" data-formula="y=\phi(\mathrm{w}\cdot\mathrm{x}+b)\tag{1}"> </p><p>其中 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation.svg" alt="[公式]" eeimg="1" data-formula="\phi"> 为非线性激活函数，<b>w</b> 和 <b>x</b> 都是 k 维，b 是 1 维。weight normalization 将 <b>w</b> 分解为 g 和<b> v </b>两部分：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_008.svg" alt="[公式]" eeimg="1" data-formula="\mathrm{w} = \frac{g}{\|\mathrm{v}\|}\mathrm{v}\tag{2}"> </p><p>经过上式的分解后，<b>w</b> 的模长等于 g，和向量 <b>v</b> 解耦了，<b>w</b> 的方向与<b> v</b> 相同，与 g 也无关。</p><p>用 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_014.svg" alt="[公式]" eeimg="1" data-formula="\nabla_\mathrm{w}L"> 表示损失函数对 weight 的梯度，根据公式 (2)，其对 <b>v</b> 和 g 的梯度可以通过链式法则得到：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_011.svg" alt="[公式]" eeimg="1" data-formula="\nabla_gL = \nabla_\mathrm{w}L\cdot\frac{\partial \mathrm{w}}{\partial g}\tag{3}"> </p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_010.svg" alt="[公式]" eeimg="1" data-formula="\nabla_\mathrm{v}L = \nabla_\mathrm{w}L\cdot\frac{\partial \mathrm{w}}{\partial \mathrm{v}}\tag{4}"> </p><p>由 (2) 易知：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_009.svg" alt="[公式]" eeimg="1" data-formula="\frac{\partial\mathrm{w}}{\partial g}=\frac{\mathrm{v}}{\|\mathrm{v}\|}\tag{5}"> </p><p>(5) 代入 (3) 得 ：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_015.svg" alt="[公式]" eeimg="1" data-formula="\nabla_gL = \nabla_\mathrm{w}L\cdot\frac{\mathrm{v}}{\|\mathrm{v}\|}\tag{6}"> </p><p><b>w</b> 对 <b>v</b> 得偏导得推导略显繁琐：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_006.svg" alt="[公式]" eeimg="1" data-formula="\begin{align*} \frac{\partial\mathrm{w}}{\partial\mathrm{v}}&=g\cdot\mathrm{d}(\frac{\mathrm{v}}{\|\mathrm{v}\|})/\mathrm{d}\mathrm{v}\\ &=g\cdot(\frac{\mathrm{d}\mathrm{v}}{\mathrm{dv}}\frac{1}{\|\mathrm{v}\|}+\mathrm{v}\frac{\mathrm{d}\|\mathrm{v}\|^{-1}}{\mathrm{dv}})\\ &=g\cdot(\mathbb{I}\frac{1}{\|\mathrm{v}\|}-\mathrm{v}\|\mathrm{v}\|^{-2}\frac{\mathrm{d}\|\mathrm{v}\|}{\mathrm{dv}})\\ &=g\cdot(\mathbb{I}\frac{1}{\|\mathrm{v}\|}-\frac{\mathrm{v}}{\|\mathrm{v}\|^{2}}\frac{\mathrm{d}((v_1^2+...+v_n^2)^{\frac{1}{2}})}{\mathrm{dv}})\\ &=g\cdot(\mathbb{I}\frac{1}{\|\mathrm{v}\|}-\frac{\mathrm{v}}{\|\mathrm{v}\|^{2}}\frac{1}{2}(v_1^2+...+v_n^2)^{-\frac{1}{2}}\frac{\mathrm{d}((v_1^2+...+v_n^2)}{\mathrm{dv}})\\ &=g\cdot(\mathbb{I}\frac{1}{\|\mathrm{v}\|}-\frac{\mathrm{v}}{\|\mathrm{v}\|^{2}}\frac{1}{2}(v_1^2+...+v_n^2)^{-\frac{1}{2}}2\mathrm{v}\\ &=g\cdot(\mathbb{I}\frac{1}{\|\mathrm{v}\|}-\frac{\mathrm{v}}{\|\mathrm{v}\|^{2}}\frac{\mathrm{v}}{\|\mathrm{v}\|})\tag{7} \end{align*}"> </p><p>把 (7) 带入 (4)，并利用 (6) 得：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_002.svg" alt="[公式]" eeimg="1" data-formula="\begin{align} \nabla_{\mathrm{v}}L&=\nabla_{\mathrm{w}}L\cdot g (\mathbb{I}\frac{1}{\|\mathrm{v}\|}-\frac{\mathrm{v}}{\|\mathrm{v}\|^2}\frac{\mathrm{v}}{\|\mathrm{v}\|})\\ &=\frac{g}{\|\mathrm{v}\|}\nabla_{\mathrm{w}}L-g\cdot\nabla_{\mathrm{w}}L\frac{\mathrm{v}}{\|\mathrm{v}\|}\cdot\frac{\mathrm{v}}{\|\mathrm{v}\|^2}\\ &=\frac{g}{\|\mathrm{v}\|}\nabla_{\mathrm{w}}L-\frac{g\nabla_gL}{\|\mathrm{v}\|^2}\mathrm{v} \end{align}\tag{8}"> </p><p>(6) 和 (8) 即为<b> v</b> 和 g 的梯度公式。</p><p>由于 v 和 w 方向相同，因此可以把公式 (8) 第二行中的</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_012.svg" alt="[公式]" eeimg="1" data-formula="\frac{\mathrm{v}}{\|\mathrm{v}\|}\cdot\frac{\mathrm{v}}{\|\mathrm{v}\|^2}"> 替换为 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_003.svg" alt="[公式]" eeimg="1" data-formula="\frac{\mathrm{w}'\mathrm{w}}{\|\mathrm{v}\|\|\mathrm{w}\|^2}"> （注意，替换前 v.v 表示向量点乘，替换后用 w'w 代表向量点乘，和论文保持一致，方便得到后面的矩阵表示。）</p><p>得到：</p><p><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_005.svg" alt="[公式]" eeimg="1" data-formula="\begin{align} \nabla_{\mathrm{v}}L &=\frac{g}{\|\mathrm{v}\|}\nabla_{\mathrm{w}}L-g\cdot\nabla_{\mathrm{w}}L\frac{\mathrm{w}'\mathrm{w}}{\|\mathrm{v}\|\|\mathrm{w}\|^2}\\ &=\frac{g}{\|\mathrm{v}\|}\nabla_{\mathrm{w}}L-\frac{g}{\|\mathrm{v}\|}\frac{\mathrm{ww'}}{\|\mathrm{w}\|^2}\nabla_\mathrm{w}L\\ &=\frac{g}{\|\mathrm{v}\|}(I-\frac{\mathrm{ww'}}{\|\mathrm{w}\|^2})\nabla_\mathrm{w}L \end{align}\tag{9}"> </p><p>(9) 最后一行，中间括号部分为一个投影矩阵，将空间中任意一个向量投影到向量 <b>w</b> 的补空间（垂直于 <b>w</b> 的超平面）。经过 weight normalization 后，损失函数对 <b>v</b> 的梯度，其实是它在 <b>w</b> 的梯度基础上，做了两个方面的改进：</p><ul><li>只保留垂直于 <b>w</b> 方向的部分</li><li>长度放缩为 g/||<b>v</b>||</li></ul><p>由于 <b>v</b> 和 <b>w</b> 的方向相同，而 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_004.svg" alt="[公式]" eeimg="1" data-formula="\nabla_{\mathrm{v}}L"> 垂直于 <b>v</b> ，根据勾股定理，随着迭代次数的增加，<b>v</b> 的模是单调递增的。当 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_013.svg" alt="[公式]" eeimg="1" data-formula="\|\nabla_{\mathrm{v}}L\|"> 很大时，经过若干次迭代，||<b>v</b>|| 会很快增大，导致 g/||<b>v</b>|| 减小，从而降低 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_013.svg" alt="[公式]" eeimg="1" data-formula="\|\nabla_{\mathrm{v}}L\|">；而当 <img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/equation_013.svg" alt="[公式]" eeimg="1" data-formula="\|\nabla_{\mathrm{v}}L\|"> 很小时，||<b>v</b>|| 会停止进一步增大。因此，weight normalization 可以使梯度值更稳定。</p><p>梯度前面的放缩项 g/||<b>v</b>||，可以使 learning rate 的选择范围更广，当 learning rate 很大时，||<b>v</b>|| 会迅速增大从而让梯度的放缩项减小，稳定训练。</p><h2>Pytorch Implementation</h2><p>pytorch 的官方实现代码为：<a href="https://link.zhihu.com/?target=https%3A//github.com/pytorch/pytorch/blob/master/torch/nn/utils/weight_norm.py" class=" external" target="_blank" rel="nofollow noreferrer" data-za-detail-view-id="1043"><span class="invisible">https://</span><span class="visible">github.com/pytorch/pyto</span><span class="invisible">rch/blob/master/torch/nn/utils/weight_norm.py</span><span class="ellipsis"></span></a></p><p>可以直接无脑调用：</p><div class="highlight"><pre><code class="language-python"><span class="kn">from</span> <span class="nn">torch.nn.utils</span> <span class="kn">import</span> <span class="n">weight_norm</span>
<span class="kn">import</span> <span class="nn">torch</span>
<span class="n">conv</span> <span class="o">=</span> <span class="n">weight_norm</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span><span class="n">name</span><span class="o">=</span><span class="s1">'weight'</span><span class="p">)</span>
<span class="n">fc</span> <span class="o">=</span> <span class="n">weight_norm</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">40</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s1">'weight'</span><span class="p">)</span>
<span class="n">conv</span><span class="o">.</span><span class="n">weight_g</span><span class="o">.</span><span class="n">shape</span> <span class="c1"># 卷积核的输出包括10个layer，这 10 个值对应原始 weight 的 10 个 norm</span>
<span class="c1"># torch.Size([10, 1, 1, 1])</span>
<span class="n">conv</span><span class="o">.</span><span class="n">weight_v</span><span class="o">.</span><span class="n">shape</span> <span class="c1"># 和原始的卷积核形状相同</span>
<span class="c1"># torch.Size([10, 3, 5, 5])</span></code></pre></div><p>看一下 pytorch 官方的实现细节：</p><div class="highlight"><pre><code class="language-python"><span class="kn">from</span> <span class="nn">torch.nn.parameter</span> <span class="kn">import</span> <span class="n">Parameter</span>
<span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">_weight_norm</span><span class="p">,</span> <span class="n">norm_except_dim</span>
<span class="k">class</span> <span class="nc">WeightNorm</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">dim</span><span class="p">):</span>
<span class="k">if</span> <span class="n">dim</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">dim</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
<span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span> <span class="c1"># 一般都是 weight</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dim</span> <span class="o">=</span> <span class="n">dim</span> <span class="c1"># 默认是 0，即输出的卷积层数、neuron 数，在weight中所在的维度</span>
<span class="k">def</span> <span class="nf">compute_weight</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">):</span>
<span class="c1"># 这个函数用于从 weight_g 和 weight_v 反推 weight</span>
<span class="c1"># 公式即为：weight = weight_g*weight_v/|weight_v|</span>
<span class="n">g</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'_g'</span><span class="p">)</span>
<span class="n">v</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'_v'</span><span class="p">)</span>
<span class="k">return</span> <span class="n">_weight_norm</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">g</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dim</span><span class="p">)</span>
<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">apply</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">dim</span><span class="p">):</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">hook</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">_forward_pre_hooks</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hook</span><span class="p">,</span> <span class="n">WeightNorm</span><span class="p">)</span> <span class="ow">and</span> <span class="n">hook</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="n">name</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"Cannot register two weight_norm hooks on "</span>
<span class="s2">"the same parameter {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
<span class="k">if</span> <span class="n">dim</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">dim</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
<span class="n">fn</span> <span class="o">=</span> <span class="n">WeightNorm</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
<span class="n">weight</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="c1"># 获取原始权重</span>
<span class="c1"># remove w from parameter list</span>
<span class="c1"># 从模型的参数列表中删除 weight</span>
<span class="k">del</span> <span class="n">module</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
<span class="c1"># add g and v as new parameters and express w as g/||v|| * v</span>
<span class="c1"># 初始化参数 weight_g 和 weight_v, 其中 weight_g 保存 neuron 的权重，</span>
<span class="c1"># weight_v 初始化为原始的 weight</span>
<span class="n">module</span><span class="o">.</span><span class="n">register_parameter</span><span class="p">(</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'_g'</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">norm_except_dim</span><span class="p">(</span><span class="n">weight</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span><span class="o">.</span><span class="n">data</span><span class="p">))</span>
<span class="n">module</span><span class="o">.</span><span class="n">register_parameter</span><span class="p">(</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'_v'</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">weight</span><span class="o">.</span><span class="n">data</span><span class="p">))</span>
<span class="c1"># 通过 fn 调用本类中的函数 compute_weight，从 weight_g 和 weight_v 重新计算 weight</span>
<span class="c1"># 并设置为新的成员变量 weight，注意这里的 weight 仅仅是成员变量，不再是网络的参数了，</span>
<span class="c1"># 不会被直接通过梯度下降法更新 </span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">fn</span><span class="o">.</span><span class="n">compute_weight</span><span class="p">(</span><span class="n">module</span><span class="p">))</span>
<span class="c1"># recompute weight before every forward()</span>
<span class="c1"># 设置前向传播的钩子，fn 默认调用本类中的 __call__ 方法</span>
<span class="n">module</span><span class="o">.</span><span class="n">register_forward_pre_hook</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>
<span class="k">return</span> <span class="n">fn</span>
<span class="k">def</span> <span class="nf">remove</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">):</span>
<span class="c1"># apply 的逆操作，重新计算 weigh</span>
<span class="n">weight</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_weight</span><span class="p">(</span><span class="n">module</span><span class="p">)</span>
<span class="c1"># 删除 weight 成员变量</span>
<span class="nb">delattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="c1"># 从计算图中删除 weight_g 和 weight_v</span>
<span class="k">del</span> <span class="n">module</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'_g'</span><span class="p">]</span>
<span class="k">del</span> <span class="n">module</span><span class="o">.</span><span class="n">_parameters</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'_v'</span><span class="p">]</span>
<span class="c1"># 将 weight 添加回参数列表</span>
<span class="n">module</span><span class="o">.</span><span class="n">register_parameter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">weight</span><span class="o">.</span><span class="n">data</span><span class="p">))</span>
<span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">,</span> <span class="n">inputs</span><span class="p">):</span>
<span class="c1"># 用于设置前向传播钩子，在运算之前，先将该模块成员变量 weight 用 weight_g 和 weight_v 更新</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_weight</span><span class="p">(</span><span class="n">module</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">weight_norm</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s1">'weight'</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
<span class="c1"># 通过 WeightNorm.apply 的静态方法，修改 module 内部的参数机制，共做了如下操作：</span>
<span class="c1"># 1) 删除了 module 中的 weight parameter，将其修改为普通的成员变量（即不直接参与权重更新） </span>
<span class="c1"># 2) 初始化了模块新的 named_parameter：weight_g 和 weight_v,用它们取代之前的 weight</span>
<span class="c1"># 3) 由于模块前向传播相关代码中，没有对 weight_g 和 weight_v 的操作，仍然利用 weight,</span>
<span class="c1"># 为了避免牵一发动全身，重新设置 模块的成员变量 weight，用 weight_g 和 weight_v 计算得到</span>
<span class="n">WeightNorm</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
<span class="k">return</span> <span class="n">module</span>
</code></pre></div><p class="ztext-empty-paragraph"><br></p><h2>相关阅读</h2><p><a href="https://zhuanlan.zhihu.com/p/75054200" class="internal" data-za-detail-view-id="1043">尹相楠：半小时学会 PyTorch Hook</a></p><p><a href="https://zhuanlan.zhihu.com/p/55393813" class="internal" data-za-detail-view-id="1043">尹相楠：Spectral Normalization 谱归一化</a></p><p></p></div></div><div class="ContentItem-time">编辑于 2020-03-19</div><div class="Post-topicsAndReviewer"><div class="TopicList Post-Topics"><div class="Tag Topic" data-za-detail-view-path-module="TopicItem" data-za-extra-module="{"card":{"content":{"type":"Topic","token":"19813032"}}}"><span class="Tag-content"><a class="TopicLink" href="https://www.zhihu.com/topic/19813032" target="_blank"><div class="Popover"><div id="Popover2-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover2-content">深度学习（Deep Learning）</div></div></a></span></div><div class="Tag Topic" data-za-detail-view-path-module="TopicItem" data-za-extra-module="{"card":{"content":{"type":"Topic","token":"20047018"}}}"><span class="Tag-content"><a class="TopicLink" href="https://www.zhihu.com/topic/20047018" target="_blank"><div class="Popover"><div id="Popover3-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover3-content">Torch (深度学习框架)</div></div></a></span></div><div class="Tag Topic" data-za-detail-view-path-module="TopicItem" data-za-extra-module="{"card":{"content":{"type":"Topic","token":"20058814"}}}"><span class="Tag-content"><a class="TopicLink" href="https://www.zhihu.com/topic/20058814" target="_blank"><div class="Popover"><div id="Popover4-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover4-content">TensorLayer（深度学习库）</div></div></a></span></div></div></div><div><div class="Sticky RichContent-actions is-fixed is-bottom" style="width: 690px; bottom: 0px; left: 189px;"><div class="ContentItem-actions" data-za-detail-view-path-module="BottomBar" data-za-extra-module="{"card":{"content":{"type":"Post","id":"114314389"}}}"><span><button aria-label="赞同 41 " type="button" disabled="disabled" class="Button VoteButton VoteButton--up"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--TriangleUp VoteButton-TriangleUp" fill="currentColor" viewBox="0 0 24 24" width="10" height="10"><path d="M2 18.242c0-.326.088-.532.237-.896l7.98-13.203C10.572 3.57 11.086 3 12 3c.915 0 1.429.571 1.784 1.143l7.98 13.203c.15.364.236.57.236.896 0 1.386-.875 1.9-1.955 1.9H3.955c-1.08 0-1.955-.517-1.955-1.9z" fill-rule="evenodd"></path></svg></span>赞同 41</button></span><div class="css-qbubgm"><button type="button" class="Button BottomActions-CommentBtn Button--plain Button--withIcon Button--withLabel"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--Comment Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M10.241 19.313a.97.97 0 0 0-.77.2 7.908 7.908 0 0 1-3.772 1.482.409.409 0 0 1-.38-.637 5.825 5.825 0 0 0 1.11-2.237.605.605 0 0 0-.227-.59A7.935 7.935 0 0 1 3 11.25C3 6.7 7.03 3 12 3s9 3.7 9 8.25-4.373 9.108-10.759 8.063z" fill-rule="evenodd"></path></svg></span>14 条评论</button></div><div class="Popover ShareMenu"><div class="ShareMenu-toggler" id="Popover5-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover5-content"><button type="button" class="Button Button--plain Button--withIcon Button--withLabel"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--Share Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M2.931 7.89c-1.067.24-1.275 1.669-.318 2.207l5.277 2.908 8.168-4.776c.25-.127.477.198.273.39L9.05 14.66l.927 5.953c.18 1.084 1.593 1.376 2.182.456l9.644-15.242c.584-.892-.212-2.029-1.234-1.796L2.93 7.89z" fill-rule="evenodd"></path></svg></span>分享</button></div></div><button type="button" class="Button ContentItem-action Button--plain Button--withIcon Button--withLabel"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--Heart Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M2 8.437C2 5.505 4.294 3.094 7.207 3 9.243 3 11.092 4.19 12 6c.823-1.758 2.649-3 4.651-3C19.545 3 22 5.507 22 8.432 22 16.24 13.842 21 12 21 10.158 21 2 16.24 2 8.437z" fill-rule="evenodd"></path></svg></span>喜欢</button><button type="button" class="Button ContentItem-action Button--plain Button--withIcon Button--withLabel"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--Star Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M5.515 19.64l.918-5.355-3.89-3.792c-.926-.902-.639-1.784.64-1.97L8.56 7.74l2.404-4.871c.572-1.16 1.5-1.16 2.072 0L15.44 7.74l5.377.782c1.28.186 1.566 1.068.64 1.97l-3.89 3.793.918 5.354c.219 1.274-.532 1.82-1.676 1.218L12 18.33l-4.808 2.528c-1.145.602-1.896.056-1.677-1.218z" fill-rule="evenodd"></path></svg></span>收藏</button><div class="Post-ActionMenuButton"><div class="Popover"><div id="Popover6-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover6-content"><button type="button" class="Button Button--plain Button--withIcon Button--withLabel"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--Settings Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M20.868 17.185a.896.896 0 0 1-.452.137c-.123 0-1.397-.26-1.617-.233-1.354.014-1.78 1.276-1.835 1.742-.055.453 0 .892.191 1.303a.8.8 0 0 1-.068.851C16.224 21.877 14.922 22 14.73 22a.548.548 0 0 1-.356-.151c-.11-.096-.685-1.138-1.069-1.468-1.304-.955-2.247-.329-2.63 0-.398.33-.672.7-.836 1.125a.632.632 0 0 1-.329.37c-1.354.426-2.918-.919-3.014-1.056a.564.564 0 0 1-.123-.356c-.014-.138.383-1.276.342-1.688-.342-1.9-1.836-1.687-2.096-1.673a3.192 3.192 0 0 0-.918.178.873.873 0 0 1-.59-.055c-.887-.462-1.136-2.332-1.109-2.51.055-.315.192-.521.438-.604.425-.164.809-.452 1.151-.85.931-1.262.343-2.25 0-2.634-.342-.356-.726-.645-1.15-.809-.138-.041-.234-.151-.33-.316-.38-1.434.613-2.552.867-2.77.255-.22.6-.055.723 0 .425.164.877.219 1.343.15C6.7 6.636 6.784 5.141 6.81 4.908c.014-.247-.11-1.29-.137-1.4a.488.488 0 0 1 .027-.315C7.317 2.178 9.071 2 9.222 2a.56.56 0 0 1 .439.178c.11.124.63 1.111 1 1.4.4.338 1.583.83 2.59.013.397-.274.959-1.29 1.082-1.413A.55.55 0 0 1 14.717 2c1.56 0 2.329 1.029 2.438 1.22a.458.458 0 0 1 .069.371c-.028.151-.329 1.152-.26 1.605.365 1.537 1.383 1.742 1.89 1.783.493.028 1.644-.356 1.809-.343a.63.63 0 0 1 .424.206c.535.31.85 1.715.905 2.14.027.233-.014.439-.11.562-.11.138-1.165.714-1.48 1.112-.855.982-.342 2.25-.068 2.606.26.37 1.22.905 1.288.96.15.137.26.302.315.494.146 1.413-.89 2.387-1.069 2.47zm-8.905-.535c.644 0 1.246-.123 1.822-.356a4.576 4.576 0 0 0 1.493-1.016 4.694 4.694 0 0 0 1-1.495c.247-.562.357-1.18.357-1.81 0-.659-.11-1.262-.356-1.825a4.79 4.79 0 0 0-1-1.481 4.542 4.542 0 0 0-1.494-1.002 4.796 4.796 0 0 0-3.631 0 4.627 4.627 0 0 0-1.48 1.002c-.424.425-.767.919-1 1.481a4.479 4.479 0 0 0-.37 1.825c0 .644.124 1.248.37 1.81a4.62 4.62 0 0 0 1 1.495c.425.426.918.768 1.48 1.016a4.677 4.677 0 0 0 1.809.356z" fill-rule="evenodd"></path></svg></span>设置</button></div></div></div><button type="button" class="Button ContentItem-action Button--plain Button--withIcon Button--withLabel"><span style="display:inline-flex;align-items:center"><svg class="Zi Zi--Deliver Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M5.171 4H18.83a1.5 1.5 0 0 1 1.455 1.136l2.597 10.386a4 4 0 0 1 .119.97V19s0 2-2.002 2H3c-2 0-2-2-2-2v-2.508a4 4 0 0 1 .12-.97L3.715 5.136A1.5 1.5 0 0 1 5.171 4zm1.074 2a1 1 0 0 0-.97.761l-2.123 8.62a.5.5 0 0 0 .486.619h4.717a1 1 0 0 1 .892.548C9.906 17.85 10.824 18.5 12 18.5c1.176 0 2.094-.65 2.753-1.952a1 1 0 0 1 .892-.548h4.717a.5.5 0 0 0 .486-.62l-2.122-8.619A1 1 0 0 0 17.755 6H6.245zM8 9c0-.552.453-1 .997-1h6.006c.55 0 .997.444.997 1 0 .552-.453 1-.997 1H8.997A.996.996 0 0 1 8 9zm-1.5 4c0-.552.445-1 .996-1h9.008c.55 0 .996.444.996 1 0 .552-.445 1-.996 1H7.496a.995.995 0 0 1-.996-1z"></path></svg></span>投稿</button></div></div><div class="Sticky--holder" style="position: static; inset: auto auto 0px 0px; display: block; float: none; margin: 0px 0px 10px; height: 54px;"></div></div></article><div class="Post-Sub Post-NormalSub"><div class="PostIndex-Contributions" data-za-detail-view-path-module="ColumnList" data-za-detail-view-path-module_name="文章被以下专栏收录" data-za-extra-module="{}"><h3 class="BlockTitle">文章被以下专栏收录</h3><ul><div class="ContentItem Column-ColumnItem"><div class="ContentItem-main"><div class="ContentItem-image"><a class="ColumnLink" href="https://www.zhihu.com/column/c_201634018"><div class="Popover"><div id="Popover7-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover7-content"><img class="Avatar Avatar--medium Avatar--round" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-eb370129b198ca81b667f0a8d27f1547_xs.jpg" srcset="https://pic4.zhimg.com/v2-eb370129b198ca81b667f0a8d27f1547_l.jpg?source=172ae18b 2x" alt="SIGAI" width="40" height="40"></div></div></a></div><div class="ContentItem-head"><h2 class="ContentItem-title"><a class="ColumnLink ColumnItem-Title" href="https://www.zhihu.com/column/c_201634018"><div class="Popover"><div id="Popover8-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover8-content">SIGAI</div></div></a> · <button class="ColumnItem-CancleButton"><span class="status">投稿中</span><span class="cancel">撤销投稿</span></button></h2><div class="ContentItem-meta">人工智能技术文章</div></div><div class="ContentItem-extra"><button type="button" class="Button FollowButton Button--primary Button--grey">已关注</button></div></div></div></ul></div><div class="Recommendations-Main" style="width: 1068px;"><h3 class="BlockTitle Recommendations-BlockTitle">推荐阅读</h3><ul class="Recommendations-List"><button class="PagingButton PagingButton-Previous" disabled="disabled" data-za-detail-view-path-module="Unknown" data-za-detail-view-path-module_name="推荐阅读" data-za-extra-module="{}"><svg class="Zi Zi--ArrowLeft" fill="#d3d3d3" viewBox="0 0 24 24" width="40" height="40"><path d="M14.782 16.78a.737.737 0 0 1-1.052 0L9.218 12.53a.758.758 0 0 1 0-1.063L13.73 7.22a.737.737 0 0 1 1.052 0c.29.294.29.77.001 1.063L11 12l3.782 3.716c.29.294.29.77 0 1.063z" fill-rule="evenodd"></path></svg></button><a href="https://zhuanlan.zhihu.com/p/100937718" class="PostItem"><div><img src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-230f9a68e227e5597dac3ae49eb1ca17_250x0.png" srcset="https://pic2.zhimg.com/v2-230f9a68e227e5597dac3ae49eb1ca17_qhd.jpg?source=172ae18b 2x" class="PostItem-TitleImage" alt="pytorch权重初始化方式与原理"><h1 class="PostItem-Title">pytorch权重初始化方式与原理</h1><div class="PostItem-Footer"><span>机器学习入坑者</span><span class="PostItem-FooterTitle"></span></div></div></a><a href="https://zhuanlan.zhihu.com/p/65439075" class="PostItem"><div><h1 class="PostItem-Title">Pytorch Batch Normalizatin layer的坑</h1><p class="PostItem-Summary">1.
注意momentum的定义Pytorch中的BN层的动量平滑和常见的动量法计算方式是相反的，默认的momentum=0.1
\hat{x}_{\text { new }}=(1-\text { momentum }) \times \hat{x}+\text {
momemtum …</p><div class="PostItem-Footer"><span>无茗</span><span class="PostItem-FooterTitle"></span></div></div></a><a href="https://zhuanlan.zhihu.com/p/312993987" class="PostItem"><div><h1 class="PostItem-Title">使用PyTorch时，最常见的4个错误</h1><p class="PostItem-Summary">这4个错误，可能大部分人都犯过，本文总结了使用PyTorch时常见的4个错误，并给出了每一种错误出现的原因并给出了详细的修复方法及代码步骤，希望能给大家一点提示。 本文转载自AI公园，作者…</p><div class="PostItem-Footer"><span>极市平台</span><span class="PostItem-FooterTitle">发表于极市CV资...</span></div></div></a><a href="https://zhuanlan.zhihu.com/p/136902153" class="PostItem"><div><h1 class="PostItem-Title">PyTorch--lr_scheduler.step()和optimizer.step()的先后顺序</h1><p class="PostItem-Summary">实
验基于PyTorch==1.2.0
resume模型的时候想恢复optimizer的学习率optimizer不会保存last_step等状态，而scheduler是根据
last_step来恢复学习率的，而scheduler的last_step默认是-1，所以不…</p><div class="PostItem-Footer"><span>醒了么</span><span class="PostItem-FooterTitle">发表于Pytho...</span></div></div></a><button class="PagingButton PagingButton-Next" data-za-detail-view-path-module="Unknown" data-za-detail-view-path-module_name="推荐阅读" data-za-extra-module="{}"><svg class="Zi Zi--ArrowRight" fill="#d3d3d3" viewBox="0 0 24 24" width="40" height="40"><path d="M9.218 16.78a.737.737 0 0 0 1.052 0l4.512-4.249a.758.758 0 0 0 0-1.063L10.27 7.22a.737.737 0 0 0-1.052 0 .759.759 0 0 0-.001 1.063L13 12l-3.782 3.716a.758.758 0 0 0 0 1.063z" fill-rule="evenodd"></path></svg></button></ul></div><div class="Comments-container" data-za-detail-view-path-module="CommentList" data-za-extra-module="{}"><div class="CommentsV2 CommentsV2--withEditor CommentsV2-withPagination"><div class="Topbar CommentTopbar"><div class="Topbar-title"><h2 class="CommentTopbar-title">14 条评论</h2></div><div class="Topbar-options"><button type="button" class="Button Button--plain Button--withIcon Button--withLabel"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Switch Button-zi" fill="currentColor" viewBox="0 0 24 24" width="1.2em" height="1.2em"><path d="M13.004 7V4.232c0-.405.35-.733.781-.733.183 0 .36.06.501.17l6.437 5.033c.331.26.376.722.1 1.033a.803.803 0 0 1-.601.264H2.75a.75.75 0 0 1-.75-.75V7.75A.75.75 0 0 1 2.75 7h10.254zm-1.997 9.999v2.768c0 .405-.35.733-.782.733a.814.814 0 0 1-.5-.17l-6.437-5.034a.702.702 0 0 1-.1-1.032.803.803 0 0 1 .6-.264H21.25a.75.75 0 0 1 .75.75v1.499a.75.75 0 0 1-.75.75H11.007z" fill-rule="evenodd"></path></svg></span>切换为时间排序</button></div></div><div><div class="CommentsV2-footer CommentEditorV2--normal"><div class="CommentEditorV2-inputWrap"><div class="InputLike CommentEditorV2-input Editable"><div style="min-height: 198px;" class="Dropzone Editable-content RichText RichText--editable RichText--clearBoth ztext"><div class="DraftEditor-root"><div class="public-DraftEditorPlaceholder-root"><div class="public-DraftEditorPlaceholder-inner" id="placeholder-dtis7" style="white-space: pre-wrap;">写下你的评论...</div></div><div class="DraftEditor-editorContainer"><div aria-describedby="placeholder-dtis7" class="notranslate public-DraftEditor-content" role="textbox" spellcheck="true" style="outline: currentcolor none medium; user-select: text; white-space: pre-wrap; overflow-wrap: break-word;" tabindex="0" contenteditable="true"><div data-contents="true"><div class="Editable-unstyled" data-block="true" data-editor="dtis7" data-offset-key="cshen-0-0"><div data-offset-key="cshen-0-0" class="public-DraftStyleDefault-block public-DraftStyleDefault-ltr"><span data-offset-key="cshen-0-0"><br data-text="true"></span></div></div></div></div></div></div></div><input multiple="" type="file" style="display: none;" accept="image/webp,image/jpg,image/jpeg,image/png,image/gif"><div></div></div><div class="CommentEditorV2-inputUpload"><div class="CommentEditorV2-popoverWrap"><div class="Popover ImagePopover"><button aria-label="插入图片" data-tooltip="插入图片" data-tooltip-position="bottom" data-tooltip-will-hide-on-click="true" id="Popover18-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover18-content" type="button" class="Button Editable-control Button--plain"><svg class="Zi Zi--AddImage" fill="currentColor" viewBox="0 0 24 24" width="24" height="24"><path d="M13.25 5.5H6.868c-.755 0-1.368.613-1.368 1.368v10.264c0 .755.613 1.368 1.368 1.368h10.264c.755 0 1.368-.613 1.368-1.368V10.75H20v6.467A2.783 2.783 0 0 1 17.217 20H6.783A2.783 2.783 0 0 1 4 17.217V6.783A2.783 2.783 0 0 1 6.783 4h6.467v1.5zM8.158 13.4a1.033 1.033 0 0 1 1.744-.078l.258.37a.782.782 0 0 0 1.303-.031l1.332-2.113a1.173 1.173 0 0 1 2.01.043l2.03 3.537A1.25 1.25 0 0 1 15.748 17H8.25a1.252 1.252 0 0 1-1.085-1.873l.992-1.728zM9.5 11a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm3.75-5.5a.75.75 0 1 1 0-1.5.75.75 0 0 1 0 1.5zM18.5 4V2.25a.75.75 0 1 1 1.5 0V4h1.75a.75.75 0 1 1 0 1.5H20v1.75a.75.75 0 1 1-1.5 0V5.5h-1.75a.75.75 0 1 1 0-1.5h1.75zm.75 7.5a.75.75 0 1 1 0-1.5.75.75 0 0 1 0 1.5z"></path></svg></button></div><label class="UploadPicture-wrapper"><input type="file" accept="image/png,image/jpeg" class="UploadPicture-input"></label><div class="Popover CommentEditorV2-inputUpLoad-Icon"><button aria-label="插入表情" data-tooltip="插入表情" data-tooltip-position="bottom" data-tooltip-will-hide-on-click="true" id="Popover11-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover11-content" type="button" class="Button Editable-control Button--plain"><svg class="Zi Zi--Emotion" fill="currentColor" viewBox="0 0 24 24" width="24" height="24"><path d="M7.523 13.5h8.954c-.228 2.47-2.145 4-4.477 4-2.332 0-4.25-1.53-4.477-4zM12 21a9 9 0 1 1 0-18 9 9 0 0 1 0 18zm0-1.5a7.5 7.5 0 1 0 0-15 7.5 7.5 0 0 0 0 15zm-3-8a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm6 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"></path></svg></button></div></div></div></div><button type="button" disabled="disabled" class="Button CommentEditorV2-singleButton Button--primary Button--blue">发布</button></div></div><div><div class="CommentListV2"><ul class="NestComment"><li class="NestComment--rootComment"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover19-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover19-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/bao-wen-jie-54"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-2d0555317b584b155c9ba54f4b61cf9d_s.jpg" srcset="https://pic1.zhimg.com/v2-2d0555317b584b155c9ba54f4b61cf9d_xs.jpg?source=06d4cd63 2x" alt="Monstarrrr" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/bao-wen-jie-54">Monstarrrr</a></span><span class="CommentItemV2-time">2020-03-19</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>您好，感谢您的介绍。我想请教一下如果理解公式9中的映射到w的垂直空间？我对于这个映射矩阵的功能不是很理解？谢谢</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li><li class="NestComment--child"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover20-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover20-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/bf28449bb_s.jpg" srcset="https://pic2.zhimg.com/bf28449bb_xs.jpg?source=06d4cd63 2x" alt="杨指北" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan">杨指北</a></span><span class="CommentItemV2-roleInfo"> (作者) </span><span class="CommentItemV2-reply">回复</span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/bao-wen-jie-54">Monstarrrr</a></span><span class="CommentItemV2-time">2020-03-19</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>中
间的矩阵是 (I-ww'/|w|^2)，用它乘以任何一个向量 v，就相当于 v- ww'v/|w|^2，而 w'v/|w| 为 v 在 w
方向的投影的长度，再乘以 w/|w|，得到 v 在 w 方向的投影，再用 v 减去这个投影，得到 v 垂直w方向的投影。</p></div></div><div class="CommentItemV2-footer"><button type="button" disabled="disabled" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>2</button><button type="button" class="Button Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li><li class="NestComment--child"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover21-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover21-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/bao-wen-jie-54"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-2d0555317b584b155c9ba54f4b61cf9d_s.jpg" srcset="https://pic1.zhimg.com/v2-2d0555317b584b155c9ba54f4b61cf9d_xs.jpg?source=06d4cd63 2x" alt="Monstarrrr" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/bao-wen-jie-54">Monstarrrr</a></span><span class="CommentItemV2-reply">回复</span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan">杨指北</a></span><span class="CommentItemV2-roleInfo"> (作者) </span><span class="CommentItemV2-time">2020-03-19</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>明白了，十分感谢，这种办法真是秒啊</p><div class="Richtext-content_img Richtext-content_img-square"><img class="comment_sticker" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-90359a720808ff45062287127cfa1039.gif" data-original="https://pic2.zhimg.com/v2-90359a720808ff45062287127cfa1039_r.gif" data-rawwidth="NaN"></div></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li><div><div class="CommentMoreReplyButton"><button type="button" class="Button Button--plain">展开其他 2 条回复</button></div></div></ul><ul class="NestComment"><li class="NestComment--rootComment"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover22-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover22-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kang-shi-fu-17-81"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/0fac65d7f4cddf1f0fb0733e2938feba_s.jpg" srcset="https://pic2.zhimg.com/0fac65d7f4cddf1f0fb0733e2938feba_xs.jpg?source=06d4cd63 2x" alt="啊哈" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kang-shi-fu-17-81">啊哈</a></span><span class="CommentItemV2-time">2020-08-04</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>您好，请问有tensorflow版本的实现吗</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li><li class="NestComment--child"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover23-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover23-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/shi-jun-feng-52-60"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-0392d701f6cf25ee7bba323295c26ccb_s.jpg" srcset="https://pic2.zhimg.com/v2-0392d701f6cf25ee7bba323295c26ccb_xs.jpg?source=06d4cd63 2x" alt="石峻峰" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/shi-jun-feng-52-60">石峻峰</a></span><span class="CommentItemV2-reply">回复</span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kang-shi-fu-17-81">啊哈</a></span><span class="CommentItemV2-time">2020-10-09</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p><a href="http://link.zhihu.com/?target=https%3A//github.com/seanpmorgan/tf-weightnorm/blob/master/normalization.py" class=" wrap external" target="_blank" rel="nofollow noreferrer" data-za-detail-view-id="1043">https://github.com/seanpmorgan/tf-weightnorm/blob/master/tf-example.py </a>我试过tensorflow1.15/2.1.0版本可用</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li></ul><ul class="NestComment"><li class="NestComment--rootCommentNoChild"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover24-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover24-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/shang-yong-91"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/da8e974dc_s_003.jpg" srcset="https://pic4.zhimg.com/da8e974dc_xs.jpg?source=06d4cd63 2x" alt="觞咏" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/shang-yong-91">觞咏</a></span><span class="CommentItemV2-time">2020-10-29</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>"由于 v 和 w 的方向相同，而 [公式] 垂直于 v ，根据勾股定理，随着迭代次数的增加，v 的模是单调递增的。"请问这句话如何理解呢</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li></ul><ul class="NestComment"><li class="NestComment--rootCommentNoChild"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover25-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover25-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/li-hong-jie-69-28"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/da8e974dc_s.jpg" srcset="https://pic2.zhimg.com/da8e974dc_xs.jpg?source=06d4cd63 2x" alt="虚妄" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/li-hong-jie-69-28">虚妄</a></span><span class="CommentItemV2-time">2020-11-24</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>有些情况下,使用weight norm 会使输出变成nan , 而权重却看不出来问题</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li></ul><ul class="NestComment"><li class="NestComment--rootCommentNoChild"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover26-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover26-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/wu-wu-wu-wu-wu-23-32"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-7025e807446478d4cc3a4dad73154426_s.jpg" srcset="https://pic2.zhimg.com/v2-7025e807446478d4cc3a4dad73154426_xs.jpg?source=06d4cd63 2x" alt="变化" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/wu-wu-wu-wu-wu-23-32">变化</a></span><span class="CommentItemV2-time">2020-12-02</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>你好，我想问一下为什么pytorch实现的weight normalization中没有参数初始化呢，论文里有很大一部分讲weight normalization的参数初始化问题</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li></ul><ul class="NestComment"><li class="NestComment--rootCommentNoChild"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover27-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover27-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/xu-yao-nu-li-de-ren-97-91"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/v2-174eb2f0d48e33a88335726891bfa60e_s.jpg" srcset="https://pic1.zhimg.com/v2-174eb2f0d48e33a88335726891bfa60e_xs.jpg?source=06d4cd63 2x" alt="需要努力的人" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/xu-yao-nu-li-de-ren-97-91">需要努力的人</a></span><span class="CommentItemV2-time">01-13</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>WN一般用在什么地方啊，全连接吗</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li></ul><ul class="NestComment"><li class="NestComment--rootComment"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover28-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover28-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kuang-jian-dong-90"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/66528be01ce92ec7aa880c3737947db8_s.jpg" srcset="https://pic2.zhimg.com/66528be01ce92ec7aa880c3737947db8_xs.jpg?source=06d4cd63 2x" alt="飞得更高" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kuang-jian-dong-90">飞得更高</a></span><span class="CommentItemV2-time">01-14</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext"><p>你好，我有点不太理解9式中的第一步是如何到第二步的，第一步中w'w是一向量的内积结果是一个标量，第二部ww'的结果是一个矩阵，这里能相等吗？</p></div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li><li class="NestComment--child"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover29-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover29-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan-45"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/da8e974dc_s_002.jpg" srcset="https://pic1.zhimg.com/da8e974dc_xs.jpg?source=06d4cd63 2x" alt="南有乔木" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan-45">南有乔木</a></span><span class="CommentItemV2-reply">回复</span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kuang-jian-dong-90">飞得更高</a></span><span class="CommentItemV2-time">01-14</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext">我大号被夹了，用小号回你一下。你没错，是我错了，公式（7）最后一步，括号里面第一项是个单位阵除以‖v‖，因此第二项也应该是一个和它维度相同的方阵，即 vv'……矩阵求导这块我不是很熟练</div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li><li class="NestComment--child"><div class="CommentItemV2"><div><div class="CommentItemV2-meta"><span class="UserLink CommentItemV2-avatar"><div class="Popover"><div id="Popover30-toggle" aria-haspopup="true" aria-expanded="false" aria-owns="Popover30-content"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kuang-jian-dong-90"><img class="Avatar UserLink-avatar" src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/66528be01ce92ec7aa880c3737947db8_s.jpg" srcset="https://pic2.zhimg.com/66528be01ce92ec7aa880c3737947db8_xs.jpg?source=06d4cd63 2x" alt="飞得更高" width="24" height="24"></a></div></div></span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/kuang-jian-dong-90">飞得更高</a></span><span class="CommentItemV2-reply">回复</span><span class="UserLink"><a class="UserLink-link" data-za-detail-view-element_name="User" target="_blank" href="https://www.zhihu.com/people/yin-xiang-nan-45">南有乔木</a></span><span class="CommentItemV2-time">01-14</span></div><div class="CommentItemV2-metaSibling"><div class="CommentRichText CommentItemV2-content"><div class="RichText ztext">谢谢！</div></div><div class="CommentItemV2-footer"><button type="button" class="Button CommentItemV2-likeBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>赞</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Reply" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M22.959 17.22c-1.686-3.552-5.128-8.062-11.636-8.65-.539-.053-1.376-.436-1.376-1.561V4.678c0-.521-.635-.915-1.116-.521L1.469 10.67a1.506 1.506 0 0 0-.1 2.08s6.99 6.818 7.443 7.114c.453.295 1.136.124 1.135-.501V17a1.525 1.525 0 0 1 1.532-1.466c1.186-.139 7.597-.077 10.33 2.396 0 0 .396.257.536.257.892 0 .614-.967.614-.967z" fill-rule="evenodd"></path></svg></span>回复</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Recommend" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M6.433 14.285l-.918 5.355c-.219 1.274.532 1.82 1.676 1.218l4.81-2.528 4.807 2.528c1.145.601 1.896.056 1.677-1.218l-.918-5.355 3.89-3.792c.925-.903.64-1.785-.64-1.97l-2.843-.414c-.538.583-4.195 4.547-5.757 6.12-.865.822-1.572.015-1.572.015l-1.92-1.906s-.405-.623.213-1.204c.62-.58 1.174-.193 1.174-.193l1.32 1.14 4.008-4.34-2.404-4.872c-.572-1.159-1.5-1.159-2.072 0L8.56 7.741l-5.376.781c-1.279.186-1.566 1.068-.64 1.971l3.89 3.792z" fill-rule="evenodd"></path></svg></span>推荐</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--EyeSlash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="14" height="14"><path d="M17.007 11.504c0 .65-.13 1.26-.36 1.83l3 3.073S23 14.136 23 11.504C23 8.008 17.255 4 11.995 4c-1.4 0-2.741.25-3.982.701l2.161 2.16c.57-.23 1.18-.36 1.831-.36a5.004 5.004 0 0 1 5.002 5.003zM2.57 4.342l2.067 2.075C3.499 7.258 1 9.119 1 11.504c0 3.336 5.79 7.503 11.005 7.503 1.55 0 3.031-.3 4.382-.84l.42.42 2.125 2.118s.782.571 1.314 0-.074-1.305-.074-1.305L3.955 3.183s-.76-.742-1.385-.19c-.626.554 0 1.35 0 1.35zm4.963 4.96l1.55 1.552c-.05.21-.08.43-.08.65 0 1.66 1.341 3.001 3.002 3.001.22 0 .44-.03.65-.08l1.551 1.551c-.67.33-1.41.53-2.2.53a5.004 5.004 0 0 1-5.003-5.002c0-.79.2-1.53.53-2.201zm4.312-.78l3.151 3.152.02-.16c0-1.66-1.34-3.001-3.001-3.001l-.17.01z" opacity=".9" fill-rule="evenodd"></path></svg></span>折叠</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Like" style="transform: rotate(180deg); margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M14.445 9h5.387s2.997.154 1.95 3.669c-.168.51-2.346 6.911-2.346 6.911s-.763 1.416-2.86 1.416H8.989c-1.498 0-2.005-.896-1.989-2v-7.998c0-.987.336-2.032 1.114-2.639 4.45-3.773 3.436-4.597 4.45-5.83.985-1.13 3.2-.5 3.037 2.362C15.201 7.397 14.445 9 14.445 9zM3 9h2a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V10a1 1 0 0 1 1-1z" fill-rule="evenodd"></path></svg></span>踩</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Report" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M19.947 3.129c-.633.136-3.927.639-5.697.385-3.133-.45-4.776-2.54-9.949-.888-.997.413-1.277 1.038-1.277 2.019L3 20.808c0 .3.101.54.304.718a.97.97 0 0 0 .73.304c.275 0 .519-.102.73-.304.202-.179.304-.418.304-.718v-6.58c4.533-1.235 8.047.668 8.562.864 2.343.893 5.542.008 6.774-.657.397-.178.596-.474.596-.887V3.964c0-.599-.42-.972-1.053-.835z" fill-rule="evenodd"></path></svg></span>举报</button><button type="button" class="Button CommentItemV2-hoverBtn Button--plain"><span style="display: inline-flex; align-items: center;"><svg class="Zi Zi--Trash" style="margin-right: 5px;" fill="currentColor" viewBox="0 0 24 24" width="16" height="16"><path d="M16.464 4s.051-2-1.479-2H9C7.194 2 7.465 4 7.465 4H4.752c-2.57 0-2.09 3.5 0 3.5l1.213 13.027S5.965 22 7.475 22h8.987c1.502 0 1.502-1.473 1.502-1.473l1.2-13.027c2.34 0 2.563-3.5 0-3.5h-2.7zM8.936 18.5l-.581-9h1.802v9H8.936zm4.824 0v-9h1.801l-.61 9H13.76z" fill-rule="evenodd"></path></svg></span>删除</button></div></div></div></div></li></ul></div></div></div></div></div></div></main><div class="CornerButtons"><div class="CornerAnimayedFlex CornerAnimayedFlex--hidden"><button data-tooltip="回到顶部" data-tooltip-position="left" data-tooltip-will-hide-on-click="true" aria-label="回到顶部" type="button" class="Button CornerButton Button--plain"><svg class="Zi Zi--BackToTop" aria-label="回到顶部" fill="currentColor" viewBox="0 0 24 24" width="24" height="24"><path d="M16.036 19.59a1 1 0 0 1-.997.995H9.032a.996.996 0 0 1-.997-.996v-7.005H5.03c-1.1 0-1.36-.633-.578-1.416L11.33 4.29a1.003 1.003 0 0 1 1.412 0l6.878 6.88c.782.78.523 1.415-.58 1.415h-3.004v7.005z"></path></svg></button></div></div></div></div><script id="js-initialData" type="text/json">{"initialState":{"common":{"ask":{}},"loading":{"global":{"count":0},"local":{"env\u002FgetIpinfo\u002F":false,"article\u002Fget\u002F":false,"brand\u002FgetUrl\u002F":false}},"club":{"tags":{},"admins":{"data":[]},"members":{"data":[]},"explore":{"candidateSyncClubs":{}},"profile":{},"checkin":{},"comments":{"paging":{},"loading":{},"meta":{},"ids":{}},"postList":{"paging":{},"loading":{},"ids":{}},"recommend":{"data":[]},"silences":{"data":[]},"application":{"profile":null}},"entities":{"users":{"2bcb30744c0f014ce6702f4d94922945":{"uid":28236180357120,"userType":"people","id":"2bcb30744c0f014ce6702f4d94922945"},"yin-xiang-nan":{"isFollowed":false,"avatarUrlTemplate":"https:\u002F\u002Fpic4.zhimg.com\u002Fbf28449bb.jpg?source=172ae18b","uid":"28236180357120","userType":"people","isFollowing":false,"urlToken":"yin-xiang-nan","id":"2bcb30744c0f014ce6702f4d94922945","description":"","name":"杨指北","isAdvertiser":false,"headline":"无业游民","gender":1,"url":"\u002Fpeople\u002F2bcb30744c0f014ce6702f4d94922945","avatarUrl":"https:\u002F\u002Fpic2.zhimg.com\u002Fbf28449bb_l.jpg?source=172ae18b","isOrg":false,"type":"people","vipInfo":{"isVip":true,"vipIcon":{"url":"https:\u002F\u002Fpic2.zhimg.com\u002Fv2-4812630bc27d642f7cafcd6cdeca3d7a_r.png?source=172ae18b","nightModeUrl":"https:\u002F\u002Fpic2.zhimg.com\u002Fv2-c9686ff064ea3579730756ac6c289978_r.png?source=172ae18b"}},"levelInfo":{"exp":150802,"level":8,"nicknameColor":{"color":"","nightModeColor":""},"levelIcon":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-ce2d9b858d70d9e107f14c57a0ceb563_l.png","iconInfo":{"url":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-ce2d9b858d70d9e107f14c57a0ceb563_l.png","nightModeUrl":"https:\u002F\u002Fpic3.zhimg.com\u002Fv2-2c1a38e54e53ce62f8b7bc346a043c61_l.png","width":93,"height":51}},"badge":[],"badgeV2":{"title":"","mergedBadges":[],"detailBadges":[],"icon":"","nightIcon":""},"exposedMedal":{"medalId":"1124316222665379841","medalName":"我的知乎 2019","avatarUrl":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-2592b0b52e1fac99f69b38e00252413b_r.png?source=172ae18b","miniAvatarUrl":"https:\u002F\u002Fpic1.zhimg.com\u002Fv2-ad363cc3088dc8de7544fd08b1c4987a_l.png?source=172ae18b","description":"参与「我的知乎 2019」即可获得"}}},"questions":{},"answers":{},"articles":{"114314389":{"trackUrl":["https:\u002F\u002Fsugar.zhihu.com\u002Fplutus_adreaper\u002Fpage_monitor_log?si=__SESSIONID__&ti=__ATOKEN__&at=view&pf=__OS__&ed=BiBUKF0xBSkqGGFQA2B_AHGhHcvz4B4D&idfa=__IDFA__&imei=__IMEI__&androidid=__ANDROIDID__&oaid=__OAID__&ci=__CREATIVEID__&zid=__ZONEID__"],"id":114314389,"title":"weight normalization 原理和实现","type":"article","articleType":"normal","excerptTitle":"","url":"https:\u002F\u002Fzhuanlan.zhihu.com\u002Fp\u002F114314389","imageUrl":"","titleImage":"","excerpt":"\u003Cimg src=\"https:\u002F\u002Fpic3.zhimg.com\u002Fv2-09e03155915360824d0a485d1a7cc582_200x112.png\" data-caption=\"a simple neural network\" data-size=\"normal\" data-rawwidth=\"256\" data-rawheight=\"349\" data-watermark=\"watermark\" data-original-src=\"v2-09e03155915360824d0a485d1a7cc582\" data-watermark-src=\"v2-985fd9f93fbbb02d524022b07b1f4b62\" data-private-watermark-src=\"\" class=\"origin_image inline-img zh-lightbox-thumb\" data-original=\"https:\u002F\u002Fpic3.zhimg.com\u002Fv2-09e03155915360824d0a485d1a7cc582_r.png\"\u002F\u003E除了最近很火的 spectral norm 之外，pytorch 还实现了 2016 年 OpenAI 的文章： \u003Cb\u003EWeight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks\u003C\u002Fb\u003E。该模块用于归一化网络权重，从而加速训练。本文介绍该模块的原理和 p…","created":1584586668,"updated":1584586826,"author":{"isFollowed":false,"avatarUrlTemplate":"https:\u002F\u002Fpic4.zhimg.com\u002Fbf28449bb.jpg?source=172ae18b","uid":"28236180357120","userType":"people","isFollowing":false,"urlToken":"yin-xiang-nan","id":"2bcb30744c0f014ce6702f4d94922945","description":"","name":"杨指北","isAdvertiser":false,"headline":"无业游民","gender":1,"url":"\u002Fpeople\u002F2bcb30744c0f014ce6702f4d94922945","avatarUrl":"https:\u002F\u002Fpic2.zhimg.com\u002Fbf28449bb_l.jpg?source=172ae18b","isOrg":false,"type":"people","vipInfo":{"isVip":true,"vipIcon":{"url":"https:\u002F\u002Fpic2.zhimg.com\u002Fv2-4812630bc27d642f7cafcd6cdeca3d7a_r.png?source=172ae18b","nightModeUrl":"https:\u002F\u002Fpic2.zhimg.com\u002Fv2-c9686ff064ea3579730756ac6c289978_r.png?source=172ae18b"}},"levelInfo":{"exp":150802,"level":8,"nicknameColor":{"color":"","nightModeColor":""},"levelIcon":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-ce2d9b858d70d9e107f14c57a0ceb563_l.png","iconInfo":{"url":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-ce2d9b858d70d9e107f14c57a0ceb563_l.png","nightModeUrl":"https:\u002F\u002Fpic3.zhimg.com\u002Fv2-2c1a38e54e53ce62f8b7bc346a043c61_l.png","width":93,"height":51}},"badge":[],"badgeV2":{"title":"","mergedBadges":[],"detailBadges":[],"icon":"","nightIcon":""},"exposedMedal":{"medalId":"1124316222665379841","medalName":"我的知乎 2019","avatarUrl":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-2592b0b52e1fac99f69b38e00252413b_r.png?source=172ae18b","miniAvatarUrl":"https:\u002F\u002Fpic1.zhimg.com\u002Fv2-ad363cc3088dc8de7544fd08b1c4987a_l.png?source=172ae18b","description":"参与「我的知乎 2019」即可获得"}},"commentPermission":"all","copyrightPermission":"need_review","state":"published","imageWidth":0,"imageHeight":0,"content":"\u003Cp\u003E除了最近很火的 spectral norm 之外，pytorch 还实现了 2016 年 OpenAI 的文章：\u003Cb\u003EWeight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks\u003C\u002Fb\u003E。该模块用于归一化网络权重，从而加速训练。本文介绍该模块的原理和 pytorch 的实现细节。\u003C\u002Fp\u003E\u003Ch2\u003EWeight Normalization\u003C\u002Fh2\u003E\u003Cp\u003E通常的梯度下降法直接将损失函数对 weight 求导得到梯度，然后以一定学习率，沿着梯度下降的方向更新权重。而 weight normalization 将每个神经元 weight 的\u003Cb\u003E方向\u003C\u002Fb\u003E和\u003Cb\u003E长度\u003C\u002Fb\u003E解耦，即把参数 weight 分为两部分：\u003Cb\u003Ev\u003C\u002Fb\u003E 和 g，其中 \u003Cb\u003Ev\u003C\u002Fb\u003E 代表方向，而 g 代表长度。然后将损失函数分别对这两部分求导，并更新它们的值。文章通过实验和理论分析，证明这种方式可以加速网络收敛。\u003C\u002Fp\u003E\u003Cfigure data-size=\"normal\"\u003E\u003Cnoscript\u003E\u003Cimg src=\"https:\u002F\u002Fpic3.zhimg.com\u002Fv2-985fd9f93fbbb02d524022b07b1f4b62_b.jpg\" data-size=\"normal\" data-rawwidth=\"256\" data-rawheight=\"349\" class=\"content_image\" width=\"256\"\u002F\u003E\u003C\u002Fnoscript\u003E\u003Cimg src=\"data:image\u002Fsvg+xml;utf8,<svg xmlns='http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg' width='256' height='349'><\u002Fsvg>\" data-size=\"normal\" data-rawwidth=\"256\" data-rawheight=\"349\" class=\"content_image lazy\" width=\"256\" data-actualsrc=\"https:\u002F\u002Fpic3.zhimg.com\u002Fv2-985fd9f93fbbb02d524022b07b1f4b62_b.jpg\"\u002F\u003E\u003Cfigcaption\u003Ea simple neural network\u003C\u002Ffigcaption\u003E\u003C\u002Ffigure\u003E\u003Cp\u003E用 \u003Cb\u003Ew\u003C\u002Fb\u003E，\u003Cb\u003Eb\u003C\u002Fb\u003E 分别表示某神经元（上图红色部分）的 weight 和 bias，其输入为 \u003Cb\u003Ex\u003C\u002Fb\u003E。其输出通常按照如下公式计算：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=y%3D%5Cphi%28%5Cmathrm%7Bw%7D%5Ccdot%5Cmathrm%7Bx%7D%2Bb%29%5Ctag%7B1%7D\" alt=\"y=\\phi(\\mathrm{w}\\cdot\\mathrm{x}+b)\\tag{1}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E其中 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cphi\" alt=\"\\phi\" eeimg=\"1\"\u002F\u003E 为非线性激活函数，\u003Cb\u003Ew\u003C\u002Fb\u003E 和 \u003Cb\u003Ex\u003C\u002Fb\u003E 都是 k 维，b 是 1 维。weight normalization 将 \u003Cb\u003Ew\u003C\u002Fb\u003E 分解为 g 和\u003Cb\u003E v \u003C\u002Fb\u003E两部分：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cmathrm%7Bw%7D+%3D+%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Cmathrm%7Bv%7D%5Ctag%7B2%7D\" alt=\"\\mathrm{w} = \\frac{g}{\\|\\mathrm{v}\\|}\\mathrm{v}\\tag{2}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E经过上式的分解后，\u003Cb\u003Ew\u003C\u002Fb\u003E 的模长等于 g，和向量 \u003Cb\u003Ev\u003C\u002Fb\u003E 解耦了，\u003Cb\u003Ew\u003C\u002Fb\u003E 的方向与\u003Cb\u003E v\u003C\u002Fb\u003E 相同，与 g 也无关。\u003C\u002Fp\u003E\u003Cp\u003E用 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cnabla_%5Cmathrm%7Bw%7DL\" alt=\"\\nabla_\\mathrm{w}L\" eeimg=\"1\"\u002F\u003E 表示损失函数对 weight 的梯度，根据公式 (2)，其对 \u003Cb\u003Ev\u003C\u002Fb\u003E 和 g 的梯度可以通过链式法则得到：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cnabla_gL+%3D+%5Cnabla_%5Cmathrm%7Bw%7DL%5Ccdot%5Cfrac%7B%5Cpartial+%5Cmathrm%7Bw%7D%7D%7B%5Cpartial+g%7D%5Ctag%7B3%7D\" alt=\"\\nabla_gL = \\nabla_\\mathrm{w}L\\cdot\\frac{\\partial \\mathrm{w}}{\\partial g}\\tag{3}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cnabla_%5Cmathrm%7Bv%7DL+%3D+%5Cnabla_%5Cmathrm%7Bw%7DL%5Ccdot%5Cfrac%7B%5Cpartial+%5Cmathrm%7Bw%7D%7D%7B%5Cpartial+%5Cmathrm%7Bv%7D%7D%5Ctag%7B4%7D\" alt=\"\\nabla_\\mathrm{v}L = \\nabla_\\mathrm{w}L\\cdot\\frac{\\partial \\mathrm{w}}{\\partial \\mathrm{v}}\\tag{4}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E由 (2) 易知：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cfrac%7B%5Cpartial%5Cmathrm%7Bw%7D%7D%7B%5Cpartial+g%7D%3D%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Ctag%7B5%7D\" alt=\"\\frac{\\partial\\mathrm{w}}{\\partial g}=\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|}\\tag{5}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E(5) 代入 (3) 得 ：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cnabla_gL+%3D+%5Cnabla_%5Cmathrm%7Bw%7DL%5Ccdot%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Ctag%7B6%7D\" alt=\"\\nabla_gL = \\nabla_\\mathrm{w}L\\cdot\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|}\\tag{6}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E\u003Cb\u003Ew\u003C\u002Fb\u003E 对 \u003Cb\u003Ev\u003C\u002Fb\u003E 得偏导得推导略显繁琐：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cbegin%7Balign%2A%7D+%5Cfrac%7B%5Cpartial%5Cmathrm%7Bw%7D%7D%7B%5Cpartial%5Cmathrm%7Bv%7D%7D%26%3Dg%5Ccdot%5Cmathrm%7Bd%7D%28%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%29%2F%5Cmathrm%7Bd%7D%5Cmathrm%7Bv%7D%5C%5C+%26%3Dg%5Ccdot%28%5Cfrac%7B%5Cmathrm%7Bd%7D%5Cmathrm%7Bv%7D%7D%7B%5Cmathrm%7Bdv%7D%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%2B%5Cmathrm%7Bv%7D%5Cfrac%7B%5Cmathrm%7Bd%7D%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E%7B-1%7D%7D%7B%5Cmathrm%7Bdv%7D%7D%29%5C%5C+%26%3Dg%5Ccdot%28%5Cmathbb%7BI%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D-%5Cmathrm%7Bv%7D%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E%7B-2%7D%5Cfrac%7B%5Cmathrm%7Bd%7D%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%7B%5Cmathrm%7Bdv%7D%7D%29%5C%5C+%26%3Dg%5Ccdot%28%5Cmathbb%7BI%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D-%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E%7B2%7D%7D%5Cfrac%7B%5Cmathrm%7Bd%7D%28%28v_1%5E2%2B...%2Bv_n%5E2%29%5E%7B%5Cfrac%7B1%7D%7B2%7D%7D%29%7D%7B%5Cmathrm%7Bdv%7D%7D%29%5C%5C+%26%3Dg%5Ccdot%28%5Cmathbb%7BI%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D-%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E%7B2%7D%7D%5Cfrac%7B1%7D%7B2%7D%28v_1%5E2%2B...%2Bv_n%5E2%29%5E%7B-%5Cfrac%7B1%7D%7B2%7D%7D%5Cfrac%7B%5Cmathrm%7Bd%7D%28%28v_1%5E2%2B...%2Bv_n%5E2%29%7D%7B%5Cmathrm%7Bdv%7D%7D%29%5C%5C+%26%3Dg%5Ccdot%28%5Cmathbb%7BI%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D-%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E%7B2%7D%7D%5Cfrac%7B1%7D%7B2%7D%28v_1%5E2%2B...%2Bv_n%5E2%29%5E%7B-%5Cfrac%7B1%7D%7B2%7D%7D2%5Cmathrm%7Bv%7D%5C%5C+%26%3Dg%5Ccdot%28%5Cmathbb%7BI%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D-%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E%7B2%7D%7D%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%29%5Ctag%7B7%7D++%5Cend%7Balign%2A%7D\" alt=\"\\begin{align*} \\frac{\\partial\\mathrm{w}}{\\partial\\mathrm{v}}&=g\\cdot\\mathrm{d}(\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|})\u002F\\mathrm{d}\\mathrm{v}\\\\ &=g\\cdot(\\frac{\\mathrm{d}\\mathrm{v}}{\\mathrm{dv}}\\frac{1}{\\|\\mathrm{v}\\|}+\\mathrm{v}\\frac{\\mathrm{d}\\|\\mathrm{v}\\|^{-1}}{\\mathrm{dv}})\\\\ &=g\\cdot(\\mathbb{I}\\frac{1}{\\|\\mathrm{v}\\|}-\\mathrm{v}\\|\\mathrm{v}\\|^{-2}\\frac{\\mathrm{d}\\|\\mathrm{v}\\|}{\\mathrm{dv}})\\\\ &=g\\cdot(\\mathbb{I}\\frac{1}{\\|\\mathrm{v}\\|}-\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^{2}}\\frac{\\mathrm{d}((v_1^2+...+v_n^2)^{\\frac{1}{2}})}{\\mathrm{dv}})\\\\ &=g\\cdot(\\mathbb{I}\\frac{1}{\\|\\mathrm{v}\\|}-\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^{2}}\\frac{1}{2}(v_1^2+...+v_n^2)^{-\\frac{1}{2}}\\frac{\\mathrm{d}((v_1^2+...+v_n^2)}{\\mathrm{dv}})\\\\ &=g\\cdot(\\mathbb{I}\\frac{1}{\\|\\mathrm{v}\\|}-\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^{2}}\\frac{1}{2}(v_1^2+...+v_n^2)^{-\\frac{1}{2}}2\\mathrm{v}\\\\ &=g\\cdot(\\mathbb{I}\\frac{1}{\\|\\mathrm{v}\\|}-\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^{2}}\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|})\\tag{7} \\end{align*}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E把 (7) 带入 (4)，并利用 (6) 得：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cbegin%7Balign%7D+%5Cnabla_%7B%5Cmathrm%7Bv%7D%7DL%26%3D%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL%5Ccdot+g+%28%5Cmathbb%7BI%7D%5Cfrac%7B1%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D-%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E2%7D%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%29%5C%5C+%26%3D%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL-g%5Ccdot%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Ccdot%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E2%7D%5C%5C+%26%3D%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL-%5Cfrac%7Bg%5Cnabla_gL%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E2%7D%5Cmathrm%7Bv%7D+%5Cend%7Balign%7D%5Ctag%7B8%7D\" alt=\"\\begin{align} \\nabla_{\\mathrm{v}}L&=\\nabla_{\\mathrm{w}}L\\cdot g (\\mathbb{I}\\frac{1}{\\|\\mathrm{v}\\|}-\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^2}\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|})\\\\ &=\\frac{g}{\\|\\mathrm{v}\\|}\\nabla_{\\mathrm{w}}L-g\\cdot\\nabla_{\\mathrm{w}}L\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|}\\cdot\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^2}\\\\ &=\\frac{g}{\\|\\mathrm{v}\\|}\\nabla_{\\mathrm{w}}L-\\frac{g\\nabla_gL}{\\|\\mathrm{v}\\|^2}\\mathrm{v} \\end{align}\\tag{8}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E(6) 和 (8) 即为\u003Cb\u003E v\u003C\u002Fb\u003E 和 g 的梯度公式。\u003C\u002Fp\u003E\u003Cp\u003E由于 v 和 w 方向相同，因此可以把公式 (8) 第二行中的\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Ccdot%5Cfrac%7B%5Cmathrm%7Bv%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5E2%7D\" alt=\"\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|}\\cdot\\frac{\\mathrm{v}}{\\|\\mathrm{v}\\|^2}\" eeimg=\"1\"\u002F\u003E 替换为 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cfrac%7B%5Cmathrm%7Bw%7D%27%5Cmathrm%7Bw%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5C%7C%5Cmathrm%7Bw%7D%5C%7C%5E2%7D\" alt=\"\\frac{\\mathrm{w}'\\mathrm{w}}{\\|\\mathrm{v}\\|\\|\\mathrm{w}\\|^2}\" eeimg=\"1\"\u002F\u003E （注意，替换前 v.v 表示向量点乘，替换后用 w'w 代表向量点乘，和论文保持一致，方便得到后面的矩阵表示。）\u003C\u002Fp\u003E\u003Cp\u003E得到：\u003C\u002Fp\u003E\u003Cp\u003E\u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cbegin%7Balign%7D+%5Cnabla_%7B%5Cmathrm%7Bv%7D%7DL+%26%3D%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL-g%5Ccdot%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL%5Cfrac%7B%5Cmathrm%7Bw%7D%27%5Cmathrm%7Bw%7D%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%5C%7C%5Cmathrm%7Bw%7D%5C%7C%5E2%7D%5C%5C+%26%3D%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Cnabla_%7B%5Cmathrm%7Bw%7D%7DL-%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%5Cfrac%7B%5Cmathrm%7Bww%27%7D%7D%7B%5C%7C%5Cmathrm%7Bw%7D%5C%7C%5E2%7D%5Cnabla_%5Cmathrm%7Bw%7DL%5C%5C+%26%3D%5Cfrac%7Bg%7D%7B%5C%7C%5Cmathrm%7Bv%7D%5C%7C%7D%28I-%5Cfrac%7B%5Cmathrm%7Bww%27%7D%7D%7B%5C%7C%5Cmathrm%7Bw%7D%5C%7C%5E2%7D%29%5Cnabla_%5Cmathrm%7Bw%7DL+%5Cend%7Balign%7D%5Ctag%7B9%7D\" alt=\"\\begin{align} \\nabla_{\\mathrm{v}}L &=\\frac{g}{\\|\\mathrm{v}\\|}\\nabla_{\\mathrm{w}}L-g\\cdot\\nabla_{\\mathrm{w}}L\\frac{\\mathrm{w}'\\mathrm{w}}{\\|\\mathrm{v}\\|\\|\\mathrm{w}\\|^2}\\\\ &=\\frac{g}{\\|\\mathrm{v}\\|}\\nabla_{\\mathrm{w}}L-\\frac{g}{\\|\\mathrm{v}\\|}\\frac{\\mathrm{ww'}}{\\|\\mathrm{w}\\|^2}\\nabla_\\mathrm{w}L\\\\ &=\\frac{g}{\\|\\mathrm{v}\\|}(I-\\frac{\\mathrm{ww'}}{\\|\\mathrm{w}\\|^2})\\nabla_\\mathrm{w}L \\end{align}\\tag{9}\" eeimg=\"1\"\u002F\u003E \u003C\u002Fp\u003E\u003Cp\u003E(9) 最后一行，中间括号部分为一个投影矩阵，将空间中任意一个向量投影到向量 \u003Cb\u003Ew\u003C\u002Fb\u003E 的补空间（垂直于 \u003Cb\u003Ew\u003C\u002Fb\u003E 的超平面）。经过 weight normalization 后，损失函数对 \u003Cb\u003Ev\u003C\u002Fb\u003E 的梯度，其实是它在 \u003Cb\u003Ew\u003C\u002Fb\u003E 的梯度基础上，做了两个方面的改进：\u003C\u002Fp\u003E\u003Cul\u003E\u003Cli\u003E只保留垂直于 \u003Cb\u003Ew\u003C\u002Fb\u003E 方向的部分\u003C\u002Fli\u003E\u003Cli\u003E长度放缩为 g\u002F||\u003Cb\u003Ev\u003C\u002Fb\u003E||\u003C\u002Fli\u003E\u003C\u002Ful\u003E\u003Cp\u003E由于 \u003Cb\u003Ev\u003C\u002Fb\u003E 和 \u003Cb\u003Ew\u003C\u002Fb\u003E 的方向相同，而 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5Cnabla_%7B%5Cmathrm%7Bv%7D%7DL\" alt=\"\\nabla_{\\mathrm{v}}L\" eeimg=\"1\"\u002F\u003E 垂直于 \u003Cb\u003Ev\u003C\u002Fb\u003E ，根据勾股定理，随着迭代次数的增加，\u003Cb\u003Ev\u003C\u002Fb\u003E 的模是单调递增的。当 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5C%7C%5Cnabla_%7B%5Cmathrm%7Bv%7D%7DL%5C%7C\" alt=\"\\|\\nabla_{\\mathrm{v}}L\\|\" eeimg=\"1\"\u002F\u003E 很大时，经过若干次迭代，||\u003Cb\u003Ev\u003C\u002Fb\u003E|| 会很快增大，导致 g\u002F||\u003Cb\u003Ev\u003C\u002Fb\u003E|| 减小，从而降低 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5C%7C%5Cnabla_%7B%5Cmathrm%7Bv%7D%7DL%5C%7C\" alt=\"\\|\\nabla_{\\mathrm{v}}L\\|\" eeimg=\"1\"\u002F\u003E；而当 \u003Cimg src=\"https:\u002F\u002Fwww.zhihu.com\u002Fequation?tex=%5C%7C%5Cnabla_%7B%5Cmathrm%7Bv%7D%7DL%5C%7C\" alt=\"\\|\\nabla_{\\mathrm{v}}L\\|\" eeimg=\"1\"\u002F\u003E 很小时，||\u003Cb\u003Ev\u003C\u002Fb\u003E|| 会停止进一步增大。因此，weight normalization 可以使梯度值更稳定。\u003C\u002Fp\u003E\u003Cp\u003E梯度前面的放缩项 g\u002F||\u003Cb\u003Ev\u003C\u002Fb\u003E||，可以使 learning rate 的选择范围更广，当 learning rate 很大时，||\u003Cb\u003Ev\u003C\u002Fb\u003E|| 会迅速增大从而让梯度的放缩项减小，稳定训练。\u003C\u002Fp\u003E\u003Ch2\u003EPytorch Implementation\u003C\u002Fh2\u003E\u003Cp\u003Epytorch 的官方实现代码为：\u003Ca href=\"https:\u002F\u002Flink.zhihu.com\u002F?target=https%3A\u002F\u002Fgithub.com\u002Fpytorch\u002Fpytorch\u002Fblob\u002Fmaster\u002Ftorch\u002Fnn\u002Futils\u002Fweight_norm.py\" class=\" external\" target=\"_blank\" rel=\"nofollow noreferrer\"\u003E\u003Cspan class=\"invisible\"\u003Ehttps:\u002F\u002F\u003C\u002Fspan\u003E\u003Cspan class=\"visible\"\u003Egithub.com\u002Fpytorch\u002Fpyto\u003C\u002Fspan\u003E\u003Cspan class=\"invisible\"\u003Erch\u002Fblob\u002Fmaster\u002Ftorch\u002Fnn\u002Futils\u002Fweight_norm.py\u003C\u002Fspan\u003E\u003Cspan class=\"ellipsis\"\u003E\u003C\u002Fspan\u003E\u003C\u002Fa\u003E\u003C\u002Fp\u003E\u003Cp\u003E可以直接无脑调用：\u003C\u002Fp\u003E\u003Cdiv class=\"highlight\"\u003E\u003Cpre\u003E\u003Ccode class=\"language-python\"\u003E\u003Cspan class=\"kn\"\u003Efrom\u003C\u002Fspan\u003E \u003Cspan class=\"nn\"\u003Etorch.nn.utils\u003C\u002Fspan\u003E \u003Cspan class=\"kn\"\u003Eimport\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Eweight_norm\u003C\u002Fspan\u003E\n\u003Cspan class=\"kn\"\u003Eimport\u003C\u002Fspan\u003E \u003Cspan class=\"nn\"\u003Etorch\u003C\u002Fspan\u003E\n\n\u003Cspan class=\"n\"\u003Econv\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Eweight_norm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Etorch\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Enn\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003EConv2d\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"mi\"\u003E3\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"mi\"\u003E10\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"mi\"\u003E5\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E),\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E\u003Cspan class=\"s1\"\u003E'weight'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n\u003Cspan class=\"n\"\u003Efc\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Eweight_norm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Etorch\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Enn\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003ELinear\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"mi\"\u003E20\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"mi\"\u003E40\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E),\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E\u003Cspan class=\"s1\"\u003E'weight'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n\n\u003Cspan class=\"n\"\u003Econv\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eweight_g\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eshape\u003C\u002Fspan\u003E \u003Cspan class=\"c1\"\u003E# 卷积核的输出包括10个layer，这 10 个值对应原始 weight 的 10 个 norm\u003C\u002Fspan\u003E\n\u003Cspan class=\"c1\"\u003E# torch.Size([10, 1, 1, 1])\u003C\u002Fspan\u003E\n\u003Cspan class=\"n\"\u003Econv\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eweight_v\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eshape\u003C\u002Fspan\u003E \u003Cspan class=\"c1\"\u003E# 和原始的卷积核形状相同\u003C\u002Fspan\u003E\n\u003Cspan class=\"c1\"\u003E# torch.Size([10, 3, 5, 5])\u003C\u002Fspan\u003E\u003C\u002Fcode\u003E\u003C\u002Fpre\u003E\u003C\u002Fdiv\u003E\u003Cp\u003E看一下 pytorch 官方的实现细节：\u003C\u002Fp\u003E\u003Cdiv class=\"highlight\"\u003E\u003Cpre\u003E\u003Ccode class=\"language-python\"\u003E\u003Cspan class=\"kn\"\u003Efrom\u003C\u002Fspan\u003E \u003Cspan class=\"nn\"\u003Etorch.nn.parameter\u003C\u002Fspan\u003E \u003Cspan class=\"kn\"\u003Eimport\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003EParameter\u003C\u002Fspan\u003E\n\u003Cspan class=\"kn\"\u003Efrom\u003C\u002Fspan\u003E \u003Cspan class=\"nn\"\u003Etorch\u003C\u002Fspan\u003E \u003Cspan class=\"kn\"\u003Eimport\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003E_weight_norm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Enorm_except_dim\u003C\u002Fspan\u003E\n\n\n\u003Cspan class=\"k\"\u003Eclass\u003C\u002Fspan\u003E \u003Cspan class=\"nc\"\u003EWeightNorm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"nb\"\u003Eobject\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Edef\u003C\u002Fspan\u003E \u003Cspan class=\"fm\"\u003E__init__\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Eif\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E \u003Cspan class=\"ow\"\u003Eis\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003ENone\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E:\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E-\u003C\u002Fspan\u003E\u003Cspan class=\"mi\"\u003E1\u003C\u002Fspan\u003E\n \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"c1\"\u003E# 一般都是 weight\u003C\u002Fspan\u003E\n \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E \u003Cspan class=\"c1\"\u003E# 默认是 0，即输出的卷积层数、neuron 数，在weight中所在的维度\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"k\"\u003Edef\u003C\u002Fspan\u003E \u003Cspan class=\"nf\"\u003Ecompute_weight\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 这个函数用于从 weight_g 和 weight_v 反推 weight\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 公式即为：weight = weight_g*weight_v\u002F|weight_v|\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Eg\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"nb\"\u003Egetattr\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E+\u003C\u002Fspan\u003E \u003Cspan class=\"s1\"\u003E'_g'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Ev\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"nb\"\u003Egetattr\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E+\u003C\u002Fspan\u003E \u003Cspan class=\"s1\"\u003E'_v'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Ereturn\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003E_weight_norm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ev\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Eg\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"nd\"\u003E@staticmethod\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Edef\u003C\u002Fspan\u003E \u003Cspan class=\"nf\"\u003Eapply\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Efor\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ek\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ehook\u003C\u002Fspan\u003E \u003Cspan class=\"ow\"\u003Ein\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003E_forward_pre_hooks\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eitems\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E():\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Eif\u003C\u002Fspan\u003E \u003Cspan class=\"nb\"\u003Eisinstance\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ehook\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003EWeightNorm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E \u003Cspan class=\"ow\"\u003Eand\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ehook\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E==\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E:\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Eraise\u003C\u002Fspan\u003E \u003Cspan class=\"ne\"\u003ERuntimeError\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"s2\"\u003E"Cannot register two weight_norm hooks on "\u003C\u002Fspan\u003E\n \u003Cspan class=\"s2\"\u003E"the same parameter {}"\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eformat\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E))\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"k\"\u003Eif\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E \u003Cspan class=\"ow\"\u003Eis\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003ENone\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E:\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E-\u003C\u002Fspan\u003E\u003Cspan class=\"mi\"\u003E1\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"n\"\u003Efn\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003EWeightNorm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"n\"\u003Eweight\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"nb\"\u003Egetattr\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E \u003Cspan class=\"c1\"\u003E# 获取原始权重\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"c1\"\u003E# remove w from parameter list\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 从模型的参数列表中删除 weight\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Edel\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003E_parameters\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E[\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E]\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"c1\"\u003E# add g and v as new parameters and express w as g\u002F||v|| * v\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 初始化参数 weight_g 和 weight_v, 其中 weight_g 保存 neuron 的权重，\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# weight_v 初始化为原始的 weight\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eregister_parameter\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E+\u003C\u002Fspan\u003E \u003Cspan class=\"s1\"\u003E'_g'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003EParameter\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Enorm_except_dim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eweight\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"mi\"\u003E2\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Edata\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E))\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eregister_parameter\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E+\u003C\u002Fspan\u003E \u003Cspan class=\"s1\"\u003E'_v'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003EParameter\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eweight\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Edata\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E))\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 通过 fn 调用本类中的函数 compute_weight，从 weight_g 和 weight_v 重新计算 weight\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 并设置为新的成员变量 weight，注意这里的 weight 仅仅是成员变量，不再是网络的参数了，\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 不会被直接通过梯度下降法更新 \u003C\u002Fspan\u003E\n \u003Cspan class=\"nb\"\u003Esetattr\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Efn\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ecompute_weight\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E))\u003C\u002Fspan\u003E \n\n \u003Cspan class=\"c1\"\u003E# recompute weight before every forward()\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 设置前向传播的钩子，fn 默认调用本类中的 __call__ 方法\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eregister_forward_pre_hook\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Efn\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"k\"\u003Ereturn\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Efn\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"k\"\u003Edef\u003C\u002Fspan\u003E \u003Cspan class=\"nf\"\u003Eremove\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# apply 的逆操作，重新计算 weigh\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Eweight\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ecompute_weight\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 删除 weight 成员变量\u003C\u002Fspan\u003E\n \u003Cspan class=\"nb\"\u003Edelattr\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 从计算图中删除 weight_g 和 weight_v\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Edel\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003E_parameters\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E[\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E+\u003C\u002Fspan\u003E \u003Cspan class=\"s1\"\u003E'_g'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E]\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Edel\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003E_parameters\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E[\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E \u003Cspan class=\"o\"\u003E+\u003C\u002Fspan\u003E \u003Cspan class=\"s1\"\u003E'_v'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E]\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 将 weight 添加回参数列表\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eregister_parameter\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003EParameter\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eweight\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Edata\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E))\u003C\u002Fspan\u003E\n\n \u003Cspan class=\"k\"\u003Edef\u003C\u002Fspan\u003E \u003Cspan class=\"fm\"\u003E__call__\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Einputs\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 用于设置前向传播钩子，在运算之前，先将该模块成员变量 weight 用 weight_g 和 weight_v 更新\u003C\u002Fspan\u003E\n \u003Cspan class=\"nb\"\u003Esetattr\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"bp\"\u003Eself\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Ecompute_weight\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E))\u003C\u002Fspan\u003E\n\n\u003Cspan class=\"k\"\u003Edef\u003C\u002Fspan\u003E \u003Cspan class=\"nf\"\u003Eweight_norm\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E\u003Cspan class=\"s1\"\u003E'weight'\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E=\u003C\u002Fspan\u003E\u003Cspan class=\"mi\"\u003E0\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E):\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 通过 WeightNorm.apply 的静态方法，修改 module 内部的参数机制，共做了如下操作：\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 1) 删除了 module 中的 weight parameter，将其修改为普通的成员变量（即不直接参与权重更新） \u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 2) 初始化了模块新的 named_parameter：weight_g 和 weight_v,用它们取代之前的 weight\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 3) 由于模块前向传播相关代码中，没有对 weight_g 和 weight_v 的操作，仍然利用 weight,\u003C\u002Fspan\u003E\n \u003Cspan class=\"c1\"\u003E# 为了避免牵一发动全身，重新设置 模块的成员变量 weight，用 weight_g 和 weight_v 计算得到\u003C\u002Fspan\u003E\n \u003Cspan class=\"n\"\u003EWeightNorm\u003C\u002Fspan\u003E\u003Cspan class=\"o\"\u003E.\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Eapply\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E(\u003C\u002Fspan\u003E\u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Ename\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E,\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Edim\u003C\u002Fspan\u003E\u003Cspan class=\"p\"\u003E)\u003C\u002Fspan\u003E\n \u003Cspan class=\"k\"\u003Ereturn\u003C\u002Fspan\u003E \u003Cspan class=\"n\"\u003Emodule\u003C\u002Fspan\u003E\n\u003C\u002Fcode\u003E\u003C\u002Fpre\u003E\u003C\u002Fdiv\u003E\u003Cp class=\"ztext-empty-paragraph\"\u003E\u003Cbr\u002F\u003E\u003C\u002Fp\u003E\u003Ch2\u003E相关阅读\u003C\u002Fh2\u003E\u003Cp\u003E\u003Ca href=\"https:\u002F\u002Fzhuanlan.zhihu.com\u002Fp\u002F75054200\" class=\"internal\"\u003E尹相楠：半小时学会 PyTorch Hook\u003C\u002Fa\u003E\u003C\u002Fp\u003E\u003Cp\u003E\u003Ca href=\"https:\u002F\u002Fzhuanlan.zhihu.com\u002Fp\u002F55393813\" class=\"internal\"\u003E尹相楠：Spectral Normalization 谱归一化\u003C\u002Fa\u003E\u003C\u002Fp\u003E\u003Cp\u003E\u003C\u002Fp\u003E","adminClosedComment":false,"topics":[{"url":"https:\u002F\u002Fwww.zhihu.com\u002Fapi\u002Fv4\u002Ftopics\u002F19813032","type":"topic","id":"19813032","name":"深度学习（Deep Learning）"},{"url":"https:\u002F\u002Fwww.zhihu.com\u002Fapi\u002Fv4\u002Ftopics\u002F20047018","type":"topic","id":"20047018","name":"Torch (深度学习框架)"},{"url":"https:\u002F\u002Fwww.zhihu.com\u002Fapi\u002Fv4\u002Ftopics\u002F20058814","type":"topic","id":"20058814","name":"TensorLayer（深度学习库）"}],"voteupCount":41,"voting":0,"commentCount":14,"contributions":[{"id":23355037,"state":"need_approved","type":"include","column":{"description":"专注于AI技术研究与机器学习框架研发，让AI所见即所得","canManage":false,"intro":"人工智能技术文章","isFollowing":true,"urlToken":"c_201634018","id":"c_201634018","articlesCount":183,"acceptSubmission":true,"title":"SIGAI","url":"https:\u002F\u002Fzhuanlan.zhihu.com\u002Fc_201634018","commentPermission":"all","created":1528796147,"updated":1599156093,"imageUrl":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-eb370129b198ca81b667f0a8d27f1547_720w.jpg?source=172ae18b","author":{"isFollowed":false,"avatarUrlTemplate":"https:\u002F\u002Fpic2.zhimg.com\u002Fv2-a8087295deb16dd4766446f9ee8638b5.jpg?source=172ae18b","uid":"957655130280108032","userType":"organization","isFollowing":false,"urlToken":"bei-jing-zhang-liang-wu-xian-ke-ji-you-xian-gong-si","id":"33fb869726de328ef68001a919f0c151","description":"张量无限（SIGAI）致力于打造世界应用最广泛的人工智能平台产品。零编程、可视化、自动化，让AI所见即所得。","name":"SIGAI","isAdvertiser":false,"headline":"微信公众号：SIGAI","gender":1,"url":"\u002Forg\u002F33fb869726de328ef68001a919f0c151","avatarUrl":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-a8087295deb16dd4766446f9ee8638b5_l.jpg?source=172ae18b","isOrg":true,"type":"people"},"followers":9758,"type":"column"}}],"isTitleImageFullScreen":false,"upvotedFollowees":[],"commercialInfo":{"isCommercial":false,"plugin":{}},"suggestEdit":{"status":false,"reason":"","tip":"","url":"","title":""},"reason":"","annotationAction":[],"canTip":false,"canOpenTipjar":false,"tipjarorsCount":0,"isLabeled":false,"hasPublishingDraft":false,"isFavorited":false,"favlistsCount":36,"isNormal":true,"status":0,"activityToppingInfo":{"state":"untopped"},"shareText":"weight normalization 原理和实现 - 来自知乎专栏，作者: 杨指北 https:\u002F\u002Fzhuanlan.zhihu.com\u002Fp\u002F114314389 （想看更多？下载 @知乎 App：http:\u002F\u002Fweibo.com\u002Fp\u002F100404711598 ）","canComment":{"status":true,"reason":""},"mcnFpShow":-1,"isVisible":true,"isLiked":false,"likedCount":4,"visibleOnlyToAuthor":false,"hasColumn":false,"republishers":[]}},"columns":{"c_201634018":{"description":"专注于AI技术研究与机器学习框架研发，让AI所见即所得","canManage":false,"intro":"人工智能技术文章","isFollowing":true,"urlToken":"c_201634018","id":"c_201634018","articlesCount":183,"acceptSubmission":true,"title":"SIGAI","url":"https:\u002F\u002Fzhuanlan.zhihu.com\u002Fc_201634018","commentPermission":"all","created":1528796147,"updated":1599156093,"imageUrl":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-eb370129b198ca81b667f0a8d27f1547_720w.jpg?source=172ae18b","author":{"isFollowed":false,"avatarUrlTemplate":"https:\u002F\u002Fpic2.zhimg.com\u002Fv2-a8087295deb16dd4766446f9ee8638b5.jpg?source=172ae18b","uid":"957655130280108032","userType":"organization","isFollowing":false,"urlToken":"bei-jing-zhang-liang-wu-xian-ke-ji-you-xian-gong-si","id":"33fb869726de328ef68001a919f0c151","description":"张量无限（SIGAI）致力于打造世界应用最广泛的人工智能平台产品。零编程、可视化、自动化，让AI所见即所得。","name":"SIGAI","isAdvertiser":false,"headline":"微信公众号：SIGAI","gender":1,"url":"\u002Forg\u002F33fb869726de328ef68001a919f0c151","avatarUrl":"https:\u002F\u002Fpic4.zhimg.com\u002Fv2-a8087295deb16dd4766446f9ee8638b5_l.jpg?source=172ae18b","isOrg":true,"type":"people"},"followers":9758,"type":"column"}},"topics":{},"roundtables":{},"favlists":{},"comments":{},"notifications":{},"ebooks":{},"activities":{},"feeds":{},"pins":{},"promotions":{},"drafts":{},"chats":{},"posts":{},"clubs":{},"clubTags":{}},"currentUser":"2bcb30744c0f014ce6702f4d94922945","account":{"lockLevel":{},"unlockTicketStatus":false,"unlockTicket":null,"challenge":[],"errorStatus":false,"message":"","isFetching":false,"accountInfo":{},"urlToken":{"loading":false}},"settings":{"socialBind":null,"inboxMsg":null,"notification":{},"email":{},"privacyFlag":null,"blockedUsers":{"isFetching":false,"paging":{"pageNo":1,"pageSize":6},"data":[]},"blockedFollowees":{"isFetching":false,"paging":{"pageNo":1,"pageSize":6},"data":[]},"ignoredTopics":{"isFetching":false,"paging":{"pageNo":1,"pageSize":6},"data":[]},"restrictedTopics":null,"laboratory":{}},"notification":{},"people":{"profileStatus":{},"activitiesByUser":{},"answersByUser":{},"answersSortByVotesByUser":{},"answersIncludedByUser":{},"votedAnswersByUser":{},"thankedAnswersByUser":{},"voteAnswersByUser":{},"thankAnswersByUser":{},"topicAnswersByUser":{},"zvideosByUser":{},"articlesByUser":{},"articlesSortByVotesByUser":{},"articlesIncludedByUser":{},"pinsByUser":{},"questionsByUser":{},"commercialQuestionsByUser":{},"favlistsByUser":{},"followingByUser":{},"followersByUser":{},"mutualsByUser":{},"followingColumnsByUser":{},"followingQuestionsByUser":{},"followingFavlistsByUser":{},"followingTopicsByUser":{},"publicationsByUser":{},"columnsByUser":{},"allFavlistsByUser":{},"brands":null,"creationsByUser":{},"creationsSortByVotesByUser":{},"creationsFeed":{},"infinity":{}},"env":{"ab":{"config":{"experiments":[{"expId":"launch-qa_art2qa_new-3","expPrefix":"qa_art2qa_new","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-qa_cl_guest-2","expPrefix":"qa_cl_guest","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-qa_column_invite-2","expPrefix":"qa_column_invite","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-qa_repost-2","expPrefix":"qa_repost","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-se_sug_v2-3","expPrefix":"se_sug_v2","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-tp_zrec-8","expPrefix":"tp_zrec","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-us_adjust_new-11","expPrefix":"us_adjust_new","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-us_noti_count-8","expPrefix":"us_noti_count","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-vd_andplay_d-2","expPrefix":"vd_andplay_d","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-vd_hookupplay_an-2","expPrefix":"vd_hookupplay_an","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-vd_ios_play-4","expPrefix":"vd_ios_play","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-vd_timeguide-2","expPrefix":"vd_timeguide","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-vd_video_replay-3","expPrefix":"vd_video_replay","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"launch-vd_zvideo_link-10","expPrefix":"vd_zvideo_link","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"zanswer-2","expPrefix":"zanswer","isDynamicallyUpdated":false,"isRuntime":false,"includeTriggerInfo":false},{"expId":"recnew_2th-3","expPrefix":"recnew_2th","isDynamicallyUpdated":false,"isRuntime":false,"includeTriggerInfo":false},{"expId":"li_video-1","expPrefix":"li_video","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"se_meta_ss-3","expPrefix":"se_meta_ss","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"qa_cdzixun-1","expPrefix":"qa_cdzixun","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"qa_fo_recom_h-2","expPrefix":"qa_fo_recom_h","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"vd_bullet_gui-1","expPrefix":"vd_bullet_gui","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"vd_video_tab-9","expPrefix":"vd_video_tab","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"li_tp_paidanswer-10","expPrefix":"li_tp_paidanswer","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"li_video_section-6","expPrefix":"li_video_section","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"tp_content-2","expPrefix":"tp_content","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"captcha_v2-2_v9","expPrefix":"captcha_v2","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"meta_ebook-1_v1","expPrefix":"meta_ebook","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"club_fn-1_v4","expPrefix":"club_fn","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"webpImg-1_v3","expPrefix":"webpImg","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"general_1-2_v1","expPrefix":"general_1","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"correct_gpu-6_v5","expPrefix":"correct_gpu","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"se_fix_ebook-1_v3","expPrefix":"se_fix_ebook","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"correct_pos-4_v2","expPrefix":"correct_pos","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"rec_new2th-2_v3","expPrefix":"rec_new2th","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"hw_aa_30-1_v2","expPrefix":"hw_aa_30","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false},{"expId":"hw_aa_50-2_v1","expPrefix":"hw_aa_50","isDynamicallyUpdated":true,"isRuntime":false,"includeTriggerInfo":false}],"params":[{"id":"se_fix_ebook","type":"Int","value":"0","chainId":"_gene_","layerId":"se_fix_ebook","key":103},{"id":"se_cvr_boost","type":"Int","value":"0","chainId":"_gene_","layerId":"se_cvr_boost","key":183},{"id":"web_heifetz_grow_ad","type":"String","value":"1","layerId":"webgw_layer_3"},{"id":"gue_cdzixun","type":"String","value":"0","layerId":"gueqa_layer_3"},{"id":"ge_item","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_945","key":2971},{"id":"ge_newcard","type":"String","value":"3","chainId":"_gene_","layerId":"geus_layer_839","key":2997},{"id":"gue_v_serial","type":"String","value":"1","layerId":"guevd_layer_695"},{"id":"show_ad","type":"Int","value":"0","chainId":"_gene_","layerId":"show_ad","key":27},{"id":"gue_bulletmb","type":"String","value":"0","layerId":"guevd_layer_812"},{"id":"web_answerlist_ad","type":"String","value":"0","layerId":"webqa_layer_1"},{"id":"tp_contents","type":"String","value":"2","chainId":"_all_","layerId":"tptp_layer_627"},{"id":"gue_video_replay","type":"String","value":"2","layerId":"guevd_layer_3"},{"id":"rec_new2th","type":"Int","value":"1","chainId":"_gene_","layerId":"Hump","key":320},{"id":"general_1","type":"Int","value":"2","chainId":"_gene_","layerId":"general_1","key":8},{"id":"se_4a","type":"Int","value":"0","chainId":"_gene_","layerId":"rtiq","key":335},{"id":"li_edu_page","type":"String","value":"old","chainId":"_all_","layerId":"lili_layer_580"},{"id":"zr_expslotpaid","type":"String","value":"1","chainId":"_all_","layerId":"zrrec_layer_11"},{"id":"club_fn","type":"Int","value":"1","layerId":"club_fn"},{"id":"qap_question_author","type":"String","value":"0","chainId":"_all_","layerId":"qapqa_layer_2"},{"id":"ge_dipin_pre","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_1000","key":3124},{"id":"pf_noti_entry_num","type":"String","value":"2","chainId":"_all_","layerId":"pfus_layer_718"},{"id":"web_login","type":"String","value":"0","layerId":"webgw_layer_759"},{"id":"ge_rec_2th","type":"String","value":"11","chainId":"_gene_","layerId":"geli_layer_965","key":3023},{"id":"li_vip_verti_search","type":"String","value":"0","chainId":"_all_","layerId":"lili_layer_2"},{"id":"gue_messrec","type":"String","value":"0","layerId":"gueqa_layer_769"},{"id":"ge_sug_rep","type":"String","value":"1","chainId":"_gene_","layerId":"gese_layer_1034","key":3158},{"id":"ge_v_rank_v3","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_1047","key":2966},{"id":"ge_prf_rec","type":"String","value":"0","chainId":"_gene_","layerId":"getop_layer_991","key":3040},{"id":"hot_card","type":"Int","value":"0","chainId":"_gene_","layerId":"hot_card","key":108},{"id":"gue_visit_n_artcard","type":"String","value":"1","layerId":"gueqa_layer_579"},{"id":"gue_q_share","type":"String","value":"0","layerId":"gueqa_layer_647"},{"id":"se_ffzx_jushen1","type":"String","value":"0","chainId":"_all_","layerId":"sese_layer_4"},{"id":"ge_emoji","type":"String","value":"0","chainId":"_gene_","layerId":"getp_layer_827","key":3209},{"id":"gue_profile_video","type":"String","value":"1","layerId":"guevd_layer_5"},{"id":"gue_andplayd","type":"String","value":"1","layerId":"guevd_layer_686"},{"id":"correct_pos","type":"Int","value":"2","chainId":"_gene_","layerId":"correct_pos","key":104},{"id":"zanswer","type":"Int","value":"1","layerId":"zanswer"},{"id":"correct_gpu","type":"Int","value":"5","chainId":"_gene_","layerId":"correct_gpu","key":66},{"id":"web_sem_ab","type":"String","value":"1","layerId":"webgw_layer_3"},{"id":"ge_usercard1","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_742","key":2740},{"id":"gue_playh_an","type":"String","value":"1","layerId":"guevd_layer_622"},{"id":"meta_ebook","type":"Int","value":"0","layerId":"meta_ebook"},{"id":"se_zp_five","type":"Int","value":"0","chainId":"_gene_","layerId":"se_zp_five","key":344},{"id":"web_answer_list_ad","type":"String","value":"1","layerId":"webqa_layer_4"},{"id":"ge_meta_ss","type":"String","value":"1","chainId":"_gene_","layerId":"gese_layer_834","key":3079},{"id":"ge_upload","type":"String","value":"0","chainId":"_gene_","layerId":"geus_layer_839","key":2892},{"id":"gue_bullet_guide","type":"String","value":"发个弹幕聊聊…","layerId":"guevd_layer_0"},{"id":"gue_iosplay","type":"String","value":"0","layerId":"guevd_layer_896"},{"id":"recnew_2th","type":"Int","value":"21","chainId":"_gene_","layerId":"recnew_2th","key":67},{"id":"web_audit_01","type":"String","value":"case1","layerId":"webre_layer_1"},{"id":"tp_dingyue_video","type":"String","value":"0","chainId":"_all_","layerId":"tptp_layer_4"},{"id":"top_test_4_liguangyi","type":"String","value":"1","chainId":"_all_","layerId":"iosus_layer_1"},{"id":"pfd_newbie","type":"Int","value":"0","chainId":"_gene_","layerId":"pfd_newbie","key":63},{"id":"hw_aa_30","type":"Int","value":"0","chainId":"_gene_","layerId":"hw_aa_30","key":361},{"id":"gue_art_ui","type":"String","value":"2","layerId":"gueqa_layer_647"},{"id":"ge_search_ui","type":"String","value":"1","chainId":"_gene_","layerId":"gese_layer_838","key":2898},{"id":"gue_zvideo_link","type":"String","value":"1","layerId":"guevd_layer_2"},{"id":"gue_video_guide","type":"String","value":"1","layerId":"guevd_layer_625"},{"id":"se_prerank","type":"Int","value":"0","chainId":"_gene_","layerId":"6XUc","key":354},{"id":"web_scl_rec","type":"String","value":"0","layerId":"webgw_layer_759"},{"id":"zr_slotpaidexp","type":"String","value":"1","chainId":"_all_","layerId":"zrrec_layer_5"},{"id":"pf_adjust","type":"String","value":"1","chainId":"_all_","layerId":"pfus_layer_9"},{"id":"captcha_v2","type":"Int","value":"1","layerId":"captcha_v2"},{"id":"pfd_newbie2","type":"Int","value":"0","chainId":"_gene_","layerId":"pfd_newbie2","key":71},{"id":"li_paid_answer_exp","type":"String","value":"0","chainId":"_all_","layerId":"lili_layer_3"},{"id":"gue_card_test","type":"String","value":"1","layerId":"gueqa_layer_2"},{"id":"Test_Punk","type":"Int","value":"0","layerId":"Test_Punk"},{"id":"ge_newbie3","type":"Int","value":"0","chainId":"_gene_","layerId":"ge_newbie3","key":180},{"id":"li_panswer_topic","type":"String","value":"1","chainId":"_all_","layerId":"lili_layer_602"},{"id":"li_video_section","type":"String","value":"1","chainId":"_all_","layerId":"lili_layer_7"},{"id":"li_sp_mqbk","type":"String","value":"0","chainId":"_all_","layerId":"lili_layer_748"},{"id":"ge_sxzx","type":"String","value":"0","chainId":"_gene_","layerId":"gere_layer_990","key":3060},{"id":"ge_sug_v2","type":"String","value":"1","chainId":"_gene_","layerId":"gese_layer_1000","key":3189},{"id":"ge_relation2","type":"String","value":"1","chainId":"_gene_","layerId":"gese_layer_815","key":2796},{"id":"webpImg","type":"Int","value":"1","layerId":"JnVt"},{"id":"hw_aa_50","type":"Int","value":"1","chainId":"_gene_","layerId":"hw_aa_50","key":362},{"id":"gue_art2qa","type":"String","value":"0","layerId":"gueqa_layer_579"},{"id":"gue_fo_recom","type":"String","value":"0","layerId":"gueqa_layer_780"},{"id":"qap_question_visitor","type":"String","value":" 0","chainId":"_all_","layerId":"qapqa_layer_2"},{"id":"ge_yuzhi_v1","type":"String","value":"1","chainId":"_gene_","layerId":"gese_layer_1029","key":3127},{"id":"ge_guess","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_938","key":2912},{"id":"ge_entity","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_946","key":3036},{"id":"se_doc_cnt","type":"Int","value":"0","chainId":"_gene_","layerId":"uD1x","key":302},{"id":"gue_vid_tab","type":"String","value":"7","layerId":"guevd_layer_900"},{"id":"ge_video","type":"String","value":"0","chainId":"_gene_","layerId":"geli_layer_856","key":2831},{"id":"gue_goods_card","type":"String","value":"0","layerId":"gueqa_layer_1"},{"id":"gue_repost","type":"String","value":"1","layerId":"gueqa_layer_671"},{"id":"web_column_auto_invite","type":"String","value":"1","layerId":"webqa_layer_1"},{"id":"ge_infinity6","type":"String","value":"0","chainId":"_gene_","layerId":"gese_layer_815","key":2817},{"id":"tp_zrec","type":"String","value":"1","chainId":"_all_","layerId":"tptp_layer_619"},{"id":"ge_rec_sup","type":"Int","value":"0","chainId":"_gene_","layerId":"ge_rec_sup","key":197},{"id":"gue_recmess","type":"String","value":"0","layerId":"gueqa_layer_795"},{"id":"web_collection_guest","type":"String","value":"1","layerId":"webqa_layer_4"},{"id":"tp_topic_style","type":"String","value":"0","chainId":"_all_","layerId":"tptp_layer_4"},{"id":"ge_newyanzhi","type":"String","value":"0","chainId":"_gene_","layerId":"geus_layer_1019","key":2788},{"id":"gue_bullet_second","type":"String","value":"1","layerId":"guevd_layer_1"},{"id":"gue_sharp","type":"String","value":"1","layerId":"guevd_layer_686"},{"id":"ge_hard_s_ma","type":"String","value":"0","chainId":"_gene_","layerId":"geli_layer_856","key":3031}],"chains":[{"chainId":"_all_"}],"encodedParams":"ClJnALcAmwu1CxsAQAEIAE8BNAzPC1YMlgvgC2wAiQxoAEIAtApYAQcMTAtDAD8AaQFSC2IBRwC0APQLdQzsCmoBNwxgC9wLLgEPCwELxQDkCtcLEikAAAADAAECAAALAQAAAAACBQAAAQAVAAABAAAAAAEBAQEAAAAAAAAAAA=="},"triggers":{}},"userAgent":{"Edge":false,"IE":false,"Wechat":false,"Weibo":false,"QQ":false,"MQQBrowser":false,"Qzone":false,"Mobile":false,"Android":false,"iOS":false,"isAppleDevice":false,"Zhihu":false,"ZhihuHybrid":false,"isBot":false,"Tablet":false,"UC":false,"Sogou":false,"Qihoo":false,"Baidu":false,"BaiduApp":false,"Safari":false,"GoogleBot":false,"AndroidDaily":false,"iOSDaily":false,"WxMiniProgram":false,"BaiduMiniProgram":false,"QQMiniProgram":false,"JDMiniProgram":false,"isWebView":false,"isMiniProgram":false,"origin":"Mozilla\u002F5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko\u002F20100101 Firefox\u002F73.0"},"appViewConfig":{},"ctx":{"path":"\u002Fp\u002F114314389","query":{},"href":"http:\u002F\u002Fzhuanlan.zhihu.com\u002Fp\u002F114314389","host":"zhuanlan.zhihu.com"},"trafficSource":"production","edition":{"beijing":false,"baidu":false,"sogou":false,"baiduBeijing":false,"sogouBeijing":false,"sogouInput":false,"baiduSearch":false,"googleSearch":false,"miniProgram":false,"xiaomi":false},"theme":"light","enableShortcut":true,"referer":"","xUDId":"AHDYFEOJYhKPToQxOOJvuf1dL5wszuk9fro=","mode":"ssr","conf":{},"xTrafficFreeOrigin":"","ipInfo":{"cityName":"里昂","countryName":"法国","regionName":"法国","countryCode":"FR"},"logged":true,"vars":{"passThroughHeaders":{}}},"me":{"columnContributions":[]},"label":{"recognizerLists":{}},"ecommerce":{},"comments":{"pagination":{},"collapsed":{},"reverse":{},"reviewing":{},"conversation":{},"parent":{}},"commentsV2":{"stickers":[],"commentWithPicPermission":{},"notificationsComments":{},"pagination":{},"collapsed":{},"reverse":{},"reviewing":{},"conversation":{},"conversationMore":{},"parent":{}},"pushNotifications":{"default":{"isFetching":false,"isDrained":false,"ids":[]},"follow":{"isFetching":false,"isDrained":false,"ids":[]},"vote_thank":{"isFetching":false,"isDrained":false,"ids":[]},"currentTab":"default","notificationsCount":{"default":0,"follow":0,"vote_thank":0}},"messages":{"data":{},"currentTab":"common","messageCount":0},"register":{"registerValidateSucceeded":null,"registerValidateErrors":{},"registerConfirmError":null,"sendDigitsError":null,"registerConfirmSucceeded":null},"login":{"loginUnregisteredError":false,"loginBindWechatError":false,"loginConfirmError":null,"sendDigitsError":null,"needSMSIdentify":false,"validateDigitsError":false,"loginConfirmSucceeded":null,"qrcodeLoginToken":"","qrcodeLoginScanStatus":0,"qrcodeLoginError":null,"qrcodeLoginReturnNewToken":false},"active":{"sendDigitsError":null,"activeConfirmSucceeded":null,"activeConfirmError":null},"switches":{},"captcha":{"captchaNeeded":false,"captchaValidated":false,"captchaBase64String":null,"captchaValidationMessage":null,"loginCaptchaExpires":false},"sms":{"supportedCountries":[]},"chat":{"chats":{},"inbox":{"recents":{"isFetching":false,"isDrained":false,"isPrevDrained":false,"result":[],"next":null,"key":null},"strangers":{"isFetching":false,"isDrained":false,"isPrevDrained":false,"result":[],"next":null,"key":null},"friends":{"isFetching":false,"isDrained":false,"isPrevDrained":false,"result":[],"next":null,"key":null},"search":{"isFetching":false,"isDrained":false,"isPrevDrained":false,"result":[],"next":null,"key":null},"config":{"newCount":0,"strangerMessageSwitch":false,"strangerMessageUnread":false,"friendCount":0}},"global":{"isChatMqttExisted":false}},"emoticons":{"emoticonGroupList":[],"emoticonGroupDetail":{}},"creator":{"currentCreatorUrlToken":null,"homeData":{"recommendQuestions":[]},"tools":{"question":{"invitationCount":{"questionFolloweeCount":0,"questionTotalCount":0},"goodatTopics":[]},"customPromotion":{"itemLists":{}},"recommend":{"recommendTimes":{}}},"explore":{"academy":{"tabs":[],"article":{}}},"rights":[],"rightsStatus":{},"levelUpperLimit":10,"account":{"growthLevel":{}},"mcn":{},"applyStatus":{},"videoSupport":{},"mcnManage":{},"tasks":{},"recentlyCreated":[]},"answers":{"voters":{},"copyrightApplicants":{},"favlists":{},"newAnswer":{},"concernedUpvoters":{},"simpleConcernedUpvoters":{},"paidContent":{},"settings":{}},"recommendation":{"homeRecommendations":[]},"shareTexts":{},"articles":{"voters":{}},"previewPost":{},"favlists":{"relations":{}},"columns":{"voters":{}},"reward":{"answer":{},"article":{},"question":{}},"video":{"data":{},"shareVideoDetail":{},"last":{}},"topstory":{"recommend":{"isFetching":false,"isDrained":false,"afterId":0,"items":[],"next":null},"follow":{"isFetching":false,"isDrained":false,"afterId":0,"items":[],"next":null},"room":{"meta":{},"isFetching":false,"afterId":0,"items":[],"next":null},"followWonderful":{"isFetching":false,"isDrained":false,"afterId":0,"items":[],"next":null},"sidebar":null,"announcement":{},"hotListCategories":[],"hotList":[],"guestFeeds":{"isFetching":false,"isDrained":false,"afterId":0,"items":[],"next":null},"followExtra":{"isNewUser":null,"isFetched":false,"followCount":0,"followers":[]},"hotDaily":{"data":[],"paging":{}},"hotHighlight":{"isFetching":false,"isDrained":false,"data":[],"paging":{}}},"readStatus":{},"column":{},"requestColumn":{"categories":[],"error":null},"articleContribution":{"contributeRequests":[],"deleteContributeIdList":[],"handledContributeIdList":[],"recommendedColumns":[],"pinnedColumns":[],"sentContributeRequestsIdList":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"c_201634018"]},"columnContribution":{"contributeRequests":[],"autoInviteEnabled":false,"recommendedContributors":[],"contributionInvitation":null},"draftHistory":{"history":{},"drafts":{}},"upload":{},"articleDraft":{"titleImage":"","titleImageSize":{},"isTitleImageFullScreen":false,"canTitleImageFullScreen":false,"title":"","titleImageUploading":false,"error":"","content":"","draftLoading":false,"updating":false,"globalLoading":false,"pendingVideo":{"resource":null,"error":null},"deleteFail":{"fail":false},"recommendTopics":[],"selectedColumn":0,"articleDisclaimers":[]},"articleDrafts":{"isDrained":false,"isLoading":false,"items":[]},"columnAutocomplete":{"users":[],"friends":[]},"columnCollection":{},"userProfit":{"permission":{"permissionStatus":{"zhiZixuan":0,"recommend":-1,"task":0,"plugin":0},"visible":false}},"mcn":{"bindInfo":{},"memberCategoryList":[],"producerList":[],"categoryList":[],"lists":{},"banners":{},"protocolStatus":{"isAgreedNew":true,"isAgreedOld":true},"probationCountdownDays":0},"zvideos":{"campaigns":{},"tagoreCategory":[],"recommendations":{},"insertable":{},"recruit":{"form":{"platform":"","nickname":"","followerCount":"","domain":"","contact":""},"submited":false,"ranking":[]},"club":{}},"republish":{}},"fetchHost":"www.zhihu.com","subAppName":"column"}</script><script src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/vendor.js"></script><script src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/column_004.js"></script><script src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/hm.js" async=""></script><script src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/zap.js"></script><script src="weight%20normalization%20%E5%8E%9F%E7%90%86%E5%92%8C%E5%AE%9E%E7%8E%B0%20-%20%E7%9F%A5%E4%B9%8E_files/push.js"></script><div><div style="display: none;">想来知乎工作？请发送邮件到 [email protected]</div></div><div><div><div class="css-8pdeid"></div></div></div><div><div><div style="left: -1179px; top: -999px;" class="Editable-languageSuggestions"><div><div class="Popover"><label class="Editable-languageSuggestionsInput Input-wrapper"><input autocomplete="off" role="combobox" aria-expanded="false" aria-autocomplete="list" aria-activedescendant="AutoComplete13-0" id="Popover12-toggle" aria-haspopup="true" aria-owns="Popover12-content" class="Input" placeholder="选择语言"><svg class="Zi Zi--Select" fill="#afbdcf" viewBox="0 0 24 24" width="24" height="24"><path d="M12 16.183l2.716-2.966a.757.757 0 0 1 1.064.001.738.738 0 0 1 0 1.052l-3.247 3.512a.758.758 0 0 1-1.064 0L8.22 14.27a.738.738 0 0 1 0-1.052.758.758 0 0 1 1.063 0L12 16.183zm0-9.365L9.284 9.782a.758.758 0 0 1-1.064 0 .738.738 0 0 1 0-1.052l3.248-3.512a.758.758 0 0 1 1.065 0L15.78 8.73a.738.738 0 0 1 0 1.052.757.757 0 0 1-1.063.001L12 6.818z" fill-rule="evenodd"></path></svg></label></div></div></div></div></div></body></html>