• 欢迎光临~

# 2 前向自动微分

[begin{gathered} {left[u, u^{prime}right]+left[v, v^{prime}right] equivleft[u+v^{prime}, u^{prime}+v^{prime}right]} \ cleft[u, u^{prime}right] equivleft[c u, c u^{prime}right] \ {left[u, u^{prime}right] cdotleft[v, v^{prime}right] equivleft[u v, u v^{prime}+u^{prime} vright]} \ {left[u, u^{prime}right] /left[v, v^{prime}right] equivleft[u / v,left(v u^{prime}-u v^{prime}right) / v^2right]} \ exp (left[u, u^{prime}right]) equivleft[e^u, u^{prime} e^uright] \ ln (left[u, u^{prime}right]) equivleft[ln u_{,} u^{prime} / uright] \ cos (left[u, u^{prime}right]) equivleft[cos u,-u^{prime} sin u^{prime}right] \ vdots quadvdots end{gathered} ]

[begin{aligned} & x \ & y_1= x^2\ & y_2=y_1 - x\ & y_3 = exp(y_2)\ & y_4 = y_3/x end{aligned} ]

[begin{aligned} & frac{dy_1}{dx} = 2x\ &frac{dy_2}{dx} = frac{dy_1}{dx} - frac{dx}{dx} = 2x-1\ & frac{dy_3}{dx} = exp(y_2)cdot frac{dy_2}{dx} \ & frac{dy_4}{dx} = frac{frac{dy_3}{dx}x - y_3}{x^2} end{aligned} ]

[(f(g(x)))' = f'(g(x))cdot g'(x) ]

[y_k = g(y_i) ]

[y'_k = g'(y_i)cdot y_i' ]

[y_k = g(y_i,cdots, y_j) ]

[begin{aligned} frac{d}{dx} y_k(x) &= frac{d}{dx} g(y_i(x),cdots, y_j(x))\ &= sum_{h=i}^jfrac{partial g}{partial y_h} frac{d y_h}{dx} end{aligned} ]

[begin{aligned} & y_1 = x\ & y_2 = x \ & y_3 = y_2 \ & y_4 = y_1cdot y_2cdot y_3 \ end{aligned} ]

[begin{aligned} frac{dy_1}{dx} &=1 \ frac{dy_2}{dx} &= 1\ frac{dy_3}{dx} &= 1cdot frac{dy_2}{dx} = 1 \ frac{dy_4}{dx} &= y_2 y_3cdot frac{dy_1}{dx} + y_1 y_3frac{dy_2}{dx} + y_1 y_2 frac{dy_3}{dx}\ &= y_2 y_3 + y_1 y_3 + y_1y_2 \ &= 3x^2 end{aligned} ]

[begin{aligned} y_1 = exp(x_2)\ y_2 = x_1 cdot y_1\ y_3 = y_2 - x_1 end{aligned} ]

[begin{aligned} & frac{d y_1}{ d x_2} = exp(x_2)\ & frac{d y_2}{d x_1} = y_1=exp(x_2) quad frac{d y_2}{dx_2} = x_1 cdot frac{dy_1}{dx_2} = x_1cdot exp(x_2) \ & frac{d y_3}{d x_1} = frac{dy_2}{dx_1} - frac{dx_1}{dx_1} =exp(x_2) -1 quad frac{dy_3}{dx_2} = frac{dy_2}{dx_2} = x_1cdot exp(x_2) \ end{aligned} ]

``````import math

class Var:
def __init__(self, val, deriv=1.0):
self.val = val
self.deriv = deriv

if isinstance(other, Var):
val = self.val + other.val
deriv = self.deriv + other.deriv
else:
val = self.val + other
deriv = self.deriv
return Var(val, deriv)

return self + other

def __sub__(self, other):
if isinstance(other, Var):
val = self.val - other.val
deriv = self.deriv - other.deriv
else:
val = self.val - other
deriv = self.deriv
return Var(val, deriv)

def __rsub__(self, other):
val = other - self.val
deriv = - self.deriv
return Var(val, deriv)

def __mul__(self, other):
if isinstance(other, Var):
val = self.val * other.val
deriv = self.val * other.deriv + self.deriv * other.val
else:
val = self.val * other
deriv = self.deriv * other
return Var(val, deriv)

def __rmul__(self, other):
return self * other

def __truediv__(self, other):
if isinstance(other, Var):
val = self.val / other.val
deriv = (self.deriv * other.val - self.val * other.deriv)/other.val**2
else:
val = self.val / other
deriv = self.deriv / other
return Var(val, deriv)

def __rtruediv__(self, other):
val = other / self.val
deriv = other * 1/self.val**2
return Var(val, deriv)

def __repr__(self):
return "value: {}t gradient: {}".format(self.val, self.deriv)

def exp(f: Var):
return Var(math.exp(f.val), math.exp(f.val) * f.deriv)
``````

``````fx = lambda x: exp(x*x - x)/x
df = fx(Var(2.0))
print(df)
``````

``````value: 3.694528049465325         deriv: 9.236320123663312
``````

# 3 反向自动微分

[begin{aligned} y_1 = exp(x_2)\ y_2 = x_1 cdot y_1\ y_3 = y_2 - x_1 end{aligned} ]

[begin{aligned} & frac{partial f}{partial y_3} = 1\ & frac{partial f}{partial y_2} = frac{partial f}{partial y_3}frac{partial y_3}{partial y_2} = 1 cdot 1 = 1\ & frac{partial f}{partial y_1} = frac{partial f}{partial y_2} frac{partial y_2}{partial y_1} = 1 cdot x_1 = x_1\ & frac{partial f}{partial x_2} = frac{partial f}{partial y_1} frac{partial y_1}{partial x_2} = x_1 cdot exp(x_2)\ & frac{partial f}{partial x_1} = frac{partial f}{partial y_2}frac{partial y_2}{x_1} + frac{partial f}{partial y_3}frac{partial y_3}{partial x_1} = 1cdot y_1 + 1cdot (-1) = exp(x_2) - 1 end{aligned} ]

# 参考

• [1] Solomon J. Numerical algorithms: methods for computer vision, machine learning, and graphics[M]. CRC press, 2015.

• [2] S&DS 631: Computation and Optimization Automatic Differentiation