diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..62e7561 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,27 @@ +name: documentation + +on: + push: + branches: + - master + tags: '*' + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: '1' + - name: build and deploy documentation + run: | + julia --project=docs/ -e ' + using Pkg + Pkg.instantiate() + include("docs/make.jl") + ' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.gitignore b/.gitignore index 9a477bf..8400fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ notebooks +!docs/src/notebooks +docs/build +docs/**/*.md diff --git a/.travis.yml b/.travis.yml index dcc95bc..5dc6db5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ dist: bionic os: - linux julia: - - 1.3 + - 1 - nightly notifications: email: false diff --git a/README.md b/README.md index 799e8cb..f79b3dc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ +!!! note + This is a clone of [the "Differentiation for Hackers" handbook](https://github.com/MikeInnes/diff-zoo), written by Mike J. Innes. + All the purpose of this clone is to render the notebooks with [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl) and host them on GitHub Pages. + The notebooks rendered by Documenter.jl are available [here](https://aviatesk.github.io/diff-zoo/dev/). + Except that, all the contents should be identical, and all the credit goes to him. + # Differentiation for Hackers [![Build Status](https://travis-ci.org/MikeInnes/diff-zoo.svg?branch=master)](https://travis-ci.org/MikeInnes/diff-zoo) +[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://aviatesk.github.io/diff-zoo/dev/) The goal of this handbook is to demystify *algorithmic differentiation*, the tool that underlies modern machine learning. It begins with a calculus-101 style diff --git a/docs/Manifest.toml b/docs/Manifest.toml new file mode 100644 index 0000000..239a77a --- /dev/null +++ b/docs/Manifest.toml @@ -0,0 +1,93 @@ +# This file is machine-generated - editing it directly is not advised + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.8.5" + +[[Documenter]] +deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] +git-tree-sha1 = "47f13b6305ab195edb73c86815962d84e31b0f48" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.27.3" + +[[IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.2" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.1" + +[[LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Literate]] +deps = ["Base64", "IOCapture", "JSON", "REPL"] +git-tree-sha1 = "501a1a74a0c825037860d36d87d703e987d39dbc" +uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +version = "2.8.1" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[Parsers]] +deps = ["Dates"] +git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "1.1.0" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 0000000..cf645b5 --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,3 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..f9b87cb --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,97 @@ +using Documenter, Literate + +const PROJECT_DIR = normpath(@__DIR__, "..") +const README_FILENAME = normpath(PROJECT_DIR, "README.md") +const SRC_DIR = normpath(PROJECT_DIR, "src") +const DOC_SRC_DIR = normpath(@__DIR__, "src") +const INDEX_FILENAME = normpath(DOC_SRC_DIR, "index.md") +const NOTEBOOKS_DIR = normpath(DOC_SRC_DIR, "notebooks") + +function generate_index() + isfile(INDEX_FILENAME) && rm(INDEX_FILENAME) + open(INDEX_FILENAME, write=true) do io + s = read(README_FILENAME, String) + + # fix links to notebooks + s = get(ENV, "CI", nothing) == "true" ? + replace(s, r"\((?:.+\/)(.+)\.ipynb\)" => s"(notebooks/\1/)") : + replace(s, r"\((?:.+\/)(.+)\.ipynb\)" => s"(notebooks/\1.html)") + + write(io, s) + end + + return relpath(INDEX_FILENAME, DOC_SRC_DIR) +end + +function postprocess(f) + return function (s) + # fix relative links + s = get(ENV, "CI", nothing) == "true" ? + replace(s, r"\(\./(.+).ipynb\)" => s"(../../\1/)") : + replace(s, r"\(\./(.+).ipynb\)" => s"(../\1.html)") + + # fix equation syntax + eqopen = false + s = replace(s, r"\$\$" => function (_) + eqopen = !eqopen + eqopen ? "```math" : "```" + end) + s = replace(s, r"{align}" => "{aligned}") + + # fix footnote syntax + footnotes = BitSet() + s = replace(s, r"\$\^(\d+)\$" => function (_s) + i = parse(Int, match(r"\$\^(\d+)\$", _s)[1]) + if i in footnotes + return "[^$i]:" + else + push!(footnotes, i) + return "[^$i]" + end + end) + + return """ + ```@setup $(first(splitext(basename(f)))) + using Pkg + Pkg.activate("$PROJECT_DIR") + Pkg.instantiate() + for f in ["utils.jl"] + cp(normpath("$SRC_DIR", f), normpath(@__DIR__, f), force = true) + end + ``` + """ * s + end +end + +function generate_notebooks() + isdir(NOTEBOOKS_DIR) && rm(NOTEBOOKS_DIR; recursive = true) + ret = [] + + for (n, f) in ["Intro" => "intro.jl", + "Back & Forth" => "backandforth.jl", + "Forward" => "forward.jl", + "Tracing" => "tracing.jl", + "Reverse" => "reverse.jl"] + out = Literate.markdown(normpath(SRC_DIR, f), NOTEBOOKS_DIR; + postprocess = postprocess(f), + credit = false, + documenter = true) + push!(ret, n => relpath(out, DOC_SRC_DIR)) + end + + return ret +end + +let + makedocs(; sitename="diff-zoo", + pages = [ + "README" => generate_index(), + "Notebooks" => Any[generate_notebooks()...] + ], + format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), + ) +end + +deploydocs(; repo = "github.com/aviatesk/diff-zoo.git", + push_preview = true, + ) diff --git a/docs/src/.gitkeep b/docs/src/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/forward.jl b/src/forward.jl index 165e66c..a1ee527 100644 --- a/src/forward.jl +++ b/src/forward.jl @@ -224,6 +224,7 @@ D(x -> x*D(y -> x*y, 1), 4) # == 8 # We can see how our definition of $\epsilon$ works out by applying it to # $f(x+\epsilon)$; let's say that $f(x) = sin(x^2)$. # +# $$ # \begin{align} # f(x + \epsilon) &= \sin((x + \epsilon)^2) \\ # &= \sin(x^2 + 2x\epsilon + \epsilon^2) \\ @@ -231,6 +232,7 @@ D(x -> x*D(y -> x*y, 1), 4) # == 8 # &= \sin(x^2)\cos(2x\epsilon) + \cos(x^2)\sin(2x\epsilon) \\ # &= \sin(x^2) + 2x\cos(x^2)\epsilon \\ # \end{align} +# $$ # # A few things have happened here. Firstly, we directly expand $(x+\epsilon)^2$ # and remove the $\epsilon^2$ term. We expand $sin(a+b)$ and then apply a *small @@ -239,10 +241,12 @@ D(x -> x*D(y -> x*y, 1), 4) # == 8 # our original definition of $\epsilon$ if we look at the Taylor expansion of # both functions). Finally we can plug this into our derivative rule. # +# $$ # \begin{align} # \frac{d}{dx} f(x) &= \frac{f(x+\epsilon)-f(x)}{\epsilon} \\ # &= 2x\cos(x^2) # \end{align} +# $$ # # This is, in my opinion, a rather nice way to derive functions by hand. # diff --git a/src/tracing.jl b/src/tracing.jl index e475934..b5ec529 100644 --- a/src/tracing.jl +++ b/src/tracing.jl @@ -290,9 +290,9 @@ gradient(x -> gradient(mysin, x)[1], 0.5) # ### Footnotes # $^1$ Systems like TensorFlow can also just provide ways to inject control flow -# into the graph. This brings us closer to a [source-to-source -# approach](./reverse.ipynb) where Python is used to build an expression in -# TensorFlows internal graph language. +# into the graph. This brings us closer to a [source-to-source +# approach](./reverse.ipynb) where Python is used to build an expression in +# TensorFlows internal graph language. # Fun fact: PyTorch and Flux's tapes are actually closer to the `Expr` format # that we originally used, in which "tracked" tensors just have pointers to