Take testing seriously

src/extra_rules.jl

@@ -246,6 +246,7 @@ end
 @ChainRules.non_differentiable Base.:(|)(a::Integer, b::Integer)
 @ChainRules.non_differentiable Base.throw(err)
 @ChainRules.non_differentiable Core.Compiler.return_type(args...)
+
 ChainRulesCore.canonicalize(::NoTangent) = NoTangent()
 
 # Disable thunking at higher order (TODO: These should go into ChainRulesCore)
@@ -262,3 +263,12 @@ Base.real(z::NoTangent) = z  # TODO should be in CRC, https://github.com/JuliaDi
 
 # Avoid https://github.com/JuliaDiff/ChainRulesCore.jl/pull/495
 ChainRulesCore._backing_error(P::Type{<:Base.Pairs}, G::Type{<:NamedTuple}, E::Type{<:AbstractDict}) = nothing
+
+# For gradient(pow_simd, 2, 3)[1] in zygote_features.jl
+ChainRulesCore.@non_differentiable Base.SimdLoop.simd_inner_length(::Any, ::Any)
+
+# This allows fill!(similar([1,2,3], ZeroTangent), false)
+function Base.convert(::Type{ZeroTangent}, x::Number)
+    iszero(x) || throw(InexactError(:convert, ZeroTangent, x))
+    ZeroTangent()
+end

src/runtime.jl

@@ -28,3 +28,9 @@ accum(x::Tangent{T}, y::Tangent) where T = _tangent(T, accum(backing(x), backing
 _tangent(::Type{T}, z) where T = Tangent{T,typeof(z)}(z)
 _tangent(::Type, ::NamedTuple{()}) = NoTangent()
 _tangent(::Type, ::NamedTuple{<:Any, <:Tuple{Vararg{AbstractZero}}}) = NoTangent()
+
+function accum(x::Tangent{T}, y::Tuple) where {T<:Tuple}
+  # @warn "gradient is both a Tangent and a Tuple" x y
+  _tangent(T, accum(backing(x), y))
+end
+accum(x::Tuple, y::Tangent{<:Tuple}) = accum(y, x)

test/Project.toml

@@ -2,11 +2,15 @@
 ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2"
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
+DiffTests = "de460e47-3fe3-5279-bb4a-814414816d5d"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
@@ -16,6 +20,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 ChainRules = "1.44.5"
 ChainRulesCore = "1.15.3"
 Combinatorics = "1"
+DiffTests = "0.1.1"
 StaticArrays = "1"
 StatsBase = "0.33"
 StructArrays = "0.6.12"

test/chainrules.jl

@@ -0,0 +1,97 @@
+
+# This file has integration tests for some rules defined in ChainRules.jl,
+# especially those which aim to support higher derivatives, as properly
+# testing those is difficult. Organised according to the files in CR.jl.
+
+using Diffractor, ForwardDiff, ChainRulesCore
+using Test, LinearAlgebra
+
+using Test: Threw, eval_test
+
+
+#####
+##### Base/array.jl
+#####
+
+
+
+
+
+#####
+##### Base/arraymath.jl
+#####
+
+
+
+
+#####
+##### Base/base.jl
+#####
+
+
+
+
+
+#####
+##### Base/indexing.jl
+#####
+
+@testset "getindex, first" begin
+    @test_broken gradient(x -> sum(abs2, gradient(first, x)[1]), [1,2,3])[1] == [0, 0, 0]  # MethodError: no method matching +(::Tuple{ZeroTangent, ZeroTangent}, ::Tuple{ZeroTangent, ZeroTangent})
+    @test_broken gradient(x -> sum(abs2, gradient(sqrt∘first, x)[1]), [1,2,3])[1] ≈ [-0.25, 0, 0]  # error() in perform_optic_transform(ff::Type{Diffractor.∂⃖recurse{2}}, args::Any)
+    @test gradient(x -> sum(abs2, gradient(x -> x[1]^2, x)[1]), [1,2,3])[1] == [8, 0, 0]
+    @test_broken gradient(x -> sum(abs2, gradient(x -> sum(x[1:2])^2, x)[1]), [1,2,3])[1] == [48, 0, 0]  # MethodError: no method matching +(::Tuple{ZeroTangent, ZeroTangent}, ::Tuple{ZeroTangent, ZeroTangent})
+end
+
+@testset "eachcol etc" begin
+    @test gradient(m -> sum(prod, eachcol(m)), [1 2 3; 4 5 6])[1] == [4 5 6; 1 2 3]
+    @test gradient(m -> sum(first, eachcol(m)), [1 2 3; 4 5 6])[1] == [1 1 1; 0 0 0]
+    @test gradient(m -> sum(first(eachcol(m))), [1 2 3; 4 5 6])[1] == [1 0 0; 1 0 0]
+    @test_skip gradient(x -> sum(sin, gradient(m -> sum(first(eachcol(m))), x)[1]), [1 2 3; 4 5 6])[1]  # MethodError: no method matching one(::Base.OneTo{Int64}), unzip_broadcast, split_bc_forwards
+end
+
+#####
+##### Base/mapreduce.jl
+#####
+
+@testset "sum" begin
+    @test gradient(x -> sum(abs2, gradient(sum, x)[1]), [1,2,3])[1] == [0,0,0]
+    @test gradient(x -> sum(abs2, gradient(x -> sum(abs2, x), x)[1]), [1,2,3])[1] == [8,16,24]
+
+    @test gradient(x -> sum(abs2, gradient(sum, x .^ 2)[1]), [1,2,3])[1] == [0,0,0]
+    @test gradient(x -> sum(abs2, gradient(sum, x .^ 3)[1]), [1,2,3])[1] == [0,0,0]
+end
+
+@testset "foldl" begin
+
+    @test gradient(x -> foldl(*, x), [1,2,3,4])[1] == [24.0, 12.0, 8.0, 6.0]
+    @test gradient(x -> foldl(*, x; init=5), [1,2,3,4])[1] == [120.0, 60.0, 40.0, 30.0]
+    @test gradient(x -> foldr(*, x), [1,2,3,4])[1] == [24, 12, 8, 6]
+
+    @test gradient(x -> foldl(*, x), (1,2,3,4))[1] == Tangent{NTuple{4,Int}}(24.0, 12.0, 8.0, 6.0)
+    @test_broken gradient(x -> foldl(*, x; init=5), (1,2,3,4))[1] == Tangent{NTuple{4,Int}}(120.0, 60.0, 40.0, 30.0)  # does not return a Tangent
+    @test gradient(x -> foldl(*, x; init=5), (1,2,3,4)) |> only |> Tuple == (120.0, 60.0, 40.0, 30.0)
+    @test_broken gradient(x -> foldr(*, x), (1,2,3,4))[1] == Tangent{NTuple{4,Int}}(24, 12, 8, 6)
+    @test gradient(x -> foldr(*, x), (1,2,3,4)) |> only |> Tuple == (24, 12, 8, 6)
+
+end
+
+
+#####
+##### LinearAlgebra/dense.jl
+#####
+
+
+@testset "dot" begin
+
+    @test gradient(x -> dot(x, [1,2,3])^2, [4,5,6])[1] == [64,128,192]
+    @test_broken gradient(x -> sum(gradient(x -> dot(x, [1,2,3])^2, x)[1]), [4,5,6])[1] == [12,24,36]  # MethodError: no method matching +(::Tuple{Tangent{ChainRules.var
+
+end
+
+
+#####
+##### LinearAlgebra/norm.jl
+#####
+
+