1. Test Modules
  2. Network Diagram
  3. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Model Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    3. Composite Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    4. Results
  4. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Network Diagram

This is a network apply the following layout:

LayerTests.java:203 executed in 0.07 seconds (0.000 gc):

    return Graphviz.fromGraph((Graph) TestUtil.toGraph(((DAGNetwork) layer).addRef())).height(400).width(600)
        .render(Format.PNG).toImage();
Logging
executing command [/bin/sh, -c, dot -Tsvg /tmp/GraphvizJava/DotEngine1023719743384061589/dotfile.dot -ooutfile.svg]

Returns

Result

Using Seed 5054916210341890048

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.01 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 1.348, 1.836, 0.588, 1.052, 0.572, 0.124 ], [ 0.648, 0.816, 0.548, -1.26, 0.04, -0.648 ], [ 1.64, -1.26, -1.82, 1.532, 1.872, 0.424 ], [ -0.464, 0.188, 1.132, -0.428, 1.428, -1.072 ], [ -1.852, 0.928, -0.756, 1.152, 0.776, 1.404 ], [ -0.632, 1.684, 1.824, 0.312, 1.416, -0.788 ], [ -1.592, -1.344, -1.964, -1.004, 1.268, 0.528 ], [ -1.136, 0.728, -1.668, -0.368, -1.596, 0.956 ], ... ],
    	[ [ 0.072, 0.428, 0.78, -0.08, 1.008, -1.324 ], [ 0.04, -0.832, 1.256, 0.636, 1.684, -1.12 ], [ 0.84, 0.924, 0.856, 1.8, -0.456, -1.772 ], [ 1.528, 0.94, -0.308, -0.52, 0.88, -1.04 ], [ -0.384, -0.204, -0.396, -1.1, 0.932, -0.72 ], [ 0.42, 1.584, -1.748, 1.252, -0.58, -1.496 ], [ -1.096, 1.676, 1.584, -1.932, 1.036, -0.58 ], [ 0.348, -0.696, 1.996, 1.764, -1.288, -0.632 ], ... ],
    	[ [ -0.168, 0.48, 0.224, -0.972, -0.572, -0.488 ], [ -0.228, 1.34, 1.772, 1.784, -1.796, -1.356 ], [ 0.536, -0.992, 0.012, 0.228, 1.748, 1.896 ], [ 0.66, 0.536, 0.356, -0.992, 0.316, -0.748 ], [ 0.976, 0.836, 0.8, 0.808, 1.992, -1.896 ], [ -1.6, -0.42, -1.808, -0.42, 1.868, 1.692 ], [ 0.924, 1.492, -1.012, -0.444, -1.724, 0.02 ], [ 1.368, -1.536, 1.96, 1.144, 1.888, -0.108 ], ... ],
    	[ [ 1.36, -0.892, 0.552, -0.592, -1.6, -0.432 ], [ -0.34, 1.624, -1.304, 1.584, 0.788, 1.0 ], [ -0.676, -1.972, -0.888, -0.364, 0.02, -0.032 ], [ 1.404, -0.548, -0.28, 0.924, 0.4, -0.544 ], [ -1.124, 1.3, 1.916, -1.452, -1.696, -1.5 ], [ 0.792, 1.156, -1.9, -1.852, -1.82, -0.16 ], [ 1.28, 1.66, 0.696, 1.36, 1.848, -1.812 ], [ 1.816, 0.552, -0.728, -1.156, -1.604, 0.528 ], ... ],
    	[ [ 1.404, -0.924, -0.44, 1.324, 0.548, 1.204 ], [ 0.176, 1.096, 1.448, -1.62, -1.24, 1.524 ], [ 0.192, -1.62, -0.696, 1.42, -1.08, 0.28 ], [ -0.136, 0.212, -0.18, 1.728, 0.58, -0.216 ], [ -0.668, 0.808, 1.212, 1.448, 0.768, 0.456 ], [ 1.92, -1.492, -0.996, -1.44, -1.076, 0.468 ], [ -0.36, -1.608, -0.756, -0.284, -0.256, -0.512 ], [ -0.384, -0.12, -1.096, 0.312, -1.788, -1.516 ], ... ],
    	[ [ -1.256, 0.284, 0.18, -0.404, 1.86, -0.476 ], [ 1.276, -1.16, 0.392, -1.064, 1.096, 1.688 ], [ 0.688, -0.048, -1.956, -1.428, -1.712, 0.644 ], [ 1.364, 0.8, -1.984, 0.708, 0.348, -0.636 ], [ -0.556, -1.576, 1.852, -1.032, -0.804, 0.7 ], [ -1.184, -1.788, 1.408, 1.196, 1.488, -0.532 ], [ -0.408, 1.852, 0.948, 1.456, -1.176, 1.72 ], [ 1.768, -1.984, -1.1, -0.744, -1.132, 0.044 ], ... ],
    	[ [ -1.516, -0.884, -0.512, -1.796, -1.124, -1.844 ], [ 1.504, 0.512, 1.504, 1.832, 1.168, -0.416 ], [ 1.344, -1.252, 0.64, -0.304, -0.956, 0.8 ], [ -1.368, -0.544, -1.732, -1.884, -1.924, 0.676 ], [ 1.936, -0.296, 1.584, 1.048, 1.068, 1.192 ], [ 0.56, 1.648, 1.804, 1.472, -1.988, 0.22 ], [ -0.36, 0.876, 0.508, -0.632, 1.912, -1.584 ], [ -1.416, -0.756, 1.404, 0.008, -0.332, -0.116 ], ... ],
    	[ [ 1.084, -0.644, -1.084, -0.032, -0.944, -0.708 ], [ -0.612, 0.852, 1.764, 1.16, 0.548, -1.64 ], [ -0.872, 0.988, -1.804, -1.264, 0.328, 1.008 ], [ -0.424, -1.82, -1.748, 0.816, -0.428, -0.868 ], [ -1.752, -0.464, 1.152, 1.208, 1.676, 0.984 ], [ -1.352, 1.296, -0.452, 0.404, 0.892, 1.308 ], [ -1.044, 1.976, -1.956, -1.656, 1.056, 0.404 ], [ 1.844, -1.86, 1.696, -1.844, 0.216, -1.756 ], ... ],
    	...
    ]
    [
    	[ [ -0.08, 1.548, -0.856, -0.54, 0.896, 1.816 ], [ -1.664, -1.86, -0.436, 0.404, -1.48, -0.512 ], [ 0.936, -1.392, -1.068, 0.348, 1.8, 1.192 ], [ 0.708, -1.112, 0.68, 1.092, 0.476, -0.352 ], [ 0.82, 0.892, 1.956, -0.716, -1.556, 0.924 ], [ 0.024, 0.236, -1.66, -0.26, -1.248, 0.82 ], [ 1.448, 1.644, -1.76, -0.048, -2.0, 0.276 ], [ -0.552, 0.68, -1.436, -0.5, -1.22, 1.028 ], ... ],
    	[ [ -1.428, 1.204, -1.524, -1.664, 0.96, 1.304 ], [ 0.452, 1.464, 1.14, 1.104, -0.036, 1.936 ], [ -1.528, 1.444, 0.484, 1.192, -1.192, -0.324 ], [ -1.596, 0.06, 1.732, 0.92, 1.944, 0.664 ], [ -1.06, 0.024, 0.504, -1.292, 0.488, -1.2 ], [ 0.364, -1.732, 0.768, 1.808, 0.552, -1.964 ], [ -1.508, -0.328, -0.676, -0.208, 1.356, -1.596 ], [ -0.34, 1.064, 1.068, -0.752, 1.432, -0.54 ], ... ],
    	[ [ -1.256, 0.372, 0.56, 0.608, 1.74, -1.808 ], [ -0.02, -0.172, -1.108, -1.904, 1.624, -1.504 ], [ -0.884, -1.572, 1.032, -1.396, 0.948, -1.556 ], [ -1.88, 1.276, 0.248, -0.684, -1.264, 0.848 ], [ 0.836, -1.352, -0.592, -1.664, 0.96, 0.456 ], [ -0.968, -0.488, -1.424, 0.464, 1.02, 1.024 ], [ -0.452, 0.676, -1.956, -0.352, -0.34, -1.548 ], [ 1.92, -1.244, 0.728, 1.28, 0.012, -0.384 ], ... ],
    	[ [ -0.692, 1.792, 0.756, -1.22, -0.632, 0.512 ], [ -1.704, 0.88, 0.132, 0.68, 0.456, -1.64 ], [ -0.232, 1.744, 0.772, 0.436, 1.316, -0.584 ], [ -1.92, -1.032, 1.58, 1.128, 0.732, -1.48 ], [ 1.044, -1.164, -1.508, 0.9, 0.984, -1.62 ], [ -0.208, 1.044, -0.348, -1.296, -0.548, 1.68 ], [ -0.8, 0.584, -0.292, -0.748, 1.42, -0.956 ], [ -0.404, 1.288, -1.632, -0.976, 1.036, 1.036 ], ... ],
    	[ [ 0.788, -1.312, 1.016, 0.512, -1.12, 1.756 ], [ 1.092, -1.452, -1.012, 0.228, -0.616, 0.328 ], [ 0.452, 1.26, -0.304, 0.128, -1.796, -0.432 ], [ -1.628, -0.984, -0.348, -0.908, -1.564, 1.74 ], [ 1.5, 0.868, -0.052, -1.152, -1.556, 0.28 ], [ 0.068, -1.908, 0.944, 0.152, 1.16, -1.976 ], [ -1.808, 0.656, -0.82, 1.468, 1.924, -1.052 ], [ -0.556, -0.164, -1.412, 0.152, -1.64, 0.32 ], ... ],
    	[ [ -0.336, -0.896, -1.852, 1.34, -1.312, -1.32 ], [ 1.756, 1.868, 0.448, -1.28, 1.216, -0.624 ], [ -0.98, 1.32, 0.212, -0.516, -1.708, 0.992 ], [ -1.844, 0.732, -0.036, 0.112, 1.332, -1.244 ], [ 0.968, -0.208, 0.028, -1.412, -0.996, -1.532 ], [ 1.092, -1.356, 1.44, 0.612, 1.496, -0.148 ], [ -0.82, -0.752, 1.124, 1.724, 1.16, 0.78 ], [ 0.624, 1.412, -0.388, -0.048, 1.64, -0.236 ], ... ],
    	[ [ -0.416, -0.532, 1.72, -1.072, 0.408, -1.148 ], [ -0.86, 0.088, -0.412, -1.048, 1.832, 0.232 ], [ -0.852, -0.436, -0.12, 0.704, -0.78, 0.212 ], [ -1.992, -1.592, -0.668, -0.008, 1.048, -1.032 ], [ 0.488, -0.468, 1.656, -0.92, 1.252, -0.764 ], [ 0.776, 0.72, 1.58, -0.868, -1.452, -0.464 ], [ 1.26, 0.968, -1.712, -1.924, 0.772, 0.252 ], [ -1.524, -1.504, -0.104, 1.652, -0.92, -1.98 ], ... ],
    	[ [ -0.328, 1.596, 0.392, 1.964, -1.344, 0.716 ], [ 0.54, -0.848, -0.184, -0.472, -0.772, 0.28 ], [ 0.68, 1.864, 1.24, -0.4, 1.456, -1.032 ], [ 1.348, -0.76, 1.364, -1.18, 1.852, -0.188 ], [ 0.6, -1.46, -0.144, 1.368, -1.056, 0.264 ], [ 1.996, 0.404, 1.42, -0.244, 1.6, 1.296 ], [ 0.676, -1.864, 1.884, 0.596, 0.22, 1.052 ], [ 1.96, -1.204, -1.796, -1.604, 1.392, 0.868 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 30.51 seconds (1.414 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11730899082640
Reset training subject: 11731457256991
Constructing line search parameters: GD
th(0)=183.7372888499562;dx=-0.03780873268760233
New Minimum: 183.7372888499562 > 183.65584357706663
WOLFE (weak): th(2.154434690031884)=183.65584357706663; dx=-0.03779836115137249 evalInputDelta=0.08144527288956738
New Minimum: 183.65584357706663 > 183.57442064897447
WOLFE (weak): th(4.308869380063768)=183.57442064897447; dx=-0.037787989615142625 evalInputDelta=0.16286820098173393
New Minimum: 183.57442064897447 > 183.2489523845803
WOLFE (weak): th(12.926608140191302)=183.2489523845803; dx=-0.037746503470223236 evalInputDelta=0.4883364653759088
New Minimum: 183.2489523845803 > 181.78876946470012
WOLFE (weak): th(51.70643256076521)=181.78876946470012; dx=-0.03755981581808594 evalInputDelta=1.948519385256077
New Minimum: 181.78876946470012 > 174.12339795694473
WOLFE (weak): th(258.53216280382605)=174.12339795694473; dx=-0.03656414834002042 evalInputDelta=9.613890893011472
New Minimum: 174.12339795694473 > 130.88041973946946
END: th(1551.1929768229563)=130.88041973946946; dx=-0.03034122660211089 evalInputDelta=52.85686911048674
Fitness changed from 183.7372888499562 to 130.88041973946946
Iteration 1 complete. Error: 130.88041973946946 Total: 3.7944; Orientation: 0.1272; Line Search: 2.8334
th(0)=130.88041973946946;dx=-0.02452823486693995
New Minimum: 130.88041973946946 > 65.30008109558787
END: th(3341.943960201201)=65.30008109558787; dx=-0.014718586399286986 evalInputDelta=65.58033864388159
Fitness changed from 130.88041973946946 to 65.30008109558787
Iteration 2 complete. Error: 65.30008109558787 Total: 0.8957; Orientation: 0.1388; Line Search: 0.6213
th(0)=65.30008109558787;dx=-0.009385193377788145
New Minimum: 65.30008109558787 > 21.599592634943086
END: th(7200.000000000001)=21.599592634943086; dx=-0.0027538311946131823 evalInputDelta=43.700488460644785
Fitness changed from 65.30008109558787 to 21.599592634943086
Iteration 3 complete. Error: 21.599592634943086 Total: 0.7756; Orientation: 0.1391; Line Search: 0.4928
th(0)=21.599592634943086;dx=-0.0017113475508818753
New Minimum: 21.599592634943086 > 7.095966643260786
END: th(15511.929768229566)=7.095966643260786; dx=-1.586487949483639E-4 evalInputDelta=14.5036259916823
Fitness changed from 21.599592634943086 to 7.095966643260786
Iteration 4 complete. Error: 7.095966643260786 Total: 0.7708; Orientation: 0.1376; Line Search: 0.4991
th(0)=7.095966643260786;dx=-5.412935360168877E-4
Armijo: th(33419.43960201202)=22.744387161413332; dx=0.0014777796473645136 evalInputDelta=-15.648420518152546
New Minimum: 7.095966643260786 > 6.485640114329949
WOLF (strong): th(16709.71980100601)=6.485640114329949; dx=4.68243055673813E-4 evalInputDelta=0.6103265289308366
New Minimum: 6.485640114329949 > 5.018182958504482
END: th(5569.906600335336)=5.018182958504482; dx=-2.0478133878665408E-4 evalInputDelta=2.0777836847563034
Fitness changed from 7.095966643260786 to 5.018182958504482
Iteration 5 complete. Error: 5.018182958504482 Total: 1.5280; Orientation: 0.1383; Line Search: 1.2529
th(0)=5.018182958504482;dx=-1.8590254748659417E-4
New Minimum: 5.018182958504482 > 3.2023491043763523
END: th(12000.000000000004)=3.2023491043763523; dx=-1.1673642820142746E-4 evalInputDelta=1.81583385412813
Fitness changed from 5.018182958504482 to 3.2023491043763523
Iteration 6 complete. Error: 3.2023491043763523 Total: 0.9308; Orientation: 0.1431; Line Search: 0.6430
th(0)=3.2023491043763523;dx=-1.03020734638275E-4
New Minimum: 3.2023491043763523 > 1.7218300843427798
END: th(25853.216280382614)=1.7218300843427798; dx=-1.1511941217377688E-5 evalInputDelta=1.4805190200335725
Fitness changed from 3.2023491043763523 to 1.7218300843427798
Iteration 7 complete. Error: 1.7218300843427798 Total: 0.7975; Orientation: 0.1393; Line Search: 0.5132
th(0)=1.7218300843427798;dx=-1.8889806503619074E-4
Armijo: th(55699.06600335337)=42.97386802301191; dx=0.001670145091195859 evalInputDelta=-41.25203793866913
Armijo: th(27849.533001676686)=9.404478120920931; dx=7.406235130798343E-4 evalInputDelta=-7.6826480365781515
New Minimum: 1.7218300843427798 > 1.406408111478516
WOLF (strong): th(9283.177667225562)=1.406408111478516; dx=1.2094246100248423E-4 evalInputDelta=0.31542197286426377
New Minimum: 1.406408111478516 > 1.373321030027622
END: th(2320.7944168063905)=1.373321030027622; dx=-1.1143793352652199E-4 evalInputDelta=0.3485090543151579
Fitness changed from 1.7218300843427798 to 1.373321030027622
Iteration 8 complete. Error: 1.373321030027622 Total: 1.5034; Orientation: 0.1353; Line Search: 1.2223
th(0)=1.373321030027622;dx=-7.104515872855238E-5
New Minimum: 1.373321030027622 > 1.1329083816841496
END: th(5000.000000000003)=1.1329083816841496; dx=-2.5119900608836505E-5 evalInputDelta=0.2404126483434723
Fitness changed from 1.373321030027622 to 1.1329083816841496
Iteration 9 complete. Error: 1.1329083816841496 Total: 0.8801; Orientation: 0.1377; Line Search: 0.5958
th(0)=1.1329083816841496;dx=-2.490054621014355E-5
New Minimum: 1.1329083816841496 > 0.8872201255827825
END: th(10772.173450159426)=0.8872201255827825; dx=-2.0714808432652458E-5 evalInputDelta=0.24568825610136713
Fitness changed from 1.1329083816841496 to 0.8872201255827825
Iteration 10 complete. Error: 0.8872201255827825 Total: 0.8110; Orientation: 0.1382; Line Search: 0.5235
th(0)=0.8872201255827825;dx=-1.839283235870789E-5
New Minimum: 0.8872201255827825 > 0.5357375976474645
END: th(23207.94416806391)=0.5357375976474645; dx=-1.1897013686250092E-5 evalInputDelta=0.351482527935318
Fitness changed from 0.8872201255827825 to 0.5357375976474645
Iteration 11 complete. Error: 0.5357375976474645 Total: 0.7988; Orientation: 0.1421; Line Search: 0.5040
th(0)=0.5357375976474645;dx=-1.3410949540523133E-5
Armijo: th(50000.00000000004)=1.018702279882525; dx=3.2729536829925535E-5 evalInputDelta=-0.4829646822350604
New Minimum: 0.5357375976474645 > 0.4888418989496903
WOLF (strong): th(25000.00000000002)=0.4888418989496903; dx=9.659293644701203E-6 evalInputDelta=0.04689569869777421
New Minimum: 0.4888418989496903 > 0.45602168923369435
END: th(8333.33333333334)=0.45602168923369435; dx=-5.720868478781687E-6 evalInputDelta=0.07971590841377019
Fitness changed from 0.5357375976474645 to 0.45602168923369435
Iteration 12 complete. Error: 0.45602168923369435 Total: 1.4529; Orientation: 0.1385; Line Search: 1.1681
th(0)=0.45602168923369435;dx=-1.0149920577544669E-5
New Minimum: 0.45602168923369435 > 0.3648492271923349
END: th(17953.62241693238)=0.3648492271923349; dx=-6.521384315130587E-9 evalInputDelta=0.09117246204135943
Fitness changed from 0.45602168923369435 to 0.3648492271923349
Iteration 13 complete. Error: 0.3648492271923349 Total: 0.8151; Orientation: 0.1432; Line Search: 0.5191
th(0)=0.3648492271923349;dx=-2.349633012236575E-5
Armijo: th(38679.90694677319)=2.4129666070715885; dx=1.2939717330162344E-4 evalInputDelta=-2.048117379879254
Armijo: th(19339.953473386595)=0.6496696064812055; dx=5.2950421589628835E-5 evalInputDelta=-0.2848203792888706
New Minimum: 0.3648492271923349 > 0.295514173477346
WOLF (strong): th(6446.651157795532)=0.295514173477346; dx=1.9859204482991114E-6 evalInputDelta=0.06933505371498894
END: th(1611.662789448883)=0.33211466562503034; dx=-1.712576747969954E-5 evalInputDelta=0.03273456156730459
Fitness changed from 0.3648492271923349 to 0.295514173477346
Iteration 14 complete. Error: 0.295514173477346 Total: 1.6088; Orientation: 0.1593; Line Search: 1.2988
th(0)=0.295514173477346;dx=-7.300169137690793E-6
New Minimum: 0.295514173477346 > 0.2738657141930889
END: th(3472.2222222222254)=0.2738657141930889; dx=-5.169343410041289E-6 evalInputDelta=0.02164845928425707
Fitness changed from 0.295514173477346 to 0.2738657141930889
Iteration 15 complete. Error: 0.2738657141930889 Total: 0.9350; Orientation: 0.1430; Line Search: 0.6266
th(0)=0.2738657141930889;dx=-4.520039227327517E-6
New Minimum: 0.2738657141930889 > 0.2422160941998623
END: th(7480.676007055159)=0.2422160941998623; dx=-3.941661282994618E-6 evalInputDelta=0.03164961999322663
Fitness changed from 0.2738657141930889 to 0.2422160941998623
Iteration 16 complete. Error: 0.2422160941998623 Total: 0.8249; Orientation: 0.1426; Line Search: 0.5291
th(0)=0.2422160941998623;dx=-3.831327062722777E-6
New Minimum: 0.2422160941998623 > 0.18666038441981148
END: th(16116.627894488833)=0.18666038441981148; dx=-3.062883083935555E-6 evalInputDelta=0.05555570978005081
Fitness changed from 0.2422160941998623 to 0.18666038441981148
Iteration 17 complete. Error: 0.18666038441981148 Total: 0.7914; Orientation: 0.1412; Line Search: 0.5002
th(0)=0.18666038441981148;dx=-3.1425012245762833E-6
New Minimum: 0.18666038441981148 > 0.14124227169950673
WOLF (strong): th(34722.22222222226)=0.14124227169950673; dx=5.264179318867316E-7 evalInputDelta=0.04541811272030474
END: th(17361.11111111113)=0.14802719977639947; dx=-1.3080416463447755E-6 evalInputDelta=0.038633184643412005
Fitness changed from 0.18666038441981148 to 0.14124227169950673
Iteration 18 complete. Error: 0.14124227169950673 Total: 1.0159; Orientation: 0.1334; Line Search: 0.7362
th(0)=0.14124227169950673;dx=-1.7313012513710024E-5
Armijo: th(37403.380035275804)=1.8928610383744813; dx=1.1097400063954203E-4 evalInputDelta=-1.7516187666749745
Armijo: th(18701.690017637902)=0.41725566671680503; dx=4.683049406291601E-5 evalInputDelta=-0.27601339501729827
New Minimum: 0.14124227169950673 > 0.09995873930078633
WOLF (strong): th(6233.896672545968)=0.09995873930078633; dx=4.068156345165324E-6 evalInputDelta=0.0412835323987204
END: th(1558.474168136492)=0.11842563884315602; dx=-1.1967720298991187E-5 evalInputDelta=0.022816632856350716
Fitness changed from 0.14124227169950673 to 0.09995873930078633
Iteration 19 complete. Error: 0.09995873930078633 Total: 1.6280; Orientation: 0.1334; Line Search: 1.3469
th(0)=0.09995873930078633;dx=-3.104349113339031E-6
New Minimum: 0.09995873930078633 > 0.09181414078208902
END: th(3357.6308113518407)=0.09181414078208902; dx=-1.7470469908194027E-6 evalInputDelta=0.00814459851869731
Fitness changed from 0.09995873930078633 to 0.09181414078208902
Iteration 20 complete. Error: 0.09181414078208902 Total: 0.8067; Orientation: 0.1401; Line Search: 0.5140
th(0)=0.09181414078208902;dx=-1.3477019186531028E-6
New Minimum: 0.09181414078208902 > 0.08309831538614754
END: th(7233.796296296306)=0.08309831538614754; dx=-1.0620494868167967E-6 evalInputDelta=0.008715825395941484
Fitness changed from 0.09181414078208902 to 0.08309831538614754
Iteration 21 complete. Error: 0.08309831538614754 Total: 0.7825; Orientation: 0.1378; Line Search: 0.4997
th(0)=0.08309831538614754;dx=-1.1234292716021505E-6
New Minimum: 0.08309831538614754 > 0.06769476185328757
END: th(15584.74168136492)=0.06769476185328757; dx=-8.533187358772926E-7 evalInputDelta=0.015403553532859965
Fitness changed from 0.08309831538614754 to 0.06769476185328757
Iteration 22 complete. Error: 0.06769476185328757 Total: 0.7753; Orientation: 0.1336; Line Search: 0.4973
th(0)=0.06769476185328757;dx=-1.0901772637228392E-6
New Minimum: 0.06769476185328757 > 0.06381044431785149
WOLF (strong): th(33576.30811351841)=0.06381044431785149; dx=8.588047422232409E-7 evalInputDelta=0.0038843175354360826
New Minimum: 0.06381044431785149 > 0.05757265054315093
END: th(16788.154056759206)=0.05757265054315093; dx=-1.156862607497991E-7 evalInputDelta=0.010122111310136642
Fitness changed from 0.06769476185328757 to 0.05757265054315093
Iteration 23 complete. Error: 0.05757265054315093 Total: 1.0316; Orientation: 0.1381; Line Search: 0.7492
th(0)=0.05757265054315093;dx=-2.3084519828228236E-6
Armijo: th(36168.98148148153)=0.20916713100323972; dx=1.0691020374343881E-5 evalInputDelta=-0.15159448046008878
Armijo: th(18084.490740740766)=0.07459768140377113; dx=4.191284195760529E-6 evalInputDelta=-0.017025030860620197
New Minimum: 0.05757265054315093 > 0.05018716985904117
END: th(6028.163580246922)=0.05018716985904117; dx=-1.4187325662837302E-7 evalInputDelta=0.007385480684109762
Fitness changed from 0.05757265054315093 to 0.05018716985904117
Iteration 24 complete. Error: 0.05018716985904117 Total: 1.3273; Orientation: 0.1333; Line Search: 1.0484
th(0)=0.05018716985904117;dx=-7.51685859437342E-7
New Minimum: 0.05018716985904117 > 0.043422601323072976
END: th(12987.284734470768)=0.043422601323072976; dx=-2.9003589753834414E-7 evalInputDelta=0.006764568535968192
Fitness changed from 0.05018716985904117 to 0.043422601323072976
Iteration 25 complete. Error: 0.043422601323072976 Total: 0.8816; Orientation: 0.1384; Line Search: 0.5976
th(0)=0.043422601323072976;dx=-9.749016558386392E-7
Armijo: th(27980.256761265344)=0.0571825457190317; dx=1.9584483411576524E-6 evalInputDelta=-0.01375994439595872
New Minimum: 0.043422601323072976 > 0.040043087760225686
WOLF (strong): th(13990.128380632672)=0.040043087760225686; dx=4.917733426595065E-7 evalInputDelta=0.0033795135628472903
New Minimum: 0.040043087760225686 > 0.040016211077495735
END: th(4663.376126877557)=0.040016211077495735; dx=-4.860099896725908E-7 evalInputDelta=0.0034063902455772413
Fitness changed from 0.043422601323072976 to 0.040016211077495735
Iteration 26 complete. Error: 0.040016211077495735 Total: 1.2980; Orientation: 0.1425; Line Search: 1.0050
th(0)=0.040016211077495735;dx=-4.7306803404210363E-7
New Minimum: 0.040016211077495735 > 0.035421867629193936
WOLFE (weak): th(10046.939300411537)=0.035421867629193936; dx=-4.4150770110016155E-7 evalInputDelta=0.004594343448301799
New Minimum: 0.035421867629193936 > 0.03114460893026061
END: th(20093.878600823075)=0.03114460893026061; dx=-4.099473681582194E-7 evalInputDelta=0.008871602147235126
Fitness changed from 0.040016211077495735 to 0.03114460893026061
Iteration 27 complete. Error: 0.03114460893026061 Total: 1.0330; Orientation: 0.1327; Line Search: 0.7542
Final threshold in iteration 27: 0.03114460893026061 (> 0.0) after 30.495s (< 30.000s)

Returns

    0.03114460893026061

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.01 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.5667107661740532, -0.06983471938064231, 0.12194971121121534, -0.6772052878376513, 0.5169281283069668, 0.8213498086033865 ], [ -0.4355698746214757, -0.6271133511221979, -0.30019464140282415, 0.5805528716977578, -0.08186648309057719, 0.555471963545645 ], [ -0.22414120124446893, -1.5480609788774187, 0.6381076642408829, 1.540559777674958, 1.604595847311169, 1.3191047245188106 ], [ -0.04049507602488336, 1.306704398936477, -0.3575825674868835, 0.35362101008803287, -0.8988193760802495, -1.5161080940521334 ], [ -1.1761728154772355, 1.1116694822783424, -0.7949626450232292, 0.6283850920100915, 2.2908318455269616, -1.0795926933454238 ], [ 0.6518944944336805, -1.0306426841290834, 0.8435657225420001, -1.3679049258289462, 1.421618737795938, -0.4056123722904308 ], [ -2.1697181768319287, 0.4583305142137782, -1.5339712961521497, -0.4130862286279377, -0.5914292069333514, -1.5097283148866938 ], [ 1.7114559153691329, -0.1742594942772755, 0.44749832453601274, -0.18963797903642246, -1.5591612341672216, 0.46496087223409654 ], ... ],
    	[ [ -0.4780470979081274, 0.030190828451868303, 1.3122131233398207, 0.7870225957575177, 1.8351039539965166, -1.2606329603000286 ], [ -0.43932925803532086, -0.5113724929933054, 1.2129694462304181, 0.03290385127886659, 0.9519580829010582, -1.3838027086703089 ], [ 0.4955441309409206, -1.1090300757749312, 0.5978206811441285, 2.8358827505585587, 0.11529635140077413, -2.4557807249274246 ], [ 0.30006120086310134, -0.6829585125142049, -0.16127834605935792, -0.48508313474895737, 0.274442275253711, -0.43416463833308255 ], [ 0.09191023591170833, 1.165436771350036, -0.6925584899567604, -0.30388530504143796, -0.2563891827089596, 0.4516692017312163 ], [ -0.8456228468369013, 0.9139609312363471, -0.8937468023742059, -1.6783763649435566, 0.9346894860887032, -0.6811786078804194 ], [ -0.9079870811405474, 3.1472912845325536, -0.7143502185687356, -1.8286556042943494, 1.4952394409545664, 0.5667862378125392 ], [ 1.1755722920025617, -2.6084313387958598, 0.0578698134166241, -0.12358094065668632, -1.6591511508007808, -0.740542735152248 ], ... ],
    	[ [ 0.0937276474758803, -1.7744983316586465, -1.1705948557671981, -1.3357185113060261, -1.3568189600100038, -0.1254873607238754 ], [ 0.19315234121873823, -1.7254808599814249, 0.013858462331101716, 2.248280967694034, -0.19487302065788017, -1.37580365409832 ], [ 1.453001244231917, 1.057109056272291, 1.018790243655333, -0.5183252160960452, 1.9216136950458071, 0.9667332845501191 ], [ 2.0749107494227803, -0.4844437025462159, 0.9563445980480333, -0.7024872874449144, 0.3503514791320146, 1.0147619064044149 ], [ 1.0079174795023942, -0.945034699103642, 1.853683858829739, 0.26340871785560466, -1.0136021935297248, -1.6035649412126196 ], [ 0.7256284122207414, -0.9307150069705984, -0.13926419914508378, 0.4786017095411711, 1.9183353535345058, 0.57416156620581 ], [ 0.07811386738563685, 0.18505598634418385, -0.8271854780012744, -1.9811524427790888, -1.5376549616829813, 0.2249350193297177 ], [ 0.19031376875355882, -1.890198972796235, 3.215191070692069, 2.0054884615059136, 0.1292065896195635, -0.5879462345706798 ], ... ],
    	[ [ 2.2829455680129502, -0.20798101733739277, -0.9707934115170457, -0.2699366663611484, -0.11369509426148591, -0.6236410650061036 ], [ -1.0974871033962068, 0.35255447666037665, -0.6273279479056124, 0.970349696957516, -0.018550153117471282, 0.3151378991658457 ], [ 1.680163069157878, -1.0816050008538154, -0.4623694888196795, -0.4510882072422809, 0.25741547372493717, 0.7961609747851965 ], [ 0.43254927071461535, -0.35369576279347764, 0.6069224853247633, 0.821053080270524, 0.9836417966703042, -0.1306959849414255 ], [ -1.4632906281490399, 0.7142522227220001, 2.0669865127365896, -2.387277534455547, 0.16783509827589543, -0.49643434340442316 ], [ 0.495993925572572, 0.09524169219702518, -2.333149782733705, -0.6781361993523459, -0.6830978911212753, -1.339358004841787 ], [ 0.8596928450011198, 1.212600467179852, 0.7478775347327482, 2.1513388903389288, 0.9826726409616509, 0.7090735912947763 ], [ 2.3174941064217522, -0.9835356725292228, 0.03794746692463416, -1.2575624896744395, -0.05253702771792747, 0.17043450440790056 ], ... ],
    	[ [ 1.8453443857170293, -1.0902677842456672, -0.7948255253806932, 0.3415076481521999, -0.010969988408296036, 4.308747514838475 ], [ -0.08816078970436919, 0.7971333452251638, 0.41822473951038586, -0.9089975536392872, -1.1823077849125898, 0.9654447707619733 ], [ 0.21363408314513105, -1.0082839611049992, 0.07166433710527065, 0.25850811429986836, 1.195287910850201, 0.46291429968484843 ], [ -1.2994002183591324, 1.3049353496063079, 0.37896988559330097, 1.6883105407301278, -0.9975915730020437, 0.22777801353481575 ], [ 0.9190866063941971, 0.7061210460654143, 0.04767243138753913, 1.91184205148411, -0.36192823368493976, 1.4640928858295663 ], [ 0.02404917344082624, -1.4278421546731714, 1.2426583325762397, -0.4457624610024233, 0.6421958883171494, -1.074166823037596 ], [ -0.6252248795490702, -0.05822749177408806, 1.4478832182981853, -0.4443149035535245, -0.25876194446434736, 0.5712980922803605 ], [ -1.6330103458723662, 1.530649281295995, -0.19198870784170938, 0.2781925836629953, -0.11230416597582701, -2.5458941549142478 ], ... ],
    	[ [ -0.954879785290077, -1.1278659960223776, 0.32138525876353596, 0.22805135785950417, 1.061478218624605, -1.7903689790284292 ], [ 1.5791976464478423, -1.0149413746872964, 0.9616171540763429, -0.841990792501799, 0.48457775667520303, 1.0307008910069653 ], [ 1.5279580978074765, 0.14281621115670867, 0.7973712828036109, -0.5321707935528167, -0.4162495540864483, 0.2495500615602843 ], [ 0.11175572745805841, 0.7066575303806354, -0.3125555737110167, -0.770503752881049, 1.5246244559494508, -0.16665552715709409 ], [ -1.714881142523988, -1.1422084202635114, 0.5499434232133822, -2.0629427521962684, -0.3546003466207827, 0.9520530496199214 ], [ -1.4709556635149315, -0.633295330302031, 1.0444854874357947, 1.7289736123874246, 0.8059692113761872, -1.385089767822286 ], [ -0.08519544419536328, -0.7903660507617429, 0.3664824993226637, 0.11115498249880629, 0.16743634413611036, 0.8066388239338249 ], [ 0.6718196501259969, -1.3097578013242246, -0.6661186149181162, 0.8309066077272724, -0.38355505804596834, 0.010280858267219934 ], ... ],
    	[ [ -2.0294320185238606, -1.0454982891224718, -1.2859587639381318, -1.9302280204275564, -0.37251486398765754, -0.9735335539618367 ], [ 0.7796545341536699, -0.7194810293062767, -0.0019064074617914616, 1.6186085603804496, 1.7327417553985376, -0.24799632907025315 ], [ 0.9080182960410539, 1.857321824583501, 1.2731995229929736, 0.1441549471124496, 1.0416611790724941, -0.08272980048046522 ], [ -1.3112386604945856, 1.2682235582345704, -1.4367805253116688, -1.304414717194554, -1.6459200066928623, 1.232617702752513 ], [ 1.0059323510718106, -0.3471149642979634, 0.128169147335087, -0.27737408755616577, -0.9450349215588194, 0.9581330768057026 ], [ 0.07531262924570603, 1.005895685860696, 0.594460680949452, 0.8384260090646969, -1.9247023094382762, -0.853535612867914 ], [ -0.7127121284359651, 2.1829237604640688, 0.3451612373407974, -1.150900957310659, 2.0958486239537564, -1.4124035203496919 ], [ -1.081141088833277, -0.33410959288616016, 1.2351169848974313, -0.4794192198306655, -1.769280961170799, -1.8153992815045832 ], ... ],
    	[ [ 0.8262197623514491, -2.374050709658967, -0.7119990943277738, -2.2883923682581337, 0.9972842134233121, -1.3985904173892896 ], [ -0.7549304630474379, -0.11820684247125078, 0.5582210964005914, 0.3698386576669123, 0.11205197907003105, -2.4482917302058103 ], [ 0.9532647966525574, 1.2106716875907761, -1.941821085036027, -0.275455952416813, -1.5537929792460603, -0.2110802967814816 ], [ 0.1873262238350275, -1.1345226421234655, 0.1587601377605647, 1.1005172621410868, -1.606253277912737, 1.0522590643236573 ], [ -1.188150805728872, 0.38676476950368044, -0.6966161174403601, 1.4201062933727, -0.08073734621185222, 0.9633602219995636 ], [ 0.9217513210977146, 0.2356059717911649, -1.2025000616879864, 0.02914371169118153, -0.023996113011277315, 0.4933327851024707 ], [ -2.3431351027032115, 0.20025969281793132, -1.5634249679346874, -2.3808236114113615, 1.2812722687424845, -0.5293727282719377 ], [ 1.7086128055579988, -1.3534601928456527, 1.3856225921815017, -1.5635153228553198, 0.5371480060825501, -0.4847634905767208 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.04 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -2.1859462221117987, -6.1237686110238645, -5.482905779973428 ], [ 4.280440699942932, -2.666163955255577, 4.955675281768226 ], [ 12.252868098180137, -5.217956053628655, 4.331423938058147 ], [ 2.4108674211627843, 7.961052763232546, 1.0561548970333197 ], [ -3.9819859488641605, 2.22085282479346, 8.450304090205723 ], [ -5.400697062275863, 19.18176010436308, 0.25200539627180907 ], [ -2.7480070540109764, -10.638790877069443, 2.666522063317766 ], [ 3.2722826318726486, 16.95355553482492, -3.6077845385859546 ], ... ],
    	[ [ -2.316198574446187, 8.016542028145018, 12.307626053447283 ], [ 8.759787754341845, -3.602380617533968, 0.5799540995473624 ], [ 19.980527334140614, -4.454676050318937, 5.550036460267096 ], [ 6.896845232398829, 5.511959456208966, -7.566581878574384 ], [ 6.370397111012638, 0.3960875056719913, -9.541513131541778 ], [ -2.3184317121993936, 8.990729788148771, 0.3279884930319047 ], [ -29.068259483196176, 7.704776486328761, 7.794519117197563 ], [ -0.5553435473407893, 7.287582377064201, -2.5418340666588257 ], ... ],
    	[ [ 15.788665928244942, 2.4567342635570775, 4.941310718961244 ], [ 14.12901133945439, -4.155944571754267, -11.004773195793558 ], [ 5.070503545155217, -7.185603837124115, -1.3778421011218676 ], [ -17.485895188360907, 13.502688071261682, 4.784141303429966 ], [ 2.8196060748860696, 7.112684502456471, 0.028781900091874462 ], [ 12.948604444256244, 6.121409774706628, 12.828187268837588 ], [ -11.324549749474096, -14.999160920759499, 0.5867072639073397 ], [ 15.604345149030792, 15.401246222670832, 4.9250830368213245 ], ... ],
    	[ [ -1.9226267362200131, 6.125562932009387, -5.263143602051179 ], [ 3.8908618119048453, -12.082455627871106, -14.035874380489316 ], [ -1.526396676027804, 3.82003136344595, 13.857463032492292 ], [ 13.313832795123185, -6.3759367256398995, -13.468797593439097 ], [ 13.693848772340068, -9.742447869382323, -14.871641179299882 ], [ -14.961071525962872, 6.860996629148734, 16.267692458332156 ], [ 9.206281961821416, -6.826246970096285, 3.5669106895204057 ], [ 13.370245241234308, -19.010379337215504, -19.041562862902218 ], ... ],
    	[ [ -3.9302200778546803, -9.633730140002239, -10.793840391621705 ], [ -8.588770537811266, 2.7491890895196116, -1.9902213067261083 ], [ 9.722645441393942, 10.5087871196406, 8.112449223362463 ], [ -4.377269897738751, 2.0575658728142088, -2.3747627811479948 ], [ -17.253709904126303, 8.180537898406925, 1.6436226773396767 ], [ -1.5419483013293056, -7.099140432587604, -5.326076599704494 ], [ 17.87528513401789, -7.223996659056671, -0.5640853881864923 ], [ -0.39784531009967317, 19.661250525636902, 12.397447253309549 ], ... ],
    	[ [ -2.3906181204344907, 4.319018135664465, 17.741643859641474 ], [ 2.7309513679787556, -13.994684743074174, -6.9867733508455805 ], [ -0.4230495913290831, -3.0290987521251656, -8.651339134035581 ], [ -1.8087573868528022, 5.057072781478753, -4.131038214216965 ], [ -5.021174782487949, 1.6631231600444878, 2.3470019320861986 ], [ 12.151904435725987, 14.53898314117202, 12.708333341238095 ], [ 2.1309855948875533, -13.421252934160202, 0.7308110964369383 ], [ 5.0089109178522975, 0.22737997011762143, 1.2792457678055238 ], ... ],
    	[ [ -1.830345065203665, 3.352380113391596, 20.123245391667865 ], [ 8.600166025208901, 0.284666168723996, 14.693087925808324 ], [ 10.030838615143274, -1.6703563881956072, -8.024726533305333 ], [ -9.029902462861164, -2.333255857540226, -0.238599156439131 ], [ 0.5094099892589541, -17.319853727217065, -14.100200835145202 ], [ 4.7541762634822335, -0.9159032522833453, -12.589185842961372 ], [ -11.3795664577777, -2.024475224860187, 6.886264396374798 ], [ 4.285381579016803, 14.95352248375922, 12.312415059811087 ], ... ],
    	[ [ -9.007120707600402, -4.496112844186458, -8.177900131997035 ], [ 25.182842774998942, 1.5522708788134492, 3.9871033195215984 ], [ -8.813613583840153, 17.286801889184947, 7.101468781896648 ], [ -16.430429792719195, 2.1883519818794466, -9.43384582271418 ], [ 18.706070554346148, -12.123662852921841, 3.1661143231845372 ], [ -3.615579153255415, 8.122248907827688, -22.458683565334304 ], [ -22.807306416441413, -8.414907399678322, 10.390167600392212 ], [ -10.944750571313547, 17.404598357288087, 5.040915312653683 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 32.56 seconds (0.719 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11761485967819
Reset training subject: 11761631486864
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=183.7372888499562}, derivative=-0.03780873268760233}
New Minimum: 183.7372888499562 > 183.73728884995245
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=183.73728884995245}, derivative=-0.03780873268760185}, evalInputDelta = -3.751665644813329E-12
New Minimum: 183.73728884995245 > 183.73728884992974
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=183.73728884992974}, derivative=-0.03780873268759896}, evalInputDelta = -2.646061147970613E-11
New Minimum: 183.73728884992974 > 183.73728884977095
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=183.73728884977095}, derivative=-0.03780873268757874}, evalInputDelta = -1.8525270206737332E-10
New Minimum: 183.73728884977095 > 183.73728884865938
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=183.73728884865938}, derivative=-0.03780873268743721}, evalInputDelta = -1.296825757890474E-9
New Minimum: 183.73728884865938 > 183.7372888408783
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=183.7372888408783}, derivative=-0.03780873268644648}, evalInputDelta = -9.07789399207104E-9
New Minimum: 183.7372888408783 > 183.73728878641106
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=183.73728878641106}, derivative=-0.037808732679511375}, evalInputDelta = -6.354514425765956E-8
New Minimum: 183.73728878641106 > 183.73728840514025
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=183.73728840514025}, derivative=-0.037808732630965625}, evalInputDelta = -4.4481595296019805E-7
New Minimum: 183.73728840514025 > 183.73728573624447
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=183.73728573624447}, derivative=-0.03780873229114538}, evalInputDelta = -3.113711727564805E-6
New Minimum: 183.73728573624447 > 183.73726705397502
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=183.73726705397502}, derivative=-0.03780872991240368}, evalInputDelta = -2.1795981183458935E-5
New Minimum: 183.73726705397502 > 183.7371362781214
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=183.7371362781214}, derivative=-0.03780871326121178}, evalInputDelta = -1.5257183480343883E-4
New Minimum: 183.7371362781214 > 183.7362208487588
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=183.7362208487588}, derivative=-0.0378085967028685}, evalInputDelta = -0.00106800119741024
New Minimum: 183.7362208487588 > 183.72981292224017
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=183.72981292224017}, derivative=-0.03780778079446544}, evalInputDelta = -0.007475927716029673
New Minimum: 183.72981292224017 > 183.68496130857193
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=183.68496130857193}, derivative=-0.037802069435644084}, evalInputDelta = -0.052327541384272536
New Minimum: 183.68496130857193 > 183.37118973903267
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=183.37118973903267}, derivative=-0.037762089923894536}, evalInputDelta = -0.3660991109235283
New Minimum: 183.37118973903267 > 181.18408533304984
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=181.18408533304984}, derivative=-0.03748223334164774}, evalInputDelta = -2.5532035169063647
New Minimum: 181.18408533304984 > 166.32988694997422
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=166.32988694997422}, derivative=-0.03552323726592014}, evalInputDelta = -17.407401899981977
New Minimum: 166.32988694997422 > 84.6715887498538
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=84.6715887498538}, derivative=-0.02181026473582701}, evalInputDelta = -99.0657001001024
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=606.796934938045}, derivative=0.07418054297482492}, evalInputDelta = 423.05964608808875
F(1789.4654922092848) = LineSearchPoint{point=PointSample{avg=123.78759502807382}, derivative=-0.02919417302126177}, evalInputDelta = -59.94969382188238
F(12526.258445464993) = LineSearchPoint{point=PointSample{avg=87.81403442016092}, derivative=0.022493184976781575}, evalInputDelta = -95.92325442979528
87.81403442016092 <= 183.7372888499562
New Minimum: 84.6715887498538 > 35.26531240713298
F(7853.845706471293) = LineSearchPoint{point=PointSample{avg=35.26531240713298}, derivative=-8.483881999699072E-18}, evalInputDelta = -148.47197644282323
Left bracket at 7853.845706471293
Converged to left
Fitness changed from 183.7372888499562 to 35.26531240713298
Iteration 1 complete. Error: 35.26531240713298 Total: 6.9647; Orientation: 0.1395; Line Search: 6.3921
F(0.0) = LineSearchPoint{point=PointSample{avg=35.26531240713298}, derivative=-0.00460471814832281}
New Minimum: 35.26531240713298 > 15.666180260827275
F(7853.845706471293) = LineSearchPoint{point=PointSample{avg=15.666180260827275}, derivative=-3.8624624766900393E-4}, evalInputDelta = -19.599132146305706
F(54976.91994529905) = LineSearchPoint{point=PointSample{avg=593.8271633042066}, derivative=0.024924585156253835}, evalInputDelta = 558.5618508970736
F(4228.9938419460805) = LineSearchPoint{point=PointSample{avg=20.595035476748173}, derivative=-0.0023332332787399917}, evalInputDelta = -14.670276930384809
F(29602.956893622562) = LineSearchPoint{point=PointSample{avg=134.30137993076747}, derivative=0.011295675938756918}, evalInputDelta = 99.03606752363449
F(2277.1505302786586) = LineSearchPoint{point=PointSample{avg=26.172275681089516}, derivative=-0.0033816109108551393}, evalInputDelta = -9.093036726043465
F(15940.05371195061) = LineSearchPoint{point=PointSample{avg=30.103240506959374}, derivative=0.00395703251395089}, evalInputDelta = -5.162071900173608
30.103240506959374 <= 35.26531240713298
New Minimum: 15.666180260827275 > 15.527304722490106
F(8572.949331038686) = LineSearchPoint{point=PointSample{avg=15.527304722490106}, derivative=-3.049318610115481E-19}, evalInputDelta = -19.738007684642874
Left bracket at 8572.949331038686
Converged to left
Fitness changed from 35.26531240713298 to 15.527304722490106
Iteration 2 complete. Error: 15.527304722490106 Total: 2.4629; Orientation: 0.1331; Line Search: 2.1830
F(0.0) = LineSearchPoint{point=PointSample{avg=15.527304722490106}, derivative=-0.0016497163828108003}
New Minimum: 15.527304722490106 > 8.947692102391606
F(8572.949331038686) = LineSearchPoint{point=PointSample{avg=8.947692102391606}, derivative=1.1474577560640064E-4}, evalInputDelta = -6.5796126200985
8.947692102391606 <= 15.527304722490106
New Minimum: 8.947692102391606 > 8.91570599991833
F(8015.436824730907) = LineSearchPoint{point=PointSample{avg=8.91570599991833}, derivative=-4.743384504624082E-20}, evalInputDelta = -6.611598722571776
Left bracket at 8015.436824730907
Converged to left
Fitness changed from 15.527304722490106 to 8.91570599991833
Iteration 3 complete. Error: 8.91570599991833 Total: 1.3194; Orientation: 0.1495; Line Search: 1.0097
F(0.0) = LineSearchPoint{point=PointSample{avg=8.91570599991833}, derivative=-6.998573438339723E-4}
New Minimum: 8.91570599991833 > 5.855895021219827
F(8015.436824730907) = LineSearchPoint{point=PointSample{avg=5.855895021219827}, derivative=-6.362218837511013E-5}, evalInputDelta = -3.0598109786985033
F(56108.05777311635) = LineSearchPoint{point=PointSample{avg=94.59078572834932}, derivative=0.0037537887443780636}, evalInputDelta = 85.67507972843099
F(4316.004444085873) = LineSearchPoint{point=PointSample{avg=6.634424605593571}, derivative=-3.5726918320227725E-4}, evalInputDelta = -2.2812813943247594
F(30212.03110860111) = LineSearchPoint{point=PointSample{avg=23.997588738974628}, derivative=0.001698259780587893}, evalInputDelta = 15.081882739056297
F(2324.002392969316) = LineSearchPoint{point=PointSample{avg=7.503590855640917}, derivative=-5.153867958015211E-4}, evalInputDelta = -1.4121151442774131
F(16268.016750785213) = LineSearchPoint{point=PointSample{avg=8.033809886231776}, derivative=5.914364923931858E-4}, evalInputDelta = -0.8818961136865546
8.033809886231776 <= 8.91570599991833
New Minimum: 5.855895021219827 > 5.830397571522816
F(8816.963787201306) = LineSearchPoint{point=PointSample{avg=5.8303975715228

...skipping 3744 bytes...

, derivative=6.086468774756972E-4}, evalInputDelta = 14.563718848943289
F(4513.265566888543) = LineSearchPoint{point=PointSample{avg=1.9206777760956566}, derivative=-5.6755244374384716E-5}, evalInputDelta = -0.3812821770369448
F(31592.8589682198) = LineSearchPoint{point=PointSample{avg=4.888473562064949}, derivative=2.759458165506562E-4}, evalInputDelta = 2.5865136089323473
F(2430.219920632292) = LineSearchPoint{point=PointSample{avg=2.065556598348264}, derivative=-8.234763367631097E-5}, evalInputDelta = -0.23640335478433716
F(17011.539444426046) = LineSearchPoint{point=PointSample{avg=2.170917259946131}, derivative=9.679909143717262E-5}, evalInputDelta = -0.13104269318647033
2.170917259946131 <= 2.3019599531326014
New Minimum: 1.793052105476475 > 1.7895876583063879
F(9132.754716632506) = LineSearchPoint{point=PointSample{avg=1.7895876583063879}, derivative=1.0164395367051604E-20}, evalInputDelta = -0.5123722948262135
Right bracket at 9132.754716632506
Converged to right
Fitness changed from 2.3019599531326014 to 1.7895876583063879
Iteration 8 complete. Error: 1.7895876583063879 Total: 2.6395; Orientation: 0.1340; Line Search: 2.3612
F(0.0) = LineSearchPoint{point=PointSample{avg=1.7895876583063879}, derivative=-8.848977748940279E-5}
New Minimum: 1.7895876583063879 > 1.418728545794043
F(9132.754716632506) = LineSearchPoint{point=PointSample{avg=1.418728545794043}, derivative=7.2746076925092135E-6}, evalInputDelta = -0.3708591125123448
1.418728545794043 <= 1.7895876583063879
New Minimum: 1.418728545794043 > 1.4162051409739873
F(8438.997767332283) = LineSearchPoint{point=PointSample{avg=1.4162051409739873}, derivative=-1.0587911840678754E-20}, evalInputDelta = -0.3733825173324006
Left bracket at 8438.997767332283
Converged to left
Fitness changed from 1.7895876583063879 to 1.4162051409739873
Iteration 9 complete. Error: 1.4162051409739873 Total: 1.2696; Orientation: 0.1354; Line Search: 0.9890
F(0.0) = LineSearchPoint{point=PointSample{avg=1.4162051409739873}, derivative=-6.078959452499871E-5}
New Minimum: 1.4162051409739873 > 1.1388958828122502
F(8438.997767332283) = LineSearchPoint{point=PointSample{avg=1.1388958828122502}, derivative=-4.931304047865707E-6}, evalInputDelta = -0.2773092581617371
F(59072.984371325976) = LineSearchPoint{point=PointSample{avg=9.374188094935821}, derivative=3.3021843881493227E-4}, evalInputDelta = 7.957982953961833
F(4544.075720871229) = LineSearchPoint{point=PointSample{avg=1.2083099323715492}, derivative=-3.071205349885018E-5}, evalInputDelta = -0.20789520860243815
F(31808.530046098604) = LineSearchPoint{point=PointSample{avg=2.831105783085947}, derivative=1.4975319265804108E-4}, evalInputDelta = 1.4149006421119599
F(2446.8100035460466) = LineSearchPoint{point=PointSample{avg=1.2872783298192485}, derivative=-4.459399551091873E-5}, evalInputDelta = -0.1289268111547388
F(17127.670024822328) = LineSearchPoint{point=PointSample{avg=1.345896090193313}, derivative=5.2579598573561186E-5}, evalInputDelta = -0.07030905078067429
1.345896090193313 <= 1.4162051409739873
New Minimum: 1.1388958828122502 > 1.1370589350886142
F(9184.01275963699) = LineSearchPoint{point=PointSample{avg=1.1370589350886142}, derivative=-5.293955920339377E-21}, evalInputDelta = -0.2791462058853731
Left bracket at 9184.01275963699
Converged to left
Fitness changed from 1.4162051409739873 to 1.1370589350886142
Iteration 10 complete. Error: 1.1370589350886142 Total: 2.4816; Orientation: 0.1366; Line Search: 2.1242
F(0.0) = LineSearchPoint{point=PointSample{avg=1.1370589350886142}, derivative=-5.022085555379433E-5}
New Minimum: 1.1370589350886142 > 0.9257466655570381
F(9184.01275963699) = LineSearchPoint{point=PointSample{avg=0.9257466655570381}, derivative=4.2034391886349636E-6}, evalInputDelta = -0.2113122695315761
0.9257466655570381 <= 1.1370589350886142
New Minimum: 0.9257466655570381 > 0.9242558661528748
F(8474.68911427819) = LineSearchPoint{point=PointSample{avg=0.9242558661528748}, derivative=-4.446922973085077E-21}, evalInputDelta = -0.21280306893573941
Left bracket at 8474.68911427819
Converged to left
Fitness changed from 1.1370589350886142 to 0.9242558661528748
Iteration 11 complete. Error: 0.9242558661528748 Total: 1.3276; Orientation: 0.1348; Line Search: 1.0477
F(0.0) = LineSearchPoint{point=PointSample{avg=0.9242558661528748}, derivative=-3.579476244184049E-5}
New Minimum: 0.9242558661528748 > 0.7603152149966319
F(8474.68911427819) = LineSearchPoint{point=PointSample{avg=0.7603152149966319}, derivative=-2.8947160618688895E-6}, evalInputDelta = -0.1639406511562429
F(59322.82379994733) = LineSearchPoint{point=PointSample{avg=5.631842271286624}, derivative=1.945055622179607E-4}, evalInputDelta = 4.70758640513375
F(4563.294138457487) = LineSearchPoint{point=PointSample{avg=0.8013341488839663}, derivative=-1.807935285262501E-5}, evalInputDelta = -0.12292171726890855
F(31943.058969202408) = LineSearchPoint{point=PointSample{avg=1.7614569647861655}, derivative=8.821310468266784E-5}, evalInputDelta = 0.8372010986332907
F(2457.158382246339) = LineSearchPoint{point=PointSample{avg=0.8480219644310621}, derivative=-2.625569573995523E-5}, evalInputDelta = -0.07623390172181266
F(17200.108675724372) = LineSearchPoint{point=PointSample{avg=0.8828375059124165}, derivative=3.0978704471356305E-5}, evalInputDelta = -0.04141836024045831
0.8828375059124165 <= 0.9242558661528748
New Minimum: 0.7603152149966319 > 0.7592359969190476
F(9220.336047875846) = LineSearchPoint{point=PointSample{avg=0.7592359969190476}, derivative=-1.6940658945086007E-21}, evalInputDelta = -0.16501986923382717
Left bracket at 9220.336047875846
Converged to left
Fitness changed from 0.9242558661528748 to 0.7592359969190476
Iteration 12 complete. Error: 0.7592359969190476 Total: 2.4833; Orientation: 0.1343; Line Search: 2.2055
F(0.0) = LineSearchPoint{point=PointSample{avg=0.7592359969190476}, derivative=-3.052556220696293E-5}
New Minimum: 0.7592359969190476 > 0.6303969740679632
F(9220.336047875846) = LineSearchPoint{point=PointSample{avg=0.6303969740679632}, derivative=2.5788535008814457E-6}, evalInputDelta = -0.1288390228510844
0.6303969740679632 <= 0.7592359969190476
New Minimum: 0.6303969740679632 > 0.6294708180134794
F(8502.066433749) = LineSearchPoint{point=PointSample{avg=0.6294708180134794}, derivative=-7.72917564369549E-21}, evalInputDelta = -0.12976517890556827
Left bracket at 8502.066433749
Converged to left
Fitness changed from 0.7592359969190476 to 0.6294708180134794
Iteration 13 complete. Error: 0.6294708180134794 Total: 1.4135; Orientation: 0.1496; Line Search: 1.1046
F(0.0) = LineSearchPoint{point=PointSample{avg=0.6294708180134794}, derivative=-2.234841082567336E-5}
New Minimum: 0.6294708180134794 > 0.5267902452762327
F(8502.066433749) = LineSearchPoint{point=PointSample{avg=0.5267902452762327}, derivative=-1.8058517968004813E-6}, evalInputDelta = -0.10268057273724662
F(59514.465036243) = LineSearchPoint{point=PointSample{avg=3.578445042089191}, derivative=1.2144950237643678E-4}, evalInputDelta = 2.9489742240757115
F(4578.035772018692) = LineSearchPoint{point=PointSample{avg=0.5524786857511961}, derivative=-1.12870328870495E-5}, evalInputDelta = -0.0769921322622833
F(32046.250404130846) = LineSearchPoint{point=PointSample{avg=1.1539529538851028}, derivative=5.5081234744693624E-5}, evalInputDelta = 0.5244821358716234
F(2465.096184933142) = LineSearchPoint{point=PointSample{avg=0.581721048205341}, derivative=-1.6392284243337433E-5}, evalInputDelta = -0.04774976980813839
F(17255.673294531993) = LineSearchPoint{point=PointSample{avg=0.6035533525735092}, derivative=1.9344475250678095E-5}, evalInputDelta = -0.0259174654399702
0.6035533525735092 <= 0.6294708180134794
New Minimum: 0.5267902452762327 > 0.5261154000716588
F(9249.46464856358) = LineSearchPoint{point=PointSample{avg=0.5261154000716588}, derivative=4.0234064994579266E-21}, evalInputDelta = -0.10335541794182057
Right bracket at 9249.46464856358
Converged to right
Fitness changed from 0.6294708180134794 to 0.5261154000716588
Iteration 14 complete. Error: 0.5261154000716588 Total: 2.6641; Orientation: 0.1499; Line Search: 2.3545
Final threshold in iteration 14: 0.5261154000716588 (> 0.0) after 32.557s (< 30.000s)

Returns

    0.5261154000716588

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.7112081689300198, -0.19428505941872773, 0.23526333349172968, -1.0056452440527253, 0.7267063172529923, 0.8205191234775993 ], [ -0.464903465297063, -0.6174718511967607, -0.33628237860372834, 0.729623457589904, 0.12148376006323264, 0.7139901779576249 ], [ -0.16926293424154706, -1.7082737046838459, 0.6422010004939792, 1.159095013359698, 1.4986585353184636, 1.2515672186145437 ], [ -0.08160240981200015, 1.2453960570796716, -0.13200078235360957, 0.5192359579179139, -0.7875681648947324, -1.4196497861203494 ], [ -1.2243264619413696, 0.8757817283526319, -0.8462001703551665, 0.194675448540426, 2.0956914014659955, -1.1845498629654472 ], [ 0.6797792101789155, -1.0694634727987626, 0.7952536373596848, -0.9914381156764162, 1.3693867624790483, -0.5591639119006196 ], [ -2.1558453585410793, 0.4585733808973906, -1.6290887526916313, -0.6886009044067403, -0.7944618001148311, -1.57968193790756 ], [ 1.777058932954429, -0.15872266140096786, 0.363105854498003, 0.2562948428780223, -1.442074755055522, 0.3977701677730034 ], ... ],
    	[ [ -0.35123882421919367, -0.051749904246491944, 1.3524087190196417, 0.5779093583949323, 1.8010771526740277, -1.3912522174540396 ], [ -0.511782091371494, -0.5434794136222756, 1.1619502509969097, 0.21041948405469404, 0.8987167882834073, -1.175099215037168 ], [ 0.555757879617418, -1.100395999948842, 0.5363431582476255, 2.895608077421084, 0.24546191455664654, -2.393699554512384 ], [ 0.1280876021556402, -0.5072309711822853, -0.24019476277157725, -0.3044692885230279, 0.1391027856072566, -0.3896238807178357 ], [ 0.16180479890306945, 1.177652953178705, -0.7946368089273337, -0.0684869442560028, -0.15563499717535578, 0.5854562340363912 ], [ -1.0356826830016284, 0.9872838944463559, -0.5970307417146994, -2.0034192114027, 1.0717371747389735, -0.48720947080189697 ], [ -0.8958733095758941, 3.0772392825482853, -0.9195177389334537, -1.466395707356459, 1.25625703746977, 0.7459801324963381 ], [ 1.1874401497089708, -2.7017078885995183, 0.21941167831833575, -0.41743794308270005, -1.6419947348965829, -0.6393252546376007 ], ... ],
    	[ [ -0.05822566585818279, -1.8578179032780904, -1.1738976699677035, -1.434652162474244, -1.331100565478614, -0.14188597872613615 ], [ 0.1024476113802005, -1.8143024648196178, 0.031083532745698743, 2.0549825341501196, -0.10876504832911546, -1.463608778572876 ], [ 1.4374143867836775, 1.1579100706700847, 0.9892642527004126, -0.40903621195154205, 1.6591208118451106, 1.0862325885309698 ], [ 2.1199378076823496, -0.6226238798000625, 1.0000008795360587, -0.9336535858934387, 0.19762530597402161, 0.8825168196589277 ], [ 0.9784120307648302, -0.9888090856396502, 1.8663341376674556, 0.25418785705936536, -1.1176395721663666, -1.64607796660838 ], [ 0.7394109319353743, -0.7778664643359622, -0.35187603102208886, 0.7011932675268464, 1.7090367550168029, 0.48162761218372613 ], [ 0.16632913909185487, 0.2203256603750855, -0.653434882802448, -1.9955795264922942, -1.2861381305926616, 0.28409696294608183 ], [ 0.14687943007576562, -1.537015047535648, 3.3024207455222894, 2.0445519435679405, -0.21698279163285222, -0.44642266399890757 ], ... ],
    	[ [ 2.1237505446913914, -0.3166993061876486, -1.0044002440770357, -0.20227336217330183, -0.06508593237792532, -0.5887289547521115 ], [ -0.9808610366219304, 0.32538065985874154, -0.6630679228302718, 1.0952553238866527, -0.08422174441765766, 0.46877476183447275 ], [ 1.557519237521379, -0.9212233023299029, -0.41888251259902104, -0.6803042214104907, 0.3413319050673618, 0.793263742534541 ], [ 0.4118073416040024, -0.28391469535568176, 0.5920893988999284, 1.0685692358703698, 0.9788658358349002, 0.15636868089660017 ], [ -1.4828402218770187, 0.9860949932446745, 1.9893969962154021, -2.269432645382202, -0.05292651161064335, -0.2458675895347993 ], [ 0.5605097479450458, 0.008276995791208898, -2.128501811552001, -0.9483160267590912, -0.7221503242203674, -1.4238061335424568 ], [ 0.7562626218249818, 1.2873427973774387, 0.6373890814311057, 2.0912021638059546, 0.6544165608923462, 0.5093026381731021 ], [ 2.2991909421502696, -0.9480185499494079, -0.009062344247130401, -1.2906622816175821, 0.11219021852931639, -0.11483679895978327 ], ... ],
    	[ [ 1.7345432650409938, -0.8764706137654834, -0.6394700391141838, 0.22149962878176022, -0.2817238852942681, 4.197963069279127 ], [ -0.04385390085012308, 0.9180748175328807, 0.5727239325511394, -1.0168773895296497, -1.0579420202244503, 0.9281210320717606 ], [ 0.25651733543026284, -0.9664621338929146, 0.16815307444225067, 0.3403115118902055, 1.132747081454382, 0.2677836533092763 ], [ -1.2772623629301056, 1.3838299786684487, 0.4416932056463215, 1.5092018328979813, -1.1154668649140507, -0.03001152174376761 ], [ 1.0386532943082027, 0.6576009330273793, 0.1206676753987158, 1.793033921131058, -0.29848046101153186, 1.365022884229096 ], [ 0.022415076932903004, -1.305258257184896, 1.155230313591883, -0.1781613507533647, 0.6476623340825113, -1.0106601058131326 ], [ -0.697546729231142, 0.02289884984896106, 1.6312990267007095, -0.570435340824031, -0.1733223749358398, 0.6347191138097564 ], [ -1.5687546669687538, 1.3641518696677248, -0.060724984831544045, 0.2239184879380135, -0.23902152745136365, -2.396332673622386 ], ... ],
    	[ [ -0.9392630466313732, -0.9812103910637303, 0.15143462180428058, 0.27845909298240173, 1.1854305702286094, -1.829819206298261 ], [ 1.4654235952125858, -0.8246725767006561, 0.9799869791928589, -0.6811875897720483, 0.2674936014828742, 0.9105234144056104 ], [ 1.5483556954020978, 0.061227630574755346, 0.9111640158055623, -0.6437797771378615, -0.16340258714438677, 0.3212267610093075 ], [ 0.06187514868963383, 0.6939793905131498, -0.1819371610254737, -0.8617452127674217, 1.577732414938805, -0.14965195900184666 ], [ -1.684885659927074, -1.1096339300162712, 0.5865197422972968, -1.887704166811486, -0.32446779349775184, 0.9523521651601031 ], [ -1.4590927187671199, -0.5528499599340487, 1.1899796348796325, 1.36717119707997, 0.7282866446271018, -1.5816353978779423 ], [ -0.07329333662675475, -0.884883016697664, 0.23284218356834746, 0.11933397602218992, 0.15504028548108728, 0.5584332951117629 ], [ 0.6714527412206819, -1.0792710532349, -0.7331236718683637, 0.9243251240420354, -0.3436986748812397, -0.033288582018976734 ], ... ],
    	[ [ -1.8908740216299738, -1.0937326104820106, -1.215154017246729, -1.8896634601300346, -0.45384696735025754, -0.7866938486896684 ], [ 0.8220437387816572, -0.8219994221961672, -0.036499964808315195, 1.5225459834959882, 1.8250467431032478, -0.5115330869165556 ], [ 0.7778595569136072, 1.9077184955751612, 1.1242868211950905, 0.11027714892814149, 0.7848174574888692, -0.3330451396013772 ], [ -1.2507441451698922, 1.1218525662880379, -1.4744906582140793, -1.0573999754588639, -1.3372520473902394, 1.2228954796173463 ], [ 0.9357604039913637, -0.2384208802007834, 0.2235241803948506, -0.5441809013133224, -0.907569284820586, 0.837905415017808 ], [ 0.014720386613572192, 0.8642794443168101, 0.49698020249259134, 0.9814454562955391, -1.8713092414680939, -0.7782419136794434 ], [ -0.7184321837863572, 2.1597817501881997, 0.4319486289853572, -1.1092977936225363, 2.063929614296207, -1.1755563357177732 ], [ -1.0449093297741439, -0.47122639085657053, 1.1765985771667358, -0.693443498593897, -1.8158143590229194, -1.8554086950911872 ], ... ],
    	[ [ 0.9630069700276123, -2.341728201819066, -0.6775147528420314, -2.1764105066978754, 1.0521998400209747, -1.4917989873836137 ], [ -0.8711057058984852, -0.3067887410884572, 0.5025177532066725, 0.5220053923670386, 0.21888511191831772, -2.0793920258742755 ], [ 0.9471415636169838, 1.1118096063911134, -2.022629792124806, -0.08452634477749721, -1.4595656846154634, -1.675860034797312E-4 ], [ 0.1030030963274207, -1.1066673746602014, 0.150802091442297, 0.7982311106145731, -1.8068803772326203, 0.9308970975946048 ], [ -1.230418652800714, 0.28368541521815777, -0.8337089300866029, 1.5457792708259033, -0.04347105773334835, 0.9646381583177961 ], [ 0.9108208592343594, 0.20965134032394167, -1.1585739057113735, 0.0075266326886469954, 0.07022695240315413, 0.40182826788298975 ], [ -2.291058387141624, 0.26909281372120525, -1.62067846287634, -2.401354385793386, 1.2846124183873917, -0.5726317104153089 ], [ 1.6917884328696327, -1.2655728290598243, 1.367557282371212, -1.1345167658936735, 0.5935040299439408, -0.34001835930236174 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.04 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -3.24858768885944, -5.309737159191295, -5.125800862543066 ], [ 5.206458993077939, -2.1749285975475483, 4.286230608294704 ], [ 10.672519330742649, -5.151972720932656, 4.264077887155562 ], [ 4.323906251024445, 8.213544959184665, 6.957902830169022E-4 ], [ -4.78705603959097, 2.6118617991238424, 8.495506975726832 ], [ -4.175823390418106, 18.950797010695435, -0.07595628549911833 ], [ -2.969841948946223, -10.411514546241465, 1.9890171427093384 ], [ 3.3232225700900235, 15.334040609005502, -2.4567625691857065 ], ... ],
    	[ [ -2.118783390490175, 8.096156705782521, 12.630369912122173 ], [ 8.97614656684406, -4.053284782642147, 1.0070871057479922 ], [ 20.161094829051958, -4.818585202746661, 5.301354471403609 ], [ 6.559181975206244, 4.5988871353082175, -6.788789073217188 ], [ 6.20760699675715, -0.20082571243596167, -9.362831198943738 ], [ -2.5290816802426948, 9.488816561760054, -0.051578281215630586 ], [ -29.228839965209108, 7.235874573783341, 8.563022104529795 ], [ -0.017220504371081698, 8.031721672584261, -3.897357800807676 ], ... ],
    	[ [ 15.65761417209554, 2.8198075328509433, 4.838033299497313 ], [ 14.082641062074334, -3.9988227962348457, -10.876015511937617 ], [ 4.762234017591104, -6.518797061545109, -1.9550974380902983 ], [ -17.017771039518053, 14.216560976694824, 4.01489142480014 ], [ 3.3814998168820987, 6.6025979007316575, -0.11590740864473457 ], [ 11.916662342475018, 4.72351859465818, 14.567601008585559 ], [ -10.858069061269374, -15.352098073745704, 0.9789171111107534 ], [ 14.304988356180258, 14.69350593828728, 6.610696602050787 ], ... ],
    	[ [ -1.7825475128293622, 5.745744846283353, -4.446235870112332 ], [ 4.167922536745883, -12.431537678039845, -13.335451711109938 ], [ -1.949890185242473, 3.7062506641123067, 14.608284586433424 ], [ 13.206780236151552, -7.1036630008826505, -12.230796807848618 ], [ 13.38864197405571, -9.899318428606879, -14.818925048826188 ], [ -14.734241761561513, 8.04417249157904, 15.202472519566804 ], [ 8.903513240893773, -6.782194251579064, 3.999397137920035 ], [ 13.738214769406165, -18.948680361124307, -18.780195540978887 ], ... ],
    	[ [ -4.378003193323202, -10.078638013053956, -10.581290852447237 ], [ -8.561008561187379, 2.936704301817698, -2.076187137320136 ], [ 9.310989560249256, 10.4699105743605, 9.187933756140215 ], [ -4.590288982678814, 2.5316807836607595, -2.1000279372230324 ], [ -17.515931985454635, 8.052348505537754, 2.179524678377531 ], [ -1.8822552203904366, -7.927531667882142, -4.319279401864795 ], [ 17.803286581183404, -6.7971363466226284, -0.2553257881554653 ], [ -0.2700256053534875, 19.85187855524891, 12.453248853786684 ], ... ],
    	[ [ -2.6215423379614093, 3.913897803790641, 17.345975074513596 ], [ 2.1733159365588897, -14.960913714607301, -6.130729168651559 ], [ 0.0031935589099050337, -2.6788499888901343, -8.94377807220356 ], [ -2.4625759110414327, 5.145697933524979, -3.5869468909195694 ], [ -4.979000496570656, 1.4021381476777115, 2.1774992130056763 ], [ 12.272422944964655, 15.794945420193068, 11.75967283997603 ], [ 2.1802203678273844, -13.94311164903863, 1.2963165329285196 ], [ 4.617819267378367, -0.6064557040381938, 1.8753307281586695 ], ... ],
    	[ [ -2.2481161073133777, 4.191163397616556, 18.82906019437581 ], [ 9.685671888083307, 1.1595285725809397, 13.130232635821086 ], [ 9.364396570876117, -2.5942066374685493, -7.710853990622516 ], [ -8.654509328167109, -2.9786779899982303, -0.31148074231494 ], [ 0.49663153991911707, -16.51150422405004, -14.605083905812233 ], [ 4.5737035278266545, -1.148719168281938, -11.86411162702228 ], [ -10.977574550806462, -1.5769979314516236, 6.089527707871771 ], [ 4.467486899107245, 15.505227721215515, 11.171360346792548 ], ... ],
    	[ [ -8.84203233533577, -4.522423119368991, -7.8387659516176065 ], [ 25.19036455317294, 1.6524284038030113, 3.4934555855253504 ], [ -7.890595034410359, 17.38038998541839, 5.909209543772689 ], [ -16.70563918725369, 2.480897511950416, -10.388192496126216 ], [ 18.995422393968454, -11.774882216692424, 3.3747577510592865 ], [ -2.9771027343042458, 7.946005115764434, -22.102103703797823 ], [ -23.64564677956608, -8.91317300170455, 10.683350112101152 ], [ -10.70304522791287, 16.941571840473024, 5.383894196271258 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 38.53 seconds (0.642 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11794123965708
Reset training subject: 11794268570692
Adding measurement 3f2e2d9e to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 183.7372888499562 < 183.7372888499562. Total: 1
th(0)=183.7372888499562;dx=-0.03780873268760233
Adding measurement 601c3cd4 to history. Total: 1
New Minimum: 183.7372888499562 > 183.65584357706663
WOLFE (weak): th(2.154434690031884)=183.65584357706663; dx=-0.03779836115137249 evalInputDelta=0.08144527288956738
Adding measurement 2bddecfe to history. Total: 2
New Minimum: 183.65584357706663 > 183.57442064897447
WOLFE (weak): th(4.308869380063768)=183.57442064897447; dx=-0.037787989615142625 evalInputDelta=0.16286820098173393
Adding measurement e591fbf to history. Total: 3
New Minimum: 183.57442064897447 > 183.2489523845803
WOLFE (weak): th(12.926608140191302)=183.2489523845803; dx=-0.037746503470223236 evalInputDelta=0.4883364653759088
Adding measurement 31ed5d13 to history. Total: 4
New Minimum: 183.2489523845803 > 181.78876946470012
WOLFE (weak): th(51.70643256076521)=181.78876946470012; dx=-0.03755981581808594 evalInputDelta=1.948519385256077
Adding measurement 3020858d to history. Total: 5
New Minimum: 181.78876946470012 > 174.12339795694473
WOLFE (weak): th(258.53216280382605)=174.12339795694473; dx=-0.03656414834002042 evalInputDelta=9.613890893011472
Adding measurement 21283ac4 to history. Total: 6
New Minimum: 174.12339795694473 > 130.88041973946946
END: th(1551.1929768229563)=130.88041973946946; dx=-0.03034122660211089 evalInputDelta=52.85686911048674
Fitness changed from 183.7372888499562 to 130.88041973946946
Iteration 1 complete. Error: 130.88041973946946 Total: 3.0781; Orientation: 0.1843; Line Search: 2.4612
Non-optimal measurement 130.88041973946946 < 130.88041973946946. Total: 7
Rejected: LBFGS Orientation magnitude: 1.470e+03, gradient 1.566e-01, dot -0.971; [a143daa5-dfa1-4b30-b3a0-faa9d3b497a3 = 1.000/1.000e+00, 2665435e-3cda-4f61-9c31-7935e3dac129 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 130.88041973946946, 174.12339795694473, 181.78876946470012, 183.2489523845803, 183.57442064897447, 183.65584357706663, 183.7372888499562
Rejected: LBFGS Orientation magnitude: 1.470e+03, gradient 1.566e-01, dot -0.971; [a143daa5-dfa1-4b30-b3a0-faa9d3b497a3 = 1.000/1.000e+00, 2665435e-3cda-4f61-9c31-7935e3dac129 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 130.88041973946946, 174.12339795694473, 181.78876946470012, 183.2489523845803, 183.57442064897447, 183.65584357706663
Rejected: LBFGS Orientation magnitude: 1.470e+03, gradient 1.566e-01, dot -0.971; [a143daa5-dfa1-4b30-b3a0-faa9d3b497a3 = 1.000/1.000e+00, 2665435e-3cda-4f61-9c31-7935e3dac129 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 130.88041973946946, 174.12339795694473, 181.78876946470012, 183.2489523845803, 183.57442064897447
Rejected: LBFGS Orientation magnitude: 1.470e+03, gradient 1.566e-01, dot -0.971; [a143daa5-dfa1-4b30-b3a0-faa9d3b497a3 = 1.000/1.000e+00, 2665435e-3cda-4f61-9c31-7935e3dac129 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 130.88041973946946, 174.12339795694473, 181.78876946470012, 183.2489523845803
LBFGS Accumulation History: 3 points
Removed measurement 21283ac4 to history. Total: 6
Removed measurement 3020858d to history. Total: 5
Removed measurement 31ed5d13 to history. Total: 4
Removed measurement e591fbf to history. Total: 3
Adding measurement 66e129b9 to history. Total: 3
th(0)=130.88041973946946;dx=-0.02452823486693995
Adding measurement 2915a19d to history. Total: 4
New Minimum: 130.88041973946946 > 65.30008109558787
END: th(3341.943960201201)=65.30008109558787; dx=-0.014718586399286986 evalInputDelta=65.58033864388159
Fitness changed from 130.88041973946946 to 65.30008109558787
Iteration 2 complete. Error: 65.30008109558787 Total: 24.8994; Orientation: 24.0627; Line Search: 0.6923
Non-optimal measurement 65.30008109558787 < 65.30008109558787. Total: 5
Rejected: LBFGS Orientation magnitude: 1.003e+03, gradient 9.688e-02, dot -0.987; [2665435e-3cda-4f61-9c31-7935e3dac129 = 1.000/1.000e+00, a143daa5-dfa1-4b30-b3a0-faa9d3b497a3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 65.30008109558787, 130.88041973946946, 183.57442064897447, 183.65584357706663, 183.7372888499562
Rejected: LBFGS Orientation magnitude: 1.003e+03, gradient 9.688e-02, dot -0.987; [a143daa5-dfa1-4b30-b3a0-faa9d3b497a3 = 1.000/1.000e+00, 2665435e-3cda-4f61-9c31-7935e3dac129 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 65.30008109558787, 130.88041973946946, 183.57442064897447, 183.65584357706663
LBFGS Accumulation History: 3 points
Removed measurement 2915a19d to history. Total: 4
Removed measurement 66e129b9 to history. Total: 3
Adding measurement 1d1ab651 to history. Total: 3
th(0)=65.30008109558787;dx=-0.009385193377788145
Adding measurement 4b975796 to history. Total: 4
New Minimum: 65.30008109558787 > 21.599592634943086
END: th(7200.000000000001)=21.599592634943086; dx=-0.0027538311946131823 evalInputDelta=43.700488460644785
Fitness changed from 65.30008109558787 to 21.599592634943086
Iteration 3 complete. Error: 21.599592634943086 Total: 10.5519; Orientation: 9.6647; Line Search: 0.7316
Final threshold in iteration 3: 21.599592634943086 (> 0.0) after 38.530s (< 30.000s)

Returns

    21.599592634943086

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.33842621746489576, -0.48319048980598683, 0.582975516560331, -1.791163770004535, 1.4980155290641228, 0.6622828533027596 ], [ -0.35301316211835454, -0.7827846651891188, -0.2616813167198063, 1.3060195863729864, 0.8356348529179504, 1.0649823590855063 ], [ 0.1807191852043431, -1.8178960708856442, 0.9812689684805322, 0.10697172771515825, 0.4358081881884341, 1.283881048586771 ], [ -0.24800428182381612, 1.191452099433323, 0.4907540141346491, 0.6045425765608167, -0.8789665151556106, -1.0417791337912305 ], [ -0.9246560890513787, -0.003203849966972247, -0.6189492764718638, -0.8075198900746956, 0.846554174059418, -1.196789350513099 ], [ 0.7862586883120604, -1.2823956442186533, 0.41819114244041933, -0.6970440981148639, 0.43779622387022005, -0.3267794557034874 ], [ -2.1093028173244197, 0.604649432518606, -1.761696075600624, -1.0872512402270718, -1.178501956633508, -1.6982662846095953 ], [ 1.5419210659905331, 0.29212952809183607, 0.1995057744157795, 0.6735203120098858, -1.1726365742782563, 1.0066872066359775 ], ... ],
    	[ [ -1.1507944085768966, -0.2558463578837048, 0.7593851255200068, -0.6675874557802712, 1.664833230691487, -1.6034130577560954 ], [ 0.44475819144708817, -0.7467997347302058, 1.0433890179233816, 0.552254869493844, 0.7316731453520585, -0.3437049109168788 ], [ 0.28921319612875723, 0.028520998474132364, -0.2444746082720014, 2.368742326910824, 0.21149897167058865, -2.287788410896849 ], [ -0.17608775305250146, 0.02266340219568519, 0.6833820517406015, 0.555595053655122, -0.6655912366059933, 0.10506033661605793 ], [ 0.9786039223344636, 1.4472854263922998, -1.0737680490866333, 0.5388237095012646, -0.31980174170650955, 1.0611296991858807 ], [ -1.4023055867963101, 1.5929887770493645, 0.6024267560066299, -2.0056685733439257, 1.4120731640846056, 0.27121684950307634 ], [ -0.8335862098012328, 2.5065772570887073, -1.1143153704491198, 0.213211249908011, 0.7929260452483262, 1.1798193041492528 ], [ 0.524183763254555, -2.3516384468305453, 0.7509441460319926, -0.6807717432680829, -1.4481723445393893, -0.4492789779343202 ], ... ],
    	[ [ -0.627318596056814, -1.3272298602445485, -1.033018806760436, -1.9305797442699377, -1.187361032941752, -0.2683218912483598 ], [ 0.11208937497134275, -1.5329736474465332, -0.4730500065804055, 1.8049271984990278, 0.18356044376750447, -1.3323178300204142 ], [ 1.2213848669564782, 1.2130845530576573, 0.7200890422053745, -0.5938429044835398, 0.25091211062876523, 0.9179943623930866 ], [ 2.221834874351157, -1.360285799958583, 0.8994491845159844, -0.33117169332469476, -0.6527136781746005, 0.6569791020918417 ], [ 1.201080606635868, -0.9025008035382981, 1.5613543162211823, 0.5166065948412758, -1.160437754076233, -1.49015577308771 ], [ 0.840645273796947, -0.24196755510846518, -0.035546899711212554, 1.5714612217385067, 1.0598091417000661, 0.06038345062538629 ], [ 0.05215931911010774, 0.5840474605766088, -0.6428075528188355, -1.4445539176188376, -1.111765665036922, -0.033657177080618106 ], [ -0.3736601599460535, -0.18569752496360914, 2.835057214713861, 1.9436304566429226, -0.9695201074860753, -0.1428224692207743 ], ... ],
    	[ [ 1.2716540502974722, 0.0947157672509196, -0.5354342692548939, -0.7856959978661637, 0.5272125650728108, -0.6985684975593343 ], [ -0.9586741270096388, -0.331784326509843, 0.21163530681164977, 0.9102810444092416, 0.18742860556735863, 0.34692713534347164 ], [ 1.7456917883316883, -0.7676374011767717, -0.8474086441362796, -1.24979593966791, 0.3667040705728719, 0.8514063356011422 ], [ 0.3194962429885462, 0.5024258022853042, 0.41941071904373706, 1.6592600859905515, 0.5845269539649807, 0.9819303514350399 ], [ -1.9157528662447247, 1.1104933434603175, 0.2706472352353773, -1.9437011622979534, -0.5669775552432189, 1.0409775540056208 ], [ 0.2984469671373029, -0.47032424345571977, -1.0988318484353834, -1.3930215345592987, -0.7447838833808371, -1.7282207320268048 ], [ 0.09371556951689544, 1.1953296707206413, 0.3216712721689524, 1.9048807092755438, 0.09637513625318384, -0.6584428190312348 ], [ 1.6862323870832725, -0.992048745831198, 0.0833302710861222, -1.1522507301297908, 0.4342645388229869, -1.09640091364603 ], ... ],
    	[ [ 0.717627131983982, -0.8402751558437433, 0.2919534568999518, 0.5588654455097088, -1.269067947520934, 3.5457818711494977 ], [ -0.9863957854353019, 0.4004977375532944, 0.18259159887883158, -1.0218529964882372, 0.07841268996371875, 0.6379828065747845 ], [ 0.42021852116801534, -0.07431846657340924, 0.9279814801714203, -0.08835790362417872, 1.6080599567877008, -0.10430334868577529 ], [ -1.1724718138335544, 1.6605251933336376, 0.8877685615545303, 0.7775552160177499, -1.8695602024311762, -0.3710406348799623 ], [ 1.3574073876358455, 0.13205779309816623, 0.6747667078063209, 1.6928593029059593, -0.7744266949310132, 1.6106612905898816 ], [ -0.5181149129065212, -1.1587956089146194, 0.9628161168932797, 0.5133020897806648, 0.9725740123588998, -1.3632657164704098 ], [ -1.190111760218554, -0.012183899794570074, 1.8074758525573422, -1.2665288502542564, 0.08479344498338126, 0.28055911235378345 ], [ -1.370836587902005, 1.0024566000069455, 0.05537159949669826, -0.09102663537225475, -0.43000866391560927, -1.7138402238164656 ], ... ],
    	[ [ 0.29158502301064915, -0.6793652840552284, 0.20144140594011894, 0.6904245466393117, 0.5423040140588022, -1.402729605027274 ], [ 0.88170125097348, 0.5304711858450618, 1.1441278362131426, -0.12301763607313987, 0.3916268740330168, -0.0063509137705886065 ], [ 0.10527960996204949, -0.26445514388123703, 1.1206207904971746, -1.4589237508888773, 0.31599882609536833, -0.5287030666722081 ], [ 0.453404022908984, -0.281265708676487, 0.5240624784792581, -0.9281421106960751, 1.8979108858768636, -0.26961696677128344 ], [ -0.7665822143667592, 0.09959295379307898, 0.7035760626518404, -1.6373746416931205, 0.6307666581103222, 1.2618540744283382 ], [ -0.3603144945112778, 0.5281406499737649, 1.7615894739009754, 0.0412743613394031, 0.8465831049719306, -2.3515298922438563 ], [ 0.1702725536699455, -0.7417990012050674, -0.55422066176559, 0.48808212559006275, -0.2448035992074024, -0.5845339400050228 ], [ 0.14484095625147947, -0.06885120514520476, -1.1870864436192214, 1.0087642349865131, -0.5257597059350054, 0.05334390636330916 ], ... ],
    	[ [ -1.1736373954818913, -1.0011078663785926, -0.1261319837704437, -1.471801673045391, -0.4585574171437377, -0.26332879456046954 ], [ 1.0753542694678373, -1.4154191563147922, 0.5056589356287028, 1.1752706458300985, 1.3460457609374452, -1.316665499585778 ], [ 0.3143887427343159, 1.5152864076252035, 0.3538752975335698, 0.5676997915314625, 0.955071182457823, -0.7910720629338097 ], [ -1.0637466881089708, 1.0049774762326924, -1.1838013650555006, 0.31913289743305195, 0.5648011129106391, 1.0925117123243822 ], [ 0.03938455402650565, 0.5683466699596614, 0.10677443695089239, -1.4917949451741879, -0.41187919591565947, 0.3389863340008022 ], [ -0.6867205678201296, -0.6431348499604727, 0.29605785179508337, 0.495334064785777, -2.0666697165706562, -0.5966752018150353 ], [ -0.361198380265025, 1.251338433990656, 0.3082204044950252, -1.2580163010795558, 1.7049402928187345, -0.7665997434871052 ], [ -0.7673945379861147, 0.18263775509283087, 1.3076847312195907, -1.0886950827236772, -2.2789515485961624, -1.9783856149712862 ], ... ],
    	[ [ 0.6106066919766246, -2.393620530927839, -1.147127804591737, -1.7962572768270677, 0.8330264007700172, -1.371975204176109 ], [ -0.31545015804079507, -1.2442050685776185, -0.06991838945605844, 1.351879943515526, -0.118714184766196, -0.7193804376371717 ], [ 1.420661852554507, 0.9605007367889871, -1.7748883427018307, 0.9486014858059133, -0.8684118642224731, 0.9174689561302766 ], [ -0.7601630179804963, -0.8687003905405519, -0.08132052385846122, -0.35896562053429193, -2.031089144140592, 0.4427139186292042 ], [ -1.3174132820330982, 0.030062183350174826, -0.9856641738831358, 0.19390521871904137, -0.1388866840510836, -0.20260736696942483 ], [ 0.890148139541702, -0.4294298244230795, -0.9107921178051182, -0.06281391745821793, 0.5454777963857051, 0.7250102260394387 ], [ -1.454091986954521, 0.14282292167528032, -1.6481148465943434, -1.4726835660190045, 1.391130366244781, -0.628246563602415 ], [ 1.156133186561834, -0.5223454668629909, 0.6261748982074103, 0.6065833738990756, 1.1229069598933241, -0.16470482153952154 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.04 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -8.951541325158695, -3.372284105318053, -2.3533839119329167 ], [ 8.861596675542291, 0.8517603216142902, 8.652893671075761 ], [ 4.0353010734018895, -4.097917271321135, 0.4917853603947932 ], [ 13.012561002153307, 3.6779657745458, -1.0741453139717154 ], [ 0.6150985148153082, 4.4680220394313475, 10.109218455130573 ], [ 0.6168045200693487, 13.695735737756813, -6.985635402608987 ], [ -1.2611768371298044, -8.590278716325617, -0.18788718688820225 ], [ -1.7588575001704758, 4.433800638427769, -3.0101201411805865 ], ... ],
    	[ [ -1.653507800119396, 2.6253616970860927, 15.603961126231429 ], [ 9.324723158879712, -2.2710234828309197, 1.9095039404398575 ], [ 15.254887852121596, -8.115025695437103, 4.5426343162480025 ], [ 5.2862854501339935, 0.020083231289432523, -3.3464319652062438 ], [ 2.397150156284913, -6.150647341574928, -7.193061363917033 ], [ -4.906604690063749, 6.67919348515319, -1.0225284926750584 ], [ -26.240253385999864, 6.417446934795596, 13.106422715139159 ], [ 0.46824598311244764, 4.226357744738706, -4.408195164515441 ], ... ],
    	[ [ 6.218972670266824, 0.282390251043296, 4.831264273601918 ], [ 17.674342758536277, -2.296116656082077, -5.9267230679166145 ], [ 2.1651524702042786, 1.3319492675638331, 2.3314988841262987 ], [ -10.586239333427274, 9.918209131840314, -0.832539043078808 ], [ -2.226526709995966, 5.077320961525585, -0.31347692345332545 ], [ 8.282904409959333, -2.4685079640134266, 9.557785213085095 ], [ -4.13548778506333, -16.51878251800242, -0.9885800283568189 ], [ 8.582473141478262, 10.973707389727792, 11.932112303136448 ], ... ],
    	[ [ -6.611949060823778, 5.697321779262951, -4.961086712891264 ], [ 2.2181425335151794, -7.625199854210232, -4.867927039093514 ], [ -4.682257876058684, -1.3727319551198274, 12.763991261334153 ], [ 15.256117424025057, -6.9983937260211455, -14.400192503760367 ], [ 12.413779778856382, -10.179473472440984, -12.66343519993714 ], [ -15.949005315811034, 14.36516114976452, 8.19065444843635 ], [ 5.16049572453961, 0.4407938837340295, 7.142394817965683 ], [ 13.414694334961235, -15.849300585016811, -11.94583329535947 ], ... ],
    	[ [ 3.817985041801314, -3.465246718383793, -8.160381430826824 ], [ -8.543846914240904, -2.773947682894399, 1.5412595426815132 ], [ 1.1243951500678078, 12.299951464167822, 12.467399360334712 ], [ -4.2821768712190496, 8.484038979522781, 2.7840463619552698 ], [ -16.776726188940675, 8.269514792198182, 3.0862291549424308 ], [ 7.632071112658819, -8.720719874556892, 0.5837018689513334 ], [ 20.1019686705685, 0.034839839933191806, 3.2800631122084454 ], [ -2.9937189828305573, 14.632737005595047, 12.176310881479742 ], ... ],
    	[ [ -2.8012733826351326, -2.342943760899832, 3.48012250985728 ], [ 7.25960544996238, -13.404950175119675, -3.0377918886060082 ], [ 4.911263678991134, 3.461505322253041, 0.534060450530251 ], [ -7.130450945114543, 3.222341156466772, -3.199457564901352 ], [ -11.725240597827868, -9.75080953686394, -5.811701513701024 ], [ 6.102418890826472, 18.89057989242059, 7.460672703090724 ], [ 5.187619722360161, -10.585289611013902, 5.625199179021581 ], [ 5.237507255866563, -7.081924469089436, 2.1329409388191927 ], ... ],
    	[ [ 0.4653281280375698, 3.0896928143157742, 11.866819316464275 ], [ 11.545511162650586, 7.946701999040034, 3.7752171736266904 ], [ 4.936955071649632, -9.352349719031356, -7.334550231915476 ], [ -6.690471535313785, -4.658978333200466, 1.8366101755040274 ], [ 0.24132295030130885, -10.435865286735512, -9.98645267865883 ], [ 7.055967161027426, 6.77558440894974, -7.636123549890557 ], [ -8.99654646109845, 5.438949143847757, 1.2052664006184282 ], [ 1.9001589229339246, 14.547266800202268, 2.0760439917916913 ], ... ],
    	[ [ -0.8592407654648345, -6.061784509299832, -4.605145370895872 ], [ 21.073778886049592, -2.0407425532324845, -3.727108802379901 ], [ -0.5203755350514413, 11.678884853506272, -5.499276322486295 ], [ -10.665811478833398, 6.44324488654901, -10.543540519581711 ], [ 12.360864220281359, -2.002532366050208, 5.690388700747252 ], [ 0.3210990816371004, 1.6296556033803482, -16.2562706613455 ], [ -20.30799315197091, -6.717346085337209, 11.006880353286608 ], [ -8.08396033311404, 13.481655975051138, 2.9446407702324024 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.13 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -1.506617118008532], [27.0, 2.1168746791316724]; valueStats=DoubleSummaryStatistics{count=44, sum=541.211824, min=0.031145, average=12.300269, max=130.880420}
Plotting 27 points for GD
Plotting 14 points for CjGD
Plotting 3 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -1.506617118008532], [35.451, 2.1168746791316724]; valueStats=DoubleSummaryStatistics{count=44, sum=541.211824, min=0.031145, average=12.300269, max=130.880420}
Plotting 27 points for GD
Plotting 14 points for CjGD
Plotting 3 points for LBFGS

Returns

Result

Model Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0042 = network_target.state();
    assert temp_18_0042 != null;
    String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0042.freeRef();
    return temp_18_0041;

Returns

    

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 19.35 seconds (0.058 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11833254173293
Reset training subject: 11833427817605
Constructing line search parameters: GD
th(0)=197.50009284723055;dx=-350.5025323910296
New Minimum: 197.50009284723055 > 164.23062077015743
WOLF (strong): th(2.154434690031884)=164.23062077015743; dx=319.61789037240277 evalInputDelta=33.26947207707312
New Minimum: 164.23062077015743 > 0.39902114614778006
END: th(1.077217345015942)=0.39902114614778006; dx=-15.442321009313467 evalInputDelta=197.10107170108276
Fitness changed from 197.50009284723055 to 0.39902114614778006
Iteration 1 complete. Error: 0.39902114614778006 Total: 0.8768; Orientation: 0.0003; Line Search: 0.4018
th(0)=0.39902114614778006;dx=-0.705795520831755
Armijo: th(2.3207944168063896)=0.44216334587904316; dx=0.7429743415138814 evalInputDelta=-0.0431421997312631
New Minimum: 0.39902114614778006 > 3.0512004227218166E-4
WOLF (strong): th(1.1603972084031948)=3.0512004227218166E-4; dx=0.018589410341063383 evalInputDelta=0.3987160261055079
END: th(0.3867990694677316)=0.1727186650079131; dx=-0.46433387710748214 evalInputDelta=0.22630248113986695
Fitness changed from 0.39902114614778006 to 3.0512004227218166E-4
Iteration 2 complete. Error: 3.0512004227218166E-4 Total: 0.6492; Orientation: 0.0005; Line Search: 0.5203
th(0)=3.0512004227218166E-4;dx=-5.42590608172533E-4
New Minimum: 3.0512004227218166E-4 > 2.048664323224683E-5
END: th(0.8333333333333335)=2.048664323224683E-5; dx=-1.4052954952330918E-4 evalInputDelta=2.8463339903993484E-4
Fitness changed from 3.0512004227218166E-4 to 2.048664323224683E-5
Iteration 3 complete. Error: 2.048664323224683E-5 Total: 0.3894; Orientation: 0.0004; Line Search: 0.2605
th(0)=2.048664323224683E-5;dx=-3.641694830878565E-5
New Minimum: 2.048664323224683E-5 > 7.273749119213132E-6
WOLF (strong): th(1.7953622416932369)=7.273749119213132E-6; dx=2.1698031082841963E-5 evalInputDelta=1.32128941130337E-5
New Minimum: 7.273749119213132E-6 > 8.380162161664847E-7
END: th(0.8976811208466184)=8.380162161664847E-7; dx=-7.359458612972232E-6 evalInputDelta=1.9648627016080345E-5
Fitness changed from 2.048664323224683E-5 to 8.380162161664847E-7
Iteration 4 complete. Error: 8.380162161664847E-7 Total: 0.5191; Orientation: 0.0004; Line Search: 0.3911
th(0)=8.380162161664847E-7;dx=-1.4888477425433545E-6
New Minimum: 8.380162161664847E-7 > 4.3218864157338577E-7
WOLF (strong): th(1.9339953473386584)=4.3218864157338577E-7; dx=1.069169820203413E-6 evalInputDelta=4.058275745930989E-7
New Minimum: 4.3218864157338577E-7 > 1.6703170774604032E-8
END: th(0.9669976736693292)=1.6703170774604032E-8; dx=-2.0983896117006963E-7 evalInputDelta=8.213130453918806E-7
Fitness changed from 8.380162161664847E-7 to 1.6703170774604032E-8
Iteration 5 complete. Error: 1.6703170774604032E-8 Total: 0.5181; Orientation: 0.0004; Line Search: 0.3890
th(0)=1.6703170774604032E-8;dx=-2.9650491427796774E-8
New Minimum: 1.6703170774604032E-8 > 1.2046610778629298E-8
WOLF (strong): th(2.083333333333334)=1.2046610778629298E-8; dx=2.5180193831662644E-8 evalInputDelta=4.656559995974734E-9
New Minimum: 1.2046610778629298E-8 > 9.606649029470043E-11
END: th(1.041666666666667)=9.606649029470043E-11; dx=-2.2351487980750623E-9 evalInputDelta=1.6607104284309333E-8
Fitness changed from 1.6703170774604032E-8 to 9.606649029470043E-11
Iteration 6 complete. Error: 9.606649029470043E-11 Total: 0.5173; Orientation: 0.0004; Line Search: 0.3893
th(0)=9.606649029470043E-11;dx=-1.7024566610153536E-10
New Minimum: 9.606649029470043E-11 > 9.390443408087296E-11
WOLF (strong): th(2.244202802116546)=9.390443408087296E-11; dx=1.6831887391341425E-10 evalInputDelta=2.1620562138274613E-12
New Minimum: 9.390443408087296E-11 > 9.526013437595842E-15
END: th(1.122101401058273)=9.526013437595842E-15; dx=-9.633960927585948E-13 evalInputDelta=9.605696428126283E-11
Fitness changed from 9.606649029470043E-11 to 9.526013437595842E-15
Iteration 7 complete. Error: 9.526013437595842E-15 Total: 0.5175; Orientation: 0.0004; Line Search: 0.3876
Low gradient: 1.2950272651072233E-7
th(0)=9.526013437595842E-15;dx=-1.677095617371094E-14
Armijo: th(2.417494184173323)=1.2124465381342722E-14; dx=1.8920663057003048E-14 evalInputDelta=-2.59845194374688E-15
New Minimum: 9.526013437595842E-15 > 3.970417164180791E-17
WOLF (strong): th(1.2087470920866614)=3.970417164180791E-17; dx=1.0748534451112082E-15 evalInputDelta=9.486309265954034E-15
END: th(0.4029156973622205)=3.9671247441131346E-15; dx=-1.0822352973487851E-14 evalInputDelta=5.5588886934827074E-15
Fitness changed from 9.526013437595842E-15 to 3.970417164180791E-17
Iteration 8 complete. Error: 3.970417164180791E-17 Total: 0.6475; Orientation: 0.0006; Line Search: 0.5188
Low gradient: 8.369876027083692E-9
th(0)=3.970417164180791E-17;dx=-7.005482470875028E-17
New Minimum: 3.970417164180791E-17 > 2.179635228394095E-18
END: th(0.8680555555555558)=2.179635228394095E-18; dx=-1.6401706774082352E-17 evalInputDelta=3.752453641341382E-17
Fitness changed from 3.970417164180791E-17 to 2.179635228394095E-18
Iteration 9 complete. Error: 2.179635228394095E-18 Total: 0.3869; Orientation: 0.0005; Line Search: 0.2579
Low gradient: 1.960526322586336E-9
th(0)=2.179635228394095E-18;dx=-3.843663461553902E-18
New Minimum: 2.179635228394095E-18 > 9.184490074409012E-19
WOLF (strong): th(1.8701690017637886)=9.184490074409012E-19; dx=2.494923104031573E-18 evalInputDelta=1.2611862209531936E-18
New Minimum: 9.184490074409012E-19 > 6.72635875225954E-20
END: th(0.9350845008818943)=6.72635875225954E-20; dx=-6.743702054164442E-19 evalInputDelta=2.1123716408714995E-18
Fitness changed from 2.179635228394095E-18 to 6.72635875225954E-20
Iteration 10 complete. Error: 6.72635875225954E-20 Total: 0.5176; Orientation: 0.0005; Line Search: 0.3886
Low gradient: 3.442807431164239E-10
th(0)=6.72635875225954E-20;dx=-1.1852923008079706E-19
New Minimum: 6.72635875225954E-20 > 4.041573095425997E-20
WOLF (strong): th(2.0145784868111027)=4.041573095425997E-20; dx=9.187565631467399E-20 evalInputDelta=2.6847856568335423E-20
New Minimum: 4.041573095425997E-20 > 8.55019284254519E-22
END: th(1.0072892434055514)=8.55019284254519E-22; dx=-1.3326798418941091E-20 evalInputDelta=6.640856823834088E-20
Fitness changed from 6.72635875225954E-20 to 8.55019284254519E-22
Iteration 11 complete. Error: 8.55019284254519E-22 Total: 0.5226; Orientation: 0.0004; Line Search: 0.3934
Zero gradient: 3.8795473954391545E-11
th(0)=8.55019284254519E-22;dx=-1.5050887993458729E-21
New Minimum: 8.55019284254519E-22 > 7.08292292305214E-22
WOLF (strong): th(2.1701388888888897)=7.08292292305214E-22; dx=1.3698653392537017E-21 evalInputDelta=1.4672699194930496E-22
New Minimum: 7.08292292305214E-22 > 1.774069559532519E-24
END: th(1.0850694444444449)=1.774069559532519E-24; dx=-6.761465800467715E-23 evalInputDelta=8.532452146949864E-22
Fitness changed from 8.55019284254519E-22 to 1.774069559532519E-24
Iteration 12 complete. Error: 1.774069559532519E-24 Total: 0.5183; Orientation: 0.0005; Line Search: 0.3891
Zero gradient: 1.7653756818042E-12
th(0)=1.774069559532519E-24;dx=-3.1165512979056437E-24
Armijo: th(2.337711252204736)=1.9689214561226738E-24; dx=3.283249781770564E-24 evalInputDelta=-1.9485189659015461E-25
New Minimum: 1.774069559532519E-24 > 1.3312296819672615E-27
WOLF (strong): th(1.168855626102368)=1.3312296819672615E-27; dx=8.336820534132347E-26 evalInputDelta=1.772738329850552E-24
END: th(0.38961854203412266)=7.6756443253526015E-25; dx=-2.0499169548789718E-24 evalInputDelta=1.006505126997259E-24
Fitness changed from 1.774069559532519E-24 to 1.3312296819672615E-27
Iteration 13 complete. Error: 1.3312296819672615E-27 Total: 0.6501; Orientation: 0.0004; Line Search: 0.5211
Zero gradient: 4.8227008779207396E-14
th(0)=1.3312296819672615E-27;dx=-2.325844375789747E-27
New Minimum: 1.3312296819672615E-27 > 9.926683342864614E-29
END: th(0.8394077028379595)=9.926683342864614E-29; dx=-6.0515558131778115E-28 evalInputDelta=1.2319628485386154E-27
Fitness changed from 1.3312296819672615E-27 to 9.926683342864614E-29
Iteration 14 complete. Error: 9.926683342864614E-29 Total: 0.3908; Orientation: 0.0005; Line Search: 0.2611
Zero gradient: 1.2567973416045572E-14
th(0)=9.926683342864614E-29;dx=-1.5795395578642822E-28
New Minimum:

...skipping 4040 bytes...

182786618934E-35
Armijo: th(4.058526479522111)=8.903452603156164E-33; dx=4.376089799436492E-37 evalInputDelta=-2.0366508544192484E-33
New Minimum: 6.866801748736916E-33 > 6.184265015979042E-33
END: th(2.0292632397610557)=6.184265015979042E-33; dx=-1.335903420911695E-35 evalInputDelta=6.8253673275787416E-34
Fitness changed from 6.866801748736916E-33 to 6.184265015979042E-33
Iteration 22 complete. Error: 6.184265015979042E-33 Total: 0.5170; Orientation: 0.0005; Line Search: 0.3879
Zero gradient: 4.367852358371841E-18
th(0)=6.184265015979042E-33;dx=-1.9078134224534456E-35
Armijo: th(4.371915118947706)=7.482143766532775E-33; dx=4.767959124219834E-36 evalInputDelta=-1.2978787505537335E-33
New Minimum: 6.184265015979042E-33 > 4.7091923291798875E-33
END: th(2.185957559473853)=4.7091923291798875E-33; dx=-5.1585978348329825E-36 evalInputDelta=1.4750726867991542E-33
Fitness changed from 6.184265015979042E-33 to 4.7091923291798875E-33
Iteration 23 complete. Error: 4.7091923291798875E-33 Total: 0.5935; Orientation: 0.0005; Line Search: 0.4650
Zero gradient: 3.854187254595651E-18
th(0)=4.7091923291798875E-33;dx=-1.4854759393487563E-35
Armijo: th(4.709502797067904)=9.435945840405922E-33; dx=1.1847371436587713E-35 evalInputDelta=-4.7267535112260346E-33
Armijo: th(2.354751398533952)=4.8897532546178984E-33; dx=-1.8492254892898242E-36 evalInputDelta=-1.805609254380109E-34
New Minimum: 4.7091923291798875E-33 > 2.6774375275002806E-33
END: th(0.7849171328446506)=2.6774375275002806E-33; dx=-8.298990420959244E-36 evalInputDelta=2.031754801679607E-33
Fitness changed from 4.7091923291798875E-33 to 2.6774375275002806E-33
Iteration 24 complete. Error: 2.6774375275002806E-33 Total: 0.6443; Orientation: 0.0006; Line Search: 0.5154
Zero gradient: 2.7866643926594897E-18
th(0)=2.6774375275002806E-33;dx=-7.765498437316284E-36
New Minimum: 2.6774375275002806E-33 > 2.403006776572836E-33
END: th(1.6910526998008797)=2.403006776572836E-33; dx=-6.930834077755816E-36 evalInputDelta=2.744307509274446E-34
Fitness changed from 2.6774375275002806E-33 to 2.403006776572836E-33
Iteration 25 complete. Error: 2.403006776572836E-33 Total: 0.3878; Orientation: 0.0005; Line Search: 0.2588
Zero gradient: 2.495579154230979E-18
th(0)=2.403006776572836E-33;dx=-6.22791531503221E-36
New Minimum: 2.403006776572836E-33 > 1.7921699190686447E-33
END: th(3.6432625991230885)=1.7921699190686447E-33; dx=-4.699761211190337E-36 evalInputDelta=6.108368575041913E-34
Fitness changed from 2.403006776572836E-33 to 1.7921699190686447E-33
Iteration 26 complete. Error: 1.7921699190686447E-33 Total: 0.3870; Orientation: 0.0005; Line Search: 0.2586
Zero gradient: 1.9047225652388432E-18
th(0)=1.7921699190686447E-33;dx=-3.627968050530039E-36
New Minimum: 1.7921699190686447E-33 > 8.68156969163682E-34
END: th(7.849171328446507)=8.68156969163682E-34; dx=-1.6038041460287625E-36 evalInputDelta=9.240129499049627E-34
Fitness changed from 1.7921699190686447E-33 to 8.68156969163682E-34
Iteration 27 complete. Error: 8.68156969163682E-34 Total: 0.3873; Orientation: 0.0005; Line Search: 0.2589
Zero gradient: 8.893813052343846E-19
th(0)=8.68156969163682E-34;dx=-7.909991061004175E-37
New Minimum: 8.68156969163682E-34 > 2.4100249187882368E-34
END: th(16.910526998008798)=2.4100249187882368E-34; dx=-2.4433011732982954E-37 evalInputDelta=6.271544772848583E-34
Fitness changed from 8.68156969163682E-34 to 2.4100249187882368E-34
Iteration 28 complete. Error: 2.4100249187882368E-34 Total: 0.3883; Orientation: 0.0004; Line Search: 0.2601
Zero gradient: 3.1470066751455535E-19
th(0)=2.4100249187882368E-34;dx=-9.903651013410674E-38
New Minimum: 2.4100249187882368E-34 > 6.492185013736681E-35
END: th(36.432625991230886)=6.492185013736681E-35; dx=-1.8543182824288233E-38 evalInputDelta=1.7608064174145687E-34
Fitness changed from 2.4100249187882368E-34 to 6.492185013736681E-35
Iteration 29 complete. Error: 6.492185013736681E-35 Total: 0.3872; Orientation: 0.0006; Line Search: 0.2580
Zero gradient: 1.0134492537186842E-19
th(0)=6.492185013736681E-35;dx=-1.0270793898629579E-38
New Minimum: 6.492185013736681E-35 > 1.0761490217957085E-35
END: th(78.49171328446506)=1.0761490217957085E-35; dx=-1.112512536292129E-39 evalInputDelta=5.416035991940972E-35
Fitness changed from 6.492185013736681E-35 to 1.0761490217957085E-35
Iteration 30 complete. Error: 1.0761490217957085E-35 Total: 0.3886; Orientation: 0.0005; Line Search: 0.2601
Zero gradient: 4.4156306536144585E-20
th(0)=1.0761490217957085E-35;dx=-1.949779406913965E-39
New Minimum: 1.0761490217957085E-35 > 2.769059275596759E-36
END: th(169.105269980088)=2.769059275596759E-36; dx=-3.3028058849114634E-40 evalInputDelta=7.992430942360327E-36
Fitness changed from 1.0761490217957085E-35 to 2.769059275596759E-36
Iteration 31 complete. Error: 2.769059275596759E-36 Total: 0.3869; Orientation: 0.0005; Line Search: 0.2585
Zero gradient: 1.9457997196171988E-20
th(0)=2.769059275596759E-36;dx=-3.78613654886237E-40
New Minimum: 2.769059275596759E-36 > 2.4738391808807342E-36
WOLF (strong): th(364.3262599123089)=2.4738391808807342E-36; dx=2.8033860036845985E-41 evalInputDelta=2.9522009471602487E-37
New Minimum: 2.4738391808807342E-36 > 1.2705787827554681E-37
END: th(182.16312995615445)=1.2705787827554681E-37; dx=-1.6217945260447083E-41 evalInputDelta=2.642001397321212E-36
Fitness changed from 2.769059275596759E-36 to 1.2705787827554681E-37
Iteration 32 complete. Error: 1.2705787827554681E-37 Total: 0.5184; Orientation: 0.0005; Line Search: 0.3890
Zero gradient: 4.534764916176609E-21
th(0)=1.2705787827554681E-37;dx=-2.0564092844986245E-41
Armijo: th(392.45856642232536)=4.008787513872691E-37; dx=-8.147248361155476E-42 evalInputDelta=-2.738208731117223E-37
Armijo: th(196.22928321116268)=4.921820968989602E-37; dx=-2.351701818485847E-41 evalInputDelta=-3.6512421862341343E-37
Armijo: th(65.40976107038756)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(16.35244026759689)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(3.270488053519378)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(0.5450813422532297)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(0.07786876317903281)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(0.009733595397379101)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(0.001081510599708789)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(1.081510599708789E-4)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(9.831914542807173E-6)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(8.193262119005978E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
WOLFE (weak): th(6.302509322312291E-8)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
WOLFE (weak): th(4.4117565256186034E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(6.30250932231229E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(5.357132923965447E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
WOLFE (weak): th(4.884444724792026E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(5.120788824378737E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
WOLFE (weak): th(5.002616774585381E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
WOLFE (weak): th(5.061702799482059E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Armijo: th(5.091245811930398E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
mu ~= nu (5.061702799482059E-7): th(0.0)=1.2705787827554681E-37
Fitness changed from 1.2705787827554681E-37 to 1.2705787827554681E-37
Static Iteration Total: 3.2418; Orientation: 0.0005; Line Search: 3.1130
Iteration 33 failed. Error: 1.2705787827554681E-37
Previous Error: 0.0 -> 1.2705787827554681E-37
Optimization terminated 33
Final threshold in iteration 33: 1.2705787827554681E-37 (> 0.0) after 19.344s (< 30.000s)

Returns

    1.2705787827554681E-37

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 40.60 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11852634865567
Reset training subject: 11852763691500
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=197.50009284723055}, derivative=-350.5025323910296}
New Minimum: 197.50009284723055 > 197.5000928121803
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=197.5000928121803}, derivative=-350.502532359925}, evalInputDelta = -3.505024892547226E-8
New Minimum: 197.5000928121803 > 197.50009260187878
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=197.50009260187878}, derivative=-350.5025321732995}, evalInputDelta = -2.4535177090001525E-7
New Minimum: 197.50009260187878 > 197.50009112976812
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=197.50009112976812}, derivative=-350.5025308669234}, evalInputDelta = -1.7174624247218162E-6
New Minimum: 197.50009112976812 > 197.50008082499386
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=197.50008082499386}, derivative=-350.5025217222767}, evalInputDelta = -1.2022236688835619E-5
New Minimum: 197.50008082499386 > 197.50000869158146
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=197.50000869158146}, derivative=-350.502457709762}, evalInputDelta = -8.415564909114437E-5
New Minimum: 197.50000869158146 > 197.49950375806367
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=197.49950375806367}, derivative=-350.5020096221569}, evalInputDelta = -5.890891668798304E-4
New Minimum: 197.49950375806367 > 197.49596924151334
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=197.49596924151334}, derivative=-350.4988730089195}, evalInputDelta = -0.004123605717211376
New Minimum: 197.49596924151334 > 197.47122851130774
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=197.47122851130774}, derivative=-350.4769167162593}, evalInputDelta = -0.028864335922804685
New Minimum: 197.47122851130774 > 197.29808679655116
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=197.29808679655116}, derivative=-350.3232226676414}, evalInputDelta = -0.20200605067938682
New Minimum: 197.29808679655116 > 196.0882212307074
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=196.0882212307074}, derivative=-349.24736432731476}, evalInputDelta = -1.4118716165231433
New Minimum: 196.0882212307074 > 187.723357704969
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=187.723357704969}, derivative=-341.71635594502743}, evalInputDelta = -9.776735142261543
New Minimum: 187.723357704969 > 134.27488934801892
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=134.27488934801892}, derivative=-288.9992972690127}, evalInputDelta = -63.22520349921163
New Minimum: 134.27488934801892 > 10.308850687088075
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=10.308850687088075}, derivative=80.02011346308703}, evalInputDelta = -187.19124216014248
10.308850687088075 <= 197.50009284723055
New Minimum: 10.308850687088075 > 0.015688650843483202
F(1.1268643501614906) = LineSearchPoint{point=PointSample{avg=0.015688650843483202}, derivative=1.2644746361090142E-13}, evalInputDelta = -197.48440419638706
Right bracket at 1.1268643501614906
Converged to right
Fitness changed from 197.50009284723055 to 0.015688650843483202
Iteration 1 complete. Error: 0.015688650843483202 Total: 2.4576; Orientation: 0.0004; Line Search: 2.0716
F(0.0) = LineSearchPoint{point=PointSample{avg=0.015688650843483202}, derivative=-0.027842298374168108}
New Minimum: 0.015688650843483202 > 2.3702162583279535E-6
F(1.1268643501614906) = LineSearchPoint{point=PointSample{avg=2.3702162583279535E-6}, derivative=1.7146783987159356E-6}, evalInputDelta = -0.015686280627224873
2.3702162583279535E-6 <= 0.015688650843483202
Converged to right
Fitness changed from 0.015688650843483202 to 2.3702162583279535E-6
Iteration 2 complete. Error: 2.3702162583279535E-6 Total: 0.3881; Orientation: 0.0004; Line Search: 0.2595
F(0.0) = LineSearchPoint{point=PointSample{avg=2.3702162583279535E-6}, derivative=-4.205912848875189E-6}
New Minimum: 2.3702162583279535E-6 > 4.53922734942658E-10
F(1.1268643501614906) = LineSearchPoint{point=PointSample{avg=4.53922734942658E-10}, derivative=-2.7884370018795134E-11}, evalInputDelta = -2.3697623355930107E-6
F(7.888050451130434) = LineSearchPoint{point=PointSample{avg=8.5310578284182E-5}, derivative=2.5235281902661545E-5}, evalInputDelta = 8.294036202585404E-5
F(0.606773111625418) = LineSearchPoint{point=PointSample{avg=5.052631759967459E-7}, derivative=-1.9412055602953824E-6}, evalInputDelta = -1.8649530823312074E-6
F(4.247411781377926) = LineSearchPoint{point=PointSample{avg=1.8172977938667754E-5}, derivative=1.164703817118297E-5}, evalInputDelta = 1.58027616803398E-5
F(0.3267239831829174) = LineSearchPoint{point=PointSample{avg=1.1952567095185025E-6}, derivative=-2.9864550781014644E-6}, evalInputDelta = -1.174959548809451E-6
F(2.2870678822804216) = LineSearchPoint{point=PointSample{avg=2.512447520663419E-6}, derivative=4.33029154654082E-6}, evalInputDelta = 1.422312623354655E-7
F(0.1759282986369555) = LineSearchPoint{point=PointSample{avg=1.6880371633833896E-6}, derivative=-3.5492817415356693E-6}, evalInputDelta = -6.821790949445639E-7
F(1.2314980904586885) = LineSearchPoint{point=PointSample{avg=2.0882458188599928E-8}, derivative=3.905049025026179E-7}, evalInputDelta = -2.3493338001393533E-6
2.0882458188599928E-8 <= 2.3702162583279535E-6
New Minimum: 4.53922734942658E-10 > 4.539226307825677E-10
F(1.1268718210986908) = LineSearchPoint{point=PointSample{avg=4.539226307825677E-10}, derivative=-1.4725055674362722E-20}, evalInputDelta = -2.369762335697171E-6
Left bracket at 1.1268718210986908
Converged to left
Fitness changed from 2.3702162583279535E-6 to 4.539226307825677E-10
Iteration 3 complete. Error: 4.539226307825677E-10 Total: 1.5540; Orientation: 0.0004; Line Search: 1.4256
F(0.0) = LineSearchPoint{point=PointSample{avg=4.539226307825677E-10}, derivative=-8.052622402701369E-10}
New Minimum: 4.539226307825677E-10 > 9.874904669187377E-14
F(1.1268718210986908) = LineSearchPoint{point=PointSample{avg=9.874904669187377E-14}, derivative=-1.956179157068486E-13}, evalInputDelta = -4.5382388173587585E-10
F(7.888102747690835) = LineSearchPoint{point=PointSample{avg=1.6328500166236942E-8}, derivative=4.830204116212462E-9}, evalInputDelta = 1.5874577535454375E-8
F(0.6067771344377566) = LineSearchPoint{point=PointSample{avg=9.682607489139233E-11}, derivative=-3.7176482823254517E-10}, evalInputDelta = -3.570965558911754E-10
F(4.247439941064297) = LineSearchPoint{point=PointSample{avg=3.478009406245855E-9}, derivative=2.229219643988948E-9}, evalInputDelta = 3.024086775463287E-9
F(0.3267261493126382) = LineSearchPoint{point=PointSample{avg=2.2895488372543735E-10}, derivative=-5.718405568664249E-10}, evalInputDelta = -2.2496774705713036E-10
F(2.2870830451884676) = LineSearchPoint{point=PointSample{avg=4.807127248601719E-10}, derivative=8.286895435628317E-10}, evalInputDelta = 2.6790094077604154E-11
F(0.17592946501449752) = LineSearchPoint{point=PointSample{avg=3.2330943962336945E-10}, derivative=-6.795736415128131E-10}, evalInputDelta = -1.3061319115919827E-10
F(1.2315062551014826) = LineSearchPoint{point=PointSample{avg=3.9891793645412085E-12}, derivative=7.455795102252013E-11}, evalInputDelta = -4.499334514180265E-10
3.9891793645412085E-12 <= 4.539226307825677E-10
New Minimum: 9.874904669187377E-14 > 9.87222655383182E-14
F(1.1271456323736944) = LineSearchPoint{point=PointSample{avg=9.87222655383182E-14}, derivative=2.755361642530793E-21}, evalInputDelta = -4.538239085170294E-10
Right bracket at 1.1271456323736944
Converged to right
Fitness changed from 4.539226307825677E-10 to 9.87222655383182E-14
Iteration 4 complete. Error: 9.87222655383182E-14 Total: 1.8498; Orientation: 0.0004; Line Search: 1.7210
Low gradient: 4.1852733086013007E-7
F(0.0) = LineSearchPoint{point=PointSample{avg=9.87222655383182E-14}, derivative=-1.751651266769048E-13}
New Minimum: 9.87222655383182E-14 > 2.3231671317558546E-17
F(1.1271456323736944) = LineSearchPoint{point=PointSample{avg=2.3231671317558546E-17}, derivative=3.419234485125625E-17}, evalInputDelta = -9.869903386700063E-14
2.3231671317558

...skipping 32557 bytes...

1891860120786E-39}, evalInputDelta = -4.793425639363758E-37
New Minimum: 9.932448707637061E-36 > 8.471595179450003E-36
F(95.00624255160115) = LineSearchPoint{point=PointSample{avg=8.471595179450003E-36}, derivative=-7.609232664829353E-40}, evalInputDelta = -3.1402644396927283E-35
F(665.043697861208) = LineSearchPoint{point=PointSample{avg=8.121787810343562E-33}, derivative=8.542466557841073E-38}, evalInputDelta = 8.081913570767184E-33
F(51.15720752778523) = LineSearchPoint{point=PointSample{avg=2.617873774962361E-35}, derivative=-3.387563527641704E-39}, evalInputDelta = -1.3695501826753674E-35
F(358.10045269449665) = LineSearchPoint{point=PointSample{avg=5.968062082017444E-33}, derivative=5.602766261272267E-38}, evalInputDelta = 5.928187842441067E-33
F(27.546188668807435) = LineSearchPoint{point=PointSample{avg=3.416778048189659E-35}, derivative=-4.7498085960066877E-39}, evalInputDelta = -5.7064590944806964E-36
F(192.82332068165204) = LineSearchPoint{point=PointSample{avg=2.08705930664746E-33}, derivative=2.621543721783386E-38}, evalInputDelta = 2.047185067071083E-33
F(14.83256312935785) = LineSearchPoint{point=PointSample{avg=3.784274013874216E-35}, derivative=-5.395479042378074E-39}, evalInputDelta = -2.0314994376351272E-36
New Minimum: 8.471595179450003E-36 > 8.380291833938311E-36
F(103.82794190550494) = LineSearchPoint{point=PointSample{avg=8.380291833938311E-36}, derivative=-7.208021697765427E-40}, evalInputDelta = -3.1493947742438973E-35
F(726.7955933385346) = LineSearchPoint{point=PointSample{avg=1.0450091598400821E-32}, derivative=1.1391257819630338E-37}, evalInputDelta = 1.0410217358824444E-32
F(55.907353333733425) = LineSearchPoint{point=PointSample{avg=1.9062783258806181E-35}, derivative=-2.6548476434785154E-39}, evalInputDelta = -2.0811456317571106E-35
F(391.35147333613395) = LineSearchPoint{point=PointSample{avg=5.9846336392278165E-33}, derivative=5.644075520750405E-38}, evalInputDelta = 5.944759399651439E-33
F(30.10395948739492) = LineSearchPoint{point=PointSample{avg=3.416778048189659E-35}, derivative=-4.7498085960066877E-39}, evalInputDelta = -5.7064590944806964E-36
F(210.72771641176445) = LineSearchPoint{point=PointSample{avg=2.0888853735576936E-33}, derivative=2.6060040606459807E-38}, evalInputDelta = 2.0490111339813164E-33
F(16.209824339366495) = LineSearchPoint{point=PointSample{avg=3.74775267566954E-35}, derivative=-5.304244567994385E-39}, evalInputDelta = -2.3967128196818897E-36
New Minimum: 8.380291833938311E-36 > 6.554224923704488E-36
F(113.46877037556547) = LineSearchPoint{point=PointSample{avg=6.554224923704488E-36}, derivative=-5.926377791130983E-40}, evalInputDelta = -3.3320014652672797E-35
F(794.2813926289583) = LineSearchPoint{point=PointSample{avg=1.0534035038295406E-32}, derivative=1.1780794679683584E-37}, evalInputDelta = 1.0494160798719028E-32
F(61.098568663766024) = LineSearchPoint{point=PointSample{avg=1.824105314920096E-35}, derivative=-2.474687462795778E-39}, evalInputDelta = -2.1633186427176327E-35
F(427.6899806463622) = LineSearchPoint{point=PointSample{avg=6.287218632712655E-33}, derivative=6.357426478844249E-38}, evalInputDelta = 6.247344393136278E-33
F(32.8992292804894) = LineSearchPoint{point=PointSample{avg=3.435038717291997E-35}, derivative=-4.768972187056788E-39}, evalInputDelta = -5.523852403457315E-36
F(230.2946049634258) = LineSearchPoint{point=PointSample{avg=2.0904603562677703E-33}, derivative=2.611996164125232E-38}, evalInputDelta = 2.050586116691393E-33
F(17.714969612571217) = LineSearchPoint{point=PointSample{avg=3.7386223411183707E-35}, derivative=-5.31614969017614E-39}, evalInputDelta = -2.4880161651935803E-36
F(124.00478728799851) = LineSearchPoint{point=PointSample{avg=1.0559035748895712E-33}, derivative=1.2557204393486535E-38}, evalInputDelta = 1.016029335313194E-33
F(9.5388297913845) = LineSearchPoint{point=PointSample{avg=3.939489701244091E-35}, derivative=-5.581891860120786E-39}, evalInputDelta = -4.793425639363758E-37
F(66.7718085396915) = LineSearchPoint{point=PointSample{avg=1.0845482162753974E-35}, derivative=-1.1393654751172255E-39}, evalInputDelta = -2.9028757413623316E-35
F(467.40265977784054) = LineSearchPoint{point=PointSample{avg=6.315009088502776E-33}, derivative=6.497547971825339E-38}, evalInputDelta = 6.275134848926399E-33
F(35.95405075214158) = LineSearchPoint{point=PointSample{avg=3.352865706331475E-35}, derivative=-4.570194966618678E-39}, evalInputDelta = -6.345582513062536E-36
F(251.67835526499107) = LineSearchPoint{point=PointSample{avg=2.0918299064504458E-33}, derivative=2.6145755593838976E-38}, evalInputDelta = 2.0519556668740686E-33
F(19.35987348192239) = LineSearchPoint{point=PointSample{avg=3.7021010029136944E-35}, derivative=-5.255396402140753E-39}, evalInputDelta = -2.853229547240343E-36
F(135.51911437345672) = LineSearchPoint{point=PointSample{avg=1.055629664853036E-33}, derivative=1.261028214167656E-38}, evalInputDelta = 1.0157554252766588E-33
F(10.42454725949667) = LineSearchPoint{point=PointSample{avg=3.939489701244091E-35}, derivative=-5.581891860120786E-39}, evalInputDelta = -4.793425639363758E-37
F(72.97183081647668) = LineSearchPoint{point=PointSample{avg=1.0388965435195518E-35}, derivative=-1.1083325341979809E-39}, evalInputDelta = -2.948527414118177E-35
F(510.8028157153368) = LineSearchPoint{point=PointSample{avg=6.386431130529296E-33}, derivative=6.814538084313045E-38}, evalInputDelta = 6.346556890952919E-33
F(39.29252428579514) = LineSearchPoint{point=PointSample{avg=2.763959127781067E-35}, derivative=-3.718447698191902E-39}, evalInputDelta = -1.223464829856662E-35
F(275.047670000566) = LineSearchPoint{point=PointSample{avg=2.0939698286108757E-33}, derivative=2.62290390985205E-38}, evalInputDelta = 2.0540955890344985E-33
Loops = 52
F(48.96517135422749) = LineSearchPoint{point=PointSample{avg=2.617873774962361E-35}, derivative=-3.387563527641704E-39}, evalInputDelta = -1.3695501826753674E-35
Left bracket at 48.96517135422749
F(74.82461365149832) = LineSearchPoint{point=PointSample{avg=1.0388965435195518E-35}, derivative=-1.1083325341979809E-39}, evalInputDelta = -2.948527414118177E-35
Left bracket at 74.82461365149832
F(82.94220924297181) = LineSearchPoint{point=PointSample{avg=9.932448707637061E-36}, derivative=-1.0244051874479842E-39}, evalInputDelta = -2.9941790868740227E-35
Left bracket at 82.94220924297181
F(90.16308850036843) = LineSearchPoint{point=PointSample{avg=9.567235325590296E-36}, derivative=-8.847062239890553E-40}, evalInputDelta = -3.030700425078699E-35
Left bracket at 90.16308850036843
F(96.19576816255599) = LineSearchPoint{point=PointSample{avg=8.380291833938311E-36}, derivative=-7.208021697765427E-40}, evalInputDelta = -3.1493947742438973E-35
Left bracket at 96.19576816255599
F(100.97935251209175) = LineSearchPoint{point=PointSample{avg=8.380291833938311E-36}, derivative=-7.208021697765427E-40}, evalInputDelta = -3.1493947742438973E-35
Left bracket at 100.97935251209175
F(105.63499481016503) = LineSearchPoint{point=PointSample{avg=6.919438305751253E-36}, derivative=-6.324648005936983E-40}, evalInputDelta = -3.2954801270626034E-35
Left bracket at 105.63499481016503
F(109.62388367635677) = LineSearchPoint{point=PointSample{avg=6.554224923704488E-36}, derivative=-5.926377791130983E-40}, evalInputDelta = -3.3320014652672797E-35
Left bracket at 109.62388367635677
F(113.27900135549825) = LineSearchPoint{point=PointSample{avg=6.554224923704488E-36}, derivative=-5.926377791130983E-40}, evalInputDelta = -3.3320014652672797E-35
Left bracket at 113.27900135549825
F(116.85335746131805) = LineSearchPoint{point=PointSample{avg=6.554224923704488E-36}, derivative=-5.926377791130983E-40}, evalInputDelta = -3.3320014652672797E-35
Left bracket at 116.85335746131805
F(120.34873645940608) = LineSearchPoint{point=PointSample{avg=1.0558122715440596E-33}, derivative=1.25066599511336E-38}, evalInputDelta = 1.0159380319676824E-33
Right bracket at 120.34873645940608
Converged to left
Fitness changed from 3.9874239576377287E-35 to 6.554224923704488E-36
Iteration 15 complete. Error: 6.554224923704488E-36 Total: 10.6082; Orientation: 0.0006; Line Search: 10.4795
Final threshold in iteration 15: 6.554224923704488E-36 (> 0.0) after 40.604s (< 30.000s)

Returns

    6.554224923704488E-36

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 29.36 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11893274616331
Reset training subject: 11893403306234
Adding measurement 1a93817a to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 197.50009284723055 < 197.50009284723055. Total: 1
th(0)=197.50009284723055;dx=-350.5025323910296
Adding measurement 2beb2e6f to history. Total: 1
New Minimum: 197.50009284723055 > 164.23062077015743
WOLF (strong): th(2.154434690031884)=164.23062077015743; dx=319.61789037240277 evalInputDelta=33.26947207707312
Adding measurement 206aa3c3 to history. Total: 2
New Minimum: 164.23062077015743 > 0.39902114614778006
END: th(1.077217345015942)=0.39902114614778006; dx=-15.442321009313467 evalInputDelta=197.10107170108276
Fitness changed from 197.50009284723055 to 0.39902114614778006
Iteration 1 complete. Error: 0.39902114614778006 Total: 0.7761; Orientation: 0.0014; Line Search: 0.3895
Non-optimal measurement 0.39902114614778006 < 0.39902114614778006. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 0.39902114614778006 < 0.39902114614778006. Total: 3
th(0)=0.39902114614778006;dx=-0.705795520831755
Non-optimal measurement 0.44216334587904316 < 0.39902114614778006. Total: 3
Armijo: th(2.3207944168063896)=0.44216334587904316; dx=0.7429743415138814 evalInputDelta=-0.0431421997312631
Adding measurement 33e01349 to history. Total: 3
New Minimum: 0.39902114614778006 > 3.0512004227218166E-4
WOLF (strong): th(1.1603972084031948)=3.0512004227218166E-4; dx=0.018589410341063383 evalInputDelta=0.3987160261055079
Non-optimal measurement 0.1727186650079131 < 3.0512004227218166E-4. Total: 4
END: th(0.3867990694677316)=0.1727186650079131; dx=-0.46433387710748214 evalInputDelta=0.22630248113986695
Fitness changed from 0.39902114614778006 to 3.0512004227218166E-4
Iteration 2 complete. Error: 3.0512004227218166E-4 Total: 0.9269; Orientation: 0.0010; Line Search: 0.7974
Non-optimal measurement 3.0512004227218166E-4 < 3.0512004227218166E-4. Total: 4
Rejected: LBFGS Orientation magnitude: 2.620e-02, gradient 2.329e-02, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3.0512004227218166E-4, 0.39902114614778006, 164.23062077015743, 197.50009284723055
LBFGS Accumulation History: 3 points
Removed measurement 33e01349 to history. Total: 3
Adding measurement 3290d97 to history. Total: 3
th(0)=3.0512004227218166E-4;dx=-5.42590608172533E-4
Adding measurement 494bdfa6 to history. Total: 4
New Minimum: 3.0512004227218166E-4 > 2.048664323224683E-5
END: th(0.8333333333333335)=2.048664323224683E-5; dx=-1.4052954952330918E-4 evalInputDelta=2.8463339903993484E-4
Fitness changed from 3.0512004227218166E-4 to 2.048664323224683E-5
Iteration 3 complete. Error: 2.048664323224683E-5 Total: 0.4468; Orientation: 0.0573; Line Search: 0.2612
Non-optimal measurement 2.048664323224683E-5 < 2.048664323224683E-5. Total: 5
Rejected: LBFGS Orientation magnitude: 6.789e-03, gradient 6.035e-03, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.048664323224683E-5, 3.0512004227218166E-4, 0.39902114614778006, 164.23062077015743, 197.50009284723055
Rejected: LBFGS Orientation magnitude: 6.789e-03, gradient 6.035e-03, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.048664323224683E-5, 3.0512004227218166E-4, 0.39902114614778006, 164.23062077015743
LBFGS Accumulation History: 3 points
Removed measurement 494bdfa6 to history. Total: 4
Removed measurement 3290d97 to history. Total: 3
Adding measurement 704011a5 to history. Total: 3
th(0)=2.048664323224683E-5;dx=-3.641694830878565E-5
Adding measurement 7d953514 to history. Total: 4
New Minimum: 2.048664323224683E-5 > 7.273749119213132E-6
WOLF (strong): th(1.7953622416932369)=7.273749119213132E-6; dx=2.1698031082841963E-5 evalInputDelta=1.32128941130337E-5
Adding measurement 6aba3427 to history. Total: 5
New Minimum: 7.273749119213132E-6 > 8.380162161664847E-7
END: th(0.8976811208466184)=8.380162161664847E-7; dx=-7.359458612972232E-6 evalInputDelta=1.9648627016080345E-5
Fitness changed from 2.048664323224683E-5 to 8.380162161664847E-7
Iteration 4 complete. Error: 8.380162161664847E-7 Total: 0.9718; Orientation: 0.4522; Line Search: 0.3901
Non-optimal measurement 8.380162161664847E-7 < 8.380162161664847E-7. Total: 6
Rejected: LBFGS Orientation magnitude: 1.374e-03, gradient 1.220e-03, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.380162161664847E-7, 7.273749119213132E-6, 2.048664323224683E-5, 0.39902114614778006, 164.23062077015743, 197.50009284723055
Rejected: LBFGS Orientation magnitude: 1.374e-03, gradient 1.220e-03, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.380162161664847E-7, 7.273749119213132E-6, 2.048664323224683E-5, 0.39902114614778006, 164.23062077015743
Rejected: LBFGS Orientation magnitude: 1.373e-03, gradient 1.220e-03, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.380162161664847E-7, 7.273749119213132E-6, 2.048664323224683E-5, 0.39902114614778006
LBFGS Accumulation History: 3 points
Removed measurement 6aba3427 to history. Total: 5
Removed measurement 7d953514 to history. Total: 4
Removed measurement 704011a5 to history. Total: 3
Adding measurement 1f9f730d to history. Total: 3
th(0)=8.380162161664847E-7;dx=-1.4888477425433545E-6
Adding measurement 6a9c19da to history. Total: 4
New Minimum: 8.380162161664847E-7 > 4.3218864157338577E-7
WOLF (strong): th(1.9339953473386584)=4.3218864157338577E-7; dx=1.069169820203413E-6 evalInputDelta=4.058275745930989E-7
Adding measurement b32944a to history. Total: 5
New Minimum: 4.3218864157338577E-7 > 1.6703170774604032E-8
END: th(0.9669976736693292)=1.6703170774604032E-8; dx=-2.0983896117006963E-7 evalInputDelta=8.213130453918806E-7
Fitness changed from 8.380162161664847E-7 to 1.6703170774604032E-8
Iteration 5 complete. Error: 1.6703170774604032E-8 Total: 1.0984; Orientation: 0.5807; Line Search: 0.3896
Non-optimal measurement 1.6703170774604032E-8 < 1.6703170774604032E-8. Total: 6
Rejected: LBFGS Orientation magnitude: 1.940e-04, gradient 1.722e-04, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.6703170774604032E-8, 4.3218864157338577E-7, 8.380162161664847E-7, 0.39902114614778006, 164.23062077015743, 197.50009284723055
Rejected: LBFGS Orientation magnitude: 1.940e-04, gradient 1.722e-04, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.6703170774604032E-8, 4.3218864157338577E-7, 8.380162161664847E-7, 0.39902114614778006, 164.23062077015743
Rejected: LBFGS Orientation magnitude: 1.940e-04, gradient 1.722e-04, dot -1.000; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.6703170774604032E-8, 4.3218864157338577E-7, 8.380162161664847E-7, 0.39902114614778006
LBFGS Accumulation History: 3 points
Removed measurement b32944a to history. Total: 5
Removed measurement 6a9c19da to history. Total: 4
Removed measurement 1f9f730d to history. Total: 3
Adding measurement 259ff8da to history. Total: 3
th(0)=1.6703170774604032E-8;dx=-2.9650491427796774E-8
Adding measurement 39911c40 to history. Total: 4
New Minimum: 1.6703170774604032E-8 > 1.2046610778629298E-8
WOLF (strong): th(2.083333333333334)=1.2046610778629298E-8; dx=2.5180193831662644E-8 evalInputDelta=4.656559995974734E-9
Adding measurement 50d1c902 to history. Total: 5
New Minimum: 1.2046610778629298E-8 > 9.606649029470043E-11
END: th(1.041666666666667)=9.606649029470043E-11; dx=-2.2351487980750623E-9 evalInputDelta=1.6607104284309333E-8
Fitness changed from 1.6703170774604032E-8 to 9.606649029470043E-11
Iteration 6 complete. Error: 9.606649029470043E-11 Total: 0.9269; Orientation: 0.4070; Line Search: 0.3912
Non-optimal measurement 9.606649029470043E-11 < 9.606649029470043E-11. Tot

...skipping 38523 bytes...

.23062077015743, 197.50009284723055
Rejected: LBFGS Orientation magnitude: 6.820e-19, gradient 4.416e-20, dot -0.242; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0761490217957085E-35, 6.492185013736681E-35, 0.39902114614778006, 164.23062077015743
LBFGS Accumulation History: 3 points
Removed measurement 60c6b197 to history. Total: 4
Removed measurement 11b148 to history. Total: 3
Adding measurement 2773a4e7 to history. Total: 3
th(0)=1.0761490217957085E-35;dx=-1.949779406913965E-39
Adding measurement e798b59 to history. Total: 4
New Minimum: 1.0761490217957085E-35 > 2.769059275596759E-36
END: th(169.105269980088)=2.769059275596759E-36; dx=-3.3028058849114634E-40 evalInputDelta=7.992430942360327E-36
Fitness changed from 1.0761490217957085E-35 to 2.769059275596759E-36
Iteration 31 complete. Error: 2.769059275596759E-36 Total: 0.7458; Orientation: 0.3576; Line Search: 0.2602
Non-optimal measurement 2.769059275596759E-36 < 2.769059275596759E-36. Total: 5
Rejected: LBFGS Orientation magnitude: 5.368e-20, gradient 1.946e-20, dot -0.422; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.769059275596759E-36, 1.0761490217957085E-35, 0.39902114614778006, 164.23062077015743, 197.50009284723055
Rejected: LBFGS Orientation magnitude: 5.368e-20, gradient 1.946e-20, dot -0.422; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.769059275596759E-36, 1.0761490217957085E-35, 0.39902114614778006, 164.23062077015743
LBFGS Accumulation History: 3 points
Removed measurement e798b59 to history. Total: 4
Removed measurement 2773a4e7 to history. Total: 3
Adding measurement 4c890ab7 to history. Total: 3
th(0)=2.769059275596759E-36;dx=-3.78613654886237E-40
Adding measurement 7dff10b3 to history. Total: 4
New Minimum: 2.769059275596759E-36 > 2.4738391808807342E-36
WOLF (strong): th(364.3262599123089)=2.4738391808807342E-36; dx=2.8033860036845985E-41 evalInputDelta=2.9522009471602487E-37
Adding measurement 1d5b1b76 to history. Total: 5
New Minimum: 2.4738391808807342E-36 > 1.2705787827554681E-37
END: th(182.16312995615445)=1.2705787827554681E-37; dx=-1.6217945260447083E-41 evalInputDelta=2.642001397321212E-36
Fitness changed from 2.769059275596759E-36 to 1.2705787827554681E-37
Iteration 32 complete. Error: 1.2705787827554681E-37 Total: 0.8168; Orientation: 0.2978; Line Search: 0.3903
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 6
Rejected: LBFGS Orientation magnitude: 4.651e-20, gradient 4.535e-21, dot -0.149; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2705787827554681E-37, 2.4738391808807342E-36, 2.769059275596759E-36, 0.39902114614778006, 164.23062077015743, 197.50009284723055
Rejected: LBFGS Orientation magnitude: 4.651e-20, gradient 4.535e-21, dot -0.149; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2705787827554681E-37, 2.4738391808807342E-36, 2.769059275596759E-36, 0.39902114614778006, 164.23062077015743
Rejected: LBFGS Orientation magnitude: 4.591e-20, gradient 4.535e-21, dot -0.150; [0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2705787827554681E-37, 2.4738391808807342E-36, 2.769059275596759E-36, 0.39902114614778006
LBFGS Accumulation History: 3 points
Removed measurement 1d5b1b76 to history. Total: 5
Removed measurement 7dff10b3 to history. Total: 4
Removed measurement 4c890ab7 to history. Total: 3
Adding measurement 42f4a7c8 to history. Total: 3
th(0)=1.2705787827554681E-37;dx=-2.0564092844986245E-41
Non-optimal measurement 4.008787513872691E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(392.45856642232536)=4.008787513872691E-37; dx=-8.147248361155476E-42 evalInputDelta=-2.738208731117223E-37
Non-optimal measurement 4.921820968989602E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(196.22928321116268)=4.921820968989602E-37; dx=-2.351701818485847E-41 evalInputDelta=-3.6512421862341343E-37
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(65.40976107038756)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(16.35244026759689)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(3.270488053519378)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(0.5450813422532297)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(0.07786876317903281)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(0.009733595397379101)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(0.001081510599708789)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(1.081510599708789E-4)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(9.831914542807173E-6)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(8.193262119005978E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
WOLFE (weak): th(6.302509322312291E-8)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
WOLFE (weak): th(4.4117565256186034E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(6.30250932231229E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(5.357132923965447E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
WOLFE (weak): th(4.884444724792026E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(5.120788824378737E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
WOLFE (weak): th(5.002616774585381E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
WOLFE (weak): th(5.061702799482059E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
Armijo: th(5.091245811930398E-7)=1.2705787827554681E-37; dx=-2.0564092844986245E-41 evalInputDelta=0.0
Non-optimal measurement 1.2705787827554681E-37 < 1.2705787827554681E-37. Total: 4
mu ~= nu (5.061702799482059E-7): th(0.0)=1.2705787827554681E-37
Fitness changed from 1.2705787827554681E-37 to 1.2705787827554681E-37
Static Iteration Total: 3.9466; Orientation: 0.5947; Line Search: 3.2239
Iteration 33 failed. Error: 1.2705787827554681E-37
Previous Error: 0.0 -> 1.2705787827554681E-37
Optimization terminated 33
Final threshold in iteration 33: 1.2705787827554681E-37 (> 0.0) after 29.360s (< 30.000s)

Returns

    1.2705787827554681E-37

Training Converged

TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -36.89599840117229], [32.0, -0.3990040882432118]; valueStats=DoubleSummaryStatistics{count=79, sum=0.814386, min=0.000000, average=0.010309, max=0.399021}
Plotting 32 points for GD
Plotting 15 points for CjGD
Plotting 32 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -36.89599840117229], [38.146, -0.3990040882432118]; valueStats=DoubleSummaryStatistics{count=79, sum=0.814386, min=0.000000, average=0.010309, max=0.399021}
Plotting 32 points for GD
Plotting 15 points for CjGD
Plotting 32 points for LBFGS

Returns

Result

Composite Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0037 = temp_18_0035.state();
    assert temp_18_0037 != null;
    String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0037.freeRef();
    return temp_18_0036;

Returns

    

We simultaneously regress this target input:

TrainingTester.java:287 executed in 0.01 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
      RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
      if (null != x)
        RefUtil.freeRef(x);
      return temp_18_0006;
    }).map(x -> {
      String temp_18_0007 = x.prettyPrint();
      x.freeRef();
      return temp_18_0007;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.848, 0.124, 1.152, 0.552, -1.792, 0.24 ], [ -0.18, 1.364, 1.516, -0.888, -0.688, 1.428 ], [ -0.08, 1.116, 0.74, -1.656, -0.792, 0.272 ], [ 0.944, -1.948, 0.996, -1.208, -1.024, -0.388 ], [ 1.432, -1.376, 0.288, 0.956, -0.84, -1.116 ], [ -0.224, -0.964, -0.624, -0.752, -0.944, 0.832 ], [ -0.38, 0.772, 0.016, -0.148, 0.556, -1.648 ], [ -0.792, -1.016, 0.068, -1.216, 1.196, 0.452 ], ... ],
    	[ [ -1.436, 1.752, -0.336, 0.692, -1.736, -1.712 ], [ -1.236, -1.032, 0.604, -1.18, 0.184, 0.792 ], [ -1.836, -1.576, 0.232, -0.924, -0.764, -1.8 ], [ 1.82, -1.04, 0.576, -1.28, -0.364, 1.116 ], [ 1.956, -0.172, 1.464, -0.788, -0.816, -0.468 ], [ 0.572, -0.896, 0.28, 1.016, -0.512, 1.3 ], [ 1.424, 0.352, 0.796, 1.076, -0.06, -1.484 ], [ -0.852, 1.06, -0.616, -1.128, 0.856, -1.816 ], ... ],
    	[ [ 1.772, -0.132, 0.02, -0.664, 1.656, -0.544 ], [ -0.868, 1.036, 0.208, 0.964, 0.956, -0.396 ], [ 1.452, -0.924, -0.736, -0.772, 0.992, -1.22 ], [ 1.72, 1.476, -0.116, 1.616, 1.3, 0.16 ], [ 1.508, -1.552, 1.724, -1.728, -1.888, -1.42 ], [ 1.12, -0.736, 0.84, 0.864, 1.832, -1.932 ], [ -1.888, 0.74, 1.492, -0.62, -1.104, -0.068 ], [ 1.072, 0.284, 1.396, 0.608, -1.684, -0.508 ], ... ],
    	[ [ -0.324, -1.952, -1.212, -1.124, 1.6, -0.104 ], [ -0.384, 1.516, -0.296, -0.728, -1.536, 0.268 ], [ 0.688, 1.304, -0.512, 1.832, -1.648, 0.484 ], [ -1.456, -1.588, -0.932, 1.764, -1.48, -0.244 ], [ 1.316, 0.12, -0.084, -0.836, 1.22, -1.344 ], [ 0.036, 0.864, -1.132, -1.28, 1.8, -0.8 ], [ -0.972, 0.944, -1.88, 0.172, -0.468, 0.444 ], [ 0.496, -1.596, -1.968, 0.3, -1.684, -1.66 ], ... ],
    	[ [ 0.18, 0.156, -0.664, -0.256, -0.976, -1.86 ], [ 0.232, 1.14, 1.312, 0.908, -1.464, -1.78 ], [ -0.608, 1.728, -1.5, 0.532, 1.24, 1.064 ], [ -0.316, 1.288, 0.936, -1.384, 0.284, 1.912 ], [ 1.648, 1.224, 0.34, -1.028, 0.732, 0.848 ], [ 1.712, 1.624, 0.64, -1.264, -0.308, 1.544 ], [ 0.396, -1.82, 1.72, -1.232, -1.036, -1.228 ], [ 0.188, 1.072, -1.452, -0.244, 1.876, -1.148 ], ... ],
    	[ [ -1.044, 1.748, -1.216, -0.62, 0.632, 1.284 ], [ 0.832, -0.964, 0.776, -1.28, 0.032, 0.092 ], [ -1.288, 1.624, 1.276, 0.476, 1.512, -0.112 ], [ -1.216, 0.86, 0.964, 0.748, 0.116, 0.916 ], [ 0.08, -1.84, 0.396, 1.312, -0.152, -0.056 ], [ -1.396, 0.412, 0.7, 0.864, 1.16, -1.256 ], [ 1.712, -1.788, -0.536, 1.404, 1.276, -0.564 ], [ 1.996, -0.556, 0.164, -1.984, -1.876, -1.256 ], ... ],
    	[ [ 0.424, 1.684, -0.504, 1.28, 0.688, 0.572 ], [ 0.556, 1.852, -1.028, 0.312, 1.7, 1.788 ], [ 1.684, 0.648, 1.364, -1.168, 0.724, 0.632 ], [ -0.412, 0.004, -0.3, 0.72, -0.772, -1.384 ], [ -1.04, -1.516, 1.404, -0.612, 1.092, -0.232 ], [ -0.64, -0.116, -0.224, -0.54, 1.304, 0.824 ], [ 0.088, -0.848, -0.044, -0.904, -1.5, -1.952 ], [ 0.164, -1.052, -0.516, -1.932, -1.856, 0.508 ], ... ],
    	[ [ -0.46, -1.916, -0.348, 0.424, -0.584, -1.876 ], [ 0.66, 1.196, -0.928, 1.02, 0.332, -1.884 ], [ 1.776, -1.296, -0.896, -1.212, 0.072, -0.404 ], [ 1.52, 0.452, -0.736, -0.968, -0.708, 0.148 ], [ -1.192, 0.172, 1.584, 0.296, -0.7, 0.064 ], [ -0.32, 1.252, -0.884, -1.78, -1.192, -0.832 ], [ -0.412, 1.904, 1.696, -0.264, 0.168, -1.888 ], [ 1.916, 0.62, -1.972, 1.716, -1.496, -0.404 ], ... ],
    	...
    ]
    [
    	[ [ 1.028, 1.36, -0.168, 1.792, 1.98, 1.436 ], [ 0.508, -1.356, 0.668, 0.244, -0.008, -0.836 ], [ 0.528, -1.796, 0.564, -0.724, -1.788, 1.504 ], [ -0.748, -1.616, 1.552, 1.388, 0.996, 0.324 ], [ -0.704, 0.4, -1.812, 1.408, 0.58, -1.532 ], [ 1.348, -1.428, 1.56, -1.448, -1.956, -1.896 ], [ 0.524, -1.26, 1.492, 0.692, -0.34, 0.044 ], [ 1.344, 1.576, -0.548, -0.736, -1.696, -0.08 ], ... ],
    	[ [ -0.292, 1.152, -1.952, 1.572, 0.64, 1.024 ], [ 1.46, 1.58, 1.204, 0.812, 1.208, -1.288 ], [ 0.404, -0.88, -1.572, 1.22, 0.024, 1.076 ], [ -0.544, -1.444, 1.48, -0.616, 0.112, -0.544 ], [ -1.62, 0.844, -0.344, -0.812, 1.048, 1.208 ], [ 1.92, -1.316, 1.816, -0.556, -1.264, 1.912 ], [ -1.856, 1.688, -0.636, -1.772, -0.764, -1.08 ], [ 0.1, 1.136, -0.82, -1.232, -0.184, 1.104 ], ... ],
    	[ [ 0.66, 1.636, -1.132, -0.356, 0.828, -1.888 ], [ -1.292, -0.328, 1.028, -0.56, -0.948, 1.928 ], [ -1.48, 0.212, -1.96, 0.7, -0.64, 1.664 ], [ 0.364, 0.436, -1.356, 1.852, 1.664, -1.108 ], [ 1.02, -1.352, -1.02, 1.844, 0.344, -1.436 ], [ 0.228, 0.352, 0.152, 1.572, 0.704, -1.1 ], [ -1.408, 1.796, 1.4, 1.536, -1.748, -1.712 ], [ -1.84, 1.508, -1.916, -0.912, -1.456, -0.712 ], ... ],
    	[ [ 0.148, 1.164, 1.212, 0.184, -0.9, -1.592 ], [ 1.812, -0.828, -0.752, 1.852, -1.672, -1.116 ], [ 0.164, 1.616, 0.74, -1.076, -1.952, -0.332 ], [ -0.768, -0.272, 1.576, 1.64, 1.284, -0.76 ], [ 0.564, 1.876, 0.976, 1.776, 1.376, 1.936 ], [ -1.14, 1.772, 0.376, 0.868, 0.904, -1.596 ], [ -1.18, -0.024, -0.096, 1.296, -1.448, 1.44 ], [ -1.944, 1.488, -0.824, 0.284, -1.148, 1.728 ], ... ],
    	[ [ -1.428, 1.112, -0.248, 1.596, 0.56, -1.284 ], [ -0.816, 0.984, -0.492, -0.02, 1.38, -0.376 ], [ 0.176, -0.856, 1.628, 1.136, -0.512, 1.36 ], [ 0.992, 1.38, -1.88, -1.592, 0.068, -1.704 ], [ 1.692, 0.3, 1.064, -0.856, 1.584, -1.384 ], [ 1.148, -0.044, 0.132, -2.0, 0.616, 0.828 ], [ 1.324, 1.052, -0.828, -0.344, -1.328, 0.008 ], [ 0.228, -0.176, -1.168, 0.112, 0.16, 0.824 ], ... ],
    	[ [ -1.636, 0.5, -0.364, -0.092, -0.284, 0.052 ], [ -0.568, -0.892, 1.456, 0.192, -1.728, -1.504 ], [ -0.212, 0.684, 0.052, -0.372, 0.852, -1.388 ], [ 0.02, -0.28, 1.372, -1.46, 1.204, -0.796 ], [ 1.78, 0.608, 1.712, 0.44, -0.768, -1.976 ], [ 1.544, 1.784, -1.376, -0.104, -0.712, -1.348 ], [ -0.956, 1.632, -0.648, 1.792, 0.368, 0.204 ], [ -0.468, -0.28, -0.992, -1.148, 0.68, 0.628 ], ... ],
    	[ [ -1.272, 1.392, -0.56, 1.788, 1.504, 0.736 ], [ -1.168, -1.948, -1.492, 1.024, 0.952, 1.3 ], [ -0.648, 0.336, 0.616, -1.12, 0.192, -0.348 ], [ 0.224, -1.932, -0.328, 0.832, -0.644, -0.896 ], [ 0.372, -1.428, -0.58, 1.776, -0.368, -1.328 ], [ 0.736, -1.1, 1.28, -0.284, -1.84, -1.012 ], [ -0.236, -1.3, -0.556, 0.864, -1.028, -0.76 ], [ -1.456, 0.656, 1.328, 0.244, -0.692, -1.884 ], ... ],
    	[ [ 1.648, 0.068, -1.256, 0.16, 1.892, 0.528 ], [ -0.74, 0.268, 1.964, 1.684, -0.14, 0.236 ], [ 0.428, -1.508, -0.26, -1.428, -1.292, 0.092 ], [ 0.268, 1.968, -1.192, 0.016, 0.028, 1.46 ], [ -0.696, -1.868, -0.148, -0.68, 0.356, 0.828 ], [ 1.744, -1.78, -0.84, -0.628, -0.98, -0.628 ], [ 1.784, 0.168, 0.996, -1.96, -1.992, -1.3 ], [ 1.988, 0.876, -1.448, 1.96, -0.1, 0.4 ], ... ],
    	...
    ]

Which produces the following output:

TrainingTester.java:308 executed in 0.01 seconds (0.000 gc):

    return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
      String temp_18_0008 = x.prettyPrint();
      x.freeRef();
      return temp_18_0008;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.9930879999999995, -1.9576160000000002, -1.14192 ], [ -8.985568, -11.899856000000002, 3.0366719999999994 ], [ 2.470224000000001, -14.664464000000002, -4.092496000000001 ], [ 9.204448000000001, -17.392496, 2.940864 ], [ 10.335616, 2.7864319999999996, 1.5711520000000005 ], [ 7.230255999999999, 1.6868800000000008, -2.715904000000001 ], [ -6.111663999999999, 9.486416000000002, -4.083664000000001 ], [ -0.8759999999999996, 5.470448000000001, -0.9024800000000001 ], ... ],
    	[ [ -8.936048000000001, -0.49555199999999966, -7.476928 ], [ 3.399984000000001, -2.3827839999999987, -15.237231999999997 ], [ -5.686432, -2.3936000000000024, 6.400527999999999 ], [ 8.735824, 8.873968000000003, 9.940144000000002 ], [ 8.906847999999998, -22.998511999999998, 13.743711999999999 ], [ 13.501599999999991, 9.550272, 2.5570720000000002 ], [ 16.358176, -11.367984000000002, -4.27872 ], [ -4.645759999999995, 0.5012000000000001, -11.667616 ], ... ],
    	[ [ 4.212320000000001, 2.519055999999999, 4.679263999999998 ], [ 22.479600000000005, 11.070751999999999, -18.914095999999997 ], [ -1.901376, -1.9589439999999996, 9.672815999999997 ], [ 10.798480000000001, -2.204735999999995, -1.8902399999999973 ], [ -0.8229439999999981, -10.753647999999997, 6.673232000000001 ], [ 9.39136, 8.273871999999995, 3.4660960000000016 ], [ 7.2151359999999976, -3.813648000000001, 0.772896 ], [ 9.297296000000001, 21.994384, -2.2930719999999996 ], ... ],
    	[ [ -12.680736000000001, 2.7979679999999987, 6.769232000000001 ], [ -5.325375999999996, 14.986607999999993, 0.6911999999999987 ], [ -14.694047999999999, 4.391264, -2.223728000000006 ], [ 28.627759999999995, -2.0284639999999996, -20.327680000000004 ], [ -6.339824000000001, 11.172992000000006, 9.47688 ], [ 3.8557759999999988, 6.213136, -14.530031999999999 ], [ -4.666400000000001, 14.897072, -1.246288 ], [ -6.510128000000002, -1.7718399999999999, 2.719728000000001 ], ... ],
    	[ [ 4.746624, -5.840048, -0.19699199999999945 ], [ 1.4760319999999998, 10.875312000000005, 0.46915200000000007 ], [ -6.005647999999997, 3.463247999999999, -11.783808000000002 ], [ -10.547120000000003, -17.148352000000006, -7.826784 ], [ -22.959664, -11.507712000000001, 15.147712000000002 ], [ -13.005056, 2.771120000000001, -3.6613599999999993 ], [ 13.615999999999998, -2.925488000000004, -5.749535999999999 ], [ 3.377711999999999, 1.350256, 8.328575999999998 ], ... ],
    	[ [ -13.410192000000002, 18.984752, -2.187456 ], [ -22.79033600000001, -6.480928000000002, 6.026496000000002 ], [ -2.956719999999998, 6.172144000000004, -15.219007999999997 ], [ -2.1928960000000006, -6.800943999999999, -2.666768000000001 ], [ -4.127280000000001, -18.404624000000002, 4.990768 ], [ 25.815696000000003, 1.3132320000000024, -11.888416 ], [ -1.1597279999999983, -15.144768000000001, 16.095679999999994 ], [ 16.058640000000008, 3.481711999999999, -11.734656000000001 ], ... ],
    	[ [ 7.880975999999999, 5.354576000000002, 0.38291199999999953 ], [ -12.650208000000001, -10.581792, 12.031952 ], [ 2.09104, -2.4951360000000005, 6.520464 ], [ 10.305823999999998, 4.998160000000002, -5.987088 ], [ 3.179696000000001, -2.3377280000000007, -2.0170879999999984 ], [ -12.04328, 13.838607999999997, 13.646256000000001 ], [ -1.9174079999999991, -0.892672, -7.432112000000001 ], [ -14.593856000000004, 3.066832000000001, 12.856815999999995 ], ... ],
    	[ [ 4.986063999999999, -9.598432, -5.563616 ], [ 16.046111999999997, 10.810431999999999, -19.720672000000004 ], [ -8.436592000000001, 0.18905600000000242, -6.581055999999999 ], [ -2.670928000000001, -7.707423999999999, -5.605247999999997 ], [ -11.269216, 10.899472000000001, -8.678016000000001 ], [ 4.260671999999998, 6.136000000000003, 0.8545440000000025 ], [ 4.770719999999998, 10.030592000000004, 1.7810080000000001 ], [ 9.492767999999996, 3.108464, -3.3934560000000014 ], ... ],
    	...
    ]
    [
    	[ [ -2.3230399999999998, 8.705232, 6.005903999999999 ], [ -12.142895999999997, -9.272896, 1.8103999999999985 ], [ 9.406991999999999, -6.095375999999997, 3.8570720000000005 ], [ 8.672399999999998, 8.352032000000001, 5.199455999999999 ], [ 7.572592, -8.683679999999999, 0.27844800000000114 ], [ 9.988879999999998, -11.404848, -7.530224000000001 ], [ 7.383056000000001, 9.933775999999998, 3.334672 ], [ -12.68416, -6.179984, 2.6051200000000008 ], ... ],
    	[ [ 18.062528, 18.523408, -7.5012479999999995 ], [ 11.044528000000003, -11.211391999999998, 3.230575999999998 ], [ -9.466975999999999, -5.390223999999998, 4.473232000000002 ], [ -10.923616, 5.5955679999999965, 10.996927999999999 ], [ 8.071968000000004, 15.963408000000001, 8.307039999999997 ], [ -16.681264, 15.514399999999998, 11.848287999999997 ], [ -8.338224000000004, -7.697648000000002, -13.21232 ], [ -5.071072000000003, 1.721936, -11.734176 ], ... ],
    	[ [ -2.7716799999999977, -5.964192000000002, -14.68224 ], [ -9.91848, 6.336688, -3.1230879999999974 ], [ 7.909871999999999, 6.518879999999997, -7.921808000000003 ], [ -8.022959999999998, -3.122848000000003, 11.536591999999999 ], [ 6.664656000000001, 3.8859040000000014, -6.274639999999999 ], [ 2.1310240000000005, -3.1968639999999984, -4.0754079999999995 ], [ -4.146767999999997, 2.3564160000000025, -15.641024000000005 ], [ -8.048624000000002, -5.179568000000002, -27.409584 ], ... ],
    	[ [ -0.009199999999998786, 10.523856, -6.796528 ], [ -7.022736000000001, -15.551552000000003, 2.730320000000002 ], [ 9.465744, -14.565727999999998, -9.532271999999999 ], [ 6.418448, 0.5471039999999987, 14.818479999999997 ], [ 3.7333119999999993, 13.540192, 19.480384 ], [ 8.13704, -6.020816000000002, 5.448432 ], [ -6.968431999999997, 7.317088, 5.73368 ], [ -3.250432000000002, 2.472176, -10.8096 ], ... ],
    	[ [ 4.693887999999999, 8.742336000000002, -2.660911999999999 ], [ -0.44924799999999854, -17.25672, -8.094239999999997 ], [ -7.586399999999998, 10.114880000000001, 4.769791999999999 ], [ -9.085792000000003, -22.430656000000003, 1.9690880000000006 ], [ 5.139888000000001, 4.640480000000001, -9.03512 ], [ -20.406831999999998, 13.649423999999996, 11.893888 ], [ -13.996448000000003, -4.707424, -2.878656000000001 ], [ 7.072560000000001, -6.975375999999999, -12.338687999999996 ], ... ],
    	[ [ -11.942544000000002, 8.340304000000001, -8.294175999999997 ], [ 4.610416000000001, 8.248431999999998, -5.842191999999999 ], [ 8.940976000000001, -9.110959999999993, -6.349311999999999 ], [ 4.966351999999997, 4.551168, 0.5513760000000014 ], [ 11.834991999999998, 2.5269920000000012, -7.532832000000003 ], [ 10.779568000000003, -7.602111999999997, -8.1728 ], [ 2.915263999999998, 4.055600000000001, -16.890335999999998 ], [ 4.671295999999999, -8.674128000000003, -12.169615999999998 ], ... ],
    	[ [ 7.3402080000000005, 23.440640000000005, -8.127775999999999 ], [ -5.525168, -11.895087999999998, 5.889312 ], [ 7.199807999999999, 6.16104, -6.9127360000000015 ], [ 8.755119999999998, 14.042224, -1.547311999999999 ], [ 7.75592, -7.794063999999997, 10.408096000000002 ], [ 15.87128, 5.628096, -8.91624 ], [ 17.049136, -1.1291199999999992, -9.448191999999999 ], [ 2.938767999999998, 15.499503999999996, -2.268048 ], ... ],
    	[ [ -0.14649600000000026, -1.6201280000000005, 9.474720000000001 ], [ 4.937168000000001, 5.594607999999997, -10.307056000000003 ], [ -3.3330079999999986, -1.6702399999999986, 18.834624000000005 ], [ -12.103183999999999, 15.212432000000002, -2.423984000000001 ], [ 11.979248000000002, -4.746304000000001, -4.189791999999999 ], [ -2.855807999999997, -6.994592, 27.746479999999995 ], [ 13.339376000000001, 2.532559999999997, -2.100512000000001 ], [ -0.24169600000000047, 8.238256000000002, 9.321264 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 30.15 seconds (0.580 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11923246016927
Reset training subject: 11923464149816
Constructing line search parameters: GD
th(0)=192.25124153084505;dx=-164.93998516692167
New Minimum: 192.25124153084505 > 176.34475041544346
WOLF (strong): th(2.154434690031884)=176.34475041544346; dx=150.11109793338343 evalInputDelta=15.906491115401593
New Minimum: 176.34475041544346 > 99.45337476847806
END: th(1.077217345015942)=99.45337476847806; dx=-7.367486923096786 evalInputDelta=92.797866762367
Fitness changed from 192.25124153084505 to 99.45337476847806
Iteration 1 complete. Error: 99.45337476847806 Total: 1.8263; Orientation: 0.1474; Line Search: 1.0380
th(0)=99.45337476847806;dx=-0.33819270428251236
Armijo: th(2.3207944168063896)=99.47405255768251; dx=0.35601196513573724 evalInputDelta=-0.02067778920445562
New Minimum: 99.45337476847806 > 99.26232537297076
WOLF (strong): th(1.1603972084031948)=99.26232537297076; dx=0.008909868699278902 evalInputDelta=0.19104939550729227
END: th(0.3867990694677316)=99.34493864914897; dx=-0.22249179366756244 evalInputDelta=0.10843611932908459
Fitness changed from 99.45337476847806 to 99.26232537297076
Iteration 2 complete. Error: 99.26232537297076 Total: 1.6194; Orientation: 0.1428; Line Search: 1.2664
th(0)=99.26232537297076;dx=-2.543951352269985E-4
New Minimum: 99.26232537297076 > 99.26219147942845
END: th(0.8333333333333335)=99.26219147942845; dx=-6.694936628272563E-5 evalInputDelta=1.3389354231208017E-4
Fitness changed from 99.26232537297076 to 99.26219147942845
Iteration 3 complete. Error: 99.26219147942845 Total: 1.0066; Orientation: 0.1466; Line Search: 0.6480
th(0)=99.26219147942845;dx=-1.822499029889473E-5
New Minimum: 99.26219147942845 > 99.26218326448894
WOLF (strong): th(1.7953622416932369)=99.26218326448894; dx=9.07370113204989E-6 evalInputDelta=8.21493951264074E-6
New Minimum: 99.26218326448894 > 99.26218124557872
END: th(0.8976811208466184)=99.26218124557872; dx=-4.57564458166215E-6 evalInputDelta=1.0233849735641343E-5
Fitness changed from 99.26219147942845 to 99.26218124557872
Iteration 4 complete. Error: 99.26218124557872 Total: 1.3577; Orientation: 0.1503; Line Search: 0.9957
th(0)=99.26218124557872;dx=-1.8043307731148038E-6
New Minimum: 99.26218124557872 > 99.26217892910853
END: th(1.9339953473386584)=99.26217892910853; dx=-5.911974124524977E-7 evalInputDelta=2.316470187224695E-6
Fitness changed from 99.26218124557872 to 99.26217892910853
Iteration 5 complete. Error: 99.26217892910853 Total: 1.0105; Orientation: 0.1479; Line Search: 0.6486
th(0)=99.26217892910853;dx=-1.4604922000511075E-6
New Minimum: 99.26217892910853 > 99.26217563755523
END: th(4.166666666666668)=99.26217563755523; dx=-1.194533869210526E-7 evalInputDelta=3.291553298367944E-6
Fitness changed from 99.26217892910853 to 99.26217563755523
Iteration 6 complete. Error: 99.26217563755523 Total: 1.0038; Orientation: 0.1431; Line Search: 0.6473
th(0)=99.26217563755523;dx=-3.7400395496735593E-6
Armijo: th(8.976811208466184)=99.26223653470572; dx=1.7307697154825187E-5 evalInputDelta=-6.0897150490291097E-5
Armijo: th(4.488405604233092)=99.26218246843563; dx=6.783828809271007E-6 evalInputDelta=-6.830880394659289E-6
New Minimum: 99.26217563755523 > 99.26217266613872
END: th(1.4961352014110307)=99.26217266613872; dx=-2.3208342851912372E-7 evalInputDelta=2.9714165066252463E-6
Fitness changed from 99.26217563755523 to 99.26217266613872
Iteration 7 complete. Error: 99.26217266613872 Total: 1.6352; Orientation: 0.1488; Line Search: 1.2761
th(0)=99.26217266613872;dx=-1.3837534783382823E-6
New Minimum: 99.26217266613872 > 99.26216951881119
END: th(3.223325578897764)=99.26216951881119; dx=-5.690914709810301E-7 evalInputDelta=3.147327532815325E-6
Fitness changed from 99.26217266613872 to 99.26216951881119
Iteration 8 complete. Error: 99.26216951881119 Total: 1.0154; Orientation: 0.1506; Line Search: 0.6535
th(0)=99.26216951881119;dx=-2.0877510545755536E-6
Armijo: th(6.944444444444446)=99.26217617868478; dx=4.005794648613242E-6 evalInputDelta=-6.659873591274845E-6
New Minimum: 99.26216951881119 > 99.26216755921178
WOLF (strong): th(3.472222222222223)=99.26216755921178; dx=9.590217985474684E-7 evalInputDelta=1.9595994160681585E-6
END: th(1.1574074074074077)=99.26216769015889; dx=-1.0721601031914816E-6 evalInputDelta=1.8286523015831335E-6
Fitness changed from 99.26216951881119 to 99.26216755921178
Iteration 9 complete. Error: 99.26216755921178 Total: 1.6164; Orientation: 0.1441; Line Search: 1.2614
th(0)=99.26216755921178;dx=-5.3930453541546494E-6
New Minimum: 99.26216755921178 > 99.26216596684608
WOLF (strong): th(2.4935586690183844)=99.26216596684608; dx=4.1158620870614676E-6 evalInputDelta=1.5923656917493645E-6
New Minimum: 99.26216596684608 > 99.26216379915161
END: th(1.2467793345091922)=99.26216379915161; dx=-6.385916319877436E-7 evalInputDelta=3.7600601672238554E-6
Fitness changed from 99.26216755921178 to 99.26216379915161
Iteration 10 complete. Error: 99.26216379915161 Total: 1.3500; Orientation: 0.1515; Line Search: 0.9879
th(0)=99.26216379915161;dx=-1.1509031406132786E-6
New Minimum: 99.26216379915161 > 99.26216086785954
END: th(2.6861046490814697)=99.26216086785954; dx=-1.0316566919084178E-6 evalInputDelta=2.931292073071745E-6
Fitness changed from 99.26216379915161 to 99.26216086785954
Iteration 11 complete. Error: 99.26216086785954 Total: 1.0067; Orientation: 0.1458; Line Search: 0.6467
th(0)=99.26216086785954;dx=-1.1980472073480374E-6
New Minimum: 99.26216086785954 > 99.26215537133419
END: th(5.787037037037038)=99.26215537133419; dx=-7.015519605595692E-7 evalInputDelta=5.49652534687084E-6
Fitness changed from 99.26216086785954 to 99.26215537133419
Iteration 12 complete. Error: 99.26215537133419 Total: 0.9990; Orientation: 0.1430; Line Search: 0.6447
th(0)=99.26215537133419;dx=-2.764587164445666E-6
Armijo: th(12.467793345091922)=99.26223580734509; dx=1.5667593909379258E-5 evalInputDelta=-8.043601090435004E-5
Armijo: th(6.233896672545961)=99.26216686326158; dx=6.451503383461074E-6 evalInputDelta=-1.1491927395468338E-5
New Minimum: 99.26215537133419 > 99.26215281840373
WOLF (strong): th(2.0779655575153204)=99.26215281840373; dx=3.074430206657225E-7 evalInputDelta=2.552930453703084E-6
END: th(0.5194913893788301)=99.2621541346416; dx=-1.99657961793393E-6 evalInputDelta=1.2366925830065156E-6
Fitness changed from 99.26215537133419 to 99.26215281840373
Iteration 13 complete. Error: 99.26215281840373 Total: 2.0806; Orientation: 0.1433; Line Search: 1.7259
th(0)=99.26215281840373;dx=-2.300760338949594E-6
New Minimum: 99.26215281840373 > 99.26215091073536
END: th(1.1192102704506124)=99.26215091073536; dx=-1.1081940638767915E-6 evalInputDelta=1.907668377043592E-6
Fitness changed from 99.26215281840373 to 99.26215091073536
Iteration 14 complete. Error: 99.26215091073536 Total: 1.0137; Orientation: 0.1474; Line Search: 0.6519
th(0)=99.26215091073536;dx=-1.1036150347162778E-6
New Minimum: 99.26215091073536 > 99.26214824950702
WOLFE (weak): th(2.411265432098766)=99.26214824950702; dx=-1.1037141896077392E-6 evalInputDelta=2.6612283363647293E-6
New Minimum: 99.26214824950702 > 99.2621455880396
WOLFE (weak): th(4.822530864197532)=99.2621455880396; dx=-1.1038133444622276E-6 evalInputDelta=5.322695756149187E-6
New Minimum: 99.2621455880396 > 99.26213493977903
WOLFE (weak): th(14.467592592592595)=99.26213493977903; dx=-1.1042099635099797E-6 evalInputDelta=1.5970956326327723E-5
New Minimum: 99.26213493977903 > 99.26208697526715
WOLFE (weak): th(57.87037037037038)=99.26208697526715; dx=-1.1059947418598947E-6 evalInputDelta=6.393546820504525E-5
New Minimum: 99.26208697526715 > 99.26182985626804
WOLFE (weak): th(289.3518518518519)=99.26182985626804; dx=-1.1155133512329635E-6 evalInputDelta=3.210544673208915E-4
New Minimum: 99.26182985626804 > 99.2601729467564
WOLFE (weak): th(1736.1111111111113)=99.2601729467564; dx=-1.1749955607447984E-6 evalInputDelta=0.0019779639789589964
New Minimum: 99.2601729467564 > 99.24570569609094
WOLFE (weak): th(12152.77777777778)=99.24570569609094; dx=-1.6023955214734392E-6 evalInputDelta=0.016445214644420503
New Minimum: 99.24570569609094 > 98.96759433540134
WOLFE (weak): th(97222.22222222223)=98.96759433540134; dx=-4.820140017039903E-6 evalInputDelta=0.29455657533401336
Armijo: th(875000.0000000001)=123.34911877366312; dx=1.4859306067075917E-4 evalInputDelta=-24.086967862927764
New Minimum: 98.96759433540134 > 97.8072332456752
WOLF (strong): th(486111.1111111112)=97.8072332456752; dx=1.1078281253217782E-5 evalInputDelta=1.454917665060151
New Minimum: 97.8072332456752 > 97.71232692894318
WOLFE (weak): th(291666.6666666667)=97.71232692894318; dx=-6.040213797988238E-6 evalInputDelta=1.5498239817921728
New Minimum: 97.71232692894318 > 97.34981512114365
END: th(388888.88888888893)=97.34981512114365; dx=-5.273824204460217E-7 evalInputDelta=1.9123357895917081
Fitness changed from 99.26215091073536 to 97.34981512114365
Iteration 15 complete. Error: 97.34981512114365 Total: 5.5549; Orientation: 0.1442; Line Search: 5.1980
th(0)=97.34981512114365;dx=-10.89457303102838
Armijo: th(837835.7127901771)=2.678051932223912E15; dx=1.2828465118864769E10 evalInputDelta=-2.6780519322238145E15
Armijo: th(418917.85639508854)=1.6535063012635634E14; dx=1.5874727301505923E9 evalInputDelta=-1.65350630126259E14
Armijo: th(139639.2854650295)=1.9951963919186223E12; dx=5.718511662436751E7 evalInputDelta=-1.9951963918212725E12
Armijo: th(34909.821366257376)=1.1155363531173325E10; dx=1019396.9148199279 evalInputDelta=-1.115536343382351E10
Armijo: th(6981.964273251475)=2.3100900297934997E8; dx=66379.51617550854 evalInputDelta=-2.3100890562953484E8
Armijo: th(1163.660712208579)=6627637.421222433; dx=11320.27523451429 evalInputDelta=-6627540.0714073125
Armijo: th(166.23724460122557)=135610.80641026984; dx=1639.2901548423254 evalInputDelta=-135513.4565951487
Armijo: th(20.779655575153196)=2021.18025584384; dx=196.0279579267654 evalInputDelta=-1923.8304407226965
Armijo: th(2.308850619461466)=98.74892393460416; dx=12.106136687282973 evalInputDelta=-1.399108813460515
New Minimum: 97.34981512114365 > 95.09995954147439
END: th(0.2308850619461466)=95.09995954147439; dx=-8.594397021087177 evalInputDelta=2.2498555796692585
Fitness changed from 97.34981512114365 to 95.09995954147439
Iteration 16 complete. Error: 95.09995954147439 Total: 3.9520; Orientation: 0.1475; Line Search: 3.5919
th(0)=95.09995954147439;dx=-6.781409597709602
New Minimum: 95.09995954147439 > 92.49259135486633
END: th(0.49742678686693864)=92.49259135486633; dx=-3.7020225294099243 evalInputDelta=2.6073681866080562
Fitness changed from 95.09995954147439 to 92.49259135486633
Iteration 17 complete. Error: 92.49259135486633 Total: 1.0564; Orientation: 0.1611; Line Search: 0.6696
th(0)=92.49259135486633;dx=-2.025253282727823
New Minimum: 92.49259135486633 > 91.3785587999312
END: th(1.071673525377229)=91.3785587999312; dx=-0.05380208294895899 evalInputDelta=1.114032554935136
Fitness changed from 92.49259135486633 to 91.3785587999312
Iteration 18 complete. Error: 91.3785587999312 Total: 1.0432; Orientation: 0.1569; Line Search: 0.6636
Final threshold in iteration 18: 91.3785587999312 (> 0.0) after 30.148s (< 30.000s)

Returns

    91.3785587999312

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    [-0.2241638335615962, -0.1445743911150573, 0.5243372657902299, 0.3689727527110974, 0.4924459525480458, -0.15469787713194666, -0.12217491153682258, 0.36362041599225325, 0.32537813006590843, 0.15016782125081743, 0.2589355053737311, 0.12902823391247548, -0.5526633045974314, -0.08702189938930784, 0.541096119253982, 0.4516133111876319, -0.024371267269455488, 0.5236103289108913, 0.08504441598412839, -0.22726019975921347, 0.07741462607770139, -0.010932709861297338, -0.029896355167897456, 0.20861427154636641, -0.2660858176051351, -0.2140051202872486, 0.4265849819126736, -0.24625608395643897, 0.3014707376426457, 0.12134868032835205, -0.08202986102932056, -0.16422645360410645, -0.4291916111275586, 0.28556854000923826, 0.04613703682181851, 0.11992131682518938, 0.1460207177395748, 0.07861368023952149, -0.3086376188079317, -0.07906507277246361, 0.18531957352304276, 0.3663573305375874, 0.6412317425537357, -0.16432059962397486, 0.06601449110520646, -0.49843598153177127, -0.3514026467570531, -0.34540793184354956, 0.6758146394714519, -0.006183311947047569, 0.5996342268630762, 0.023303135857293475, -0.2784015809571215, -0.2618299838841229, -0.34839830700187424, -0.04573398699430063, 0.20502921656740886, 0.62969285554249, -0.1148097129790754, -0.3996300032426399, -0.21606783208791958, -0.1620638947094436, 0.29448809490586914, 0.10284900123437907, 0.3646618250023639, -0.5571958041981239, -0.07419267267039695, -0.31390459507639606, 0.3272970994235589, 0.2632059301744981, -0.0957107165208684, -0.15949159650133704, 0.18739329082515307, -0.2644543902135249, -0.20463578158359713, -0.4830199424918746, -0.15746676887058392, -0.12799604866267428, 0.28707162908264594, 0.021962679417517676, 0.006238368171560538, -0.03314948118211422, 0.26286441094068447, 0.2460300272086, 0.16095620635459346, 0.37242305930279945, 0.37819289008528106, -0.16962092333726864, 0.3349462374095419, 0.5556557518413898, 0.317485222120215, 0.4152022030203376, -0.35553055133446754, 0.2827440373074167, 0.27724511588741896, 0.5193387606560035, 0.0926693659646356, -0.2201680676094299, -0.19099510989514465, -0.1907054621416922, -0.12450269207977133, -0.014998833383643393, 0.3631487172568108, 0.12281371457810297, 0.12555159857663672, -0.4572924705241088, -0.0783879005754165, -0.5290880822897426, 0.3241020837296231, 0.6133108328520325, -0.5424528297173283, -0.23861781736633705, 0.05856429145446834, 0.047900017954712666, 0.31444115558163244, -0.19963249638231345, -0.46759568617625646, -0.05878509717098515, -0.035382267413519644, 0.2102713587919218, 0.3627179683529266, 0.05905813882363618, 0.024265115227454843, -0.6340892368915175, -0.07579535649120774, 0.10377637379444525, -0.29692426726878496, -0.18942662613448602, 0.2917900456141696, -0.2790328072903183, -0.12139443448090145, 0.23466395022179964, -0.3255162846406099, 0.3258501349236952, 0.12889132239689896, 0.15361183132180845, -0.3245806867581374, -0.04146219162090943, 0.012969131376871335, -0.2529134918632474, 0.42167635518435137, -0.1366313166832249, -0.477077990749958, -0.3532772555163074, 0.11911478618366571, 0.3570108224820101, 0.5022634433059051, 0.23328534188980427, -0.09180905643753035, -0.335817748183294, -0.4272452715078133, 0.0673589976681596, -0.08792476658544249, 0.4754917110635236, 0.22746687117156209, 0.4361259536539634, -0.6348351202532958, -0.3231108752735931, -0.1258776850591966, 0.25706542337675115, 0.03634866030141394, 0.5744165214050383, -1.116, -0.628, 0.048, -1.724, 0.7, 1.556, -1.104, 0.016, 0.028, -2.0, 0.656, -0.636, -1.34, 1.288, -1.808, -1.424, 0.028, 0.184, -1.16, 0.0, -1.72, -0.768, 1.144, 0.0, 1.916, -0.504, 1.764, -0.472, 1.628, -1.028, -0.556, -0.876, -1.664, -0.808, 0.0, -1.456, 1.764, 0.0, 1.66, 1.556, 1.66, -1.552, -0.968, 1.82, 0.784, 0.0, 1.972, -0.892, 1.156, 0.0, -0.804, 1.288, 1.596, 0.0, -1.856, -1.808, 0.688, 1.972, 0.82, 0.0, -0.608, 0.0, -0.684, -1.54, 1.168, 0.56, 0.672, 0.996, -0.628, -0.768, -1.724, 1.652, -0.636, -1.256, -0.068, 0.08, -1.54, 1.556, -0.384, 1.032, 0.972, -1.856, -0.892, 1.256, 0.072, 0.048, 1.652, -0.124, -0.408, 1.66, 0.636, -1.16, -1.484, -0.888, -1.256, 1.912, 1.876, 1.552, 1.552, 0.0, 1.24, 0.812, -0.26, -0.368, 0.344, -1.104, -0.032, 1.512, -1.832, 0.08, 1.324, -0.368, 1.836, -0.26, -0.488, -1.76, 0.488, -0.848, -0.176, 1.524, 0.0, -0.804, 0.148, 1.144, 1.876, -1.228, -0.892, -1.456, 1.66, -1.764, 1.168, -1.536, 0.0, -1.524, 1.836, 1.108, 1.144, -1.492, 0.82, 0.3, -0.848, -0.784, -0.924, -0.712, 0.092, 0.0, -2.0, 1.288, -0.464, -0.852, 0.644, 0.184, 0.0, 0.0, -1.176, 0.688, -0.464, 0.496, 0.048, -0.804, -1.552, -1.76]

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.5045806939998326, -0.37706447081217764, 1.62507962107632, -1.3382464066833881, 0.26066824479435996, -0.23192082961027258 ], [ 1.1079814800362076, 0.4062911842651563, 0.9447678520805292, 1.0633919384706463, 1.4850114078195848, -0.523593721474528 ], [ 0.001688000349667699, -1.2743024469508994, -1.1548691110388107, -1.941942095083263, -0.2026248387589505, -0.04230017125821859 ], [ -2.047223140065391, -0.7800452339039465, -0.8709873489794889, 0.12817862955626574, -1.9177321708900434, 1.8879166484627976 ], [ -1.539877226188438, 1.0817382412853134, 1.5143085950623207, -0.54223184636445, 0.07121035396782013, -1.007983859309243 ], [ -0.26771552961922584, -0.4249656450066239, -0.9406234681827633, -1.3788506365331263, -1.0346291319081973, 2.0053547518351977 ], [ -1.2451993405451012, 0.3007619424563785, 1.9048980448657655, -1.5886805594665376, 0.5198389561369203, -1.6450438434594012 ], [ 0.1717515832010258, -1.9375041321533228, 0.835798044132619, -0.9743070470874624, 0.41391374265972014, -0.33018854131302944 ], ... ],
    	[ [ -1.6964335269947195, 0.5535886853931341, 2.017938217627337, 0.31627583167943357, 0.4029301222280602, 0.907917978671627 ], [ -0.9600608957870466, -1.159912675190278, 1.880673250969411, -1.1304366887587987, 2.0702101251829994, -0.8055963216980678 ], [ 0.30829219897159094, 0.6522086354605028, -1.3639868862156734, 0.2865025111226924, -1.8559557814058993, -1.756754293652818 ], [ 1.4935985122339297, -0.5999567654387148, 2.0645286989562206, -1.9121563481700654, 1.1002194033028168, -0.6134190419649916 ], [ -0.1445359550666373, 0.9358860927642636, -1.171465342682021, -0.004319083991238771, 1.7165712628676664, 0.5581440391154363 ], [ -0.7038225173698023, 2.0468470651386226, -1.0321629660087357, -1.818611590869557, 1.9145344757355918, -0.4464732165453307 ], [ -0.7185841206794363, 2.168595614041127, 0.0543068909141283, 0.9466958349982788, 0.860154290004384, 1.0044026929669216 ], [ 1.577786565381773, 0.33669426934769403, -0.33922836625359654, 1.5665004365320285, 0.9535663633485737, -1.8351083280140261 ], ... ],
    	[ [ -0.4244499157641889, 0.40468979001730193, -0.5034584039900079, -0.16557429869213983, 1.379011490683625, -1.0340465958561984 ], [ -0.49570275975148415, -0.8071891549226874, 0.9375467351393553, -1.7070932081287613, -1.8779099314466514, 1.8345274458047018 ], [ 1.8171940665721942, 1.0771833825120367, -0.4040775381223567, -0.5308999953969981, 0.8461312736131489, -0.3559351279145501 ], [ 0.637516029716854, 0.6147949929363622, -0.09647472096009263, 0.4412479645699366, 1.588088185911497, -1.3374298550908712 ], [ -0.3910495984328249, -1.443694324813038, 0.9692980743760067, -1.0345065804214504, -1.3779780845845724, -0.35917553650865613 ], [ -0.029450178666329454, -2.1603056177702147, -0.6888916142399903, -1.8269318796619554, -0.005209714411919024, -0.3608999525995222 ], [ -1.7020562581778096, -1.978537286313561, 0.29827893495067204, 0.6074416155971772, -1.220050048551598, -1.3556883210844077 ], [ -1.4043004806752666, -0.8394670953822649, -1.1515895133965763, 0.11670409966604686, -1.3726068237450217, -1.473603612428886 ], ... ],
    	[ [ -0.5554152770971824, -0.5214155402866681, 1.6615985101014485, 0.538556829166855, 1.7389692988723386, 0.6001737169180488 ], [ -1.0323989160661349, -0.16556787947662838, 1.8586054678023896, 1.328100254657225, -0.668844227609004, 0.41085433324724485 ], [ 1.833762640137832, 0.4888580731131603, 0.5089868503558529, -1.2732024892672624, -0.6640226660130566, -1.6349511370543304 ], [ 0.5998430333506956, 0.6842561392010317, -0.30198448230418373, 1.336050283556807, -0.8459435131369573, 0.2517524708291089 ], [ -1.099499554590935, 1.7462548485992124, 0.7726111398474358, -0.859835945246054, 1.0397825758320396, 1.221323931414631 ], [ -0.3809877122645243, 1.6270672659391578, -0.5899973985689727, -0.8937477553678471, 1.8620115783287803, 0.7642741418122225 ], [ -1.2202565708982607, 1.1669076191374215, -0.08326437193829883, -0.6169837320998045, -0.6281912943706628, -0.6995238864443043 ], [ 1.3511725032433097, -1.5594555652279347, -0.6006132460161708, -0.21761475829885224, -0.9385741018901226, -0.1140517858417017 ], ... ],
    	[ [ -1.102802703001341, -0.38456505402922897, 1.0291502645439763, -0.08216053367977848, -1.572223061899237, -0.6259928508058544 ], [ 1.1900742281975536, 1.439837533126262, -0.568864746678091, -1.0373147003023866, -0.10063995115152122, 0.7207185088003363 ], [ -0.1096141936232309, 1.556331045027649, -1.0197750931039762, 1.523991660847614, 1.8724873891653042, -0.35686426063939064 ], [ 0.8310424033813626, 1.1611721830622748, -0.41320380273198354, -0.9412303242811536, -2.2668754828386124, -1.172959478677658 ], [ 0.14315675531771258, 1.586891066869732, -1.5696883661052887, -0.6677295689216327, -1.4885975116363466, -2.1766290767091645 ], [ -1.3076100297437185, 0.13608612247072874, 0.4707539311908044, -1.8683332231630012, -1.4307674937013508, 1.8054645156902835 ], [ -1.4498507448375089, -0.35855649933031003, -0.4104689395997573, 0.7661851797046744, -1.4869662162360022, -0.8929742229383885 ], [ -1.5863557776365413, -0.8106515423751659, 1.069833782742313, 1.0092605133224746, 0.013319003889508746, 1.0061761061116519 ], ... ],
    	[ [ -0.2328295450947635, 1.5674951385317322, -1.2911474399286986, 1.7885270295388878, -0.4344119133925991, 1.0898465027498574 ], [ -1.52648622805512, -1.0427490221110776, -0.9064480108772593, -0.4809505797595836, -0.6073963437149024, 1.0192391393842457 ], [ -1.306950289946596, 0.6575236693753447, 0.5736522577069749, 1.257001613377184, -0.9815955812556332, 0.4369884631447168 ], [ 0.32938576738691033, 0.5780500924743104, -1.6075018107861594, 1.3079973072146094, 0.562461560839982, -0.3616056847108358 ], [ 0.28077582202061596, 0.9708219151049674, -0.12924746112798802, -1.6717222176529702, 0.7206337539959722, -0.7029740618335188 ], [ 0.7249553644021316, 0.14172151632052854, 0.02440543235607742, 1.9808398240019685, -0.2670722284390538, 0.3658271081263491 ], [ -1.3958303279238962, 0.8524224972412348, 0.4631622275587008, -1.8883442043722676, -0.6215902086447275, 0.7512015563974824 ], [ -1.5882188943013555, 0.23697841331599379, -0.501014238482672, -0.8004966017319958, 0.761112153296158, -1.3145787967903404 ], ... ],
    	[ [ 1.189858264307208, 1.9298666804003897, -1.0405920806926388, 0.4479030046359841, 0.29473585606984504, 0.3388121445282992 ], [ -0.8998201155816695, 0.01359306673154536, -1.0975903133575433, 1.3724410604224586, 0.42928410579280923, -0.21609369181748478 ], [ -0.8641844349870533, -1.8462506739208335, 0.3420834005167808, -0.3560089756626089, -2.25614683561896, 0.5898122785937862 ], [ -0.831875495688175, 0.6604516668929977, 0.010433326761221511, -0.8465481595723495, 1.3790109205247463, -1.0093926926805303 ], [ -0.09625487998045175, -1.279726325962133, -1.8705978715606904, -0.2152915423348131, 1.4117248974813463, -1.9847308292776542 ], [ 1.7765301031738154, -0.005023915248526014, -0.3029413901219484, 1.432009815379886, 1.0984001964023322, 0.968889584307046 ], [ -0.15025660298792806, 1.1312311750163575, -0.8133590618880792, -0.9292712691657418, 0.9505936211743204, -0.4479753686735228 ], [ -1.1565229506113883, 1.4629927022951015, 2.2393680998312813, -0.26432534752622217, -1.2235226178220346, -0.31099966366171733 ], ... ],
    	[ [ 1.2446375482166319, 0.9082380356967704, 0.4945815950063667, 0.059940783925852927, 1.3265389707523287, -0.6858350325306615 ], [ -1.1507033927431671, 0.11331305402734909, -0.4843162106617018, 0.10394019944695404, 1.6214200085823849, -1.601565922303355 ], [ 1.4896006907894563, -1.5971963763003552, 0.42233270097729086, 1.5493699073200535, 0.30259189501220224, 1.6226592277842082 ], [ 0.7527762210252419, 0.8970782655570481, -1.871163143588112, 1.273927485338703, 0.48006540087093574, 1.9812902581342622 ], [ 1.4697182789820327, -0.8034537044773499, -1.5746453732352097, -0.2540968614986636, 1.2352979412138174, 0.9214851202161769 ], [ -0.23653268747339493, 1.0149934842285804, -0.8410045375605377, -1.5811700521926315, 1.2512564285307994, 0.5839783028547749 ], [ 0.8211487043382254, 0.2380194112090193, -1.8241604859673792, 0.3925788784787617, -1.7755360855215996, 0.3253172157114719 ], [ 1.5183885760990332, 0.2544964299495784, -0.8611587070893867, 1.6800496100739277, 0.2728480924519626, -1.8409860764026813 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.03 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 1.1419058043242274, 0.10123692429585486, -0.8547776763760256 ], [ -0.24256984882170074, -1.958301561131475, 0.05195735236424808 ], [ -0.44300631585639483, -0.07199149899978047, -1.5759449638218443 ], [ -0.6802857214581514, 0.36612091328616336, -0.5768336889353409 ], [ -2.4383087215440296, 1.9887437275269824, 1.1246887799240821 ], [ -0.6817273552275842, -0.145379777006234, 1.4137031570057927 ], [ -0.14009789432311368, -0.9799552318345289, 0.16851291459276874 ], [ -0.8136414129562405, -1.1504778387014665, 1.6879233862921195 ], ... ],
    	[ [ -1.9763963480358113, -1.840617155343325, 0.6150031729974721 ], [ 2.7430337925355808, 0.615802099082252, -2.1105069477925635 ], [ -4.05021491810014, -0.007777721504310061, 4.052764015240707 ], [ 1.0841079936668065, -0.8428648779990895, 2.141516511216705 ], [ 0.9597103801518722, -0.11803915133795577, 2.2683387014869103 ], [ 1.752169478415024, -1.7667103534355357, 0.36970183419026537 ], [ -1.9689186515469805, -3.1514949438599875, 2.818035260540428 ], [ -3.164829235504957, -3.5116235751636684, 2.554596098094751 ], ... ],
    	[ [ -1.0393330944251706, -1.636678229240204, 0.9622670995113367 ], [ 3.631835070690884, -1.722920761933335, -2.381632794262762 ], [ -1.6669312723706426, -2.012385069450377, 3.467163474895818 ], [ -0.1507667432643909, -0.7596744748315895, -1.4743854305100907 ], [ 0.8173183552216101, -3.277470242270726, 1.4331824204453587 ], [ 3.651043073891678, 0.09655522610777602, -2.1250938616990727 ], [ 3.3286883608516833, -1.7108289027483636, -1.0597196693691673 ], [ -0.18342529480466285, 5.126394543826586, 1.5337707917370917 ], ... ],
    	[ [ -1.2921931370442308, -0.8517964935234944, -2.0342338385613363 ], [ -0.8253577066707074, -0.24150209255431176, -0.07410662505468449 ], [ -0.036719008831989884, -2.5402823881805743, 3.3922858215026563 ], [ 4.580775017354816, 1.1720076456547885, -3.712847029584252 ], [ -3.633798449304919, 0.9980300394917292, 0.16760027337081793 ], [ -0.8937090384648632, -0.7041571433620121, -3.642989911191869 ], [ -8.314082401473406, -3.6490377543296355, 2.4145172401506176 ], [ -3.5054895732289935, -1.7809785352772751, 1.1551389460917956 ], ... ],
    	[ [ -1.0187902449095294, 1.643189185179662, 2.568997553726753 ], [ 5.5767980181924575, 0.8181544063250771, -0.609100665067904 ], [ -0.3564778852793845, -3.2668188880120304, 1.137175965296594 ], [ 1.3094288075994975, -0.4752650904012032, 1.2370132361706825 ], [ 1.5901609304273636, -3.1328996782941156, 2.591378576514034 ], [ 5.08655512519914, -0.24641424376917356, -3.4161895202612462 ], [ 0.6066751362520865, -1.3553605929313564, -1.801221020412864 ], [ -5.140669404907531, -2.1807796976019267, 2.6110987416572446 ], ... ],
    	[ [ -1.808963966195929, 0.054468505416355464, 0.22263147937070635 ], [ 0.8654129205040546, -0.6820645954669058, -2.5762829488076955 ], [ 5.543043123609707, 2.8859931571506023, -4.342027158739286 ], [ -1.9375988304421843, -0.5300373668895539, 1.4477019296065883 ], [ -0.8130983901720173, 0.39398666298463497, 1.6761525882864285 ], [ 4.430720714854972, 2.8361374361451226, -2.849979132529808 ], [ -2.368689597984703, -2.1404882754949712, 3.079266167860807 ], [ -1.1985565703489034, 1.1236438837600538, -1.1810641630971581 ], ... ],
    	[ [ 1.0698713118261691, 4.2603947089223535, 0.2166543219482755 ], [ -3.4782285919152733, -2.141260491575234, 0.38153144432012076 ], [ 0.5998352116333477, 3.093276786443609, -0.6157706694713567 ], [ 0.0590247580499013, 1.4624961980765685, 4.670835408598934 ], [ 5.229613058303395, 3.5851261335091618, 0.5025213488751327 ], [ 0.521697546652227, -1.9497191032324446, 4.307269785073298 ], [ 3.7766226631433555, 0.12848802162119502, -3.665137657023844 ], [ -2.5210318817280677, -1.5042996136922553, 2.091616346989799 ], ... ],
    	[ [ 0.9691619414275717, 1.383460868506093, 0.986973147908833 ], [ 2.832382454516373, 4.286876629945852, -2.8801844058540786 ], [ -2.7619607775899064, -1.56691319869639, 1.7972274815169753 ], [ -1.6523042599268563, 1.5092014768708886, -0.8600559671569069 ], [ -0.665656876220136, 2.687108747589894, -1.4137896690727565 ], [ 0.7006170731013058, 0.4657538455953806, -0.3995056927230205 ], [ 1.5242094944906075, -1.6878420122100881, 2.3207618856647074 ], [ 0.08571576667287485, 1.3589849369738227, 2.958907823260397 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 31.68 seconds (0.269 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11953473302873
Reset training subject: 11953700975671
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=192.25124153084505}, derivative=-164.93998516692167}
New Minimum: 192.25124153084505 > 192.25124151435105
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=192.25124151435105}, derivative=-164.93998515228972}, evalInputDelta = -1.6493999055455788E-8
New Minimum: 192.25124151435105 > 192.25124141538706
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=192.25124141538706}, derivative=-164.9399850644975}, evalInputDelta = -1.1545799338819052E-7
New Minimum: 192.25124141538706 > 192.25124072263912
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=192.25124072263912}, derivative=-164.93998444994884}, evalInputDelta = -8.082059252956242E-7
New Minimum: 192.25124072263912 > 192.25123587340363
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=192.25123587340363}, derivative=-164.9399801481134}, evalInputDelta = -5.6574414202259504E-6
New Minimum: 192.25123587340363 > 192.25120192875883
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=192.25120192875883}, derivative=-164.93995003526106}, evalInputDelta = -3.960208621833772E-5
New Minimum: 192.25120192875883 > 192.25096431641862
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=192.25096431641862}, derivative=-164.93973924529828}, evalInputDelta = -2.7721442643269256E-4
New Minimum: 192.25096431641862 > 192.2493010385399
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=192.2493010385399}, derivative=-164.93826371556455}, evalInputDelta = -0.0019404923051524747
New Minimum: 192.2493010385399 > 192.23765851001582
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=192.23765851001582}, derivative=-164.92793500765677}, evalInputDelta = -0.013583020829230463
New Minimum: 192.23765851001582 > 192.15618122506953
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=192.15618122506953}, derivative=-164.8556340635998}, evalInputDelta = -0.09506030577551883
New Minimum: 192.15618122506953 > 191.58684055109285
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=191.58684055109285}, derivative=-164.34952800870164}, evalInputDelta = -0.6644009797522017
New Minimum: 191.58684055109285 > 187.6504712903456
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=187.6504712903456}, derivative=-160.8068127459816}, evalInputDelta = -4.6007702404994575
New Minimum: 187.6504712903456 > 162.49755652659667
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=162.49755652659667}, derivative=-136.0091348325733}, evalInputDelta = -29.753685004248382
New Minimum: 162.49755652659667 > 104.07903366577021
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=104.07903366577021}, derivative=37.50950386102966}, evalInputDelta = -88.17220786507484
104.07903366577021 <= 192.25124153084505
New Minimum: 104.07903366577021 > 99.26778071153835
F(1.127679658064655) = LineSearchPoint{point=PointSample{avg=99.26778071153835}, derivative=0.01170347348119979}, evalInputDelta = -92.9834608193067
Right bracket at 1.127679658064655
Converged to right
Fitness changed from 192.25124153084505 to 99.26778071153835
Iteration 1 complete. Error: 99.26778071153835 Total: 6.3494; Orientation: 0.1584; Line Search: 5.5217
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26778071153835}, derivative=-0.009944159917372079}
New Minimum: 99.26778071153835 > 99.26218347685295
F(1.127679658064655) = LineSearchPoint{point=PointSample{avg=99.26218347685295}, derivative=1.716570867745445E-5}, evalInputDelta = -0.005597234685396302
99.26218347685295 <= 99.26778071153835
Converged to right
Fitness changed from 99.26778071153835 to 99.26218347685295
Iteration 2 complete. Error: 99.26218347685295 Total: 1.0520; Orientation: 0.1583; Line Search: 0.6731
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26218347685295}, derivative=-2.4330024474986715E-6}
New Minimum: 99.26218347685295 > 99.26218148917258
F(1.127679658064655) = LineSearchPoint{point=PointSample{avg=99.26218148917258}, derivative=-1.0922547084139189E-6}, evalInputDelta = -1.9876803776242014E-6
F(7.893757606452586) = LineSearchPoint{point=PointSample{avg=99.2622013137031}, derivative=6.95223168273125E-6}, evalInputDelta = 1.783685014800085E-5
F(0.6072121235732758) = LineSearchPoint{point=PointSample{avg=99.26218221869006}, derivative=-1.7110613569645474E-6}, evalInputDelta = -1.2581628965335767E-6
F(4.250484865012931) = LineSearchPoint{point=PointSample{avg=99.26218387551174}, derivative=2.6205851736425815E-6}, evalInputDelta = 3.986587842064182E-7
F(0.3269603742317639) = LineSearchPoint{point=PointSample{avg=99.26218274490844}, derivative=-2.0442649371362535E-6}, evalInputDelta = -7.319445103348698E-7
New Minimum: 99.26218148917258 > 99.26218102237837
F(2.288722619622347) = LineSearchPoint{point=PointSample{avg=99.26218102237837}, derivative=2.881601213804595E-7}, evalInputDelta = -2.454474582691546E-6
99.26218102237837 <= 99.26218347685295
New Minimum: 99.26218102237837 > 99.26218098745821
F(2.0463561416253198) = LineSearchPoint{point=PointSample{avg=99.26218098745821}, derivative=4.033096556374516E-16}, evalInputDelta = -2.4893947454529552E-6
Right bracket at 2.0463561416253198
Converged to right
Fitness changed from 99.26218347685295 to 99.26218098745821
Iteration 3 complete. Error: 99.26218098745821 Total: 3.4966; Orientation: 0.1623; Line Search: 3.1103
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26218098745821}, derivative=-1.999069902019866E-6}
New Minimum: 99.26218098745821 > 99.26217857776535
F(2.0463561416253198) = LineSearchPoint{point=PointSample{avg=99.26217857776535}, derivative=-3.560361408475473E-7}, evalInputDelta = -2.409692854143941E-6
F(14.324492991377237) = LineSearchPoint{point=PointSample{avg=99.26223472648462}, derivative=9.502166339759405E-6}, evalInputDelta = 5.373902641281347E-5
F(1.1018840762597875) = LineSearchPoint{point=PointSample{avg=99.26217927213912}, derivative=-1.1143594147200329E-6}, evalInputDelta = -1.7153190867702506E-6
F(7.713188533818512) = LineSearchPoint{point=PointSample{avg=99.26218945204084}, derivative=4.193903483927597E-6}, evalInputDelta = 8.464582634815088E-6
F(0.5933221949091163) = LineSearchPoint{point=PointSample{avg=99.26217994268984}, derivative=-1.522687331786007E-6}, evalInputDelta = -1.0447683678194153E-6
F(4.153255364363814) = LineSearchPoint{point=PointSample{avg=99.26217960969502}, derivative=1.335608082310173E-6}, evalInputDelta = -1.377763183540992E-6
99.26217960969502 <= 99.26218098745821
New Minimum: 99.26217857776535 > 99.2621784988261
F(2.489789968721767) = LineSearchPoint{point=PointSample{avg=99.2621784988261}, derivative=2.0447508754408003E-15}, evalInputDelta = -2.4886321057238092E-6
Right bracket at 2.489789968721767
Converged to right
Fitness changed from 99.26218098745821 to 99.2621784988261
Iteration 4 complete. Error: 99.2621784988261 Total: 3.4026; Orientation: 0.1579; Line Search: 3.0211
F(0.0) = LineSearchPoint{point=PointSample{avg=99.2621784988261}, derivative=-2.4349334183379145E-6}
New Minimum: 99.2621784988261 > 99.26217612864059
F(2.489789968721767) = LineSearchPoint{point=PointSample{avg=99.26217612864059}, derivative=5.310093434260004E-7}, evalInputDelta = -2.370185512745593E-6
99.26217612864059 <= 99.2621784988261
New Minimum: 99.26217612864059 > 99.26217601028895
F(2.044028926531828) = LineSearchPoint{point=PointSample{avg=99.26217601028895}, derivative=7.065288160296515E-16}, evalInputDelta = -2.4885371487926022E-6
Right bracket at 2.044028926531828
Converged to right
Fitness changed from 99.2621784988261 to 99.26217601028895
Iteration 5 complete. Error: 99.26217601028895 Total: 1.7159; Orientation: 0.1574; Line Search: 1.3334
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26217601028895}, derivative=-2.0011557413687167E-6}
New Minimum: 99.26217601028895 > 99.26217360116962
F(2.044028926531828) = LineSearchPoint{point=PointSample{avg=99.26217360116962}, derivative=-3.560705132651078E-7}, evalInputDelta = -2.409119332469345E-6
F(14.308202485722797) = LineSearchPoint{point=PointSample{avg=99.26222976109096}, derivative=9.514440773866464E-6}, evalInputDelta = 5.375080201019955E-5
F(1.1006309604402151) = LineSearchPoint{point=PointSample{avg=99.26217429523275}, derivative=-1.1153406180581186E-6}, evalInputDelta = -1.715056200168874E-6
F(7.704416723081506) = LineSearchPoint{point=PointSample{avg=99.2621844789621}, derivative=4.199550098085107E-6}, evalInputDelta = 8.46867314407973E-6
F(0.592647440237039) = LineSearchPoint{point=PointSample{avg=99.26217496564882}, derivative=-1.524178367137258E-6}, evalInputDelta = -1.0446401290664653E-6
F(4.148532081659273) = LineSearchPoint{point=PointSample{avg=99.26217463407593}, derivative=1.3376858713575413E-6}, evalInputDelta = -1.3762130208760937E-6
99.26217463407593 <= 99.26217601028895
New Minimum: 99.26217360116962 > 99.26217352240327
F(2.4864488215977594) = LineSearchPoint{point=PointSample{avg=99.26217352240327}, derivative=1.928756277819665E-15}, evalInputDelta = -2.4878856805798932E-6
Right bracket at 2.4864488215977594
Converged to right
Fitness changed from 99.26217601028895 to 99.26217352240327
Iteration 6 complete. Error: 99.26217352240327 Total: 3.3170; Orientation: 0.1565; Line Search: 2.9384
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26217352240327}, derivative=-2.4368772711986302E-6}
New Minimum: 99.26217352240327 > 99.26217115248136
F(2.4864488215977594) = LineSearchPoint{point=PointSample{avg=99.26217115248136}, derivative=5.306068429887679E-7}, evalInputDelta = -2.3699219156014806E-6
99.26217115248136 <= 99.26217352240327
New Minimum: 99.26217115248136 > 99.26217103452893
F(2.0418544417412705) = LineSearchPoint{point=PointSample{avg=99.26217103452893}, derivative=6.754033198333714E-16}, evalInputDelta = -2.4878743403178305E-6
Right bracket at 2.0418544417412705
Converged to right
Fitness changed from 99.26217352240327 to 99.26217103452893
Iteration 7 complete. Error: 99.26217103452893 Total: 1.7605; Orientation: 0.1570; Line Search: 1.3812
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26217103452893}, derivative=-2.003223133194128E-6}
New Minimum: 99.26217103452893 > 99.26216862578931
F(2.0418544417412705) = LineSearchPoint{point=PointSample{avg=99.26216862578931}, derivative=-3.5614154434072313E-7}, evalInputDelta = -2.4087396184313548E-6
F(14.292981092188894) = LineSearchPoint{point=PointSample{avg=99.26222479846925}, derivative=9.526347911407494E-6}, evalInputDelta = 5.376394031486598E-5
F(1.0994600840145303) = LineSearchPoint{point=PointSample{avg=99.2621693196152}, derivative=-1.1163330464273125E-6}, evalInputDelta = -1.7149137363503542E-6
F(7.6962205881017125) = LineSearchPoint{point=PointSample{avg=99.26217950723787}, derivative=4.205007451646857E-6}, evalInputDelta = 8.47270894155372E-6
F(0.5920169683155163) = LineSearchPoint{point=PointSample{avg=99.26216998994755}, derivative=-1.5256669324934235E-6}, evalInputDelta = -1.0445813813930727E-6
F(4.144118778208615) = LineSearchPoint{point=PointSample{avg=99.26216965960799}, derivative=1.3396702651634584E-6}, evalInputDelta = -1.374920941543678E-6
99.26216965960799 <= 99.26217103452893
New Minimum: 99.26216862578931 > 99.26216854717079
F(2.483356067318922) = LineSearchPoint{point=PointSample{avg=99.26216854717079}, derivative=1.8323198827088685E-15}, evalInputDelta = -2.4873581452311555E-6
Right bracket at 2.483356067318922
Converged to right
Fitness changed from 99.26217103452893 to 99.26216854717079
Iteration 8 complete. Error: 99.26216854717079 Total: 3.7754; Orientation: 0.1715; Line Search: 3.3685
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26216854717079}, derivative=-2.43884032423315E-6}
New Minimum: 99.26216854717079 > 99.26216617730815
F(2.483356067318922) = LineSearchPoint{point=PointSample{avg=99.26216617730815}, derivative=5.302435975847827E-7}, evalInputDelta = -2.3698626421264635E-6
99.26216617730815 <= 99.26216854717079
New Minimum: 99.26216617730815 > 99.2621660597271
F(2.0398577729315632) = LineSearchPoint{point=PointSample{avg=99.2621660597271}, derivative=6.486976415558642E-16}, evalInputDelta = -2.487443694576541E-6
Right bracket at 2.0398577729315632
Converged to right
Fitness changed from 99.26216854717079 to 99.2621660597271
Iteration 9 complete. Error: 99.2621660597271 Total: 1.7628; Orientation: 0.1648; Line Search: 1.3684
F(0.0) = LineSearchPoint{point=PointSample{avg=99.2621660597271}, derivative=-2.0052712685001417E-6}
New Minimum: 99.2621660597271 > 99.2621636511451
F(2.0398577729315632) = LineSearchPoint{point=PointSample{avg=99.2621636511451}, derivative=-3.562482795943591E-7}, evalInputDelta = -2.4085819916308537E-6
F(14.279004410520942) = LineSearchPoint{point=PointSample{avg=99.26221983887234}, derivative=9.537889579805085E-6}, evalInputDelta = 5.3779145247290217E-5
F(1.0983849546554572) = LineSearchPoint{point=PointSample{avg=99.26216434481478}, derivative=-1.1173358124939165E-6}, evalInputDelta = -1.7149123152648826E-6
F(7.6886946825882) = LineSearchPoint{point=PointSample{avg=99.26217453653467}, derivative=4.210276901983522E-6}, evalInputDelta = 8.476807579427259E-6
F(0.5914380525067846) = LineSearchPoint{point=PointSample{avg=99.26216501512226}, derivative=-1.527152176676485E-6}, evalInputDelta = -1.0446048293033527E-6
F(4.140066367547492) = LineSearchPoint{point=PointSample{avg=99.26216468582766}, derivative=1.341562368000313E-6}, evalInputDelta = -1.37389943688504E-6
99.26216468582766 <= 99.2621660597271
New Minimum: 99.2621636511451 > 99.26216357264889
F(2.4805404266246995) = LineSearchPoint{point=PointSample{avg=99.26216357264889}, derivative=1.7537466713018362E-15}, evalInputDelta = -2.4870782056041207E-6
Right bracket at 2.4805404266246995
Converged to right
Fitness changed from 99.2621660597271 to 99.26216357264889
Iteration 10 complete. Error: 99.26216357264889 Total: 3.3268; Orientation: 0.1593; Line Search: 2.9444
F(0.0) = LineSearchPoint{point=PointSample{avg=99.26216357264889}, derivative=-2.4408274027975684E-6}
New Minimum: 99.26216357264889 > 99.26216120262552
F(2.4805404266246995) = LineSearchPoint{point=PointSample{avg=99.26216120262552}, derivative=5.299346659047863E-7}, evalInputDelta = -2.3700233668932924E-6
99.26216120262552 <= 99.26216357264889
New Minimum: 99.26216120262552 > 99.26216108538087
F(2.0380531685250065) = LineSearchPoint{point=PointSample{avg=99.26216108538087}, derivative=6.291097928923716E-16}, evalInputDelta = -2.4872680199905517E-6
Right bracket at 2.0380531685250065
Converged to right
Fitness changed from 99.26216357264889 to 99.26216108538087
Iteration 11 complete. Error: 99.26216108538087 Total: 1.7247; Orientation: 0.1579; Line Search: 1.3440
Final threshold in iteration 11: 99.26216108538087 (> 0.0) after 31.684s (< 30.000s)

Returns

    99.26216108538087

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    [-0.002385923170503648, -3.906988296937473E-4, 0.013356175540637286, 0.0072406561031338225, 0.018991255749121985, -6.689293623345174E-4, 6.080338911588887E-4, 0.006594803786556378, 0.010438885362612976, 0.006048753163376481, 0.00362590855106683, 0.00492947154633473, -0.017649078860141665, -0.003118645006282898, 0.018620866329399344, 0.006843575415461881, 0.004297398652249717, 0.018343637046097983, -6.675331411294767E-4, -0.012079097845260737, 0.0014556331468395481, 0.004356499843590527, -0.0011359360443530147, 0.009224798783219504, -0.015378521040806102, -0.0044170589557258, 0.013809506312862773, -0.008406927288592448, 0.010560233021438022, 0.0032268119710232145, -0.0024644768595518647, -0.006966255776543825, -0.008917471085626363, 0.009193745171226276, 0.0034358384246775397, 5.337062638482573E-4, 0.0031320540894041126, 0.0030705704520795473, -0.004374816444868412, 0.002075520374847953, 0.004589069490911097, 0.0077911574523579046, 0.01896886931929555, 3.715290870370512E-4, 2.0898406322771427E-4, -0.013647061574115849, -0.00886745932996903, -0.010111688442723666, 0.014935841670366386, -1.2641806469390512E-6, 0.01696319371320203, 0.006612141816330764, -0.007198235722576391, -0.006986194195664784, -0.008785609150590871, -0.0016331942846661797, 0.004651844171298251, 0.015669413981820465, -0.0050856736501431864, -0.009852162024405425, 3.1697799229314335E-4, -0.0031744344334619923, 0.0030186228524583716, 0.003015444356656089, 0.011447498120881135, -0.015543399254237988, -0.0012906321555183422, -0.011989331694396815, 0.009262047641069315, 0.005313238707546042, -0.006964265410223139, -0.0033412848994150784, 0.004439186963969032, -0.008052294470155019, -0.0070694412960120275, -0.01693236623002081, -0.012354055754760687, -0.0014592299199264406, 0.009483894430293158, -5.093924975441704E-4, 0.006669569765962394, -3.570988265717443E-4, 0.0073947930707064236, 0.009636745315559806, 0.005853375960808566, 0.019917446935998645, 0.01719870243435625, -0.00654221408885112, 0.011030291149479177, 0.01681990346031329, 0.013793524229836401, 0.015111484992039135, -0.008958550568094073, 0.01169911030198822, 0.008785091943599388, 0.012575109219707628, 0.001801408612733868, -0.004757575677236084, -0.0018699570066939768, -0.005337154513778043, -0.005119228053548007, 0.0038749952355044727, 0.006955982695040286, 0.003088043173424413, -0.0037307109129097295, -0.0033540997200524215, 0.001702602124779971, -0.01222473845972613, 0.007395870137899565, 0.02052665279281574, -0.016155438469150504, -0.007451526508862169, 0.005579018209169743, -0.006793781721368601, 0.010408491744485502, -0.012308305549267482, -0.012385176994896243, 0.003121491501551325, 0.001284435942508123, 0.0029797266327791887, 0.007364642617495056, -0.0019724515925711047, 0.007184352753127218, -0.01916924766466342, -0.0021671204164812933, 0.001834904697534345, -0.006001014989076671, -0.0091042961938264, 0.013019798786060065, -0.012180896978561884, -0.0025704378214214995, 0.008525921387081005, -0.017584614226114164, 0.008979889172789865, 0.003991239667982349, 9.448769421026095E-5, -0.011140410860744646, -5.624251220725335E-4, 0.002276252086325077, -0.013376523703302143, 0.013147906883139202, -0.005169342850059434, -0.011450300592305125, -0.008543803011962342, 0.004367728307251681, 0.01186227022673105, 0.013838353687120148, 0.007760855573492047, -0.0019051220447855594, -0.007291752373148659, -0.012023529409941487, 0.0046978039891323225, -0.0052163297128392724, 0.011778993375233056, 0.007589838916958467, 0.011830510752705331, -0.01640916071900085, -0.014286469487778699, 0.0028555093650149983, 0.007716911325269532, 0.003006185275626648, 0.016449827103374382, -1.116, -0.628, 0.048, -1.724, 0.7, 1.556, -1.104, 0.016, 0.028, -2.0, 0.656, -0.636, -1.34, 1.288, -1.808, -1.424, 0.028, 0.184, -1.16, 0.0, -1.72, -0.768, 1.144, 0.0, 1.916, -0.504, 1.764, -0.472, 1.628, -1.028, -0.556, -0.876, -1.664, -0.808, 0.0, -1.456, 1.764, 0.0, 1.66, 1.556, 1.66, -1.552, -0.968, 1.82, 0.784, 0.0, 1.972, -0.892, 1.156, 0.0, -0.804, 1.288, 1.596, 0.0, -1.856, -1.808, 0.688, 1.972, 0.82, 0.0, -0.608, 0.0, -0.684, -1.54, 1.168, 0.56, 0.672, 0.996, -0.628, -0.768, -1.724, 1.652, -0.636, -1.256, -0.068, 0.08, -1.54, 1.556, -0.384, 1.032, 0.972, -1.856, -0.892, 1.256, 0.072, 0.048, 1.652, -0.124, -0.408, 1.66, 0.636, -1.16, -1.484, -0.888, -1.256, 1.912, 1.876, 1.552, 1.552, 0.0, 1.24, 0.812, -0.26, -0.368, 0.344, -1.104, -0.032, 1.512, -1.832, 0.08, 1.324, -0.368, 1.836, -0.26, -0.488, -1.76, 0.488, -0.848, -0.176, 1.524, 0.0, -0.804, 0.148, 1.144, 1.876, -1.228, -0.892, -1.456, 1.66, -1.764, 1.168, -1.536, 0.0, -1.524, 1.836, 1.108, 1.144, -1.492, 0.82, 0.3, -0.848, -0.784, -0.924, -0.712, 0.092, 0.0, -2.0, 1.288, -0.464, -0.852, 0.644, 0.184, 0.0, 0.0, -1.176, 0.688, -0.464, 0.496, 0.048, -0.804, -1.552, -1.76]

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.5760095839072977, -0.33596841055842525, 1.6519338397996615, -1.2840494131701792, 0.21998585580240532, -0.10404281885579907 ], [ 1.1158950616512073, 0.38799019796047746, 0.9879248347057541, 1.3279349690124822, 1.5319045003094196, -0.548048936978576 ], [ -0.18789696326746672, -1.359984730039424, -1.303897115403728, -1.7599873949874887, 0.003912269592903844, 0.03209228594601774 ], [ -1.9639220880326997, -1.0119264699995378, -1.071939951470913, 0.4079799771590188, -1.8598995394656785, 1.6800649237044982 ], [ -1.7640408499969218, 1.083861447001084, 1.347914182357855, -0.5200071943823312, 0.24396598045705611, -1.0559141390826339 ], [ -0.2599529273738162, -0.8158940302315772, -1.092008970637461, -1.3799422924719411, -1.28398420975952, 1.956071352864742 ], [ -1.2999810911977476, 0.3360147871622928, 1.9679589282323877, -1.7039482459544315, 0.5159433042381717, -1.6718880278544468 ], [ 0.1280243473165102, -1.9278769731557281, 0.8599615632692259, -0.9799235596532493, 0.3199389784045135, -0.13995252105229722 ], ... ],
    	[ [ -1.8559967022675965, 0.4040098232686238, 1.9159571363762775, 0.0600685182650686, 0.4879646938795941, 1.0880423764738816 ], [ -0.6999559714167558, -1.3598809716328575, 1.7280285119700616, -1.0479467512634757, 1.9758549204309261, -1.0118714086659377 ], [ 0.20417882368035695, 0.7159640439480032, -1.3559076494601432, 0.08401753021208404, -1.463872129619042, -1.6278000924475642 ], [ 1.5398094774417228, -0.644021842686556, 1.975899604763431, -1.539946588961529, 0.9838539943190202, -0.8040051513110135 ], [ -0.5478020789406379, 0.7681342463452608, -1.1839678588979718, 0.17198135743741247, 1.7799226994270636, 0.7080727040620506 ], [ -0.8920577139858994, 1.9040408728537237, -1.152056100845598, -1.655921966115536, 1.7640092954070974, -0.48400089622767767 ], [ -1.0398996959750664, 1.9760479034756548, 0.023946454639073328, 0.8119680160745161, 0.8039065598354539, 1.1879811926651807 ], [ 1.632020243927992, 0.29604652624976363, -0.2880214170724046, 1.2560668306130673, 0.9360835577892718, -1.8559268942623803 ], ... ],
    	[ [ -0.49191129827524366, 0.36798288598461, -0.4678435460345899, -0.5278558127510685, 1.408055758459477, -0.7839066556594613 ], [ -0.4360460018344803, -0.48804405003151835, 0.97206608145966, -1.895920638981624, -1.97581534247843, 1.5640192293313961 ], [ 1.7760821711231822, 0.976047519652877, -0.25193557628244195, -0.8279787666265529, 0.8680403647413762, 0.2238151658438604 ], [ 0.4358659455663144, 0.727994925742523, 0.19183349723088475, 0.6679203765813057, 1.3759066625963132, -1.7718561771997963 ], [ -0.5397734803413846, -1.6119969409221988, 0.9961658287087353, -1.1958960325208299, -1.3198585138877783, 0.1800685875557853 ], [ -0.28012622916044455, -1.9639063552646772, -0.41999240466721977, -1.935831546201378, -0.211927835990525, -0.5718048353116733 ], [ -1.7238916311697923, -1.9320131784695118, 0.30805226584750856, 0.47207397981945526, -1.243934070845028, -1.187980763322504 ], [ -1.4999670843150785, -0.6679739051900114, -1.111936354048227, -0.15178935544568556, -1.3079208151081507, -1.3120288168120355 ], ... ],
    	[ [ -0.42794906117765824, -0.6719162990121572, 1.6119914599713863, 0.3680305454691937, 1.531951464242299, 0.6320108074052395 ], [ -1.2080450809418237, 0.1479908794887644, 1.9238643543201588, 1.2079142884197343, -0.7839660808401995, 0.1318885241556322 ], [ 1.9278274824231445, 0.4240825792615038, 0.8761545066870885, -1.4039007119985205, -0.8799705614886281, -1.29615240527967 ], [ 0.48793821236604185, 1.1999362446520982, -0.11610177972598476, 1.5838128397387528, -0.6760907610900725, -0.1879785293726023 ], [ -0.84392899528787, 1.8919621426500082, 1.0359839242701363, -1.1718943694673707, 1.2120666643010156, 1.659910598461743 ], [ -0.3840742832001754, 1.6640572538305864, -0.48008624113345516, -1.0519548462587853, 1.559964860420773, 0.5200476606993599 ], [ -0.947964998280971, 1.083942466793332, -0.15198493362457877, -0.6359357336771873, -0.8079136690928317, -0.7919964769987906 ], [ 1.0680708297140504, -1.6798183438604544, -0.49199476087977284, -0.44789022012036733, -1.0479399631214004, -0.004013100434239934 ], ... ],
    	[ [ -0.9239725702018072, -0.3160050919492405, 1.180010430202455, -0.12809405696001527, -1.5838959869835592, -0.508022144938202 ], [ 1.0319102684847687, 1.8477434089631732, -0.34404700911859665, -1.383978142310537, 0.06811990307058674, 0.8358941291475849 ], [ 0.19992987494466102, 1.2200967434666814, -0.8639576212807264, 1.379920022265802, 1.275913104585992, -0.440103627704472 ], [ 0.8839629156114388, 1.5157689533061582, -0.6599044519571694, -0.556114945456627, -1.963926756462977, -1.6320290533349522 ], [ 0.2240783355418054, 1.2961204878198427, -1.8799527196633214, -0.4479787150704951, -1.3359375401256433, -1.6879195670529656 ], [ -1.3081191922780995, -0.07204572373159543, 0.21594192717074434, -1.4159171559080175, -1.6120272565174436, 1.2320423215791094 ], [ -1.831844724213464, -0.4319105561351089, -0.5440117788207167, 0.76388313169843, -1.3160950436277405, -0.6320184999682339 ], [ -1.528004243799594, -1.1278794842544728, 0.9359447879389969, 0.9240867207801722, -0.30792387717365416, 1.247948136554543 ], ... ],
    	[ [ 0.12385704692077591, 1.6558423470895662, -1.2960578486936658, 1.8239637473582369, -0.5120101751516567, 0.8038182859136701 ], [ -1.4839625528359266, -1.127900654471664, -0.9559516382410838, -0.5800658233124277, -0.4720765044569436, 1.119920728178882 ], [ -1.1920815736737005, 0.5599412665895391, 0.24404822230750614, 1.256086563156402, -1.06394235300624, 0.23596755832765642 ], [ 0.5719936653490271, 0.5280608104920007, -1.9199408506604143, 1.5158725108759157, 0.811924338030281, -0.3080457489415355 ], [ 0.2881502247642644, 0.8359401231096909, -0.19591042994894187, -1.6320100710599397, 1.159973913873103, -0.3718371795865039 ], [ 0.5439183175938123, 0.1320005318047297, 0.06387948407667988, 1.8399498452685998, -0.3400976980943654, 0.18808948898945124 ], [ -1.303931987580159, 1.127875870960892, 0.5920265653235024, -1.9200404881331397, -0.5119000587981282, 0.6440184004582313 ], [ -1.7479374619729677, 0.40797997721250967, -0.3599536322279408, -1.083802164554287, 0.7720040162507397, -1.1479326168247859 ], ... ],
    	[ [ 0.9159375241848867, 1.936054612356883, -1.0920828759186316, 0.5359571610170992, 0.3359357814449424, 0.5039533270636665 ], [ -0.6558543915268824, -0.42795903067070823, -1.1959369219347158, 1.499927806674037, 0.18803040208006153, 0.11603309774206899 ], [ -1.3880239619461894, -1.5920776397997307, 0.5159108269811442, -0.3480605092841432, -1.9840075461056157, 0.5041179633510163 ], [ -0.920027208921766, 0.9439190996877502, 0.1440271192891847, -0.8919562625176513, 1.4640055186792564, -0.9239437734052155 ], [ 0.17612498662064727, -1.4279069799356316, -1.8799059299070802, -0.41191806132346237, 1.2479789286724334, -1.7799195206774212 ], [ 1.591835617959582, 0.4358580280468006, -0.10015477051134083, 1.2319234050090602, 1.3320130364558191, 0.6959339549882795 ], [ 0.07205384763205051, 0.7801092686316127, -0.951883701592169, -0.9239705556528095, 0.6199275551092807, -0.224063936857875 ], [ -1.2519531642680433, 1.3439458668464936, 1.9157108169570913, -0.4520495525557478, -1.2239762851557794, -0.3919427102615249 ], ... ],
    	[ [ 1.0040432569406847, 0.9559684583520399, 0.48793752413557884, -0.12002028303127123, 1.5600156122701467, -0.6478937236767935 ], [ -0.927906839780023, -0.059959417774179374, -0.5398796550729983, -0.10781418674009234, 1.5279799406817565, -1.6799025504558016 ], [ 1.6160095392570375, -1.8159086110494491, 0.46401582355149434, 1.2560056904590027, 0.22395945121949928, 1.5760376774140707 ], [ 0.6879610762790168, 0.8919759046706145, -1.9680878253244438, 1.407891630218803, 0.579949964118179, 1.675897141243162 ], [ 1.5159655028181942, -0.683988175836402, -1.7039776034709875, -0.19197738069645692, 1.4439792109645582, 0.9479300740181466 ], [ -0.13602516254487018, 0.7961281944923658, -0.7559414103984821, -1.5000011641484798, 1.0360165791988927, 0.700065085615519 ], [ 0.5120577153633799, 0.3679400982376578, -1.7359972901210494, 0.3719169644400575, -1.6359558751750332, 0.3680192392545678 ], [ 1.4080297883943749, -0.06803209971711795, -0.8040545940918803, 1.8759842394855692, 0.17197844134129733, -1.61189436862259 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.03 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.036803242970667845, -0.012643665632752167, -0.0185677582299601 ], [ 0.021431028371427234, -0.043411318078402275, -0.014199874258426105 ], [ -0.019745391710686903, -0.0035673628646961607, -0.04274928683890954 ], [ -0.04466668760414057, 0.03764683465302142, -0.04925088349392589 ], [ -0.08618265311568228, 0.06169152412454791, 0.051467388822351016 ], [ -0.015602591918104322, -0.0028866234149535134, 0.05161129120718637 ], [ -0.022838685843424515, -0.06055711598255361, 0.015266682316305047 ], [ -0.02155171247632417, -0.06367172788003403, 0.04984069817473826 ], ... ],
    	[ [ -0.050008378196959205, -0.061512592567277646, 0.0026870948915708264 ], [ 0.07681346466712771, -0.005351822633037561, 0.0052469632015668165 ], [ -0.11000289431467213, 0.07296981968340442, 0.058525028003067275 ], [ -0.0048295589494250155, -0.09130380974795474, 0.1244649114612128 ], [ 0.004460152680870656, 0.043961781451458855, 0.00249492186443862 ], [ 0.006498048174669903, -0.10202448779151073, 0.0350090288619678 ], [ -0.04245767450829054, -0.0899574916204406, 0.08287308870531807 ], [ -0.0656011852786454, -0.12138022712253253, 0.10284831333895578 ], ... ],
    	[ [ -0.004707379883805928, -0.06600398202026489, 0.041525108245523974 ], [ 0.05744064976725547, -0.11022848137775322, -0.021255415095711473 ], [ -0.0276610329614732, -0.02621462407667873, -0.003674053453294733 ], [ -0.08470738406382286, -0.056218049951104, -0.01613653192329384 ], [ 0.07487055016110275, -0.08178745263082873, -0.0025473222446613557 ], [ 0.03249637104609119, -0.021496148695926882, -0.03120785547464977 ], [ 0.07298383344513738, -0.037440328575172585, -0.03147811277395984 ], [ -0.044239693246174334, 0.13116626260446654, 0.060253532324753716 ], ... ],
    	[ [ -0.028453001427176206, -0.04260380079024002, -0.08151599171735878 ], [ -0.08086051789438835, -0.06928734769354061, -0.001780396996382569 ], [ 0.10269248195338547, -0.08106068244198308, 0.09885277070410461 ], [ 0.021547481423506196, 0.0015358956660841139, -0.043424351720833206 ], [ -0.08576195270662823, -0.001505613568967484, -0.037047253907053095 ], [ -0.07332636282818543, -0.11944449710079048, -0.04322835682317902 ], [ -0.23500536931919885, -0.13102487282206995, 0.0740180521302486 ], [ -0.06652158490298232, -0.07260077314178147, -2.938021731087203E-4 ], ... ],
    	[ [ -0.062290235540694155, 0.026296842763119294, 0.11237920552850875 ], [ 0.15593550883404877, -5.861437071844465E-4, 0.022784652120148536 ], [ 0.03233592893503956, -0.057796404026982696, 0.022589438495426998 ], [ 0.07908927341230235, 0.008927511293514694, 0.11383422756581284 ], [ 0.126839926068645, -0.10463637232223372, 0.04226677638721025 ], [ 0.13396656256111872, -0.05968487696540355, -0.08977916677288296 ], [ -0.023080450750541066, -0.020678830790564925, -0.07119480206151223 ], [ -0.12953574811173138, -0.05403640316601777, 0.01264475384226031 ], ... ],
    	[ [ -0.07010985313636223, 0.003447417606843988, -0.00993293103350713 ], [ 0.09646480189553353, 0.012157189740666293, -0.1080453852917817 ], [ 0.1571879145291342, 0.03648163723330288, -0.05689163536461283 ], [ -0.04116275017628154, 0.07143344338515466, -8.526595349755854E-6 ], [ 0.03664284946740679, 0.03879028272259981, -0.015348556113459103 ], [ 0.049190640977274165, 0.011021424592603333, -0.0029365379510914434 ], [ -0.0511499293219292, -1.5847419758985553E-4, 0.07169079311416103 ], [ -0.0695356529326521, -0.03942432976984518, 0.03557891706314989 ], ... ],
    	[ [ 0.0304263289387118, 0.13318406493876297, 0.01673755373784027 ], [ -0.0856544511504502, -0.01807674625015476, -0.06445876722661595 ], [ -0.004497962661602228, 0.08889065213695425, -0.00954863010743072 ], [ -0.02068484190391679, 0.09077595884267907, 0.16992028262368342 ], [ 0.18678170647792988, 0.1326975282819175, 0.06950669456725048 ], [ 0.07429919228961054, -0.038318674677029214, 0.12500369356460023 ], [ 0.06368963291661787, -0.028246617237895545, -0.06121425244513673 ], [ -0.04352807034895425, -0.04154736429820055, 0.03457221848573886 ], ... ],
    	[ [ 0.044639144197065495, 0.06212257549499075, 0.021576845881102782 ], [ 0.04901942980077645, 0.07168926699149944, -0.010140408526698261 ], [ -0.016388126609521374, -0.0197015096219141, 0.05878462646052312 ], [ -0.030674460210966932, 0.08400610722627912, -0.02783046125211268 ], [ -0.028753152883711938, 0.08728958768060072, -0.06787100620250558 ], [ 0.0568997807292664, 0.02836846056012818, -0.01012025238099443 ], [ 0.06486348827279584, -0.06537938984884745, 0.0470820465559252 ], [ -0.03768408133337548, 0.07387609631458329, 0.08806076683050208 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 41.83 seconds (0.387 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 11985229580452
Reset training subject: 11985452264242
Adding measurement 6a7ee500 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 192.25124153084505 < 192.25124153084505. Total: 1
th(0)=192.25124153084505;dx=-164.93998516692167
Adding measurement 24b89a47 to history. Total: 1
New Minimum: 192.25124153084505 > 176.34475041544346
WOLF (strong): th(2.154434690031884)=176.34475041544346; dx=150.11109793338343 evalInputDelta=15.906491115401593
Adding measurement ab8e3dc to history. Total: 2
New Minimum: 176.34475041544346 > 99.45337476847806
END: th(1.077217345015942)=99.45337476847806; dx=-7.367486923096786 evalInputDelta=92.797866762367
Fitness changed from 192.25124153084505 to 99.45337476847806
Iteration 1 complete. Error: 99.45337476847806 Total: 2.1150; Orientation: 0.1969; Line Search: 1.2506
Non-optimal measurement 99.45337476847806 < 99.45337476847806. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 99.45337476847806 < 99.45337476847806. Total: 3
th(0)=99.45337476847806;dx=-0.33819270428251236
Non-optimal measurement 99.47405255768251 < 99.45337476847806. Total: 3
Armijo: th(2.3207944168063896)=99.47405255768251; dx=0.35601196513573724 evalInputDelta=-0.02067778920445562
Adding measurement 33387ca9 to history. Total: 3
New Minimum: 99.45337476847806 > 99.26232537297076
WOLF (strong): th(1.1603972084031948)=99.26232537297076; dx=0.008909868699278902 evalInputDelta=0.19104939550729227
Non-optimal measurement 99.34493864914897 < 99.26232537297076. Total: 4
END: th(0.3867990694677316)=99.34493864914897; dx=-0.22249179366756244 evalInputDelta=0.10843611932908459
Fitness changed from 99.45337476847806 to 99.26232537297076
Iteration 2 complete. Error: 99.26232537297076 Total: 1.9469; Orientation: 0.1813; Line Search: 1.5404
Non-optimal measurement 99.26232537297076 < 99.26232537297076. Total: 4
Rejected: LBFGS Orientation magnitude: 1.796e-02, gradient 1.595e-02, dot -1.000; [c0711a23-0d0b-4211-8429-e97adfa151ad = 1.000/1.000e+00, 0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00, 910f7025-f804-4678-9d44-1639e9fd1ecf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 99.26232537297076, 99.45337476847806, 176.34475041544346, 192.25124153084505
LBFGS Accumulation History: 3 points
Removed measurement 33387ca9 to history. Total: 3
Adding measurement d33f97d to history. Total: 3
th(0)=99.26232537297076;dx=-2.543951352269985E-4
Adding measurement 711e6dbd to history. Total: 4
New Minimum: 99.26232537297076 > 99.26219147942845
END: th(0.8333333333333335)=99.26219147942845; dx=-6.694936628272563E-5 evalInputDelta=1.3389354231208017E-4
Fitness changed from 99.26232537297076 to 99.26219147942845
Iteration 3 complete. Error: 99.26219147942845 Total: 6.0023; Orientation: 4.8335; Line Search: 0.9461
Non-optimal measurement 99.26219147942845 < 99.26219147942845. Total: 5
Rejected: LBFGS Orientation magnitude: 4.815e-03, gradient 4.269e-03, dot -1.000; [c0711a23-0d0b-4211-8429-e97adfa151ad = 1.000/1.000e+00, 0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00, 910f7025-f804-4678-9d44-1639e9fd1ecf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 99.26219147942845, 99.26232537297076, 99.45337476847806, 176.34475041544346, 192.25124153084505
Rejected: LBFGS Orientation magnitude: 4.817e-03, gradient 4.269e-03, dot -1.000; [910f7025-f804-4678-9d44-1639e9fd1ecf = 1.000/1.000e+00, 0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00, c0711a23-0d0b-4211-8429-e97adfa151ad = 1.000/1.000e+00]
Orientation rejected. Popping history element from 99.26219147942845, 99.26232537297076, 99.45337476847806, 176.34475041544346
LBFGS Accumulation History: 3 points
Removed measurement 711e6dbd to history. Total: 4
Removed measurement d33f97d to history. Total: 3
Adding measurement 29f00b5e to history. Total: 3
th(0)=99.26219147942845;dx=-1.8224990298894727E-5
Adding measurement 30eaa44c to history. Total: 4
New Minimum: 99.26219147942845 > 99.26218326448894
WOLF (strong): th(1.7953622416932369)=99.26218326448894; dx=9.07370113204989E-6 evalInputDelta=8.21493951264074E-6
Adding measurement 2e5fccd1 to history. Total: 5
New Minimum: 99.26218326448894 > 99.26218124557872
END: th(0.8976811208466184)=99.26218124557872; dx=-4.57564458166215E-6 evalInputDelta=1.0233849735641343E-5
Fitness changed from 99.26219147942845 to 99.26218124557872
Iteration 4 complete. Error: 99.26218124557872 Total: 12.6903; Orientation: 11.1679; Line Search: 1.2756
Non-optimal measurement 99.26218124557872 < 99.26218124557872. Total: 6
Rejected: LBFGS Orientation magnitude: 1.643e-03, gradient 1.343e-03, dot -0.997; [910f7025-f804-4678-9d44-1639e9fd1ecf = 1.000/1.000e+00, 0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00, c0711a23-0d0b-4211-8429-e97adfa151ad = 1.000/1.000e+00]
Orientation rejected. Popping history element from 99.26218124557872, 99.26218326448894, 99.26219147942845, 99.45337476847806, 176.34475041544346, 192.25124153084505
Rejected: LBFGS Orientation magnitude: 1.644e-03, gradient 1.343e-03, dot -0.997; [910f7025-f804-4678-9d44-1639e9fd1ecf = 1.000/1.000e+00, c0711a23-0d0b-4211-8429-e97adfa151ad = 1.000/1.000e+00, 0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 99.26218124557872, 99.26218326448894, 99.26219147942845, 99.45337476847806, 176.34475041544346
Rejected: LBFGS Orientation magnitude: 1.657e-03, gradient 1.343e-03, dot -0.998; [c0711a23-0d0b-4211-8429-e97adfa151ad = 1.000/1.000e+00, 910f7025-f804-4678-9d44-1639e9fd1ecf = 1.000/1.000e+00, 0cc4cf9a-72bb-4c1e-9f50-24e8e8e795ca = 1.000/1.000e+00]
Orientation rejected. Popping history element from 99.26218124557872, 99.26218326448894, 99.26219147942845, 99.45337476847806
LBFGS Accumulation History: 3 points
Removed measurement 2e5fccd1 to history. Total: 5
Removed measurement 30eaa44c to history. Total: 4
Removed measurement 29f00b5e to history. Total: 3
Adding measurement 7bf4b0fd to history. Total: 3
th(0)=99.26218124557872;dx=-1.8043307731148038E-6
Adding measurement 6588b64b to history. Total: 4
New Minimum: 99.26218124557872 > 99.26217892910853
END: th(1.9339953473386584)=99.26217892910853; dx=-5.911974124524977E-7 evalInputDelta=2.316470187224695E-6
Fitness changed from 99.26218124557872 to 99.26217892910853
Iteration 5 complete. Error: 99.26217892910853 Total: 19.0755; Orientation: 18.0147; Line Search: 0.8391
Final threshold in iteration 5: 99.26217892910853 (> 0.0) after 41.830s (< 30.000s)

Returns

    99.26217892910853

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    [-0.0025959962621000914, -2.990509369782924E-4, 0.013445605426321056, 0.00710311297502334, 0.01907488521067041, -6.06226531784571E-4, 7.394390789051465E-4, 0.006670465145325242, 0.010362491165083004, 0.006299609256501047, 0.003448795379980757, 0.004942532540996053, -0.01777331162030474, -0.003009954510367857, 0.018823756986874194, 0.006927234087607977, 0.004212420919524617, 0.018341749889425732, -7.585528628704106E-4, -0.012086341683080159, 0.0015423303488163202, 0.004183035864370998, -0.0014172480287546604, 0.009204134227319836, -0.015126340357895273, -0.004090472306775416, 0.013627754894189026, -0.008401069435558543, 0.010613303663340925, 0.0031904032439733572, -0.002471684369893829, -0.006963294731011068, -0.008888761914290201, 0.009264047378148073, 0.003636946764880435, 5.87598656498634E-4, 0.0032602817139644052, 0.0031992271300668135, -0.00451632587213262, 0.002057152395972031, 0.004459679948930675, 0.0075560539704979945, 0.01883549473509907, 5.07251591791419E-4, 1.9901883616768882E-4, -0.013874148778306079, -0.008853357086152808, -0.009920076051669936, 0.014760070483394087, 1.2797942664737128E-5, 0.01693855866729928, 0.006641555482066569, -0.006900188829544331, -0.006845721986895779, -0.008839504824472488, -0.0015298150380774773, 0.0044831864942206, 0.015597986329969789, -0.0052320366269512095, -0.009711784052128409, 3.167797000282542E-4, -0.003037688545512892, 0.003121989094084779, 0.0029393933855865356, 0.011632344350002821, -0.015578543931909775, -0.0010192138483927685, -0.012012626915703865, 0.009266246375374196, 0.005263791523944224, -0.007063477088873058, -0.003388389307260447, 0.004460320055072012, -0.007917844017283647, -0.007113450315189353, -0.01704260277019951, -0.012585794893348223, -0.0013553155154539013, 0.00938681680717398, -6.84412674035592E-4, 0.006676419199301275, -3.737567153368975E-4, 0.007302641385612056, 0.00939849390898254, 0.00573576507402472, 0.019895157442924697, 0.017145930083569492, -0.006590032567229909, 0.011121496107384743, 0.016897747188025555, 0.013885323465289956, 0.01494255852565246, -0.008976394474498052, 0.011732748564384792, 0.008771206189689993, 0.012677402572244598, 0.0018351059050823355, -0.0047041872879241145, -0.00180271343816679, -0.005497036237155602, -0.005310764917075314, 0.0036936648576222147, 0.006937991150654454, 0.0029280243284397285, -0.0037991475254087843, -0.003534183525021722, 0.0019178648589926583, -0.012098070814258852, 0.007308242644541112, 0.02026240042863708, -0.016163032806790704, -0.007453465033691801, 0.005708343287075306, -0.0066268793534122435, 0.010465634338433281, -0.012336037100659988, -0.012294680592731595, 0.002959378967936339, 0.0011880326863344412, 0.003067626217853351, 0.007361277488633839, -0.0018418558464838146, 0.006933159275136251, -0.019184051068118393, -0.002265792441029, 0.0017050955094464848, -0.00591365080962223, -0.009123459914572831, 0.013129318994343787, -0.012134457592009734, -0.0026554566525767626, 0.008643484778838697, -0.017503067045535656, 0.009022022411905483, 0.0038757306730175696, 2.1900093820842968E-4, -0.011105853883472763, -4.632709786220573E-4, 0.0022544246378041304, -0.013514601471687766, 0.013229802000566364, -0.0052671412786377166, -0.011392761035238752, -0.008556049069098449, 0.004391907132162832, 0.011714325629112263, 0.01405865944591628, 0.007879950250211425, -0.0017920788980407645, -0.0071808033952950965, -0.011965208156191157, 0.004602788766283863, -0.005301253287264553, 0.011900500425392573, 0.007555771421032245, 0.011941892534563408, -0.016202298478661566, -0.014313352785201702, 0.002924561688204204, 0.007849105404001737, 0.003039333972387399, 0.016466951527218512, -1.116, -0.628, 0.048, -1.724, 0.7, 1.556, -1.104, 0.016, 0.028, -2.0, 0.656, -0.636, -1.34, 1.288, -1.808, -1.424, 0.028, 0.184, -1.16, 0.0, -1.72, -0.768, 1.144, 0.0, 1.916, -0.504, 1.764, -0.472, 1.628, -1.028, -0.556, -0.876, -1.664, -0.808, 0.0, -1.456, 1.764, 0.0, 1.66, 1.556, 1.66, -1.552, -0.968, 1.82, 0.784, 0.0, 1.972, -0.892, 1.156, 0.0, -0.804, 1.288, 1.596, 0.0, -1.856, -1.808, 0.688, 1.972, 0.82, 0.0, -0.608, 0.0, -0.684, -1.54, 1.168, 0.56, 0.672, 0.996, -0.628, -0.768, -1.724, 1.652, -0.636, -1.256, -0.068, 0.08, -1.54, 1.556, -0.384, 1.032, 0.972, -1.856, -0.892, 1.256, 0.072, 0.048, 1.652, -0.124, -0.408, 1.66, 0.636, -1.16, -1.484, -0.888, -1.256, 1.912, 1.876, 1.552, 1.552, 0.0, 1.24, 0.812, -0.26, -0.368, 0.344, -1.104, -0.032, 1.512, -1.832, 0.08, 1.324, -0.368, 1.836, -0.26, -0.488, -1.76, 0.488, -0.848, -0.176, 1.524, 0.0, -0.804, 0.148, 1.144, 1.876, -1.228, -0.892, -1.456, 1.66, -1.764, 1.168, -1.536, 0.0, -1.524, 1.836, 1.108, 1.144, -1.492, 0.82, 0.3, -0.848, -0.784, -0.924, -0.712, 0.092, 0.0, -2.0, 1.288, -0.464, -0.852, 0.644, 0.184, 0.0, 0.0, -1.176, 0.688, -0.464, 0.496, 0.048, -0.804, -1.552, -1.76]

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.5760138812850272, -0.33596703480120577, 1.6519381374487778, -1.284045688959007, 0.21998451461082702, -0.10403806153583943 ], [ 1.115898625306524, 0.3879903507462168, 0.987929841816568, 1.3279465308602567, 1.5319104495210947, -0.5480496792253557 ], [ -0.18790842948480033, -1.3599911651401202, -1.3039076095353619, -1.759981612671208, 0.003923235309961449, 0.032092806748517624 ], [ -1.9639202927251131, -1.011939658314347, -1.0719509711738726, 0.407991075031273, -1.8599003954003652, 1.6800563918453446 ], [ -1.7640478957228776, 1.083865987450001, 1.347908556161946, -0.520004181336998, 0.2439735001713304, -1.0559144665260753 ], [ -0.2599521636782997, -0.8159105188204303, -1.0920159282655646, -1.3799420571887657, -1.2839954689551978, 1.9560694717922054 ], [ -1.2999849454427332, 0.3360172187512756, 1.9679630000378492, -1.703953403784044, 0.5159440515376545, -1.671892574308519 ], [ 0.1280220509173455, -1.927878948552147, 0.8599649159859754, -0.9799270521305211, 0.319936172117554, -0.13994710550522263 ], ... ],
    	[ [ -1.856003506295857, 0.4040044016196348, 1.91595649536205, 0.06005867784383196, 0.48797148718650474, 1.0880496668101745 ], [ -0.6999480908916239, -1.3598912715586227, 1.7280222402578946, -1.0479439876146361, 1.9758552070775428, -1.0118828617576034 ], [ 0.2041705271084494, 0.7159662701737707, -1.3559105178134323, 0.084008181215841, -1.463862801377765, -1.627798465121716 ], [ 1.5398181902738144, -0.6440211061950799, 1.9759009981580027, -1.5399329818987613, 0.9838519017508469, -0.8040123465665434 ], [ -0.5478224744983963, 0.7681234236212576, -1.1839706401972583, 0.17198782214981384, 1.779927637700558, 0.7080796565745925 ], [ -0.8920654484825038, 1.904032633654578, -1.152060800292521, -1.655914650534211, 1.7640046254540813, -0.48400219304211856 ], [ -1.0399145898266429, 1.9760414491272895, 0.02394737084969404, 0.8119643438503987, 0.8039070161433732, 1.187989718467376 ], [ 1.6320224010826352, 0.2960428770039294, -0.28801752597487, 1.256053782629188, 0.9360840985884107, -1.8559293338423528 ], ... ],
    	[ [ -0.4919151812078671, 0.36798199033682216, -0.4678456531130137, -0.5278731460930582, 1.4080571901084629, -0.7838995112111439 ], [ -0.4360446505690921, -0.4880308319434861, 0.9720659671654651, -1.8959289885775295, -1.9758260992130465, 1.5640053907501996 ], [ 1.7760814069403017, 0.9760449978729691, -0.25192829438538883, -0.8279924210603818, 0.8680421800642152, 0.2238443462923869 ], [ 0.43585984997730093, 0.7280020330218802, 0.19184944672500595, 0.667933207593063, 1.3759018524209083, -1.7718753576579598 ], [ -0.5397833668664529, -1.6120059974058116, 0.9961645840163299, -1.1959045768948888, -1.3198591644428854, 0.18008955629009069 ], [ -0.2801353708088324, -1.9638996166157638, -0.4199829058642033, -1.9358401789975619, -0.21193728734864667, -0.5718195118377887 ], [ -1.7238969703283074, -1.932014602561748, 0.3080502333230761, 0.4720663376243194, -1.2439341958119357, -1.1879788136528502 ], [ -1.4999755611440793, -0.6679681243313015, -1.1119392827368275, -0.1518062217643701, -1.3079224700780243, -1.3120250992464588 ], ... ],
    	[ [ -0.42794330811323034, -0.6719253133595576, 1.6119896149171942, 0.36802437862348997, 1.53194430411663, 0.632011660618956 ], [ -1.2080545579407869, 0.14800467122392355, 1.923870369007214, 1.2079130014768868, -0.7839713741619699, 0.13187803162912903 ], [ 1.9278374963690807, 0.4240808984237611, 0.8761695623678735, -1.4039103950985783, -0.8799783638970272, -1.296137768212838 ], [ 0.4879312472530705, 1.1999569767591771, -0.11609492593891046, 1.5838272069744415, -0.6760830188599305, -0.18799631129736388 ], [ -0.8439179671892895, 1.8919691591725496, 1.0359990566571025, -1.1719076753501432, 1.2120758905999443, 1.6599310148684088 ], [ -0.38407391797972906, 1.664057625823305, -0.4800799500695706, -1.0519614222363207, 1.5599552171139095, 0.5200367989095221 ], [ -0.9479558973060765, 1.083941956049414, -0.15198955843134002, -0.6359376554099188, -0.807926137979361, -0.7920046127818049 ], [ 1.0680584833941693, -1.6798268570879564, -0.4919905066346734, -0.44790464361700727, -1.0479477800350339, -0.004008544025039392 ], ... ],
    	[ [ -0.9239643446804977, -0.31600287768059787, 1.1800184437605392, -0.12809403112042206, -1.5838985295430195, -0.5080183299287349 ], [ 1.0319044274087734, 1.8477663820056522, -0.34403820262359575, -1.383994203575928, 0.06812436565231679, 0.8359005525323023 ], [ 0.19994355087382704, 1.220082230740546, -0.8639495538282975, 1.3799206635987842, 1.2758908934726503, -0.4401075926772854 ], [ 0.8839635862495155, 1.5157884085045639, -0.6599219265007685, -0.5561012111192094, -1.9639203755030736, -1.6320527132014602 ], [ 0.22407995276006726, 1.2961081460550514, -1.879969244424207, -0.44797260995663096, -1.335934954376372, -1.6879059408716661 ], [ -1.308117310824205, -0.07205276544101685, 0.2159302107017892, -1.4159024551927684, -1.6120361936615637, 1.2320181229817437 ], [ -1.831864430702357, -0.43191490631464535, -0.5440186203173671, 0.7638855990053101, -1.3160887707064017, -0.6320047847573667 ], [ -1.5279981352263035, -1.127892783680991, 0.9359441389912226, 0.924085199980257, -0.3079367400080364, 1.2479621232012792 ], ... ],
    	[ [ 0.12387487435027524, 1.6558505331760784, -1.296057313277103, 1.8239658511150112, -0.5120154891837202, 0.8038095187581153 ], [ -1.483961037165817, -1.1279084797044427, -0.9559553472643961, -0.5800713268461127, -0.47206974007697383, 1.119924974611243 ], [ -1.1920776880359258, 0.559939510749485, 0.24403320375476337, 1.2560864775684155, -1.0639482328350294, 0.23596132865164365 ], [ 0.5720046074643852, 0.5280579655305917, -1.9199525950921748, 1.515885428309195, 0.811937479053691, -0.3080417151915813 ], [ 0.28815041823963905, 0.8359372163746672, -0.19591314034511473, -1.6320085602017878, 1.1599929805452467, -0.37182466502682904 ], [ 0.5439144112871919, 0.1319981122651878, 0.06388731406408678, 1.8399492882344577, -0.3400990079522548, 0.1880834542914197 ], [ -1.3039285791807498, 1.1278908452407816, 0.5920339780982818, -1.9200415452756452, -0.5118928959359875, 0.6440131384013338 ], [ -1.747947704716017, 0.40798580245061494, -0.35994888676587616, -1.0838179764668445, 0.772006734230557, -1.1479302276546435 ], ... ],
    	[ [ 0.9159268125623036, 1.9360550976317885, -1.092085805161282, 0.5359615755946523, 0.3359381215822215, 0.5039630890873813 ], [ -0.6558457854944938, -0.4279773636374489, -1.1959405062408284, 1.4999370633815647, 0.18801949310209334, 0.11604506503216798 ], [ -1.3880457806212032, -1.5920669038341815, 0.515917130501706, -0.34806152253435185, -1.9839992026317073, 0.5041110961075114 ], [ -0.9200318453817614, 0.9439337660548361, 0.1440319121429448, -0.891958094950547, 1.464009941086565, -0.9239427076890958 ], [ 0.17613484275099195, -1.427916341276075, -1.8799085400994124, -0.41192698300291825, 1.2479730412928314, -1.7799144035664507 ], [ 1.591831725053226, 0.43588011313289254, -0.10014296109890382, 1.2319181835160542, 1.3320247771296014, 0.6959247211954283 ], [ 0.07206121187776687, 0.7800926994730212, -0.9518914931522584, -0.9239704907309813, 0.6199139134900804, -0.2240562151384211 ], [ -1.2519589731260488, 1.343943380892521, 1.9157041820519591, -0.45205437627557266, -1.2239788297232668, -0.39194913419556876 ], ... ],
    	[ [ 1.0040328058982637, 0.9559701844284543, 0.48793932825090636, -0.1200275131330661, 1.560028621361134, -0.6478933161959458 ], [ -0.9279022859033444, -0.05996710871785246, -0.5398850070702669, -0.10782526920587447, 1.5279774799563437, -1.6799076135597635 ], [ 1.6160168058920938, -1.8159197227843524, 0.464020400871658, 1.255994939707306, 0.22395701495488451, 1.5760359193755262 ], [ 0.687958301936787, 0.8919771203989062, -1.9680923233160552, 1.4078983885862868, 0.5799535188327819, 1.6758890648383007 ], [ 1.5159680928004118, -0.6839836242045616, -1.703983896830894, -0.1919737841054208, 1.4439885305322346, 0.9479347997083646 ], [ -0.13601836615691168, 0.7961194584201415, -0.7559422386356436, -1.4999999392508503, 1.0360050128291203, 0.7000693059950728 ], [ 0.5120418968768481, 0.3679448361295011, -1.73599689294195, 0.37192003241771165, -1.6359528035040918, 0.3680220508259533 ], [ 1.4080266286210397, -0.06803967600295974, -0.8040512047683693, 1.8759925228558654, 0.17197384460997936, -1.6118874372994045 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.03 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.03630428068452121, -0.01228925203928055, -0.01853503987593015 ], [ 0.022294077240347075, -0.04518727264429808, -0.014050000354374453 ], [ -0.01912006089869021, -0.0029746411074936313, -0.04234071546359821 ], [ -0.04537482193630606, 0.038602974306490415, -0.04992842727106185 ], [ -0.08529721419397754, 0.0623952379542127, 0.052107453401757195 ], [ -0.01543840185728194, -0.0023566233850538664, 0.051829716596292964 ], [ -0.024045335726216036, -0.059227128120979376, 0.01560726919691864 ], [ -0.02291718027736219, -0.0650989422230197, 0.04948032364003263 ], ... ],
    	[ [ -0.05128121618987409, -0.06232280297861008, 0.0030834512308480387 ], [ 0.07507314087702463, -0.004452890657308138, 0.005047858955540614 ], [ -0.10840426313867117, 0.07397606102220967, 0.05768792872000865 ], [ -0.0069531397688254795, -0.09268786698236622, 0.12499529620647544 ], [ 0.005646626712844102, 0.04409691514830363, 0.0015799065031629474 ], [ 0.006494794336283744, -0.10130539763513834, 0.035225646032307566 ], [ -0.04213308218993964, -0.08946423994448745, 0.08157882683642266 ], [ -0.06543667106131727, -0.12162810255450489, 0.10177791068786704 ], ... ],
    	[ [ -0.005930586287496393, -0.06546119726811699, 0.043141136747509454 ], [ 0.055686733741709384, -0.11025398904062776, -0.023234180756528385 ], [ -0.02655565102923274, -0.026834895026003027, -0.004335551564362931 ], [ -0.08478153397454255, -0.05550274376746975, -0.014685893235206181 ], [ 0.07349563209242194, -0.08128504841373226, -0.0028736382807104245 ], [ 0.03323262259308145, -0.02130509522578928, -0.02996402887155529 ], [ 0.07087688760648819, -0.0373454502464665, -0.0320444778820981 ], [ -0.0440293869828521, 0.13113912926254156, 0.05897591759778541 ], ... ],
    	[ [ -0.028804645250603535, -0.04299899076051406, -0.08254560770941068 ], [ -0.07967512228086075, -0.06856572714631622, -0.002879428136124325 ], [ 0.10336397563235007, -0.08111386057068423, 0.09937013095967802 ], [ 0.021835769076438985, 0.0024512035731058097, -0.045732105327951085 ], [ -0.08620952114600046, -7.939409652809032E-4, -0.038430918584559645 ], [ -0.07387288925120682, -0.1181460706253737, -0.04295626691173775 ], [ -0.23512355144923994, -0.1309797817700078, 0.07396037310740278 ], [ -0.06764852143789916, -0.07498975658169266, -8.700677724280384E-4 ], ... ],
    	[ [ -0.06357401024033832, 0.026914376466418186, 0.11244648304299888 ], [ 0.15701403221209592, -0.0015434647567956654, 0.022825689457976205 ], [ 0.0320515990516832, -0.056685113253106705, 0.022091694564055412 ], [ 0.08023138048172371, 0.009824886519030707, 0.11379331153003411 ], [ 0.12543179025166948, -0.1035592152211803, 0.0429548748698116 ], [ 0.13206607849296462, -0.059647316229661324, -0.09061064176138173 ], [ -0.021398661734695044, -0.020276423194210337, -0.07262171222481617 ], [ -0.12975279392002792, -0.05498253286042405, 0.012344815843218709 ], ... ],
    	[ [ -0.06908466238252949, 0.003316000826537266, -0.009158260137596934 ], [ 0.09588360604855332, 0.012208173337977777, -0.10762801423798408 ], [ 0.1580000308521838, 0.036699911155784734, -0.05878984069766759 ], [ -0.03927391118323606, 0.07318219147607916, -0.0016982617366851837 ], [ 0.03760182561383016, 0.039104264257608774, -0.016304152857063593 ], [ 0.04766907241778498, 0.011116308781867808, -0.004168211001663539 ], [ -0.04990039477987706, -4.747430068967536E-4, 0.07212591790367476 ], [ -0.07023238146245812, -0.03941882919833886, 0.03542685000672596 ], ... ],
    	[ [ 0.03226119020548197, 0.13328805924712483, 0.016715843627328232 ], [ -0.08598786485866325, -0.017876810397646537, -0.06381720979425087 ], [ -0.0053709956241068755, 0.08788987256240427, -0.01000130021923335 ], [ -0.020719767451565094, 0.09056257581023469, 0.17102133713666132 ], [ 0.18721094152415146, 0.13225789446228964, 0.06888244333131574 ], [ 0.0728974239770904, -0.03788730840159309, 0.12513395818183973 ], [ 0.06382497727189979, -0.025632075473169962, -0.06135210520779609 ], [ -0.04479765502864132, -0.04188703393998777, 0.03248165652395373 ], ... ],
    	[ [ 0.04353800486051207, 0.061590162609065115, 0.022211031537435637 ], [ 0.04958742371350708, 0.07147773463515004, -0.010748643566576297 ], [ -0.01711580472433286, -0.021075274096722434, 0.05886544983339251 ], [ -0.029717160312915884, 0.08440479407997463, -0.028370245053157574 ], [ -0.02775878626030181, 0.0871930924985127, -0.06766383115613403 ], [ 0.05736397525841838, 0.029879942796542665, -0.00967920695438952 ], [ 0.0654848230571173, -0.0633200556362725, 0.04610198695405196 ], [ -0.03581271312439993, 0.07408033177694848, 0.08844642798529505 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, 1.9608443041883556], [18.0, 1.997619524699876]; valueStats=DoubleSummaryStatistics{count=34, sum=3354.574401, min=91.378559, average=98.663953, max=99.453375}
Plotting 18 points for GD
Plotting 11 points for CjGD
Plotting 5 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, 1.9608443041883556], [39.715, 1.997619524699876]; valueStats=DoubleSummaryStatistics{count=34, sum=3354.574401, min=91.378559, average=98.663953, max=99.453375}
Plotting 18 points for GD
Plotting 11 points for CjGD
Plotting 5 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 21.599592634943086 }, "CjGD": { "type": "NonConverged", "value": 0.5261154000716588 }, "GD": { "type": "NonConverged", "value": 0.03114460893026061 } }, "model":{ "LBFGS": { "type": "Converged", "value": 1.2705787827554681E-37 }, "CjGD": { "type": "Converged", "value": 6.554224923704488E-36 }, "GD": { "type": "Converged", "value": 1.2705787827554681E-37 } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": 99.26217892910853 }, "CjGD": { "type": "NonConverged", "value": 99.26216108538087 }, "GD": { "type": "NonConverged", "value": 91.3785587999312 } }}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 21.599592634943086 }, "CjGD": { "type": "NonConverged", "value": 0.5261154000716588 }, "GD": { "type": "NonConverged", "value": 0.03114460893026061 } }, "model":{ "LBFGS": { "type": "Converged", "value": 1.2705787827554681E-37 }, "CjGD": { "type": "Converged", "value": 6.554224923704488E-36 }, "GD": { "type": "Converged", "value": 1.2705787827554681E-37 } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": 99.26217892910853 }, "CjGD": { "type": "NonConverged", "value": 99.26216108538087 }, "GD": { "type": "NonConverged", "value": 91.3785587999312 } }}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "297.694",
      "gc_time": "4.394"
    },
    "created_on": 1586746361781,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "BandReduceTest",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.conv.ConvolutionLayerTest.BandReduceTest",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/conv/ConvolutionLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 21.599592634943086
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 0.5261154000716588
        },
        "GD": {
          "type": "NonConverged",
          "value": 0.03114460893026061
        }
      },
      "model": {
        "LBFGS": {
          "type": "Converged",
          "value": 1.2705787827554681E-37
        },
        "CjGD": {
          "type": "Converged",
          "value": 6.554224923704488E-36
        },
        "GD": {
          "type": "Converged",
          "value": 1.2705787827554681E-37
        }
      },
      "complete": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 99.26217892910853
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 99.26216108538087
        },
        "GD": {
          "type": "NonConverged",
          "value": 91.3785587999312
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/conv/ConvolutionLayer/BandReduceTest/trainingTest/202004135241",
    "id": "738ce686-b826-4010-9d00-13d1898aa597",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "ConvolutionLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.conv.ConvolutionLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/conv/ConvolutionLayer.java",
      "javaDoc": ""
    }
  }