1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 5247335493752290304

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.008, 1.896, 0.236 ], [ 1.616, -1.744, 1.688 ], [ 1.496, -0.876, 1.436 ], [ -0.364, -1.628, 0.304 ], [ 1.284, -0.664, -0.236 ], [ 1.932, 1.564, 1.352 ], [ -1.408, -0.024, 0.256 ], [ -0.252, -1.304, -0.204 ], ... ],
    	[ [ -1.576, 0.304, 0.244 ], [ -1.564, -1.14, -0.452 ], [ -0.764, 1.636, 0.772 ], [ -1.644, 1.628, 1.1 ], [ -1.64, -1.412, 0.444 ], [ 0.168, -0.88, 0.544 ], [ -1.992, 1.72, 0.384 ], [ 0.564, -1.304, -1.824 ], ... ],
    	[ [ -1.316, -0.26, -1.5 ], [ 1.688, 0.088, -1.912 ], [ -0.872, 0.368, -0.108 ], [ 0.844, -0.236, -0.572 ], [ -0.856, 1.612, -0.092 ], [ 1.5, 1.04, -1.84 ], [ -1.232, -1.12, -1.572 ], [ -1.768, 0.732, -1.556 ], ... ],
    	[ [ 1.356, 1.596, -0.352 ], [ 1.304, -1.144, 1.88 ], [ 1.38, -0.108, 0.168 ], [ -1.052, -0.368, 0.456 ], [ -0.316, 1.82, -1.36 ], [ -1.924, 0.236, 1.84 ], [ -0.08, 1.848, -1.372 ], [ -0.352, 1.22, -1.336 ], ... ],
    	[ [ 1.88, -1.528, 1.636 ], [ -0.928, 0.252, -1.368 ], [ 1.0, -0.664, -0.796 ], [ -0.08, -1.488, 0.552 ], [ -1.512, -1.032, -1.644 ], [ 1.676, -1.012, -0.028 ], [ -0.844, -1.764, 0.276 ], [ 0.524, -0.72, 0.616 ], ... ],
    	[ [ -0.568, -0.672, 0.356 ], [ -0.172, -0.46, -1.736 ], [ 0.108, 1.608, -1.096 ], [ 1.988, 0.992, 0.476 ], [ -0.7, -1.972, 1.368 ], [ 0.268, 1.528, 1.24 ], [ 1.84, -1.024, -1.468 ], [ 1.12, 0.716, 1.536 ], ... ],
    	[ [ -0.628, 0.136, 1.132 ], [ 1.488, 0.308, -0.756 ], [ -0.7, 0.048, -1.248 ], [ 0.444, -1.544, 0.364 ], [ -1.66, -1.292, -1.644 ], [ -1.104, 0.484, -0.744 ], [ 1.944, 0.94, 0.796 ], [ 0.884, -1.408, -0.536 ], ... ],
    	[ [ -1.312, -0.56, -1.72 ], [ -1.6, 0.348, -0.828 ], [ -1.508, -0.168, -0.652 ], [ 0.728, 0.468, 1.904 ], [ -1.608, 1.4, 1.636 ], [ 1.02, -1.216, 0.84 ], [ 0.516, -0.656, 0.156 ], [ 1.136, -0.764, -0.936 ], ... ],
    	...
    ]
    [
    	[ [ 0.812, 1.3, -0.892 ], [ 0.512, 1.824, -0.588 ], [ 0.628, 0.224, -1.188 ], [ -0.916, -1.38, 1.212 ], [ -0.568, -0.468, 1.472 ], [ -1.356, 0.992, 0.944 ], [ -1.588, -0.5, -1.724 ], [ -0.664, -0.32, -1.796 ], ... ],
    	[ [ 1.64, -1.448, -1.836 ], [ -0.5, -1.032, -1.388 ], [ -1.248, 1.796, -0.26 ], [ -0.984, -1.14, -0.284 ], [ -1.616, -0.008, 1.764 ], [ -0.6, -1.736, -1.164 ], [ -0.016, -1.508, -1.764 ], [ -0.368, 1.252, 1.168 ], ... ],
    	[ [ 1.204, -0.852, 0.872 ], [ -0.86, 0.308, 1.328 ], [ -0.916, -0.148, 1.068 ], [ 0.004, 1.324, 0.068 ], [ 0.624, -0.988, 0.392 ], [ 0.204, 0.836, 0.836 ], [ 1.228, -1.076, 1.972 ], [ -1.304, -0.968, -0.048 ], ... ],
    	[ [ -1.248, -0.28, 1.936 ], [ -0.116, 1.356, -1.12 ], [ -1.124, -1.784, 1.668 ], [ 1.916, 1.156, 1.616 ], [ -0.616, 1.764, 0.376 ], [ -0.172, -0.18, 0.176 ], [ -0.932, -0.424, -1.84 ], [ 1.252, -1.036, -0.016 ], ... ],
    	[ [ -0.176, 1.784, -1.744 ], [ -0.428, 1.12, 1.824 ], [ -0.896, 0.416, -1.872 ], [ 1.024, 1.584, 0.392 ], [ 0.404, 0.524, 1.864 ], [ -1.36, 1.504, 1.648 ], [ 0.404, 1.944, 0.736 ], [ 0.772, 0.648, -1.344 ], ... ],
    	[ [ 0.616, -1.124, -0.4 ], [ 0.58, -1.864, -0.624 ], [ -1.348, -1.548, -0.692 ], [ 1.788, 1.796, -1.396 ], [ 0.364, -1.064, 0.612 ], [ -0.432, 0.136, -0.456 ], [ -1.656, 1.792, -0.424 ], [ 1.152, 1.724, 0.12 ], ... ],
    	[ [ -0.46, 0.768, -1.688 ], [ -0.616, -1.76, -0.18 ], [ -1.888, -0.164, -0.488 ], [ -1.828, -1.824, 0.512 ], [ 0.504, 1.276, -0.116 ], [ -1.752, -1.94, 0.708 ], [ 0.452, -0.88, 1.728 ], [ -0.808, -0.448, 1.036 ], ... ],
    	[ [ -1.636, -1.624, -0.72 ], [ 0.128, -0.264, -1.232 ], [ 1.04, -0.412, -1.868 ], [ -0.592, 0.728, -1.8 ], [ 1.424, -0.62, 1.82 ], [ -0.548, 0.276, 1.424 ], [ -0.528, -1.056, -1.988 ], [ -0.432, -1.36, -0.384 ], ... ],
    	...
    ]
    [
    	[ [ 0.956, 0.824, 1.7 ], [ 0.924, -0.856, -1.212 ], [ 0.008, -0.264, -0.88 ], [ -1.236, 0.904, 0.148 ], [ 1.192, 1.292, 0.128 ], [ 0.016, 1.032, -1.904 ], [ -0.828, -0.432, 1.752 ], [ 0.044, 0.628, -0.408 ], ... ],
    	[ [ -1.584, 0.144, -0.968 ], [ 1.352, -1.576, -0.476 ], [ 1.156, -0.72, -0.032 ], [ -0.248, 0.104, 0.024 ], [ 0.804, -0.808, 0.456 ], [ -0.836, 0.34, -0.016 ], [ -1.996, -0.404, 1.676 ], [ -1.684, 0.796, -0.576 ], ... ],
    	[ [ 0.928, -1.004, 0.496 ], [ -0.716, -0.78, -0.876 ], [ 1.252, 0.62, -1.4 ], [ 0.852, 1.62, 1.564 ], [ -0.024, -0.704, 0.76 ], [ 0.64, -0.56, 0.008 ], [ 1.952, 1.044, 0.216 ], [ -0.644, 1.372, 0.088 ], ... ],
    	[ [ -0.64, -1.06, 1.344 ], [ -0.652, 1.136, 0.504 ], [ -0.108, -1.492, 0.408 ], [ 1.12, 0.82, 0.664 ], [ 0.532, -1.176, 1.5 ], [ -0.804, -1.636, -1.232 ], [ -1.504, 1.928, -0.588 ], [ 1.088, 1.268, 1.132 ], ... ],
    	[ [ -1.648, -0.096, 0.564 ], [ -1.12, -0.904, 1.888 ], [ 1.952, -0.868, -0.128 ], [ -1.436, -0.544, 1.564 ], [ -0.72, 0.464, -0.228 ], [ -1.132, -1.768, -0.568 ], [ 0.672, -1.804, 0.2 ], [ 0.356, -0.556, 1.492 ], ... ],
    	[ [ 0.376, 1.216, -0.62 ], [ -0.612, 0.516, -0.076 ], [ -0.76, 1.452, 0.524 ], [ -0.76, -1.732, -0.02 ], [ 1.364, 1.824, -1.032 ], [ -0.048, 0.5, 0.632 ], [ -1.568, -0.748, 0.192 ], [ -1.28, -0.18, 1.568 ], ... ],
    	[ [ 0.836, -1.096, -0.38 ], [ 0.68, 0.296, 1.732 ], [ 0.596, -1.604, -0.424 ], [ 1.316, -0.524, 0.828 ], [ -1.888, 0.004, 0.128 ], [ -1.316, 0.728, -1.1 ], [ -1.64, 0.184, -0.616 ], [ 0.132, 1.612, 0.276 ], ... ],
    	[ [ -1.58, -1.68, 0.544 ], [ 1.224, 0.636, -0.448 ], [ -0.304, -0.444, -0.096 ], [ -0.128, 0.504, 0.872 ], [ -0.208, -1.864, 0.736 ], [ 1.6, 0.436, 1.08 ], [ 1.456, 0.096, -0.752 ], [ 1.02, -0.448, -0.536 ], ... ],
    	...
    ]
    [
    	[ [ -1.104, -0.532, 1.18 ], [ -0.424, -1.46, 0.208 ], [ -0.24, -0.532, -1.12 ], [ 0.316, 0.204, -0.308 ], [ -0.144, 0.56, -0.148 ], [ 0.016, 1.112, 0.304 ], [ -1.924, -1.232, 1.544 ], [ 1.316, 1.116, -0.74 ], ... ],
    	[ [ 0.192, 1.148, -1.376 ], [ -0.14, 1.844, 1.084 ], [ -1.612, 1.968, 0.304 ], [ 0.408, 0.68, -1.552 ], [ -1.856, -1.192, -1.62 ], [ 0.696, -0.436, 0.668 ], [ 1.804, 1.972, 0.872 ], [ -0.468, -1.12, 1.928 ], ... ],
    	[ [ -1.816, -0.024, 0.776 ], [ 0.288, -0.508, -1.832 ], [ 1.604, 0.796, -1.752 ], [ 1.352, 1.352, -1.752 ], [ 0.772, -1.44, -1.216 ], [ 1.868, -0.876, -0.868 ], [ -1.648, 0.608, -0.732 ], [ 0.252, 0.388, 1.92 ], ... ],
    	[ [ 0.58, 1.204, 0.22 ], [ 0.108, 0.472, -1.6 ], [ -1.1, 1.76, 1.176 ], [ -0.512, 0.54, -0.6 ], [ 1.264, -0.996, 1.0 ], [ 0.068, 1.46, 1.792 ], [ -1.256, -1.392, 0.792 ], [ 0.732, 0.156, 0.808 ], ... ],
    	[ [ 0.604, 1.764, 0.212 ], [ 0.864, -0.356, -0.104 ], [ 1.544, 0.504, 1.492 ], [ 1.728, -0.104, -1.768 ], [ 1.448, -0.736, -1.564 ], [ 0.308, 0.32, -1.908 ], [ 1.336, -0.884, 1.84 ], [ -0.732, 1.34, 0.1 ], ... ],
    	[ [ 0.284, 1.288, 1.0 ], [ -0.716, -0.64, -1.904 ], [ -1.56, -0.792, -0.176 ], [ -0.84, 0.168, 1.228 ], [ 0.356, 0.988, 0.688 ], [ 0.284, -0.156, -0.868 ], [ -0.352, -0.268, 0.348 ], [ 1.18, -0.94, 0.912 ], ... ],
    	[ [ -1.308, 0.048, 0.488 ], [ -1.428, -0.852, -0.168 ], [ -0.412, -0.924, -1.596 ], [ 0.716, -0.168, 1.584 ], [ -0.44, -1.96, 0.376 ], [ -1.996, 1.096, 1.136 ], [ -0.036, -1.34, -1.984 ], [ 0.796, -1.396, 0.836 ], ... ],
    	[ [ -0.064, -1.728, -1.568 ], [ -0.592, -0.824, -0.032 ], [ 0.528, -1.908, 0.36 ], [ 0.604, 1.648, 1.644 ], [ 0.052, 1.704, -1.776 ], [ -1.18, -1.604, -0.584 ], [ 0.848, -0.572, 0.116 ], [ -1.584, 0.244, -1.096 ], ... ],
    	...
    ]
    [
    	[ [ -1.508, -1.156, -1.204 ], [ 1.64, 0.46, 0.816 ], [ 0.868, -1.292, 0.428 ], [ -0.396, -1.352, -1.04 ], [ 1.428, -1.792, 1.584 ], [ 1.796, -0.94, 0.372 ], [ -0.436, -1.08, 1.42 ], [ 1.344, -0.692, 1.992 ], ... ],
    	[ [ -1.092, -1.488, 1.308 ], [ -1.416, -0.4, 0.72 ], [ 1.684, 1.8, -1.116 ], [ -0.048, 1.324, 1.148 ], [ -0.476, 0.64, -1.856 ], [ 0.796, -0.8, 0.016 ], [ -1.484, 1.236, -0.452 ], [ -0.08, 0.616, -1.212 ], ... ],
    	[ [ -0.3, 1.044, 0.096 ], [ 0.96, 0.2, 1.864 ], [ -0.316, 1.668, 0.152 ], [ -0.64, -0.048, 1.232 ], [ 0.52, 1.728, 0.884 ], [ 0.756, -1.008, -1.364 ], [ 0.708, -0.716, -0.056 ], [ 1.928, 1.432, -1.224 ], ... ],
    	[ [ 0.812, -1.344, 1.336 ], [ -1.484, -1.492, -0.424 ], [ 1.844, -0.088, 1.392 ], [ 1.74, 1.464, -0.756 ], [ -0.788, 0.84, -0.596 ], [ -1.96, -0.328, -1.632 ], [ 0.884, -1.812, -1.26 ], [ -1.856, -1.168, -1.852 ], ... ],
    	[ [ 0.624, -1.9, -1.356 ], [ -0.62, -0.956, -0.92 ], [ 0.088, -0.896, -0.008 ], [ 0.0, -0.4, 0.788 ], [ -1.536, -0.644, 0.352 ], [ -0.984, 0.844, -1.328 ], [ -0.744, -0.676, 1.852 ], [ 0.548, 0.68, -0.08 ], ... ],
    	[ [ -1.768, 0.536, -0.972 ], [ -0.648, -1.636, 1.536 ], [ -1.02, 1.544, -1.08 ], [ 1.52, 1.46, 0.54 ], [ -0.496, 1.88, -0.944 ], [ -1.308, 0.92, -1.468 ], [ -0.988, 1.42, -1.548 ], [ 1.072, -1.808, 1.184 ], ... ],
    	[ [ -1.796, -1.304, 1.628 ], [ 1.892, -0.596, -0.336 ], [ -1.976, 0.92, 1.192 ], [ 1.5, -0.228, 0.712 ], [ 1.792, 0.212, -1.536 ], [ 1.272, 1.32, -0.18 ], [ -1.364, 1.232, 0.66 ], [ -0.176, -1.064, -1.24 ], ... ],
    	[ [ 0.416, -1.036, 0.624 ], [ 0.252, -0.768, -1.304 ], [ -0.156, -1.952, -1.372 ], [ 1.5, 0.164, 0.952 ], [ 0.172, 1.944, 0.408 ], [ 1.468, -0.096, 1.848 ], [ 1.248, -1.348, 1.788 ], [ -1.34, 0.776, 1.04 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 31.85 seconds (2.033 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 9051095022198
Reset training subject: 9051750162306
Low gradient: 5.968633241133871E-7
Constructing line search parameters: GD
th(0)=1.2824849796180558E-6;dx=-3.5624582767167093E-13
New Minimum: 1.2824849796180558E-6 > 1.282484212109783E-6
WOLFE (weak): th(2.154434690031884)=1.282484212109783E-6; dx=-3.5624572107328415E-13 evalInputDelta=7.675082728432215E-13
New Minimum: 1.282484212109783E-6 > 1.2824834446017444E-6
WOLFE (weak): th(4.308869380063768)=1.2824834446017444E-6; dx=-3.562456144749216E-13 evalInputDelta=1.5350163114818331E-12
New Minimum: 1.2824834446017444E-6 > 1.2824803745719604E-6
WOLFE (weak): th(12.926608140191302)=1.2824803745719604E-6; dx=-3.562451880813714E-13 evalInputDelta=4.605046095402818E-12
New Minimum: 1.2824803745719604E-6 > 1.2824665594833229E-6
WOLFE (weak): th(51.70643256076521)=1.2824665594833229E-6; dx=-3.5624326931043826E-13 evalInputDelta=1.8420134732986828E-11
New Minimum: 1.2824665594833229E-6 > 1.2823928802672965E-6
WOLFE (weak): th(258.53216280382605)=1.2823928802672965E-6; dx=-3.5623303586555697E-13 evalInputDelta=9.2099350759348E-11
New Minimum: 1.2823928802672965E-6 > 1.2819324331198441E-6
WOLFE (weak): th(1551.1929768229563)=1.2819324331198441E-6; dx=-3.561690768347332E-13 evalInputDelta=5.52546498211716E-10
New Minimum: 1.2819324331198441E-6 > 1.2786196542930127E-6
WOLFE (weak): th(10858.350837760694)=1.2786196542930127E-6; dx=-3.5570857181317157E-13 evalInputDelta=3.8653253250431524E-9
New Minimum: 1.2786196542930127E-6 > 1.251725720970531E-6
WOLFE (weak): th(86866.80670208555)=1.251725720970531E-6; dx=-3.5194778080353286E-13 evalInputDelta=3.0759258647524925E-8
New Minimum: 1.251725720970531E-6 > 1.0190925256240185E-6
END: th(781801.26031877)=1.0190925256240185E-6; dx=-3.1756340585822705E-13 evalInputDelta=2.6339245399403735E-7
Fitness changed from 1.2824849796180558E-6 to 1.0190925256240185E-6
Iteration 1 complete. Error: 1.0190925256240185E-6 Total: 10.2838; Orientation: 0.8012; Line Search: 8.1669
Low gradient: 5.32053810359237E-7
th(0)=1.0190925256240185E-6;dx=-2.8308125711777927E-13
New Minimum: 1.0190925256240185E-6 > 5.980584575626843E-7
END: th(1684339.7559414052)=5.980584575626843E-7; dx=-2.1685833829481787E-13 evalInputDelta=4.210340680613342E-7
Fitness changed from 1.0190925256240185E-6 to 5.980584575626843E-7
Iteration 2 complete. Error: 5.980584575626843E-7 Total: 1.9993; Orientation: 0.4598; Line Search: 1.2575
Low gradient: 4.07587229096997E-7
th(0)=5.980584575626843E-7;dx=-1.661273493229572E-13
New Minimum: 5.980584575626843E-7 > 1.4713194949573892E-7
END: th(3628800.0)=1.4713194949573892E-7; dx=-8.239916526419483E-14 evalInputDelta=4.5092650806694537E-7
Fitness changed from 5.980584575626843E-7 to 1.4713194949573892E-7
Iteration 3 complete. Error: 1.4713194949573892E-7 Total: 2.2037; Orientation: 0.4749; Line Search: 1.4501
Low gradient: 2.0216326563210884E-7
th(0)=1.4713194949573892E-7;dx=-4.0869985971039385E-14
New Minimum: 1.4713194949573892E-7 > 1.08401841543946E-9
WOLF (strong): th(7818012.6031877)=1.08401841543946E-9; dx=3.508078669750212E-15 evalInputDelta=1.4604793108029946E-7
END: th(3909006.30159385)=3.073945037224316E-8; dx=-1.86809536506428E-14 evalInputDelta=1.1639249912349576E-7
Fitness changed from 1.4713194949573892E-7 to 1.08401841543946E-9
Iteration 4 complete. Error: 1.08401841543946E-9 Total: 2.7267; Orientation: 0.5704; Line Search: 1.8285
Low gradient: 1.7352700841971578E-8
th(0)=1.08401841543946E-9;dx=-3.011162265109642E-16
New Minimum: 1.08401841543946E-9 > 3.121044403096727E-11
WOLF (strong): th(8421698.779707026)=3.121044403096727E-11; dx=5.1093517566470434E-17 evalInputDelta=1.0528079714084928E-9
END: th(4210849.389853513)=1.8683888324795112E-10; dx=-1.2501135447224928E-16 evalInputDelta=8.971795321915089E-10
Fitness changed from 1.08401841543946E-9 to 3.121044403096727E-11
Iteration 5 complete. Error: 3.121044403096727E-11 Total: 2.6826; Orientation: 0.4714; Line Search: 1.9275
Low gradient: 2.94441297823179E-9
th(0)=3.121044403096727E-11;dx=-8.669567786380028E-18
New Minimum: 3.121044403096727E-11 > 2.109826016493789E-12
WOLF (strong): th(9072000.0)=2.109826016493789E-12; dx=2.2540876244588047E-18 evalInputDelta=2.910061801447348E-11
END: th(4536000.0)=4.272709787817989E-12; dx=-3.2077400809522625E-18 evalInputDelta=2.693773424314928E-11
Fitness changed from 3.121044403096727E-11 to 2.109826016493789E-12
Iteration 6 complete. Error: 2.109826016493789E-12 Total: 2.8363; Orientation: 0.4610; Line Search: 1.8429
Low gradient: 7.655473743403381E-10
th(0)=2.109826016493789E-12;dx=-5.860627823593603E-19
New Minimum: 2.109826016493789E-12 > 2.69338065564385E-13
WOLF (strong): th(9772515.753984625)=2.69338065564385E-13; dx=2.0939663062393026E-19 evalInputDelta=1.840487950929404E-12
New Minimum: 2.69338065564385E-13 > 2.1787708540048867E-13
END: th(4886257.876992312)=2.1787708540048867E-13; dx=-1.8833307587162161E-19 evalInputDelta=1.8919489310933004E-12
Fitness changed from 2.109826016493789E-12 to 2.1787708540048867E-13
Iteration 7 complete. Error: 2.1787708540048867E-13 Total: 3.5287; Orientation: 0.5116; Line Search: 2.7189
Low gradient: 2.4601100099639216E-10
th(0)=2.1787708540048867E-13;dx=-6.052141261125116E-20
New Minimum: 2.1787708540048867E-13 > 4.6524787816537134E-14
WOLF (strong): th(1.0527123474633783E7)=4.6524787816537134E-14; dx=2.7966973975739954E-20 evalInputDelta=1.7135229758395153E-13
New Minimum: 4.6524787816537134E-14 > 1.575991515403785E-14
END: th(5263561.737316892)=1.575991515403785E-14; dx=-1.6277219320909262E-20 evalInputDelta=2.0211717024645082E-13
Fitness changed from 2.1787708540048867E-13 to 1.575991515403785E-14
Iteration 8 complete. Error: 1.575991515403785E-14 Total: 2.8747; Orientation: 0.6578; Line Search: 1.9214
Zero gradient: 6.616459936744845E-11
th(0)=1.575991515403785E-14;dx=-4.3777542094548936E-21
New Minimum: 1.575991515403785E-14 > 5.2106219482133696E-15
WOLF (strong): th(1.1340000000000002E7)=5.2106219482133696E-15; dx=2.517208670535315E-21 evalInputDelta=1.054929320582448E-14
New Minimum: 5.2106219482133696E-15 > 7.116586682540409E-16
END: th(5670000.000000001)=7.116586682540409E-16; dx=-9.302727692343157E-22 evalInputDelta=1.504825648578381E-14
Fitness changed from 1.575991515403785E-14 to 7.116586682540409E-16
Iteration 9 complete. Error: 7.116586682540409E-16 Total: 2.7017; Orientation: 0.4584; Line Search: 1.9677
Final threshold in iteration 9: 7.116586682540409E-16 (> 0.0) after 31.838s (< 30.000s)

Returns

    7.116586682540409E-16

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 50.78 seconds (1.736 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 9083010266081
Reset training subject: 9083275312799
Low gradient: 5.968633241133871E-7
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.2824849796180558E-6}, derivative=-3.56245827671671E-13}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.2824849796180558E-6}, derivative=-3.56245827671671E-13}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.2824849796180558E-6}, derivative=-3.56245827671671E-13}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.2824849796180558E-6}, derivative=-3.56245827671671E-13}, evalInputDelta = 0.0
New Minimum: 1.2824849796180558E-6 > 1.2824849796180556E-6
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.2824849796180556E-6}, derivative=-3.56245827671671E-13}, evalInputDelta = -2.1175823681357508E-22
New Minimum: 1.2824849796180556E-6 > 1.2824849796180192E-6
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.2824849796180192E-6}, derivative=-3.562458276716664E-13}, evalInputDelta = -3.663417496874849E-20
New Minimum: 1.2824849796180192E-6 > 1.2824849796174267E-6
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.2824849796174267E-6}, derivative=-3.5624582767161403E-13}, evalInputDelta = -6.291337215731316E-19
New Minimum: 1.2824849796174267E-6 > 1.2824849796138438E-6
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.2824849796138438E-6}, derivative=-3.562458276711268E-13}, evalInputDelta = -4.212083088458822E-18
New Minimum: 1.2824849796138438E-6 > 1.282484979588729E-6
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.282484979588729E-6}, derivative=-3.5624582766761914E-13}, evalInputDelta = -2.932682173278564E-17
New Minimum: 1.282484979588729E-6 > 1.2824849794126861E-6
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.2824849794126861E-6}, derivative=-3.562458276431757E-13}, evalInputDelta = -2.0536970256714634E-16
New Minimum: 1.2824849794126861E-6 > 1.2824849781804349E-6
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.2824849781804349E-6}, derivative=-3.562458274719801E-13}, evalInputDelta = -1.4376209522549673E-15
New Minimum: 1.2824849781804349E-6 > 1.2824849695549931E-6
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.2824849695549931E-6}, derivative=-3.56245826274025E-13}, evalInputDelta = -1.0063062697989204E-14
New Minimum: 1.2824849695549931E-6 > 1.282484909176615E-6
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=1.282484909176615E-6}, derivative=-3.562458178881261E-13}, evalInputDelta = -7.044144079174856E-14
New Minimum: 1.282484909176615E-6 > 1.2824844865280404E-6
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=1.2824844865280404E-6}, derivative=-3.5624575918694454E-13}, evalInputDelta = -4.930900154502635E-13
New Minimum: 1.2824844865280404E-6 > 1.282481527989834E-6
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=1.282481527989834E-6}, derivative=-3.562453482785482E-13}, evalInputDelta = -3.451628221809471E-12
New Minimum: 1.282481527989834E-6 > 1.2824608183178791E-6
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=1.2824608183178791E-6}, derivative=-3.56242471919737E-13}, evalInputDelta = -2.416130017669989E-11
New Minimum: 1.2824608183178791E-6 > 1.2823158552961384E-6
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=1.2823158552961384E-6}, derivative=-3.56222337408099E-13}, evalInputDelta = -1.6912432191744534E-10
New Minimum: 1.2823158552961384E-6 > 1.2813013435599614E-6
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=1.2813013435599614E-6}, derivative=-3.560813958264596E-13}, evalInputDelta = -1.1836360580944121E-9
New Minimum: 1.2813013435599614E-6 > 1.2742110027709873E-6
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=1.2742110027709873E-6}, derivative=-3.550948047550624E-13}, evalInputDelta = -8.273976847068553E-9
New Minimum: 1.2742110027709873E-6 > 1.2251294440992223E-6
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=1.2251294440992223E-6}, derivative=-3.4818866725531966E-13}, evalInputDelta = -5.735553551883355E-8
New Minimum: 1.2251294440992223E-6 > 9.085490491097619E-7
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=9.085490491097619E-7}, derivative=-2.9984570475700374E-13}, evalInputDelta = -3.739359305082939E-7
New Minimum: 9.085490491097619E-7 > 1.5021554107644094E-8
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=1.5021554107644094E-8}, derivative=3.8555032730976614E-14}, evalInputDelta = -1.2674634255104118E-6
1.5021554107644094E-8 <= 1.2824849796180558E-6
New Minimum: 1.5021554107644094E-8 > 1.9351402703638552E-33
F(7200000.000000137) = LineSearchPoint{point=PointSample{avg=1.9351402703638552E-33}, derivative=1.1198275795262659E-26}, evalInputDelta = -1.2824849796180558E-6
Right bracket at 7200000.000000137
Converged to right
Fitness changed from 1.2824849796180558E-6 to 1.9351402703638552E-33
Iteration 1 complete. Error: 1.9351402703638552E-33 Total: 17.7562; Orientation: 0.5360; Line Search: 16.4264
Zero gradient: 2.3184886542529377E-20
F(0.0) = LineSearchPoint{point=PointSample{avg=1.9351402703638552E-33}, derivative=-5.375389639899673E-40}
New Minimum: 1.9351402703638552E-33 > 5.388397786213182E-34
F(7200000.000000137) = LineSearchPoint{point=PointSample{avg=5.388397786213182E-34}, derivative=-2.6484067611055285E-40}, evalInputDelta = -1.396300491742537E-33
F(5.040000000000096E7) = LineSearchPoint{point=PointSample{avg=6.581366943013957E-32}, derivative=3.111872953241999E-39}, evalInputDelta = 6.387852915977571E-32
F(3876923.076923151) = LineSearchPoint{point=PointSample{avg=1.4773619437272196E-33}, derivative=-4.690105611866786E-40}, evalInputDelta = -4.5777832663663555E-34
F(2.7138461538462058E7) = LineSearchPoint{point=PointSample{avg=1.5647648967308603E-32}, derivative=1.4834524029151143E-39}, evalInputDelta = 1.3712508696944749E-32
F(2087573.9644970815) = LineSearchPoint{point=PointSample{avg=1.7956879659416886E-33}, derivative=-5.1776910970764106E-40}, evalInputDelta = -1.394523044221666E-34
F(1.461301775147957E7) = LineSearchPoint{point=PointSample{avg=3.153822208636365E-33}, derivative=5.520300634117417E-40}, evalInputDelta = 1.2186819382725098E-33
F(1124078.2885753515) = LineSearchPoint{point=PointSample{avg=1.8959087079409554E-33}, derivative=-5.320563938509244E-40}, evalInputDelta = -3.9231562422899744E-35
New Minimum: 5.388397786213182E-34 > 4.375219919785259E-34
F(7868548.0200274605) = LineSearchPoint{point=PointSample{avg=4.375219919785259E-34}, derivative=-2.337691925188739E-40}, evalInputDelta = -1.4976182783853293E-33
F(5.5079836140192226E7) = LineSearchPoint{point=PointSample{avg=7.854903374682226E-32}, derivative=3.399050516669672E-39}, evalInputDelta = 7.661389347645841E-32
F(4236910.472322479) = LineSearchPoint{point=PointSample{avg=1.4068366111909348E-33}, derivative=-4.575717686018052E-40}, evalInputDelta = -5.283036591729204E-34
F(2.9658373306257352E7) = LineSearchPoint{point=PointSample{avg=2.0153080875127547E-32}, derivative=1.702407993452547E-39}, evalInputDelta = 1.8217940604763692E-32
F(2281413.3312505656) = LineSearchPoint{point=PointSample{avg=1.7783983903796392E-33}, derivative=-5.15261563221579E-40}, evalInputDelta = -1.5674187998421596E-34
F(1.596989331875396E7) = LineSearchPoint{point=PointSample{avg=4.184654494111501E-33}, derivative=6.743058874514938E-40}, evalInputDelta = 2.249514223747646E-33
F(1228453.332211843) = LineSearchPoint{point=PointSample{avg=1.8878644082668285E-33}, derivative=-5.309250893803222E-40}, evalInputDelta = -4.7275862097026703E-35
New Minimum: 4.375219919785259E-34 > 3.754239766548985E-34
F(8599173.325482901) = LineSearchPoint{point=PointSample{avg=3.754239766548985E-34}, derivative=-2.077749914305257E-40}, evalInputDelta = -1.5597162937089566E-33
F(6.0194213278380305E7) = LineSearchPoint{point=PointSample{avg=1.0949681901209705E-31}, derivative=4.026815574521961E-39}, evalInputDelta = 1.075616787417332E-31
F(4630324.098336946) = LineSearchPoint{point=PointSample{avg=1.2806201646159212E-33}, derivative=-4.366005170441324E-40}, evalInputDelta = -6.54520105747934E-34
F(3.2412268688358624E7) = LineSearchPoint{point=PointSample{avg=2.2064327668495168E-32}, derivative=1.783825548957239E-39}, evalInputDelta = 2.0129187398131313E-32
F(2493251.437566048) = LineSearchPoint{point=PointSample{avg=1.7568683618659896E-33}, derivative=-5.121322017265953E-40}, evalInputDelta = -1.7827190849786561E-34
F(1.7452760062962335E7) = LineSearchPoint{point=PointSample{avg=6.364374448486367E-33}, derivative=8.715255651879027E-40}, evalInputDelta = 4.4292341781225124E-33
F(1342520.0048432564) = LineSearchPoint{point=PointSample{avg=1.8864945655166703E-33}, derivative=-5.307326566014163E-40}, evalInputDelta = -4.864570484718485E-35
New Minimum: 3.754239766548985E-34 > 1.8197310265073056E-34
F(9397640.033902794) = LineSearchPoint{point=PointSample{avg=1.8197310265073056E-34}, derivative=-1.3401237180334591E-40}, evalInputDelta = -1.7531671677131247E-33
F(6.578348023731956E7) = LineSearchPoint{point=PointSample{avg=1.2717150394026873E-31}, derivative=4.340084123287866E-39}, evalInputDelta = 1.2523636366990489E-31
F(5060267.710563043) = LineSearchPoint{point=PointSample{avg=1.2657772877529925E-33}, derivative=-4.340771167646632E-40}, evalInputDelta = -6.693629826108627E-34
F(3.5421873973941304E7) = LineSearchPoint{point=PointSample{avg=3.2886765334044665E-32}, derivative=2.190665820093466E-39}, evalInputDelta = 3.0951625063680807E-32
F(2724759.5364570236) = LineSearchPoint{point=PointSample{avg=1.7419530620001992E-33}, derivative=-5.099505542785295E-40}, evalInputDelta = -1.9318720836365597E-34
F(1.9073316755199164E7) = LineSearchPoint{point=PointSample{avg=8.613511721386625E-33}, derivative=1.0681751466586653E-39}, evalInputDelta = 6.67837145102277E-33
F(1467178.2119383973) = LineSearchPoint{point=PointSample{avg=1.8501340273202416E-33}, derivative=-5.255558692751003E-40}, evalInputDelta = -8.500624304361363E-35
New Minimum: 1.8197310265073056E-34 > 1.5781816387456688E-34
F(1.027024748356878E7) = LineSearchPoint{point=PointSample{avg=1.5781816387456688E-34}, derivative=-1.1785664665969372E-40}, evalInputDelta = -1.777322106489288E-33
F(7.189173238498145E7) = LineSearchPoint{point=PointSample{avg=1.56577884858111E-31}, derivative=4.824692822750348E-39}, evalInputDelta = 1.5464274458774715E-31
F(5530133.260383189) = LineSearchPoint{point=PointSample{avg=1.2038414131635789E-33}, derivative=-4.234844025331609E-40}, evalInputDelta = -7.312988572002763E-34
F(3.871093282268232E7) = LineSearchPoint{point=PointSample{avg=3.5618959991062134E-32}, derivative=2.2768984290701287E-39}, evalInputDelta = 3.3683819720698276E-32
F(2977764.0632832553) = LineSearchPoint{point=PointSample{avg=1.615029145183293E-33}, derivative=-4.9050104107339114E-40}, evalInputDelta = -3.2011112518056235E-34
F(2.0844348442982785E7) = LineSearchPoint{point=PointSample{avg=1.0250222157903259E-32}, derivative=1.1804338670607971E-39}, evalInputDelta = 8.315081887539404E-33
F(1603411.4186909834) = LineSearchPoint{point=PointSample{avg=1.8451295376702412E-33}, derivative=-5.248515182301145E-40}, evalInputDelta = -9.001073269361394E-35
New Minimum: 1.5781816387456688E-34 > 9.438199746520949E-35
F(1.1223879930836884E7) = LineSearchPoint{point=PointSample{avg=9.438199746520949E-35}, derivative=-8.234402005089904E-41}, evalInputDelta = -1.8407582728986456E-33
F(7.856715951585819E7) = LineSearchPoint{point=PointSample{avg=1.885341561895053E-31}, derivative=5.290579915432075E-39}, evalInputDelta = 1.8659901591914145E-31
F(6043627.655066014) = LineSearchPoint{point=PointSample{avg=8.670341065227722E-34}, derivative=-3.4694822831527116E-40}, evalInputDelta = -1.068106163841083E-33
F(4.23053935854621E7) = LineSearchPoint{point=PointSample{avg=5.097189425394762E-32}, derivative=2.7313957333957888E-39}, evalInputDelta = 4.903675398358377E-32
F(3254261.045035546) = LineSearchPoint{point=PointSample{avg=1.5979734979714798E-33}, derivative=-4.8790106322340115E-40}, evalInputDelta = -3.371667723923754E-34
F(2.2779827315248825E7) = LineSearchPoint{point=PointSample{avg=1.1814878568214565E-32}, derivative=1.2839719623010273E-39}, evalInputDelta = 9.87973829785071E-33
F(1752294.4088652942) = LineSearchPoint{point=PointSample{avg=1.8163822110842414E-33}, derivative=-5.207344761779387E-40}, evalInputDelta = -1.187580592796138E-34
F(1.226606086205706E7) = LineSearchPoint{point=PointSample{avg=1.3085142039852569E-33}, derivative=2.2426632742844033E-40}, evalInputDelta = -6.266260663785983E-34
1.3085142039852569E-33 <= 1.9351402703638552E-33
F(8655079.877105612) = LineSearchPoint{point=PointSample{avg=3.7455254350948884E-34}, derivative=-2.0760040875101486E-40}, evalInputDelta = -1.5605877268543665E-33
Left bracket at 8655079.877105612
F(1.0390895733403841E7) = LineSearchPoint{point=PointSample{avg=1.3840605016508763E-34}, derivative=-1.080568332462677E-40}, evalInputDelta = -1.7967342201987677E-33
Left bracket at 1.0390895733403841E7
F(1.1000616720664294E7) = LineSearchPoint{point=PointSample{avg=1.2782418401222252E-34}, derivative=-1.022004114314842E-40}, evalInputDelta = -1.8073160863516326E-33
Left bracket at 1.1000616720664294E7
F(1.1396764004597535E7) = LineSearchPoint{point=PointSample{avg=6.904863239434671E-34}, derivative=6.421388329101566E-41}, evalInputDelta = -1.244653946420388E-33
Right bracket at 1.1396764004597535E7
F(1.1243903605802104E7) = LineSearchPoint{point=PointSample{avg=9.438199746520949E-35}, derivative=-8.234402005089904E-41}, evalInputDelta = -1.8407582728986456E-33
Left bracket at 1.1243903605802104E7
Converged to left
Fitness changed from 1.9351402703638552E-33 to 9.438199746520949E-35
Iteration 2 complete. Error: 9.438199746520949E-35 Total: 33.0262; Orientation: 0.5294; Line Search: 32.1758
Final threshold in iteration 2: 9.438199746520949E-35 (> 0.0) after 50.782s (< 30.000s)

Returns

    9.438199746520949E-35

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 186.90 seconds (3.610 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 9133885010899
Reset training subject: 9134177985744
Adding measurement 6398f466 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.2824849796180558E-6 < 1.2824849796180558E-6. Total: 1
th(0)=1.2824849796180558E-6;dx=-3.5624582767167093E-13
Adding measurement 3f4b14e0 to history. Total: 1
New Minimum: 1.2824849796180558E-6 > 1.282484212109783E-6
WOLFE (weak): th(2.154434690031884)=1.282484212109783E-6; dx=-3.562457210732841E-13 evalInputDelta=7.675082728432215E-13
Adding measurement 39080b92 to history. Total: 2
New Minimum: 1.282484212109783E-6 > 1.2824834446017444E-6
WOLFE (weak): th(4.308869380063768)=1.2824834446017444E-6; dx=-3.5624561447492154E-13 evalInputDelta=1.5350163114818331E-12
Adding measurement 791ae45d to history. Total: 3
New Minimum: 1.2824834446017444E-6 > 1.2824803745719604E-6
WOLFE (weak): th(12.926608140191302)=1.2824803745719604E-6; dx=-3.562451880813714E-13 evalInputDelta=4.605046095402818E-12
Adding measurement 687c239b to history. Total: 4
New Minimum: 1.2824803745719604E-6 > 1.2824665594833229E-6
WOLFE (weak): th(51.70643256076521)=1.2824665594833229E-6; dx=-3.5624326931043826E-13 evalInputDelta=1.8420134732986828E-11
Adding measurement 54d34bf to history. Total: 5
New Minimum: 1.2824665594833229E-6 > 1.2823928802672965E-6
WOLFE (weak): th(258.53216280382605)=1.2823928802672965E-6; dx=-3.5623303586555697E-13 evalInputDelta=9.2099350759348E-11
Adding measurement eca55d2 to history. Total: 6
New Minimum: 1.2823928802672965E-6 > 1.2819324331198441E-6
WOLFE (weak): th(1551.1929768229563)=1.2819324331198441E-6; dx=-3.561690768347332E-13 evalInputDelta=5.52546498211716E-10
Adding measurement 5e740f1a to history. Total: 7
New Minimum: 1.2819324331198441E-6 > 1.2786196542930127E-6
WOLFE (weak): th(10858.350837760694)=1.2786196542930127E-6; dx=-3.5570857181317157E-13 evalInputDelta=3.8653253250431524E-9
Adding measurement 7021b28d to history. Total: 8
New Minimum: 1.2786196542930127E-6 > 1.251725720970531E-6
WOLFE (weak): th(86866.80670208555)=1.251725720970531E-6; dx=-3.5194778080353286E-13 evalInputDelta=3.0759258647524925E-8
Adding measurement 5a61ee7b to history. Total: 9
New Minimum: 1.251725720970531E-6 > 1.0190925256240185E-6
END: th(781801.26031877)=1.0190925256240185E-6; dx=-3.1756340585822705E-13 evalInputDelta=2.6339245399403735E-7
Fitness changed from 1.2824849796180558E-6 to 1.0190925256240185E-6
Iteration 1 complete. Error: 1.0190925256240185E-6 Total: 13.0041; Orientation: 0.5441; Line Search: 11.6121
Non-optimal measurement 1.0190925256240185E-6 < 1.0190925256240185E-6. Total: 10
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00, cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00, 84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00, 666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6, 1.2823928802672965E-6, 1.2824665594833229E-6, 1.2824803745719604E-6, 1.2824834446017444E-6, 1.282484212109783E-6, 1.2824849796180558E-6
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00, c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00, 84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00, 666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6, 1.2823928802672965E-6, 1.2824665594833229E-6, 1.2824803745719604E-6, 1.2824834446017444E-6, 1.282484212109783E-6
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00, 84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00, c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00, cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6, 1.2823928802672965E-6, 1.2824665594833229E-6, 1.2824803745719604E-6, 1.2824834446017444E-6
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00, de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00, cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00, 666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6, 1.2823928802672965E-6, 1.2824665594833229E-6, 1.2824803745719604E-6
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00, 666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00, c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00, 84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6, 1.2823928802672965E-6, 1.2824665594833229E-6
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00, 666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00, c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00, 84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6, 1.2823928802672965E-6
Rejected: LBFGS Orientation magnitude: 3.831e+00, gradient 5.321e-07, dot -1.000; [666c72c4-2b38-41e1-b4c1-f04ccf98a333 = 1.000/1.000e+00, 84bd493c-1f5a-4498-b054-99790beeb38e = 1.000/1.000e+00, c99bcea3-19be-4b7a-9867-1f5dec12d3a7 = 1.000/1.000e+00, cc5d186b-2e33-4af0-942a-ac3118ac0d40 = 1.000/1.000e+00, de9d2d78-4d56-41a1-83c5-fdafa0dca067 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0190925256240185E-6, 1.251725720970531E-6, 1.2786196542930127E-6, 1.2819324331198441E-6
LBFGS Accumulation History: 3 points
Removed measurement 5a61ee7b to history. Total: 9
Removed measurement 7021b28d to history. Total: 8
Removed measurement 5e740f1a to history. Total: 7
Removed measurement eca55d2 to history. Total: 6
Removed measurement 54d34bf to history. Total: 5
Removed measurement 687c239b to history. Total: 4
Removed measurement 791ae45d to history. Total: 3
Adding measurement 19dd47b8 to history. Total: 3
th(0)=1.0190925256240185E-6;dx=-2.8308125711777927E-13
Adding measurement 52fa4170 to history. Total: 4
New Minimum: 1.0190925256240185E-6 > 5.980584575626843E-7
END: th(1684339.7559414052)=5.980584575626843E-7; dx=-2.1685833829481787E-13 evalInputDelta=4.210340680613342E-7
Fitness changed from 1.0190925256240185E-6 to 5.980584575626843E-7
Iteration 2 complete. Error: 5.980584575626843E-7 Total: 173.8928; Orientation: 171.7903; Line Search: 1.8300
Final threshold in iteration 2: 5.980584575626843E-7 (> 0.0) after 186.898s (< 30.000s)

Returns

    5.980584575626843E-7

Training Converged

TrainingTester.java:432 executed in 0.15 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -34.02511083565022], [9.0, -5.9917863836641105]; valueStats=DoubleSummaryStatistics{count=13, sum=0.000003, min=0.000000, average=0.000000, max=0.000001}
Plotting 9 points for GD
Plotting 2 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -34.02511083565022], [173.893, -5.9917863836641105]; valueStats=DoubleSummaryStatistics{count=13, sum=0.000003, min=0.000000, average=0.000000, max=0.000001}
Plotting 9 points for GD
Plotting 2 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 5.980584575626843E-7 }, "CjGD": { "type": "Converged", "value": 9.438199746520949E-35 }, "GD": { "type": "Converged", "value": 7.116586682540409E-16 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 5.980584575626843E-7 }, "CjGD": { "type": "Converged", "value": 9.438199746520949E-35 }, "GD": { "type": "Converged", "value": 7.116586682540409E-16 } }, "model":null, "complete":null}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "272.715",
      "gc_time": "7.647"
    },
    "created_on": 1586743680604,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Asymmetric",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandAvgReducerLayerTest.Asymmetric",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 5.980584575626843E-7
        },
        "CjGD": {
          "type": "Converged",
          "value": 9.438199746520949E-35
        },
        "GD": {
          "type": "Converged",
          "value": 7.116586682540409E-16
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayer/Asymmetric/trainingTest/202004130800",
    "id": "7fa87bec-761b-4f6b-aba2-9d231c4a099c",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "BandAvgReducerLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandAvgReducerLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayer.java",
      "javaDoc": ""
    }
  }