1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 7448013347972440064

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.216, -0.416, 1.472 ], [ 1.384, 0.788, -1.888 ], [ 0.464, 0.78, 0.296 ], [ 1.244, -0.408, -1.768 ], [ 1.244, 0.08, -1.248 ], [ -1.412, -2.0, 1.368 ], [ 0.604, -1.456, -0.992 ], [ -1.732, -0.572, -1.548 ], ... ],
    	[ [ 0.004, -0.204, -1.736 ], [ 0.416, -1.476, -1.112 ], [ -1.164, 1.496, -0.86 ], [ 1.36, -1.216, 0.74 ], [ -0.248, 1.52, -0.432 ], [ -1.2, 0.54, 1.672 ], [ 1.78, -1.54, 0.312 ], [ -0.848, 0.512, -0.724 ], ... ],
    	[ [ -0.532, -1.316, -0.368 ], [ 1.612, 1.532, 1.62 ], [ 0.236, 1.64, 0.752 ], [ -1.444, 0.096, -1.284 ], [ 0.868, -1.956, -0.408 ], [ 0.332, 0.344, 0.568 ], [ 1.464, -0.176, 1.964 ], [ -1.504, -0.748, 1.308 ], ... ],
    	[ [ -0.592, -1.756, -1.072 ], [ -0.196, -1.472, -0.952 ], [ -1.98, -1.564, -0.828 ], [ 0.284, 0.92, 0.844 ], [ 1.044, -1.808, 1.644 ], [ -1.084, 1.288, -1.464 ], [ 0.644, -0.524, -0.576 ], [ -1.236, 0.676, -2.0 ], ... ],
    	[ [ -1.776, -0.18, 0.016 ], [ 0.484, -1.908, 1.8 ], [ -1.792, 1.856, -0.636 ], [ -1.724, 1.256, -0.652 ], [ 1.944, -1.496, 1.544 ], [ 0.212, 0.46, -1.308 ], [ -1.004, 0.752, 0.2 ], [ -1.204, 1.22, -0.152 ], ... ],
    	[ [ 1.768, 0.072, -1.248 ], [ 1.868, -1.108, 1.944 ], [ -0.032, 0.232, -0.92 ], [ 1.228, -0.072, -1.884 ], [ -1.2, 1.26, 1.12 ], [ -1.996, 1.888, 0.42 ], [ 0.78, -0.76, 1.82 ], [ -0.104, 0.192, 0.952 ], ... ],
    	[ [ -0.732, 0.54, 1.728 ], [ -1.46, -1.028, -1.648 ], [ -0.096, -1.92, -0.908 ], [ -1.312, -0.816, -0.232 ], [ -1.596, 1.34, 0.528 ], [ -0.1, -0.38, -1.076 ], [ 0.128, -0.12, 0.96 ], [ 0.82, -0.708, 1.284 ], ... ],
    	[ [ 1.804, -0.824, 1.168 ], [ 0.696, 0.712, 0.448 ], [ 0.056, 1.5, -1.3 ], [ -1.0, -1.568, 0.676 ], [ 0.984, 1.004, -1.112 ], [ -1.24, -0.632, 1.38 ], [ 0.948, -0.256, 0.136 ], [ -1.212, 1.952, 0.212 ], ... ],
    	...
    ]
    [
    	[ [ 1.78, 1.004, -0.348 ], [ 1.948, -1.456, 0.488 ], [ 1.472, -1.396, 0.26 ], [ 0.652, -1.12, 1.804 ], [ -0.632, -1.116, 0.972 ], [ -0.608, -1.108, -0.708 ], [ -1.272, 0.66, 1.216 ], [ 0.016, -1.364, -0.572 ], ... ],
    	[ [ -1.536, -0.372, 1.168 ], [ 0.504, 1.992, 1.076 ], [ -0.34, 0.372, 1.436 ], [ 0.212, -1.412, -1.996 ], [ -0.468, 1.572, 0.288 ], [ 0.608, -0.88, 0.324 ], [ 1.332, 1.576, -1.644 ], [ -0.204, 1.936, 1.22 ], ... ],
    	[ [ 0.004, 0.916, -0.18 ], [ 0.752, 1.712, 1.132 ], [ -0.872, -1.844, -1.452 ], [ 0.632, 1.116, -0.392 ], [ -1.112, 0.996, 1.812 ], [ -1.62, -1.364, 1.284 ], [ -1.86, -1.248, 0.012 ], [ 1.792, -0.548, -1.772 ], ... ],
    	[ [ 0.344, 1.14, 1.448 ], [ -0.732, -1.632, -0.268 ], [ -1.256, 1.404, -0.908 ], [ -1.128, 1.884, -1.58 ], [ 0.116, -0.368, -1.028 ], [ -1.44, -1.96, 1.0 ], [ -1.54, 0.224, 0.516 ], [ 0.48, 0.512, -0.712 ], ... ],
    	[ [ -1.144, 0.428, 1.212 ], [ 1.896, -0.892, 0.2 ], [ 0.396, -0.008, 0.416 ], [ -0.66, -0.304, 0.988 ], [ 1.728, 0.024, -1.976 ], [ 1.384, -0.304, 0.1 ], [ -1.812, 0.752, 0.52 ], [ -1.068, 0.688, 1.148 ], ... ],
    	[ [ 1.004, -0.532, -1.748 ], [ 0.2, -1.58, 1.964 ], [ 0.864, 0.3, 1.872 ], [ -0.76, -2.0, 1.244 ], [ 0.588, -1.948, 0.032 ], [ -0.648, -1.872, -1.756 ], [ 0.204, 1.2, 0.672 ], [ 1.356, 1.308, -1.464 ], ... ],
    	[ [ 1.44, -0.792, -0.016 ], [ 0.468, -0.604, -0.9 ], [ 1.912, -1.556, 0.8 ], [ 1.148, 1.72, 0.616 ], [ 0.568, 0.016, -1.724 ], [ 0.96, -0.404, -0.804 ], [ -1.892, -0.936, 1.676 ], [ 1.02, -0.736, 1.78 ], ... ],
    	[ [ -0.784, -1.92, 1.188 ], [ 0.96, 0.788, 0.596 ], [ 0.264, 0.444, -0.976 ], [ 0.544, 0.448, 1.172 ], [ -1.188, -0.828, -1.908 ], [ -0.14, 0.228, 1.888 ], [ 0.488, -1.208, -0.052 ], [ -1.248, -0.372, -1.9 ], ... ],
    	...
    ]
    [
    	[ [ 0.164, -0.196, 1.196 ], [ -1.96, -0.472, 0.324 ], [ 0.184, -1.64, -0.392 ], [ -1.232, -1.172, -0.996 ], [ 1.244, 0.824, 0.812 ], [ 1.044, -1.12, -1.756 ], [ -1.66, -1.116, -1.324 ], [ 0.924, 1.476, -0.04 ], ... ],
    	[ [ 1.256, -1.812, -0.632 ], [ -0.536, -0.756, 0.368 ], [ -1.36, -1.996, -0.82 ], [ 1.944, 1.156, 1.7 ], [ -1.36, 1.276, -1.184 ], [ -0.132, 1.82, 1.752 ], [ -1.472, -0.34, 0.028 ], [ 0.64, -1.904, -0.616 ], ... ],
    	[ [ 1.472, 1.728, 0.548 ], [ -1.432, -1.768, 1.26 ], [ -0.84, -0.596, -0.036 ], [ 0.408, -1.724, -1.64 ], [ 1.128, 1.672, -0.8 ], [ 0.224, 1.216, -0.612 ], [ -0.688, -0.992, 1.976 ], [ 1.368, 0.168, 0.28 ], ... ],
    	[ [ 1.56, -0.976, 0.284 ], [ 0.268, -1.444, 1.956 ], [ -0.732, 0.356, 1.112 ], [ 0.764, 1.38, -1.152 ], [ 1.396, 1.016, 0.076 ], [ -0.632, 1.144, -0.86 ], [ 0.024, -0.584, 0.48 ], [ 0.676, -0.228, -1.988 ], ... ],
    	[ [ -1.352, -0.752, -0.488 ], [ -0.836, 0.432, -1.104 ], [ 0.148, -1.704, -1.464 ], [ -1.508, 0.448, 1.352 ], [ -1.92, -1.86, 1.468 ], [ 1.264, -1.564, -0.788 ], [ -0.536, 1.544, 1.768 ], [ 0.544, 1.936, -0.88 ], ... ],
    	[ [ 1.524, 1.524, -0.716 ], [ -1.316, -1.748, 1.228 ], [ -1.168, 0.424, -1.98 ], [ -0.908, -0.324, 1.736 ], [ 0.264, 0.236, 0.136 ], [ 1.116, -1.436, 0.216 ], [ 1.312, -1.176, -0.776 ], [ -0.404, 1.956, 1.032 ], ... ],
    	[ [ 1.988, 1.42, -1.296 ], [ 0.164, 0.132, 0.212 ], [ 1.424, 1.232, -1.664 ], [ 1.964, -1.712, 0.364 ], [ -1.648, 1.06, 1.928 ], [ -1.124, 1.284, -0.72 ], [ 1.316, -0.072, -0.856 ], [ -1.704, -1.088, 1.212 ], ... ],
    	[ [ -0.352, 0.512, 1.364 ], [ -0.676, -1.644, -1.452 ], [ -1.536, -1.5, 1.076 ], [ -1.756, -0.46, 1.164 ], [ 0.788, -0.904, -0.612 ], [ 1.196, -1.776, -1.648 ], [ -1.908, 0.872, 1.952 ], [ -0.548, 1.8, 1.028 ], ... ],
    	...
    ]
    [
    	[ [ 0.516, -0.824, 0.608 ], [ -0.116, -1.72, -0.544 ], [ 0.752, -1.38, 1.588 ], [ 0.444, -1.328, -0.372 ], [ -0.14, -1.44, 0.44 ], [ 1.532, -0.588, -0.904 ], [ 0.976, 1.028, -1.808 ], [ 0.476, 1.492, -0.46 ], ... ],
    	[ [ -0.904, 1.376, -1.992 ], [ -0.056, -1.228, -0.804 ], [ -1.344, 0.26, -1.98 ], [ 0.672, 0.472, -1.844 ], [ 1.628, -1.648, 1.408 ], [ -0.864, -1.096, 0.496 ], [ -1.36, 1.836, 1.704 ], [ -0.904, 1.028, 0.588 ], ... ],
    	[ [ 0.716, -0.68, -0.872 ], [ -0.356, -0.42, 1.388 ], [ 1.092, 0.032, -1.74 ], [ -1.18, -0.452, 0.084 ], [ 0.66, -1.052, 0.612 ], [ -0.156, 1.572, -0.736 ], [ -1.804, 0.164, 0.172 ], [ 1.168, -1.216, 1.02 ], ... ],
    	[ [ 0.284, -1.436, -1.22 ], [ -0.112, 1.288, 2.0 ], [ 1.568, 0.312, 1.388 ], [ -1.688, 1.524, 0.4 ], [ 0.288, 0.66, 0.352 ], [ 1.636, 0.164, 0.612 ], [ -0.996, 0.924, 0.468 ], [ 1.244, -0.216, -1.996 ], ... ],
    	[ [ -0.616, 0.344, -1.544 ], [ 1.44, -1.164, -1.84 ], [ 0.256, 1.888, 1.168 ], [ 1.264, -1.104, 1.008 ], [ -1.98, -1.36, -0.936 ], [ 1.868, 0.416, -0.128 ], [ -0.516, 1.608, 0.812 ], [ -0.248, -0.688, 0.356 ], ... ],
    	[ [ -0.372, -1.44, -0.348 ], [ 1.456, 1.168, -1.976 ], [ 1.172, 1.92, 0.772 ], [ 1.94, -0.604, -0.452 ], [ -1.704, -1.96, -2.0 ], [ 1.552, 1.928, 1.444 ], [ 0.448, -0.132, 0.968 ], [ 1.652, 0.892, -0.86 ], ... ],
    	[ [ -0.74, 0.832, 1.084 ], [ -1.456, 0.384, -0.72 ], [ -0.44, -0.8, -1.368 ], [ -1.56, 0.648, -0.888 ], [ -1.108, 1.248, -1.28 ], [ -1.336, -1.028, 0.888 ], [ -0.948, -0.276, 1.844 ], [ -1.092, -1.436, -1.832 ], ... ],
    	[ [ 1.612, -0.956, -1.812 ], [ -0.996, 0.332, 1.288 ], [ -0.084, -1.184, -1.908 ], [ 1.196, 0.452, -0.108 ], [ -1.284, 0.428, -0.412 ], [ 1.948, 0.884, 0.232 ], [ -0.908, -0.956, -0.36 ], [ -1.692, -1.604, 0.264 ], ... ],
    	...
    ]
    [
    	[ [ 0.532, -0.656, 1.588 ], [ -0.032, -1.2, 0.656 ], [ 1.06, 0.54, -0.584 ], [ -1.916, -0.16, -1.932 ], [ -0.588, 1.188, 1.204 ], [ 0.32, -0.42, -0.564 ], [ -1.996, -0.812, -0.528 ], [ 0.364, -1.86, -1.112 ], ... ],
    	[ [ -1.972, -0.192, 0.428 ], [ -1.644, -0.184, 1.52 ], [ 0.18, -0.148, -0.152 ], [ 1.148, -1.38, -0.108 ], [ 1.288, 1.096, 1.844 ], [ 0.072, 1.888, 1.844 ], [ 1.076, 1.208, 0.852 ], [ -1.232, 0.396, 1.208 ], ... ],
    	[ [ 0.908, 1.62, 0.984 ], [ 0.752, -0.26, 1.028 ], [ -0.9, 0.088, 1.384 ], [ 0.08, -1.456, -1.04 ], [ -0.476, 1.444, -0.164 ], [ -1.416, 0.324, -0.664 ], [ -0.472, -0.924, -1.992 ], [ -1.812, 1.72, 0.168 ], ... ],
    	[ [ 0.688, 0.312, 1.9 ], [ 0.744, -1.64, -1.3 ], [ -0.72, -0.468, 1.616 ], [ 0.2, 0.564, 0.74 ], [ 0.172, 1.368, -1.68 ], [ -1.58, 1.36, 1.388 ], [ 1.976, -0.38, 1.304 ], [ 1.856, -1.532, 0.476 ], ... ],
    	[ [ 1.82, 0.72, 1.576 ], [ -1.056, 0.364, 1.052 ], [ 0.628, -1.396, -0.632 ], [ 1.964, 1.352, 0.472 ], [ -1.552, 0.152, 1.64 ], [ 1.004, 1.212, -1.288 ], [ -0.64, -0.368, -0.396 ], [ 0.332, -0.98, 0.476 ], ... ],
    	[ [ -1.804, -1.536, 1.94 ], [ -0.88, -0.94, 0.816 ], [ -0.032, 0.82, -0.252 ], [ 0.104, 0.868, -1.316 ], [ -0.256, -1.436, -0.492 ], [ 1.912, -1.74, 1.392 ], [ 1.58, 0.868, 1.696 ], [ 0.332, -1.24, -0.936 ], ... ],
    	[ [ -0.132, 1.948, -1.032 ], [ 1.608, -1.036, -1.54 ], [ 1.98, -0.928, -1.028 ], [ 1.12, 0.58, 1.564 ], [ -0.968, -0.66, 1.836 ], [ -1.668, 1.308, -1.108 ], [ -0.716, -1.58, -1.296 ], [ 1.384, -1.644, -0.288 ], ... ],
    	[ [ -1.996, -1.648, 0.26 ], [ 0.024, -0.312, 0.016 ], [ -0.196, 1.308, -0.096 ], [ -1.324, 0.596, 1.548 ], [ 1.26, -1.336, 1.952 ], [ 0.416, 1.724, -0.624 ], [ -0.728, -1.812, -0.24 ], [ 0.992, 0.608, -0.624 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 2.44 seconds (0.027 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 2149557172621
Reset training subject: 2149665167486
Constructing line search parameters: GD
th(0)=1.0133364868352146E-4;dx=-2.7022306232821585E-5
New Minimum: 1.0133364868352146E-4 > 5.9567429570013994E-5
END: th(2.154434690031884)=5.9567429570013994E-5; dx=-8.71764286243199E-6 evalInputDelta=4.176621911350747E-5
Fitness changed from 1.0133364868352146E-4 to 5.9567429570013994E-5
Iteration 1 complete. Error: 5.9567429570013994E-5 Total: 0.3362; Orientation: 0.0140; Line Search: 0.1365
th(0)=5.9567429570013994E-5;dx=-1.5884647440162567E-5
New Minimum: 5.9567429570013994E-5 > 3.60502149627943E-5
END: th(4.641588833612779)=3.60502149627943E-5; dx=-1.872444288993729E-6 evalInputDelta=2.3517214607219696E-5
Fitness changed from 5.9567429570013994E-5 to 3.60502149627943E-5
Iteration 2 complete. Error: 3.60502149627943E-5 Total: 0.1101; Orientation: 0.0036; Line Search: 0.0725
th(0)=3.60502149627943E-5;dx=-9.613390473375416E-6
New Minimum: 3.60502149627943E-5 > 2.6733679828794265E-5
WOLF (strong): th(10.000000000000002)=2.6733679828794265E-5; dx=2.3089230987979574E-7 evalInputDelta=9.316535134000033E-6
END: th(5.000000000000001)=2.6742371901870383E-5; dx=-2.378674569535673E-7 evalInputDelta=9.307843060923916E-6
Fitness changed from 3.60502149627943E-5 to 2.6733679828794265E-5
Iteration 3 complete. Error: 2.6733679828794265E-5 Total: 0.1570; Orientation: 0.0039; Line Search: 0.1259
th(0)=2.6733679828794265E-5;dx=-7.128980941666924E-6
New Minimum: 2.6733679828794265E-5 > 2.1523204953647716E-5
WOLF (strong): th(10.772173450159421)=2.1523204953647716E-5; dx=1.1599644016182465E-7 evalInputDelta=5.210474875146549E-6
New Minimum: 2.1523204953647716E-5 > 2.1412620385300817E-5
END: th(5.386086725079711)=2.1412620385300817E-5; dx=-7.493379564421952E-8 evalInputDelta=5.321059443493448E-6
Fitness changed from 2.6733679828794265E-5 to 2.1412620385300817E-5
Iteration 4 complete. Error: 2.1412620385300817E-5 Total: 0.1509; Orientation: 0.0051; Line Search: 0.1160
th(0)=2.1412620385300817E-5;dx=-5.7100317719262124E-6
New Minimum: 2.1412620385300817E-5 > 1.1018285534684462E-5
WOLF (strong): th(11.60397208403195)=1.1018285534684462E-5; dx=1.1556450777246307E-8 evalInputDelta=1.0394334850616355E-5
New Minimum: 1.1018285534684462E-5 > 1.0998630517633502E-5
END: th(5.801986042015975)=1.0998630517633502E-5; dx=-4.780554624657682E-9 evalInputDelta=1.0413989867667316E-5
Fitness changed from 2.1412620385300817E-5 to 1.0998630517633502E-5
Iteration 5 complete. Error: 1.0998630517633502E-5 Total: 0.1938; Orientation: 0.0124; Line Search: 0.1345
th(0)=1.0998630517633502E-5;dx=-2.932968317821513E-6
New Minimum: 1.0998630517633502E-5 > 5.335188902222398E-6
WOLF (strong): th(12.500000000000004)=5.335188902222398E-6; dx=7.211259680637351E-10 evalInputDelta=5.663441615411104E-6
New Minimum: 5.335188902222398E-6 > 5.333499839821343E-6
END: th(6.250000000000002)=5.333499839821343E-6; dx=-1.8064864070103785E-10 evalInputDelta=5.6651306778121585E-6
Fitness changed from 1.0998630517633502E-5 to 5.333499839821343E-6
Iteration 6 complete. Error: 5.333499839821343E-6 Total: 0.1369; Orientation: 0.0043; Line Search: 0.1063
th(0)=5.333499839821343E-6;dx=-1.4222665577249019E-6
New Minimum: 5.333499839821343E-6 > 4.5621866415028E-6
WOLF (strong): th(13.465216812699278)=4.5621866415028E-6; dx=2.395583776728676E-11 evalInputDelta=7.713131983185432E-7
New Minimum: 4.5621866415028E-6 > 4.5621164503018005E-6
END: th(6.732608406349639)=4.5621164503018005E-6; dx=-3.0564074933309654E-12 evalInputDelta=7.713833895195426E-7
Fitness changed from 5.333499839821343E-6 to 4.5621164503018005E-6
Iteration 7 complete. Error: 4.5621164503018005E-6 Total: 0.1060; Orientation: 0.0031; Line Search: 0.0825
th(0)=4.5621164503018005E-6;dx=-1.216564336950441E-6
New Minimum: 4.5621164503018005E-6 > 1.5745319833134392E-6
WOLF (strong): th(14.50496510503994)=1.5745319833134392E-6; dx=2.852276358110436E-13 evalInputDelta=2.9875844669883614E-6
New Minimum: 1.5745319833134392E-6 > 1.5745310056066349E-6
END: th(7.25248255251997)=1.5745310056066349E-6; dx=-1.7179345119740162E-14 evalInputDelta=2.9875854446951656E-6
Fitness changed from 4.5621164503018005E-6 to 1.5745310056066349E-6
Iteration 8 complete. Error: 1.5745310056066349E-6 Total: 0.1126; Orientation: 0.0034; Line Search: 0.0801
th(0)=1.5745310056066349E-6;dx=-4.198749310798298E-7
New Minimum: 1.5745310056066349E-6 > 1.1054351782983455E-6
WOLF (strong): th(15.625000000000007)=1.1054351782983455E-6; dx=2.7790117092175236E-15 evalInputDelta=4.690958273082894E-7
New Minimum: 1.1054351782983455E-6 > 1.1054351621927103E-6
WOLF (strong): th(7.8125000000000036)=1.1054351621927103E-6; dx=5.05274864435021E-16 evalInputDelta=4.690958434139246E-7
END: th(2.604166666666668)=1.1598944619587805E-6; dx=-3.8070965213194496E-8 evalInputDelta=4.1463654364785433E-7
Fitness changed from 1.5745310056066349E-6 to 1.1054351621927103E-6
Iteration 9 complete. Error: 1.1054351621927103E-6 Total: 0.3283; Orientation: 0.0033; Line Search: 0.3041
th(0)=1.1054351621927103E-6;dx=-2.947827091293043E-7
New Minimum: 1.1054351621927103E-6 > 8.85957033839683E-7
END: th(5.610507005291367)=8.85957033839683E-7; dx=0.0 evalInputDelta=2.1947812835302728E-7
Fitness changed from 1.1054351621927103E-6 to 8.85957033839683E-7
Iteration 10 complete. Error: 8.85957033839683E-7 Total: 0.0798; Orientation: 0.0041; Line Search: 0.0529
th(0)=8.85957033839683E-7;dx=-2.3625520923565972E-7
New Minimum: 8.85957033839683E-7 > 2.691185822338108E-7
END: th(12.087470920866618)=2.691185822338108E-7; dx=0.0 evalInputDelta=6.168384516058722E-7
Fitness changed from 8.85957033839683E-7 to 2.691185822338108E-7
Iteration 11 complete. Error: 2.691185822338108E-7 Total: 0.0709; Orientation: 0.0049; Line Search: 0.0442
th(0)=2.691185822338108E-7;dx=-7.176495940150778E-8
New Minimum: 2.691185822338108E-7 > 6.844450600359171E-8
END: th(26.041666666666682)=6.844450600359171E-8; dx=0.0 evalInputDelta=2.0067407623021908E-7
Fitness changed from 2.691185822338108E-7 to 6.844450600359171E-8
Iteration 12 complete. Error: 6.844450600359171E-8 Total: 0.0715; Orientation: 0.0038; Line Search: 0.0506
th(0)=6.844450600359171E-8;dx=-1.8251869296647023E-8
New Minimum: 6.844450600359171E-8 > 4.467484302494995E-8
END: th(56.105070052913675)=4.467484302494995E-8; dx=0.0 evalInputDelta=2.3769662978641755E-8
Fitness changed from 6.844450600359171E-8 to 4.467484302494995E-8
Iteration 13 complete. Error: 4.467484302494995E-8 Total: 0.0606; Orientation: 0.0029; Line Search: 0.0410
th(0)=4.467484302494995E-8;dx=-1.1913291473319987E-8
New Minimum: 4.467484302494995E-8 > 4.221875353020247E-9
END: th(120.8747092086662)=4.221875353020247E-9; dx=0.0 evalInputDelta=4.04529676719297E-8
Fitness changed from 4.467484302494995E-8 to 4.221875353020247E-9
Iteration 14 complete. Error: 4.221875353020247E-9 Total: 0.2056; Orientation: 0.0036; Line Search: 0.1874
th(0)=4.221875353020247E-9;dx=-1.1258334925743115E-9
New Minimum: 4.221875353020247E-9 > 0.0
END: th(260.41666666666686)=0.0; dx=0.0 evalInputDelta=4.221875353020247E-9
Fitness changed from 4.221875353020247E-9 to 0.0
Iteration 15 complete. Error: 0.0 Total: 0.0684; Orientation: 0.0029; Line Search: 0.0500
Zero gradient: 0.0
th(0)=0.0;dx=0.0 (ERROR: Starting derivative negative)
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.2318; Orientation: 0.0054; Line Search: 0.2099
Iteration 16 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 16
Final threshold in iteration 16: 0.0 (> 0.0) after 2.421s (< 30.000s)

Returns

    0.0

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 2.37 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 2151984647870
Reset training subject: 2152000692619
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821588E-5}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821588E-5}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821585E-5}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821585E-5}, evalInputDelta = 0.0
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821585E-5}, evalInputDelta = 0.0
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821588E-5}, evalInputDelta = 0.0
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.0133364868352146E-4}, derivative=-2.7022306232821585E-5}, evalInputDelta = 0.0
New Minimum: 1.0133364868352146E-4 > 1.0133345794827164E-4
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.0133345794827164E-4}, derivative=-2.7022280602644093E-5}, evalInputDelta = -1.9073524981920876E-10
New Minimum: 1.0133345794827164E-4 > 1.0133148702967293E-4
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.0133148702967293E-4}, derivative=-2.7022018291879872E-5}, evalInputDelta = -2.1616538485282532E-9
New Minimum: 1.0133148702967293E-4 > 1.0131826329787448E-4
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.0131826329787448E-4}, derivative=-2.702025498524509E-5}, evalInputDelta = -1.5385385646984645E-8
New Minimum: 1.0131826329787448E-4 > 1.0122476817002735E-4
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.0122476817002735E-4}, derivative=-2.7007784730501453E-5}, evalInputDelta = -1.0888051349411398E-7
New Minimum: 1.0122476817002735E-4 > 1.0057163737542396E-4
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.0057163737542396E-4}, derivative=-2.692051341084856E-5}, evalInputDelta = -7.620113080975016E-7
New Minimum: 1.0057163737542396E-4 > 9.606083046188966E-5
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=9.606083046188966E-5}, derivative=-2.63098728101722E-5}, evalInputDelta = -5.272818221631806E-6
New Minimum: 9.606083046188966E-5 > 6.851511891170503E-5
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=6.851511891170503E-5}, derivative=-1.3685099701897917E-5}, evalInputDelta = -3.281852977181643E-5
New Minimum: 6.851511891170503E-5 > 4.3531917246279284E-5
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=4.3531917246279284E-5}, derivative=2.7394775254831043E-6}, evalInputDelta = -5.780173143724218E-5
4.3531917246279284E-5 <= 1.0133364868352146E-4
New Minimum: 4.3531917246279284E-5 > 4.158648016906833E-5
F(8.797068519397541) = LineSearchPoint{point=PointSample{avg=4.158648016906833E-5}, derivative=1.6232973842105697E-6}, evalInputDelta = -5.974716851445313E-5
Right bracket at 8.797068519397541
New Minimum: 4.158648016906833E-5 > 4.0932815989928405E-5
F(8.298553685946155) = LineSearchPoint{point=PointSample{avg=4.0932815989928405E-5}, derivative=9.994504323442472E-7}, evalInputDelta = -6.040083269359306E-5
Right bracket at 8.298553685946155
New Minimum: 4.0932815989928405E-5 > 4.0691803717625893E-5
F(8.002569634398052) = LineSearchPoint{point=PointSample{avg=4.0691803717625893E-5}, derivative=6.289840827593109E-7}, evalInputDelta = -6.064184496589557E-5
Right bracket at 8.002569634398052
New Minimum: 4.0691803717625893E-5 > 4.05980416019247E-5
F(7.820535130265957) = LineSearchPoint{point=PointSample{avg=4.05980416019247E-5}, derivative=4.011534710013365E-7}, evalInputDelta = -6.0735607081596765E-5
Right bracket at 7.820535130265957
New Minimum: 4.05980416019247E-5 > 4.056034633208583E-5
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=4.056034633208583E-5}, derivative=2.5800911591412133E-7}, evalInputDelta = -6.077330235143563E-5
Right bracket at 7.706135457632501
Converged to right
Fitness changed from 1.0133364868352146E-4 to 4.056034633208583E-5
Iteration 1 complete. Error: 4.056034633208583E-5 Total: 1.5283; Orientation: 0.0036; Line Search: 1.4792
F(0.0) = LineSearchPoint{point=PointSample{avg=4.056034633208583E-5}, derivative=-1.0816092156938547E-5}
New Minimum: 4.056034633208583E-5 > 2.6670112014433774E-5
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=2.6670112014433774E-5}, derivative=3.146028588917278E-8}, evalInputDelta = -1.3890234317652057E-5
2.6670112014433774E-5 <= 4.056034633208583E-5
Converged to right
Fitness changed from 4.056034633208583E-5 to 2.6670112014433774E-5
Iteration 2 complete. Error: 2.6670112014433774E-5 Total: 0.0545; Orientation: 0.0023; Line Search: 0.0400
F(0.0) = LineSearchPoint{point=PointSample{avg=2.6670112014433774E-5}, derivative=-7.112029804346267E-6}
New Minimum: 2.6670112014433774E-5 > 2.1333422196789797E-5
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=2.1333422196789797E-5}, derivative=2.348719529099504E-11}, evalInputDelta = -5.336689817643976E-6
2.1333422196789797E-5 <= 2.6670112014433774E-5
Converged to right
Fitness changed from 2.6670112014433774E-5 to 2.1333422196789797E-5
Iteration 3 complete. Error: 2.1333422196789797E-5 Total: 0.0573; Orientation: 0.0031; Line Search: 0.0407
F(0.0) = LineSearchPoint{point=PointSample{avg=2.1333422196789797E-5}, derivative=-5.688912254672985E-6}
New Minimum: 2.1333422196789797E-5 > 1.0666837060095228E-5
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=1.0666837060095228E-5}, derivative=1.3642420481418822E-14}, evalInputDelta = -1.066658513669457E-5
1.0666837060095228E-5 <= 2.1333422196789797E-5
Converged to right
Fitness changed from 2.1333422196789797E-5 to 1.0666837060095228E-5
Iteration 4 complete. Error: 1.0666837060095228E-5 Total: 0.0532; Orientation: 0.0027; Line Search: 0.0386
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0666837060095228E-5}, derivative=-2.8444900813756036E-6}
New Minimum: 1.0666837060095228E-5 > 4.26668395997846E-6
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=4.26668395997846E-6}, derivative=0.0}, evalInputDelta = -6.400153100116768E-6
4.26668395997846E-6 <= 1.0666837060095228E-5
Converged to right
Fitness changed from 1.0666837060095228E-5 to 4.26668395997846E-6
Iteration 5 complete. Error: 4.26668395997846E-6 Total: 0.3074; Orientation: 0.0028; Line Search: 0.2919
F(0.0) = LineSearchPoint{point=PointSample{avg=4.26668395997846E-6}, derivative=-1.1377823231000734E-6}
New Minimum: 4.26668395997846E-6 > 1.0667027794397654E-6
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=1.0667027794397654E-6}, derivative=0.0}, evalInputDelta = -3.199981180538695E-6
1.0667027794397654E-6 <= 4.26668395997846E-6
Converged to right
Fitness changed from 4.26668395997846E-6 to 1.0667027794397654E-6
Iteration 6 complete. Error: 1.0667027794397654E-6 Total: 0.2367; Orientation: 0.0039; Line Search: 0.2178
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0667027794397654E-6}, derivative=-2.844540745172708E-7}
New Minimum: 1.0667027794397654E-6 > 0.0
F(7.706135457632501) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=0.0}, evalInputDelta = -1.0667027794397654E-6
0.0 <= 1.0667027794397654E-6
F(3.8530677288162507) = LineSearchPoint{point=PointSample{avg=2.522065339386851E-7}, derivative=-1.3831478292147815E-7}, evalInputDelta = -8.144962455010804E-7
Left bracket at 3.8530677288162507
Converged to right
Fitness changed from 1.0667027794397654E-6 to 0.0
Iteration 7 complete. Error: 0.0 Total: 0.0975; Orientation: 0.0037; Line Search: 0.0778
Zero gradient: 0.0
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=0.0}
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0331; Orientation: 0.0023; Line Search: 0.0185
Iteration 8 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 8
Final threshold in iteration 8: 0.0 (> 0.0) after 2.368s (< 30.000s)

Returns

    0.0

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 24.00 seconds (0.083 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 2154358665965
Reset training subject: 2154371651343
Adding measurement 7b609d3c to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.0133364868352146E-4 < 1.0133364868352146E-4. Total: 1
th(0)=1.0133364868352146E-4;dx=-2.7022306232821588E-5
Adding measurement 32c02044 to history. Total: 1
New Minimum: 1.0133364868352146E-4 > 5.9567429570013994E-5
END: th(2.154434690031884)=5.9567429570013994E-5; dx=-8.71764286243199E-6 evalInputDelta=4.176621911350747E-5
Fitness changed from 1.0133364868352146E-4 to 5.9567429570013994E-5
Iteration 1 complete. Error: 5.9567429570013994E-5 Total: 0.1040; Orientation: 0.0158; Line Search: 0.0529
Non-optimal measurement 5.9567429570013994E-5 < 5.9567429570013994E-5. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 5.9567429570013994E-5 < 5.9567429570013994E-5. Total: 2
th(0)=5.9567429570013994E-5;dx=-1.5884647440162567E-5
Adding measurement 18ef4cb2 to history. Total: 2
New Minimum: 5.9567429570013994E-5 > 3.60502149627943E-5
END: th(4.641588833612779)=3.60502149627943E-5; dx=-1.872444288993729E-6 evalInputDelta=2.3517214607219696E-5
Fitness changed from 5.9567429570013994E-5 to 3.60502149627943E-5
Iteration 2 complete. Error: 3.60502149627943E-5 Total: 0.0649; Orientation: 0.0070; Line Search: 0.0422
Non-optimal measurement 3.60502149627943E-5 < 3.60502149627943E-5. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 3.60502149627943E-5 < 3.60502149627943E-5. Total: 3
th(0)=3.60502149627943E-5;dx=-9.613390473375416E-6
Adding measurement 6c539b20 to history. Total: 3
New Minimum: 3.60502149627943E-5 > 2.6733679828794265E-5
WOLF (strong): th(10.000000000000002)=2.6733679828794265E-5; dx=2.3089230987979574E-7 evalInputDelta=9.316535134000033E-6
Non-optimal measurement 2.6742371901870383E-5 < 2.6733679828794265E-5. Total: 4
END: th(5.000000000000001)=2.6742371901870383E-5; dx=-2.3786745695356733E-7 evalInputDelta=9.307843060923916E-6
Fitness changed from 3.60502149627943E-5 to 2.6733679828794265E-5
Iteration 3 complete. Error: 2.6733679828794265E-5 Total: 0.0912; Orientation: 0.0046; Line Search: 0.0705
Non-optimal measurement 2.6733679828794265E-5 < 2.6733679828794265E-5. Total: 4
Rejected: LBFGS Orientation magnitude: 4.755e-03, gradient 2.670e-03, dot -0.825; [91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.6733679828794265E-5, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
LBFGS Accumulation History: 3 points
Removed measurement 6c539b20 to history. Total: 3
Adding measurement 2ee913a to history. Total: 3
th(0)=2.6733679828794265E-5;dx=-7.128980941666924E-6
Adding measurement 2ed9c1c2 to history. Total: 4
New Minimum: 2.6733679828794265E-5 > 2.1523204953647716E-5
WOLF (strong): th(10.772173450159421)=2.1523204953647716E-5; dx=1.1599644016182465E-7 evalInputDelta=5.210474875146549E-6
Adding measurement 262d8bf4 to history. Total: 5
New Minimum: 2.1523204953647716E-5 > 2.1412620385300817E-5
END: th(5.386086725079711)=2.1412620385300817E-5; dx=-7.493379564421954E-8 evalInputDelta=5.321059443493448E-6
Fitness changed from 2.6733679828794265E-5 to 2.1412620385300817E-5
Iteration 4 complete. Error: 2.1412620385300817E-5 Total: 0.6120; Orientation: 0.4681; Line Search: 0.1300
Non-optimal measurement 2.1412620385300817E-5 < 2.1412620385300817E-5. Total: 6
Rejected: LBFGS Orientation magnitude: 5.212e-03, gradient 2.390e-03, dot -0.716; [2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.1412620385300817E-5, 2.1523204953647716E-5, 2.6733679828794265E-5, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 1.159e-02, gradient 2.390e-03, dot -0.619; [3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.1412620385300817E-5, 2.1523204953647716E-5, 2.6733679828794265E-5, 3.60502149627943E-5, 5.9567429570013994E-5
Rejected: LBFGS Orientation magnitude: 2.081e-02, gradient 2.390e-03, dot -0.662; [7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.1412620385300817E-5, 2.1523204953647716E-5, 2.6733679828794265E-5, 3.60502149627943E-5
LBFGS Accumulation History: 3 points
Removed measurement 262d8bf4 to history. Total: 5
Removed measurement 2ed9c1c2 to history. Total: 4
Removed measurement 2ee913a to history. Total: 3
Adding measurement 4d67c7e6 to history. Total: 3
th(0)=2.1412620385300817E-5;dx=-5.7100317719262124E-6
Adding measurement 6974080e to history. Total: 4
New Minimum: 2.1412620385300817E-5 > 1.1018285534684462E-5
WOLF (strong): th(11.60397208403195)=1.1018285534684462E-5; dx=1.1556450777246307E-8 evalInputDelta=1.0394334850616355E-5
Adding measurement 1416a49 to history. Total: 5
New Minimum: 1.1018285534684462E-5 > 1.0998630517633502E-5
END: th(5.801986042015975)=1.0998630517633502E-5; dx=-4.780554624657682E-9 evalInputDelta=1.0413989867667316E-5
Fitness changed from 2.1412620385300817E-5 to 1.0998630517633502E-5
Iteration 5 complete. Error: 1.0998630517633502E-5 Total: 2.7519; Orientation: 2.4203; Line Search: 0.3173
Non-optimal measurement 1.0998630517633502E-5 < 1.0998630517633502E-5. Total: 6
Rejected: LBFGS Orientation magnitude: 7.504e-03, gradient 1.713e-03, dot -0.449; [2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0998630517633502E-5, 1.1018285534684462E-5, 2.1412620385300817E-5, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 1.172e-02, gradient 1.713e-03, dot -0.518; [023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0998630517633502E-5, 1.1018285534684462E-5, 2.1412620385300817E-5, 3.60502149627943E-5, 5.9567429570013994E-5
Rejected: LBFGS Orientation magnitude: 1.448e-02, gradient 1.713e-03, dot -0.728; [023ef49f-987d-4637-b1d5-215f77669440 = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 1.000/1.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.0998630517633502E-5, 1.1018285534684462E-5, 2.1412620385300817E-5, 3.60502149627943E-5
LBFGS Accumulation History: 3 points
Removed measurement 1416a49 to history. Total: 5
Removed measurement 6974080e to history. Total: 4
Removed measurement 4d67c7e6 to history. Total: 3
Adding measurement 53be1334 to history. Total: 3
th(0)=1.0998630517633502E-5;dx=-2.932968317821513E-6
Adding measurement 23d8963a to history. Total: 4
New Minimum: 1.0998630517633502

...skipping 12297 bytes...

08E-7 Total: 1.3523; Orientation: 1.3034; Line Search: 0.0372
Non-optimal measurement 2.691185822338108E-7 < 2.691185822338108E-7. Total: 5
Rejected: LBFGS Orientation magnitude: 1.920e-03, gradient 2.679e-04, dot -0.453; [91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.691185822338108E-7, 8.85957033839683E-7, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 2.573e-03, gradient 2.679e-04, dot -0.517; [7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00]
Orientation rejected. Popping history element from 2.691185822338108E-7, 8.85957033839683E-7, 3.60502149627943E-5, 5.9567429570013994E-5
LBFGS Accumulation History: 3 points
Removed measurement 76d9ce8 to history. Total: 4
Removed measurement 11d8d4b2 to history. Total: 3
Adding measurement 4aa593e5 to history. Total: 3
th(0)=2.691185822338108E-7;dx=-7.176495940150778E-8
Adding measurement 403c042 to history. Total: 4
New Minimum: 2.691185822338108E-7 > 6.844450600359171E-8
END: th(26.041666666666682)=6.844450600359171E-8; dx=0.0 evalInputDelta=2.0067407623021908E-7
Fitness changed from 2.691185822338108E-7 to 6.844450600359171E-8
Iteration 12 complete. Error: 6.844450600359171E-8 Total: 1.1068; Orientation: 1.0148; Line Search: 0.0699
Non-optimal measurement 6.844450600359171E-8 < 6.844450600359171E-8. Total: 5
Rejected: LBFGS Orientation magnitude: 4.157e-04, gradient 1.351e-04, dot -0.564; [3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 6.844450600359171E-8, 2.691185822338108E-7, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 6.684e-04, gradient 1.351e-04, dot -0.663; [023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 1.000/1.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 6.844450600359171E-8, 2.691185822338108E-7, 3.60502149627943E-5, 5.9567429570013994E-5
LBFGS Accumulation History: 3 points
Removed measurement 403c042 to history. Total: 4
Removed measurement 4aa593e5 to history. Total: 3
Adding measurement 776e16dd to history. Total: 3
th(0)=6.844450600359171E-8;dx=-1.8251869296647023E-8
Adding measurement 339ff690 to history. Total: 4
New Minimum: 6.844450600359171E-8 > 4.467484302494995E-8
END: th(56.105070052913675)=4.467484302494995E-8; dx=0.0 evalInputDelta=2.3769662978641755E-8
Fitness changed from 6.844450600359171E-8 to 4.467484302494995E-8
Iteration 13 complete. Error: 4.467484302494995E-8 Total: 1.0466; Orientation: 0.9997; Line Search: 0.0352
Non-optimal measurement 4.467484302494995E-8 < 4.467484302494995E-8. Total: 5
Rejected: LBFGS Orientation magnitude: 5.330e-04, gradient 1.091e-04, dot -0.454; [023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.467484302494995E-8, 6.844450600359171E-8, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 7.555e-04, gradient 1.091e-04, dot -0.544; [3a561929-44ca-4524-99b2-d26388629bf4 = 0.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.467484302494995E-8, 6.844450600359171E-8, 3.60502149627943E-5, 5.9567429570013994E-5
LBFGS Accumulation History: 3 points
Removed measurement 339ff690 to history. Total: 4
Removed measurement 776e16dd to history. Total: 3
Adding measurement 2c4ff507 to history. Total: 3
th(0)=4.467484302494995E-8;dx=-1.1913291473319987E-8
Adding measurement 14bd1197 to history. Total: 4
New Minimum: 4.467484302494995E-8 > 4.221875353020247E-9
END: th(120.8747092086662)=4.221875353020247E-9; dx=0.0 evalInputDelta=4.04529676719297E-8
Fitness changed from 4.467484302494995E-8 to 4.221875353020247E-9
Iteration 14 complete. Error: 4.221875353020247E-9 Total: 1.5440; Orientation: 1.4894; Line Search: 0.0395
Non-optimal measurement 4.221875353020247E-9 < 4.221875353020247E-9. Total: 5
Rejected: LBFGS Orientation magnitude: 4.101e-04, gradient 3.355e-05, dot -0.510; [2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 0.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00]
Orientation rejected. Popping history element from 4.221875353020247E-9, 4.467484302494995E-8, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 4.303e-04, gradient 3.355e-05, dot -0.549; [023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 0.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.221875353020247E-9, 4.467484302494995E-8, 3.60502149627943E-5, 5.9567429570013994E-5
LBFGS Accumulation History: 3 points
Removed measurement 14bd1197 to history. Total: 4
Removed measurement 2c4ff507 to history. Total: 3
Adding measurement 3abf6bba to history. Total: 3
th(0)=4.221875353020247E-9;dx=-1.1258334925743115E-9
Adding measurement 422462b6 to history. Total: 4
New Minimum: 4.221875353020247E-9 > 0.0
END: th(260.41666666666686)=0.0; dx=0.0 evalInputDelta=4.221875353020247E-9
Fitness changed from 4.221875353020247E-9 to 0.0
Iteration 15 complete. Error: 0.0 Total: 1.2646; Orientation: 1.2143; Line Search: 0.0372
Non-optimal measurement 0.0 < 0.0. Total: 5
Rejected: LBFGS Orientation magnitude: 0.000e+00, gradient 0.000e+00, dot NaN; [91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 3a561929-44ca-4524-99b2-d26388629bf4 = 0.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00]
Orientation rejected. Popping history element from 0.0, 4.221875353020247E-9, 3.60502149627943E-5, 5.9567429570013994E-5, 1.0133364868352146E-4
Rejected: LBFGS Orientation magnitude: 0.000e+00, gradient 0.000e+00, dot NaN; [3a561929-44ca-4524-99b2-d26388629bf4 = 0.000e+00, 91eff417-3de1-4577-85ff-150610db1a16 = 0.000e+00, 023ef49f-987d-4637-b1d5-215f77669440 = 0.000e+00, 2ddb149e-e211-4196-b527-858faefd994d = 0.000e+00, 7dd729c3-9251-48a2-9a36-1d301350703d = 0.000e+00]
Orientation rejected. Popping history element from 0.0, 4.221875353020247E-9, 3.60502149627943E-5, 5.9567429570013994E-5
LBFGS Accumulation History: 3 points
Removed measurement 422462b6 to history. Total: 4
Removed measurement 3abf6bba to history. Total: 3
Adding measurement 2409913e to history. Total: 3
th(0)=0.0;dx=0.0 (ERROR: Starting derivative negative)
Non-optimal measurement 0.0 < 0.0. Total: 4
Fitness changed from 0.0 to 0.0
Static Iteration Total: 1.6918; Orientation: 1.6286; Line Search: 0.0321
Iteration 16 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 16
Final threshold in iteration 16: 0.0 (> 0.0) after 23.999s (< 30.000s)

Returns

    0.0

Training Converged

TrainingTester.java:432 executed in 0.15 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging 
Plotting range=[1.0, -8.374494592983503], [15.0, -4.224991139999263]; valueStats=DoubleSummaryStatistics{count=34, sum=0.000442, min=0.000000, average=0.000013, max=0.000060}
Plotting 15 points for GD
Plotting 7 points for CjGD
Plotting 15 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging 
Plotting range=[0.0, -8.374494592983503], [22.203, -4.224991139999263]; valueStats=DoubleSummaryStatistics{count=34, sum=0.000442, min=0.000000, average=0.000013, max=0.000060}
Plotting 15 points for GD
Plotting 7 points for CjGD
Plotting 15 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":null, "complete":null}OK
 
  {
    "result": "OK",
    "performance": {
      "execution_time": "29.790",
      "gc_time": "0.371"
    },
    "created_on": 1586736781128,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Float",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandReducerLayerTest.Float",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/BandReducerLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "Converged",
          "value": 0.0
        },
        "CjGD": {
          "type": "Converged",
          "value": 0.0
        },
        "GD": {
          "type": "Converged",
          "value": 0.0
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/BandReducerLayer/Float/trainingTest/202004131301",
    "id": "ce69fa24-1dfa-43d6-9aa1-1cc9559c87e4",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "BandReducerLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandReducerLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/BandReducerLayer.java",
      "javaDoc": ""
    }
  }