1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 7904812361917746176

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.04 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 1.696 ], [ 1.372 ], [ 0.884 ], [ 0.66 ], [ 1.344 ], [ -1.676 ], [ -0.68 ], [ 0.568 ], ... ],
    	[ [ -0.292 ], [ -0.18 ], [ 0.144 ], [ -0.096 ], [ 0.628 ], [ 0.6 ], [ -1.684 ], [ 1.928 ], ... ],
    	[ [ 1.088 ], [ 0.688 ], [ -0.568 ], [ 1.06 ], [ 1.724 ], [ -0.344 ], [ -0.476 ], [ 1.156 ], ... ],
    	[ [ -0.092 ], [ -1.908 ], [ -0.744 ], [ -1.592 ], [ -1.16 ], [ 1.38 ], [ -0.424 ], [ -1.144 ], ... ],
    	[ [ -1.96 ], [ 0.608 ], [ 0.784 ], [ 1.08 ], [ 0.364 ], [ -0.328 ], [ 1.428 ], [ -1.516 ], ... ],
    	[ [ -1.176 ], [ 0.98 ], [ -0.536 ], [ 1.036 ], [ -1.172 ], [ -0.484 ], [ 1.096 ], [ -1.984 ], ... ],
    	[ [ -1.436 ], [ 1.992 ], [ -1.204 ], [ -0.956 ], [ 0.696 ], [ 0.544 ], [ 0.216 ], [ -0.864 ], ... ],
    	[ [ 0.752 ], [ 1.212 ], [ -0.172 ], [ 1.144 ], [ -1.628 ], [ -0.744 ], [ -1.096 ], [ -1.808 ], ... ],
    	...
    ]
    [
    	[ [ 1.74 ], [ -0.204 ], [ 0.756 ], [ -0.248 ], [ -0.14 ], [ -1.12 ], [ 1.556 ], [ -1.468 ], ... ],
    	[ [ 0.452 ], [ -1.5 ], [ 1.868 ], [ -1.128 ], [ -0.392 ], [ -0.012 ], [ 0.568 ], [ -0.52 ], ... ],
    	[ [ 0.024 ], [ 1.924 ], [ -1.62 ], [ -0.336 ], [ -1.28 ], [ 0.168 ], [ 1.78 ], [ -1.42 ], ... ],
    	[ [ -1.868 ], [ -0.836 ], [ -1.252 ], [ -0.324 ], [ 1.988 ], [ 0.624 ], [ -0.364 ], [ 0.104 ], ... ],
    	[ [ -1.564 ], [ -1.844 ], [ -0.872 ], [ -1.34 ], [ 1.488 ], [ 1.832 ], [ -1.996 ], [ 0.836 ], ... ],
    	[ [ 0.22 ], [ 1.388 ], [ -0.78 ], [ -1.472 ], [ 1.772 ], [ -0.824 ], [ -0.456 ], [ -1.22 ], ... ],
    	[ [ -1.62 ], [ 0.684 ], [ -0.468 ], [ 1.696 ], [ -1.236 ], [ 1.936 ], [ -1.052 ], [ -0.2 ], ... ],
    	[ [ 0.032 ], [ 0.084 ], [ -0.732 ], [ 0.34 ], [ 1.744 ], [ 1.096 ], [ -1.264 ], [ 0.308 ], ... ],
    	...
    ]
    [
    	[ [ -1.848 ], [ -0.176 ], [ -1.884 ], [ 0.596 ], [ 0.316 ], [ 0.908 ], [ 1.612 ], [ -0.652 ], ... ],
    	[ [ -1.412 ], [ -0.468 ], [ -0.608 ], [ 1.044 ], [ 0.62 ], [ -1.468 ], [ 1.988 ], [ -1.904 ], ... ],
    	[ [ 1.548 ], [ -1.936 ], [ 0.52 ], [ -1.896 ], [ -1.08 ], [ -0.836 ], [ 1.232 ], [ -1.996 ], ... ],
    	[ [ 1.248 ], [ -0.808 ], [ -0.632 ], [ -1.112 ], [ -0.808 ], [ -1.036 ], [ -0.24 ], [ -0.34 ], ... ],
    	[ [ 1.44 ], [ -1.164 ], [ -0.284 ], [ -0.656 ], [ 0.724 ], [ 1.988 ], [ -1.664 ], [ 1.12 ], ... ],
    	[ [ -1.576 ], [ 1.44 ], [ -1.72 ], [ 0.784 ], [ -0.172 ], [ -1.908 ], [ 1.832 ], [ -0.452 ], ... ],
    	[ [ -0.124 ], [ 0.288 ], [ 1.58 ], [ 1.332 ], [ -0.884 ], [ -1.368 ], [ -0.508 ], [ -1.012 ], ... ],
    	[ [ -0.864 ], [ -1.544 ], [ -1.192 ], [ -1.996 ], [ 1.356 ], [ 0.204 ], [ -1.432 ], [ -0.26 ], ... ],
    	...
    ]
    [
    	[ [ -0.768 ], [ -1.6 ], [ -0.952 ], [ -1.68 ], [ 0.256 ], [ 1.008 ], [ 0.956 ], [ -0.72 ], ... ],
    	[ [ -0.38 ], [ -1.112 ], [ 0.192 ], [ 1.232 ], [ 1.66 ], [ 0.1 ], [ 1.152 ], [ -0.18 ], ... ],
    	[ [ 0.84 ], [ -1.144 ], [ 1.832 ], [ 1.744 ], [ 0.188 ], [ 1.524 ], [ 0.416 ], [ 0.072 ], ... ],
    	[ [ 1.0 ], [ -0.564 ], [ -1.032 ], [ -1.624 ], [ -1.22 ], [ -1.848 ], [ 1.736 ], [ 1.904 ], ... ],
    	[ [ -0.644 ], [ 0.912 ], [ 0.416 ], [ 1.28 ], [ -0.24 ], [ -0.544 ], [ -0.072 ], [ -0.228 ], ... ],
    	[ [ -0.164 ], [ 0.692 ], [ -1.344 ], [ -1.972 ], [ 0.232 ], [ 1.232 ], [ -1.364 ], [ 0.992 ], ... ],
    	[ [ 1.704 ], [ 0.82 ], [ -1.44 ], [ 0.568 ], [ -0.62 ], [ 1.436 ], [ 1.976 ], [ -0.356 ], ... ],
    	[ [ -1.5 ], [ 0.304 ], [ 0.444 ], [ -0.488 ], [ 0.112 ], [ 1.664 ], [ 0.1 ], [ 1.504 ], ... ],
    	...
    ]
    [
    	[ [ -1.032 ], [ -1.716 ], [ -0.356 ], [ -1.908 ], [ 1.072 ], [ -1.648 ], [ 1.232 ], [ 0.032 ], ... ],
    	[ [ 0.28 ], [ 0.764 ], [ 0.512 ], [ -1.148 ], [ 0.688 ], [ 0.728 ], [ 1.192 ], [ -0.288 ], ... ],
    	[ [ 1.24 ], [ 1.404 ], [ 0.112 ], [ -1.356 ], [ 0.324 ], [ 1.844 ], [ -0.016 ], [ 1.596 ], ... ],
    	[ [ 0.904 ], [ -0.148 ], [ 0.26 ], [ -1.116 ], [ -1.944 ], [ -0.88 ], [ 0.82 ], [ 0.544 ], ... ],
    	[ [ 1.404 ], [ -0.828 ], [ -0.344 ], [ -0.96 ], [ -1.652 ], [ -0.576 ], [ -1.06 ], [ 0.184 ], ... ],
    	[ [ -1.152 ], [ 1.528 ], [ 0.132 ], [ -0.208 ], [ -1.772 ], [ -0.2 ], [ 1.996 ], [ 1.4 ], ... ],
    	[ [ -0.38 ], [ 0.064 ], [ -0.104 ], [ -0.24 ], [ -0.516 ], [ 1.916 ], [ -1.376 ], [ 0.692 ], ... ],
    	[ [ 1.28 ], [ 0.136 ], [ -1.008 ], [ 0.084 ], [ 0.36 ], [ 0.228 ], [ -1.16 ], [ -1.232 ], ... ],
    	...
    ]
    [
    	[ [ 0.76 ], [ 0.788 ], [ 0.54 ], [ 1.188 ], [ -1.876 ], [ -1.888 ], [ 1.588 ], [ -0.8 ], ... ],
    	[ [ -1.648 ], [ -0.68 ], [ -0.72 ], [ -0.452 ], [ -0.8 ], [ -1.372 ], [ -1.928 ], [ 1.312 ], ... ],
    	[ [ -1.184 ], [ 0.988 ], [ 1.24 ], [ 1.904 ], [ -1.12 ], [ -0.336 ], [ -1.764 ], [ -0.46 ], ... ],
    	[ [ 0.456 ], [ -1.356 ], [ -1.048 ], [ 1.92 ], [ -1.692 ], [ -0.42 ], [ -0.024 ], [ 0.212 ], ... ],
    	[ [ -1.704 ], [ 0.568 ], [ -0.604 ], [ 0.464 ], [ -1.336 ], [ -0.836 ], [ -1.96 ], [ -0.732 ], ... ],
    	[ [ -1.944 ], [ 0.9 ], [ -0.812 ], [ 0.972 ], [ 1.928 ], [ 1.896 ], [ -0.856 ], [ 0.716 ], ... ],
    	[ [ -1.752 ], [ 0.856 ], [ -0.96 ], [ -0.892 ], [ -1.52 ], [ 1.52 ], [ -0.308 ], [ 1.304 ], ... ],
    	[ [ -1.604 ], [ -1.084 ], [ -0.748 ], [ -0.116 ], [ 1.092 ], [ 0.76 ], [ 0.548 ], [ 0.256 ], ... ],
    	...
    ]
    [
    	[ [ 0.508 ], [ -1.888 ], [ 1.404 ], [ -0.356 ], [ -0.068 ], [ 1.448 ], [ 1.008 ], [ 1.796 ], ... ],
    	[ [ 1.076 ], [ 1.168 ], [ 0.904 ], [ 1.88 ], [ 1.616 ], [ 0.3 ], [ 1.672 ], [ 0.304 ], ... ],
    	[ [ 0.86 ], [ 0.896 ], [ -1.66 ], [ -1.732 ], [ -0.676 ], [ -0.956 ], [ -1.24 ], [ -0.544 ], ... ],
    	[ [ 0.828 ], [ 1.584 ], [ 1.78 ], [ -1.26 ], [ -1.912 ], [ -0.408 ], [ -1.924 ], [ 1.96 ], ... ],
    	[ [ -1.148 ], [ 1.204 ], [ -1.628 ], [ -1.732 ], [ 1.24 ], [ -0.772 ], [ -0.34 ], [ -0.392 ], ... ],
    	[ [ 1.732 ], [ -1.356 ], [ 0.56 ], [ 1.444 ], [ -0.48 ], [ -0.824 ], [ 1.892 ], [ 0.576 ], ... ],
    	[ [ -1.928 ], [ -0.064 ], [ 0.544 ], [ 0.22 ], [ 1.456 ], [ 1.08 ], [ -1.212 ], [ -1.632 ], ... ],
    	[ [ -1.856 ], [ 0.968 ], [ -1.752 ], [ -0.856 ], [ 0.46 ], [ 0.516 ], [ -0.712 ], [ -1.372 ], ... ],
    	...
    ]
    [
    	[ [ -1.784 ], [ 0.996 ], [ -0.116 ], [ -1.156 ], [ -0.324 ], [ 1.632 ], [ 0.368 ], [ 0.292 ], ... ],
    	[ [ 1.496 ], [ 1.288 ], [ -1.804 ], [ 1.028 ], [ -0.916 ], [ 0.912 ], [ 1.276 ], [ -0.824 ], ... ],
    	[ [ 1.2 ], [ 1.7 ], [ 1.196 ], [ -0.172 ], [ 1.996 ], [ 0.072 ], [ 0.4 ], [ -1.76 ], ... ],
    	[ [ -1.032 ], [ -0.228 ], [ -1.844 ], [ 1.228 ], [ -1.18 ], [ -0.028 ], [ -0.748 ], [ -1.064 ], ... ],
    	[ [ 1.092 ], [ -0.784 ], [ 0.248 ], [ -0.76 ], [ -0.604 ], [ 1.852 ], [ -1.348 ], [ -1.136 ], ... ],
    	[ [ 1.732 ], [ -0.152 ], [ 1.36 ], [ 1.3 ], [ -1.868 ], [ -0.788 ], [ -1.296 ], [ -0.208 ], ... ],
    	[ [ -0.856 ], [ -0.796 ], [ 0.528 ], [ 0.86 ], [ 0.772 ], [ -0.5 ], [ -1.964 ], [ 1.088 ], ... ],
    	[ [ -0.248 ], [ 1.084 ], [ 0.22 ], [ -0.544 ], [ -0.592 ], [ -1.928 ], [ 1.796 ], [ -0.952 ], ... ],
    	...
    ]
    [
    	[ [ -1.9 ], [ 0.588 ], [ -1.228 ], [ 0.344 ], [ -0.808 ], [ 1.436 ], [ 1.904 ], [ 0.996 ], ... ],
    	[ [ 0.964 ], [ -1.088 ], [ -1.744 ], [ -1.692 ], [ -0.008 ], [ 1.308 ], [ -1.536 ], [ 0.704 ], ... ],
    	[ [ 1.06 ], [ -0.256 ], [ 1.868 ], [ 1.276 ], [ 1.244 ], [ -0.776 ], [ -0.728 ], [ 1.5 ], ... ],
    	[ [ -0.388 ], [ -1.18 ], [ -1.16 ], [ -0.86 ], [ 0.308 ], [ 1.408 ], [ 1.356 ], [ 1.988 ], ... ],
    	[ [ -1.688 ], [ -1.736 ], [ 1.596 ], [ 1.98 ], [ -1.144 ], [ -1.348 ], [ -0.276 ], [ 1.488 ], ... ],
    	[ [ -1.8 ], [ 0.732 ], [ -1.992 ], [ 0.832 ], [ 1.6 ], [ 1.632 ], [ 1.2 ], [ -1.744 ], ... ],
    	[ [ -0.824 ], [ 1.904 ], [ 1.872 ], [ 1.636 ], [ 0.012 ], [ -0.696 ], [ 0.204 ], [ 1.352 ], ... ],
    	[ [ 0.7 ], [ 1.044 ], [ -1.992 ], [ 0.636 ], [ -0.964 ], [ -0.164 ], [ 1.84 ], [ -1.784 ], ... ],
    	...
    ]
    [
    	[ [ -1.016 ], [ -1.164 ], [ -1.42 ], [ 1.828 ], [ -1.504 ], [ 0.448 ], [ 0.94 ], [ 0.936 ], ... ],
    	[ [ -1.764 ], [ 0.58 ], [ -0.884 ], [ 1.62 ], [ 1.908 ], [ -0.492 ], [ 1.196 ], [ 0.14 ], ... ],
    	[ [ -1.228 ], [ 1.224 ], [ -0.464 ], [ -1.024 ], [ -0.476 ], [ 1.496 ], [ 0.2 ], [ -0.084 ], ... ],
    	[ [ -1.08 ], [ -1.12 ], [ -0.244 ], [ 0.536 ], [ -0.872 ], [ 0.888 ], [ -1.976 ], [ 0.132 ], ... ],
    	[ [ -0.572 ], [ 0.572 ], [ -1.732 ], [ 1.18 ], [ -0.512 ], [ 0.404 ], [ -0.264 ], [ 1.336 ], ... ],
    	[ [ -0.544 ], [ 1.336 ], [ 1.732 ], [ 1.568 ], [ -0.476 ], [ 1.444 ], [ -0.124 ], [ -1.2 ], ... ],
    	[ [ -0.104 ], [ 1.604 ], [ -0.872 ], [ -1.612 ], [ -1.14 ], [ -0.4 ], [ -0.048 ], [ 1.636 ], ... ],
    	[ [ -1.112 ], [ -0.036 ], [ 0.168 ], [ 1.496 ], [ 0.676 ], [ 1.872 ], [ 1.512 ], [ -1.164 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 31.34 seconds (3.775 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 6421973731411
Reset training subject: 6422448376448
Constructing line search parameters: GD
th(0)=5.336992190537778;dx=-5.929991322819743E-6
New Minimum: 5.336992190537778 > 5.3369794147664065
WOLFE (weak): th(2.154434690031884)=5.3369794147664065; dx=-5.92998422516473E-6 evalInputDelta=1.277577137148711E-5
New Minimum: 5.3369794147664065 > 5.336966639010326
WOLFE (weak): th(4.308869380063768)=5.336966639010326; dx=-5.929977127509726E-6 evalInputDelta=2.5551527452094547E-5
New Minimum: 5.336966639010326 > 5.336915536138919
WOLFE (weak): th(12.926608140191302)=5.336915536138919; dx=-5.9299487368897085E-6 evalInputDelta=7.665439885862213E-5
New Minimum: 5.336915536138919 > 5.336685576245292
WOLFE (weak): th(51.70643256076521)=5.336685576245292; dx=-5.929820979099526E-6 evalInputDelta=3.066142924863513E-4
New Minimum: 5.336685576245292 > 5.335459207154008
WOLFE (weak): th(258.53216280382605)=5.335459207154008; dx=-5.929139604218582E-6 evalInputDelta=0.001532983383770059
New Minimum: 5.335459207154008 > 5.327797593184935
WOLFE (weak): th(1551.1929768229563)=5.327797593184935; dx=-5.9248810112127625E-6 evalInputDelta=0.00919459735284267
New Minimum: 5.327797593184935 > 5.272796477736841
WOLFE (weak): th(10858.350837760694)=5.272796477736841; dx=-5.894219141570834E-6 evalInputDelta=0.06419571280093717
New Minimum: 5.272796477736841 > 4.834302441168797
WOLFE (weak): th(86866.80670208555)=4.834302441168797; dx=-5.643813872828419E-6 evalInputDelta=0.5026897493689813
New Minimum: 4.834302441168797 > 1.7077200105291426
END: th(781801.26031877)=1.7077200105291426; dx=-3.3543942728976156E-6 evalInputDelta=3.6292721800086354
Fitness changed from 5.336992190537778 to 1.7077200105291426
Iteration 1 complete. Error: 1.7077200105291426 Total: 10.3273; Orientation: 0.4174; Line Search: 8.7135
th(0)=1.7077200105291426;dx=-1.897466678365716E-6
New Minimum: 1.7077200105291426 > 0.007050823867345869
END: th(1684339.7559414052)=0.007050823867345869; dx=-1.2192303284046038E-7 evalInputDelta=1.7006691866617967
Fitness changed from 1.7077200105291426 to 0.007050823867345869
Iteration 2 complete. Error: 0.007050823867345869 Total: 2.7530; Orientation: 0.4810; Line Search: 1.9863
th(0)=0.007050823867345869;dx=-7.834248741495442E-9
Armijo: th(3628800.0)=0.007278255242010981; dx=7.959596721359349E-9 evalInputDelta=-2.274313746651116E-4
New Minimum: 0.007050823867345869 > 4.5125272751014615E-7
WOLF (strong): th(1814400.0)=4.5125272751014615E-7; dx=6.267398993196373E-11 evalInputDelta=0.007050372614618359
END: th(604800.0)=0.0031086800398173243; dx=-5.201941164352948E-9 evalInputDelta=0.003942143827528544
Fitness changed from 0.007050823867345869 to 4.5125272751014615E-7
Iteration 3 complete. Error: 4.5125272751014615E-7 Total: 3.2316; Orientation: 0.4920; Line Search: 2.4372
Low gradient: 7.08090332835944E-7
th(0)=4.5125272751014615E-7;dx=-5.013919194557174E-13
New Minimum: 4.5125272751014615E-7 > 3.4402019379103964E-8
END: th(1303002.1005312833)=3.4402019379103964E-8; dx=-1.384392948778223E-13 evalInputDelta=4.168507081310422E-7
Fitness changed from 4.5125272751014615E-7 to 3.4402019379103964E-8
Iteration 4 complete. Error: 3.4402019379103964E-8 Total: 2.6618; Orientation: 0.5008; Line Search: 1.8705
Low gradient: 1.9551078225198268E-7
th(0)=3.4402019379103964E-8;dx=-3.822446597678206E-14
New Minimum: 3.4402019379103964E-8 > 1.0772059780412938E-8
WOLF (strong): th(2807232.9265690087)=1.0772059780412938E-8; dx=2.138941151796213E-14 evalInputDelta=2.3629959598691024E-8
New Minimum: 1.0772059780412938E-8 > 1.6682846067962915E-9
END: th(1403616.4632845044)=1.6682846067962915E-9; dx=-8.417527229410089E-15 evalInputDelta=3.273373477230767E-8
Fitness changed from 3.4402019379103964E-8 to 1.6682846067962915E-9
Iteration 5 complete. Error: 1.6682846067962915E-9 Total: 2.7673; Orientation: 0.5007; Line Search: 1.9456
Low gradient: 4.305403074169701E-8
th(0)=1.6682846067962915E-9;dx=-1.853649563106985E-15
New Minimum: 1.6682846067962915E-9 > 7.714148021826179E-10
WOLF (strong): th(3024000.0)=7.714148021826179E-10; dx=1.260481702912766E-15 evalInputDelta=8.968698046136736E-10
New Minimum: 7.714148021826179E-10 > 4.2708085933982716E-11
END: th(1512000.0)=4.2708085933982716E-11; dx=-2.9658393009711056E-16 evalInputDelta=1.6255765208623088E-9
Fitness changed from 1.6682846067962915E-9 to 4.2708085933982716E-11
Iteration 6 complete. Error: 4.2708085933982716E-11 Total: 3.1815; Orientation: 0.4949; Line Search: 2.3889
Low gradient: 6.8886449186713315E-9
th(0)=4.2708085933982716E-11;dx=-4.7453428815536405E-17
New Minimum: 4.2708085933982716E-11 > 2.8001761615897522E-11
WOLF (strong): th(3257505.2513282085)=2.8001761615897522E-11; dx=3.842423427343171E-17 evalInputDelta=1.4706324318085194E-11
New Minimum: 2.8001761615897522E-11 > 3.8655646442570857E-13
END: th(1628752.6256641042)=3.8655646442570857E-13; dx=-4.51459727105195E-18 evalInputDelta=4.232152946955701E-11
Fitness changed from 4.2708085933982716E-11 to 3.8655646442570857E-13
Iteration 7 complete. Error: 3.8655646442570857E-13 Total: 2.8246; Orientation: 0.4909; Line Search: 2.0336
Low gradient: 6.553679750302359E-10
th(0)=3.8655646442570857E-13;dx=-4.29507182695233E-19
New Minimum: 3.8655646442570857E-13 > 3.4847608086914735E-13
WOLF (strong): th(3509041.1582112615)=3.4847608086914735E-13; dx=4.0780302942962697E-19 evalInputDelta=3.8080383556561223E-14
New Minimum: 3.4847608086914735E-13 > 2.467730803898811E-16
END: th(1754520.5791056307)=2.467730803898811E-16; dx=-1.0852076632980929E-20 evalInputDelta=3.863096913453187E-13
Fitness changed from 3.8655646442570857E-13 to 2.467730803898811E-16
Iteration 8 complete. Error: 2.467730803898811E-16 Total: 3.5781; Orientation: 0.4886; Line Search: 2.7922
Final threshold in iteration 8: 2.467730803898811E-16 (> 0.0) after 31.326s (< 30.000s)

Returns

    2.467730803898811E-16

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 40.05 seconds (1.031 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 6453391973580
Reset training subject: 6453704545833
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=5.336992190537778}, derivative=-5.929991322819743E-6}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=5.336992190537778}, derivative=-5.929991322819743E-6}, evalInputDelta = 0.0
New Minimum: 5.336992190537778 > 5.3369921905377735
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=5.3369921905377735}, derivative=-5.929991322819741E-6}, evalInputDelta = -4.440892098500626E-15
New Minimum: 5.3369921905377735 > 5.336992190537749
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=5.336992190537749}, derivative=-5.929991322819728E-6}, evalInputDelta = -2.930988785010413E-14
New Minimum: 5.336992190537749 > 5.336992190537575
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=5.336992190537575}, derivative=-5.9299913228196395E-6}, evalInputDelta = -2.0339285811132868E-13
New Minimum: 5.336992190537575 > 5.336992190536354
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=5.336992190536354}, derivative=-5.929991322818955E-6}, evalInputDelta = -1.4237500067793007E-12
New Minimum: 5.336992190536354 > 5.336992190527811
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=5.336992190527811}, derivative=-5.929991322814216E-6}, evalInputDelta = -9.967138225874805E-12
New Minimum: 5.336992190527811 > 5.336992190468012
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=5.336992190468012}, derivative=-5.929991322781005E-6}, evalInputDelta = -6.976552668902514E-11
New Minimum: 5.336992190468012 > 5.3369921900494175
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=5.3369921900494175}, derivative=-5.929991322548435E-6}, evalInputDelta = -4.883604631800154E-10
New Minimum: 5.3369921900494175 > 5.336992187119256
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=5.336992187119256}, derivative=-5.929991320920563E-6}, evalInputDelta = -3.418522354081688E-9
New Minimum: 5.336992187119256 > 5.336992166608124
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=5.336992166608124}, derivative=-5.929991309525506E-6}, evalInputDelta = -2.3929653814036556E-8
New Minimum: 5.336992166608124 > 5.336992023030201
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=5.336992023030201}, derivative=-5.929991229759975E-6}, evalInputDelta = -1.675075766982559E-7
New Minimum: 5.336992023030201 > 5.336991017984799
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=5.336991017984799}, derivative=-5.929990671401377E-6}, evalInputDelta = -1.172552979156194E-6
New Minimum: 5.336991017984799 > 5.336983982669634
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=5.336983982669634}, derivative=-5.929986762891266E-6}, evalInputDelta = -8.207868144260999E-6
New Minimum: 5.336983982669634 > 5.336934735593312
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=5.336934735593312}, derivative=-5.929959403320247E-6}, evalInputDelta = -5.745494446607324E-5
New Minimum: 5.336934735593312 > 5.336590012421074
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=5.336590012421074}, derivative=-5.929767886323257E-6}, evalInputDelta = -4.0217811670384407E-4
New Minimum: 5.336590012421074 > 5.334177261954405
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=5.334177261954405}, derivative=-5.928427267344284E-6}, evalInputDelta = -0.0028149285833727333
New Minimum: 5.334177261954405 > 5.3173032838982754
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=5.3173032838982754}, derivative=-5.9190429344914296E-6}, evalInputDelta = -0.019688906639502513
New Minimum: 5.3173032838982754 > 5.199933922822505
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=5.199933922822505}, derivative=-5.853352604521577E-6}, evalInputDelta = -0.13705826771527274
New Minimum: 5.199933922822505 > 4.415024175831826
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=4.415024175831826}, derivative=-5.393520294732502E-6}, evalInputDelta = -0.9219680147059517
New Minimum: 4.415024175831826 > 0.7177691933429292
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.7177691933429292}, derivative=-2.1746941262089454E-6}, evalInputDelta = -4.619222997194849
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=62.89553339176756}, derivative=2.0357089053455946E-5}, evalInputDelta = 57.55854120122978
F(613786.6638277846) = LineSearchPoint{point=PointSample{avg=2.317806421742956}, derivative=-3.907908216952402E-6}, evalInputDelta = -3.019185768794822
F(4296506.646794492) = LineSearchPoint{point=PointSample{avg=10.266372323163917}, derivative=8.22459041825178E-6}, evalInputDelta = 4.9293801326261395
F(330500.511291884) = LineSearchPoint{point=PointSample{avg=3.5570538149489095}, derivative=-4.841177342737337E-6}, evalInputDelta = -1.7799383755888685
New Minimum: 0.7177691933429292 > 0.43434868091454637
F(2313503.579043188) = LineSearchPoint{point=PointSample{avg=0.43434868091454637}, derivative=1.6917065377572175E-6}, evalInputDelta = -4.902643509623232
0.43434868091454637 <= 5.336992190537778
New Minimum: 0.43434868091454637 > 5.083827032008331E-31
F(1799999.9999999993) = LineSearchPoint{point=PointSample{avg=5.083827032008331E-31}, derivative=-1.694921698134763E-21}, evalInputDelta = -5.336992190537778
Left bracket at 1799999.9999999993
Converged to left
Fitness changed from 5.336992190537778 to 5.083827032008331E-31
Iteration 1 complete. Error: 5.083827032008331E-31 Total: 20.1376; Orientation: 0.4952; Line Search: 18.7391
Zero gradient: 7.515781198406112E-19
F(0.0) = LineSearchPoint{point=PointSample{avg=5.083827032008331E-31}, derivative=-5.648696702231497E-37}
New Minimum: 5.083827032008331E-31 > 1.3396831577612382E-32
F(1799999.9999999993) = LineSearchPoint{point=PointSample{avg=1.3396831577612382E-32}, derivative=-1.6596070137604378E-38}, evalInputDelta = -4.949858716232207E-31
F(1.2599999999999994E7) = LineSearchPoint{point=PointSample{avg=1.8137803259333096E-29}, derivative=3.3728276031751434E-36}, evalInputDelta = 1.7629420556132263E-29
F(969230.7692307688) = LineSearchPoint{point=PointSample{avg=1.1141243560130211E-31}, derivative=-2.4023671293712893E-37}, evalInputDelta = -3.96970267599531E-31
F(6784615.384615381) = LineSearchPoint{point=PointSample{avg=4.0728902897930774E-30}, derivative=1.5963113384060503E-36}, evalInputDelta = 3.564507586592244E-30
F(521893.4911242601) = LineSearchPoint{point=PointSample{avg=2.776985820496922E-31}, derivative=-4.042112471814112E-37}, evalInputDelta = -2.3068412115114092E-31
F(3653254.4378698207) = LineSearchPoint{point=PointSample{avg=5.303919428515392E-31}, derivative=5.7484076262941175E-37}, evalInputDelta = 2.200923965070606E-32
F(281019.57214383234) = LineSearchPoint{point=PointSample{avg=4.003737770182944E-31}, derivative=-4.949392727839911E-37}, evalInputDelta = -1.0800892618253874E-31
F(1967137.0050068265) = LineSearchPoint{point=PointSample{avg=2.1147263275892063E-32}, derivative=5.715140707534727E-38}, evalInputDelta = -4.872354399249411E-31
2.1147263275892063E-32 <= 5.083827032008331E-31
F(1786396.107553003) = LineSearchPoint{point=PointSample{avg=1.3756383563518048E-32}, derivative=-1.7878538466858612E-38}, evalInputDelta = -4.94626319637315E-31
Left bracket at 1786396.107553003
F(1829464.0194955086) = LineSearchPoint{point=PointSample{avg=1.525797703857224E-32}, derivative=2.7577348400801726E-38}, evalInputDelta = -4.931247261622609E-31
Right bracket at 1829464.0194955086
Converged to left
Fitness changed from 5.083827032008331E-31 to 1.3396831577612382E-32
Iteration 2 complete. Error: 1.3396831577612382E-32 Total: 8.9008; Orientation: 0.5178; Line Search: 8.0818
Zero gradient: 1.2200560814802457E-19
F(0.0) = LineSearchPoint{point=PointSample{avg=1.3396831577612382E-32}, derivative=-1.4885368419569478E-38}
New Minimum: 1.3396831577612382E-32 > 8.236722055330554E-33
F(1786396.107553003) = LineSearchPoint{point=PointSample{avg=8.236722055330554E-33}, derivative=-5.696015554528913E-40}, evalInputDelta = -5.160109522281829E-33
F(1.250477275287102E7) = LineSearchPoint{point=PointSample{avg=5.133273530912899E-31}, derivative=9.148321639396584E-38}, evalInputDelta = 4.999305215136775E-31
New Minimum: 8.236722055330554E-33 > 6.625200911634591E-33
F(961905.5963746938) = LineSearchPoint{point=PointSample{avg=6.625200911634591E-33}, derivative=-3.81712162259869E-39}, evalInputDelta = -6.771630665977791E-33
F(6733339.174622856) = LineSearchPoint{point=PointSample{avg=1.196574267861826E-31}, derivative=4.4388182991867766E-38}, evalInputDelta = 1.0626059520857021E-31
F(517949.16727868124) = LineSearchPoint{point=PointSample{avg=8.224053158172822E-33}, derivative=-9.259178223279951E-39}, evalInputDelta = -5.172778419439561E-33
F(3625644.1709507685) = LineSearchPoint{point=PointSample{avg=2.2353656681255162E-32}, derivative=1.782777243806652E-38}, evalInputDelta = 8.956825103642779E-33
F(278895.7054577514) = LineSearchPoint{point=PointSample{avg=1.1405508305295053E-32}, derivative=-1.2704099149405735E-38}, evalInputDelta = -1.991323272317329E-33
F(1952269.9382042598) = LineSearchPoint{point=PointSample{avg=1.184541828033222E-32}, derivative=1.1708274503097549E-38}, evalInputDelta = -1.5514132972801623E-33
1.184541828033222E-32 <= 1.3396831577612382E-32
New Minimum: 6.625200911634591E-33 > 6.548459879086104E-33
F(1092752.0298413537) = LineSearchPoint{point=PointSample{avg=6.548459879086104E-33}, derivative=-3.731853808655928E-39}, evalInputDelta = -6.848371698526278E-33
Left bracket at 1092752.0298413537
New Minimum: 6.548459879086104E-33 > 6.392077470488995E-33
F(1300496.1052251295) = LineSearchPoint{point=PointSample{avg=6.392077470488995E-33}, derivative=-3.558096052419617E-39}, evalInputDelta = -7.004754107123388E-33
Left bracket at 1300496.1052251295
F(1452403.4556827894) = LineSearchPoint{point=PointSample{avg=7.866167035252293E-33}, derivative=-1.1809245022200495E-39}, evalInputDelta = -5.53066454236009E-33
7.866167035252293E-33 > 6.392077470488995E-33
Fitness changed from 1.3396831577612382E-32 to 6.392077470488995E-33
Iteration 3 complete. Error: 6.392077470488995E-33 Total: 11.0107; Orientation: 0.5569; Line Search: 10.1298
Final threshold in iteration 3: 6.392077470488995E-33 (> 0.0) after 40.050s (< 30.000s)

Returns

    6.392077470488995E-33

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 193.28 seconds (3.639 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 6493536312316
Reset training subject: 6493827601402
Adding measurement 6f07c7a5 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 5.336992190537778 < 5.336992190537778. Total: 1
th(0)=5.336992190537778;dx=-5.929991322819743E-6
Adding measurement 4d1cbb43 to history. Total: 1
New Minimum: 5.336992190537778 > 5.3369794147664065
WOLFE (weak): th(2.154434690031884)=5.3369794147664065; dx=-5.92998422516473E-6 evalInputDelta=1.277577137148711E-5
Adding measurement 6955b8c0 to history. Total: 2
New Minimum: 5.3369794147664065 > 5.336966639010326
WOLFE (weak): th(4.308869380063768)=5.336966639010326; dx=-5.929977127509726E-6 evalInputDelta=2.5551527452094547E-5
Adding measurement 23e2c3ea to history. Total: 3
New Minimum: 5.336966639010326 > 5.336915536138919
WOLFE (weak): th(12.926608140191302)=5.336915536138919; dx=-5.9299487368897085E-6 evalInputDelta=7.665439885862213E-5
Adding measurement c3156af to history. Total: 4
New Minimum: 5.336915536138919 > 5.336685576245292
WOLFE (weak): th(51.70643256076521)=5.336685576245292; dx=-5.929820979099526E-6 evalInputDelta=3.066142924863513E-4
Adding measurement c241b75 to history. Total: 5
New Minimum: 5.336685576245292 > 5.335459207154008
WOLFE (weak): th(258.53216280382605)=5.335459207154008; dx=-5.929139604218582E-6 evalInputDelta=0.001532983383770059
Adding measurement 166368b8 to history. Total: 6
New Minimum: 5.335459207154008 > 5.327797593184935
WOLFE (weak): th(1551.1929768229563)=5.327797593184935; dx=-5.9248810112127625E-6 evalInputDelta=0.00919459735284267
Adding measurement 5dc890f to history. Total: 7
New Minimum: 5.327797593184935 > 5.272796477736841
WOLFE (weak): th(10858.350837760694)=5.272796477736841; dx=-5.894219141570834E-6 evalInputDelta=0.06419571280093717
Adding measurement 1bcbb108 to history. Total: 8
New Minimum: 5.272796477736841 > 4.834302441168797
WOLFE (weak): th(86866.80670208555)=4.834302441168797; dx=-5.643813872828419E-6 evalInputDelta=0.5026897493689813
Adding measurement 52ce34ba to history. Total: 9
New Minimum: 4.834302441168797 > 1.7077200105291426
END: th(781801.26031877)=1.7077200105291426; dx=-3.3543942728976156E-6 evalInputDelta=3.6292721800086354
Fitness changed from 5.336992190537778 to 1.7077200105291426
Iteration 1 complete. Error: 1.7077200105291426 Total: 12.0296; Orientation: 0.6102; Line Search: 10.5295
Non-optimal measurement 1.7077200105291426 < 1.7077200105291426. Total: 10
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00, 984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, 3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935, 5.335459207154008, 5.336685576245292, 5.336915536138919, 5.336966639010326, 5.3369794147664065, 5.336992190537778
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00, 2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00, 8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00, 984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935, 5.335459207154008, 5.336685576245292, 5.336915536138919, 5.336966639010326, 5.3369794147664065
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00, 3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, 984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00, 8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00, e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935, 5.335459207154008, 5.336685576245292, 5.336915536138919, 5.336966639010326
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00, 2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, 8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00, e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935, 5.335459207154008, 5.336685576245292, 5.336915536138919
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00, 2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00, 984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, 3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, 8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935, 5.335459207154008, 5.336685576245292
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00, e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00, 2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00, 984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935, 5.335459207154008
Rejected: LBFGS Orientation magnitude: 2.479e+03, gradient 1.377e-03, dot -1.000; [984a1b2c-1ce0-4d77-a6d7-794ab93aff10 = 1.000/1.000e+00, bb7a0d25-24b9-4c7d-bfbe-daeeac6dc521 = 1.000/1.000e+00, 3691eed8-5bcf-4efc-a9c3-76d5c13bdee7 = 1.000/1.000e+00, 2dfaf828-531f-4a21-b77d-c7e4a64197c1 = 1.000/1.000e+00, e90f473b-d772-47cf-b2d7-e0ba0c316e01 = 1.000/1.000e+00, 8ea549c0-f4ed-4971-b36a-2fcc95339540 = 1.000/1.000e+00, e06223d3-d893-432c-8f4e-34504b0686e3 = 1.000/1.000e+00, 8787f91f-0a18-45fe-bb5e-b2fb680e4bcf = 1.000/1.000e+00, 80c4ffcf-3f65-41c3-a35d-9a859cd99387 = 1.000/1.000e+00, 370b7a68-afdc-444c-bd3c-c0d0ac1b92fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.7077200105291426, 4.834302441168797, 5.272796477736841, 5.327797593184935
LBFGS Accumulation History: 3 points
Removed measurement 52ce34ba to history. Total: 9
Removed measurement 1bcbb108 to history. Total: 8
Removed measurement 5dc890f to history. Total: 7
Removed measurement 166368b8 to history. Total: 6
Removed measurement c241b75 to history. Total: 5
Removed measurement c3156af to history. Total: 4
Removed measurement 23e2c3ea to history. Total: 3
Adding measurement 2b2ec4aa to history. Total: 3
th(0)=1.7077200105291426;dx=-1.897466678365716E-6
Adding measurement 7eb7b173 to history. Total: 4
New Minimum: 1.7077200105291426 > 0.007050823867345869
END: th(1684339.7559414052)=0.007050823867345869; dx=-1.2192303284046038E-7 evalInputDelta=1.7006691866617967
Fitness changed from 1.7077200105291426 to 0.007050823867345869
Iteration 2 complete. Error: 0.007050823867345869 Total: 181.2445; Orientation: 179.0985; Line Search: 1.8547
Final threshold in iteration 2: 0.007050823867345869 (> 0.0) after 193.274s (< 30.000s)

Returns

    0.007050823867345869

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 2.9870819962066966 ], [ -0.16430280290405452 ], [ 2.0350411447982033 ], [ 1.569938115446798 ], [ -0.06924602994217455 ], [ -0.8777891230955769 ], [ -0.3222853730800338 ], [ -0.0936737755114729 ], ... ],
    	[ [ -0.14528206314905556 ], [ 0.0380318431067731 ], [ 0.8399663552328828 ], [ -2.3787768048628783 ], [ 0.409260443224927 ], [ -0.9510459629998156 ], [ 0.3329180037933034 ], [ 2.530228893507864 ], ... ],
    	[ [ 0.17161189095827073 ], [ 0.7845358865120259 ], [ -1.7739273054847335 ], [ 0.7937626849698357 ], [ 0.020569531279143183 ], [ -0.2903033896834742 ], [ -0.547139690659791 ], [ -0.6668332844350483 ], ... ],
    	[ [ -0.18988523919531575 ], [ -0.0929720536036247 ], [ -0.5935908578135617 ], [ -0.5047449617561053 ], [ 0.05428776151979764 ], [ 0.03986542093153128 ], [ -0.9169348055158196 ], [ -1.177416642217524 ], ... ],
    	[ [ -2.9239951817983876 ], [ 0.3418528093091229 ], [ -0.9981243558858239 ], [ 0.22919315369069257 ], [ -0.7914004272744279 ], [ -0.7210814094272768 ], [ 0.10764553572538782 ], [ -1.4135620312480572 ], ... ],
    	[ [ -1.714143587370189 ], [ 0.8468056314283825 ], [ -0.1625232748896176 ], [ 0.7383454559728919 ], [ 1.5948741358435419 ], [ 0.9847834247248347 ], [ 0.05790568556060505 ], [ -2.7939188839624327 ], ... ],
    	[ [ -1.2261459758099527 ], [ 0.1633583609850101 ], [ -0.9684722002984798 ], [ 1.7079062723400247 ], [ 1.3179945950189678 ], [ 0.981151087401275 ], [ -1.136264339935325 ], [ -0.7991949553510368 ], ... ],
    	[ [ 0.5323953156831055 ], [ -0.7522727614576256 ], [ 0.6065352997326063 ], [ 2.425680982152849 ], [ -0.4107431600957612 ], [ 1.1622180835457994 ], [ -2.00376448663018 ], [ -1.0043532493936875 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.04 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 3.314163992413393 ], [ 1.059394394191891 ], [ 1.5380822895964066 ], [ 0.36387623089359566 ], [ 1.0855079401156509 ], [ -2.7635782461911536 ], [ 0.8514292538399325 ], [ -0.8953475510229458 ], ... ],
    	[ [ 0.20143587370188892 ], [ -1.6599363137864538 ], [ 1.9999327104657656 ], [ -1.1655536097257566 ], [ 0.2905208864498541 ], [ 0.5499080740003688 ], [ -0.9941639924133933 ], [ 1.3604577870157275 ], ... ],
    	[ [ 1.0592237819165415 ], [ 2.493071773024052 ], [ -2.1878546109694668 ], [ 0.6475253699396715 ], [ 0.3371390625582862 ], [ -0.12060677936694836 ], [ 1.1977206186804181 ], [ -0.2776665688700967 ], ... ],
    	[ [ -1.8237704783906317 ], [ -2.6259441072072494 ], [ -1.8751817156271233 ], [ -1.8374899235122106 ], [ 0.6805755230395951 ], [ 1.9557308418630626 ], [ -0.6738696110316392 ], [ -1.046833284435048 ], ... ],
    	[ [ -3.383990363596775 ], [ -1.2282943813817542 ], [ -0.02024871177164811 ], [ -0.22961369261861483 ], [ 1.7731991454511442 ], [ 1.4618371811454463 ], [ -0.6087089285492242 ], [ -0.6871240624961144 ], ... ],
    	[ [ -0.9642871747403777 ], [ 2.181611262856765 ], [ -1.2530465497792354 ], [ -0.35530908805421624 ], [ 0.5937482716870837 ], [ -1.2344331505503305 ], [ 0.73581137112121 ], [ -3.0798377679248654 ], ... ],
    	[ [ -2.8922919516199053 ], [ 2.6187167219700203 ], [ -1.5129444005969594 ], [ 0.6718125446800494 ], [ -0.5160108099620644 ], [ 2.37430217480255 ], [ -0.7925286798706498 ], [ -1.0863899107020734 ], ... ],
    	[ [ 0.804790631366211 ], [ 1.2154544770847493 ], [ -0.8589294005347872 ], [ 1.447361964305698 ], [ 0.18651367980847777 ], [ 0.35243616709159886 ], [ -2.27552897326036 ], [ -1.3847064987873754 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.14 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging 
Plotting range=[1.0, -32.194357970114275], [8.0, 0.23241666737761246]; valueStats=DoubleSummaryStatistics{count=13, sum=3.429542, min=0.000000, average=0.263811, max=1.707720}
Plotting 8 points for GD
Plotting 3 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging 
Plotting range=[0.0, -32.194357970114275], [181.244, 0.23241666737761246]; valueStats=DoubleSummaryStatistics{count=13, sum=3.429542, min=0.000000, average=0.263811, max=1.707720}
Plotting 8 points for GD
Plotting 3 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.007050823867345869 }, "CjGD": { "type": "Converged", "value": 6.392077470488995E-33 }, "GD": { "type": "Converged", "value": 2.467730803898811E-16 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.007050823867345869 }, "CjGD": { "type": "Converged", "value": 6.392077470488995E-33 }, "GD": { "type": "Converged", "value": 2.467730803898811E-16 } }, "model":null, "complete":null}OK
 
  {
    "result": "OK",
    "performance": {
      "execution_time": "267.091",
      "gc_time": "8.709"
    },
    "created_on": 1586741052242,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Double_Add",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.SumInputsLayerTest.Double_Add",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/SumInputsLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 0.007050823867345869
        },
        "CjGD": {
          "type": "Converged",
          "value": 6.392077470488995E-33
        },
        "GD": {
          "type": "Converged",
          "value": 2.467730803898811E-16
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/SumInputsLayer/Double_Add/trainingTest/202004132412",
    "id": "0b2a60b8-b546-4c1f-b1d9-f318a0a926bf",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "SumInputsLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.SumInputsLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/SumInputsLayer.java",
      "javaDoc": ""
    }
  }