Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 6795976218900453376
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.04 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.86, 1.732, 1.564 ], [ -0.772, -0.88, -1.084 ], [ -0.1, 0.624, 1.428 ], [ -1.028, 0.46, -0.228 ], [ -0.012, -0.268, -1.056 ], [ -1.644, -1.976, -0.396 ], [ 1.496, -1.68, -1.128 ], [ -1.308, 1.088, 1.66 ], ... ],
[ [ -1.448, -0.596, 0.152 ], [ 0.684, 1.628, -1.832 ], [ -0.652, 1.448, -1.48 ], [ 1.652, 0.356, -0.056 ], [ -0.376, 0.164, -0.78 ], [ -1.848, 0.968, 1.52 ], [ -0.68, -1.54, -1.224 ], [ 1.748, -0.688, -1.328 ], ... ],
[ [ 1.112, -0.312, 0.332 ], [ 1.312, -1.44, 1.104 ], [ -1.332, 1.296, 1.788 ], [ -0.068, -1.04, 0.248 ], [ -0.32, 1.252, -1.112 ], [ -0.988, 0.136, 1.948 ], [ 0.58, 0.776, -0.408 ], [ 1.156, 1.352, -0.396 ], ... ],
[ [ -1.936, -1.492, -1.696 ], [ -0.028, 1.1, 1.176 ], [ 1.168, -1.304, -0.92 ], [ 1.96, 1.18, 1.912 ], [ 1.592, -1.2, 0.468 ], [ 0.888, 0.584, 1.208 ], [ -1.904, -0.368, 1.592 ], [ 1.06, -0.936, -0.76 ], ... ],
[ [ 1.56, 0.316, -0.124 ], [ 1.296, 1.936, 1.068 ], [ -0.596, -0.652, 1.264 ], [ -0.98, -1.38, -0.828 ], [ -1.6, -1.384, -1.384 ], [ -1.104, 0.736, -0.948 ], [ 0.824, 0.588, 1.168 ], [ 0.416, -0.2, 1.828 ], ... ],
[ [ -0.568, 0.736, 0.236 ], [ 0.6, -1.116, -1.804 ], [ 0.88, 0.916, 1.376 ], [ 1.032, -0.18, 0.308 ], [ 0.092, 0.648, 0.976 ], [ 0.08, -0.492, 1.492 ], [ 0.332, -0.424, 0.628 ], [ -1.824, 0.776, -0.228 ], ... ],
[ [ 0.32, 1.432, 0.752 ], [ -1.508, -0.82, 0.96 ], [ -1.708, -1.844, 1.492 ], [ 0.204, 1.364, 0.376 ], [ -1.956, -1.704, 1.7 ], [ 0.356, 1.416, -0.56 ], [ -1.024, -0.308, -0.228 ], [ 0.796, 1.6, 1.868 ], ... ],
[ [ -1.084, 0.168, 0.516 ], [ -1.336, 1.388, -1.212 ], [ -1.088, 1.232, -0.524 ], [ 1.192, -0.928, 1.3 ], [ -0.676, -0.896, 1.428 ], [ 1.452, -1.672, 0.348 ], [ 0.508, -0.972, 0.328 ], [ 0.412, -1.42, 0.38 ], ... ],
...
]
[
[ [ 1.204, 1.996, -0.932 ], [ 0.4, 0.572, -0.184 ], [ -0.14, 1.34, 1.064 ], [ -0.896, -1.708, 1.564 ], [ 1.756, 1.48, 0.916 ], [ -0.644, 1.664, -0.368 ], [ 1.784, 0.616, -0.788 ], [ -0.188, -0.264, 0.516 ], ... ],
[ [ -1.504, -1.184, 1.224 ], [ -1.544, -1.056, 1.416 ], [ -1.776, -1.552, -1.996 ], [ 0.484, 1.06, -0.92 ], [ -1.332, -0.92, -1.984 ], [ 0.292, -1.88, -0.708 ], [ 0.004, -1.916, -0.856 ], [ -1.544, 0.992, -1.948 ], ... ],
[ [ -0.52, -1.052, -1.228 ], [ -1.988, 0.056, -0.152 ], [ 1.58, 1.132, 1.964 ], [ -0.732, -1.344, -1.352 ], [ -0.112, 0.388, -1.576 ], [ 0.088, -0.192, -1.696 ], [ 1.736, 0.024, 1.724 ], [ 1.34, -0.2, 0.332 ], ... ],
[ [ 1.492, -0.172, -0.28 ], [ -0.416, -1.756, -1.432 ], [ 0.892, 0.648, 0.404 ], [ 1.376, 1.168, -0.5 ], [ 0.212, -1.38, 1.616 ], [ 0.376, -0.216, -1.148 ], [ -0.864, 1.004, -1.296 ], [ 1.384, -1.88, 0.076 ], ... ],
[ [ -0.856, 1.88, 0.088 ], [ -0.776, -0.964, 1.12 ], [ -0.352, -0.88, 1.308 ], [ -0.86, 0.216, -1.716 ], [ -1.632, 0.32, 1.34 ], [ -1.672, 1.312, -0.336 ], [ 0.496, 1.416, -0.008 ], [ -0.596, 1.46, 1.1 ], ... ],
[ [ -1.716, -0.456, 0.128 ], [ -0.396, -0.736, 0.432 ], [ 1.724, -1.676, 1.076 ], [ 0.088, 0.552, 1.944 ], [ -1.248, -0.272, -0.764 ], [ -1.592, 1.884, -0.868 ], [ 0.644, 0.012, 1.256 ], [ -1.636, -0.568, 0.552 ], ... ],
[ [ 1.208, 0.572, 0.272 ], [ -1.412, -1.936, 1.912 ], [ 1.776, 0.168, -0.656 ], [ 0.896, -1.196, 1.132 ], [ -0.236, 0.116, -1.6 ], [ 1.652, -0.228, 1.732 ], [ -1.208, -1.704, -0.024 ], [ -0.192, 1.868, -0.268 ], ... ],
[ [ -0.372, 0.248, 0.54 ], [ -0.648, 0.12, 1.376 ], [ 0.388, -0.84, 0.224 ], [ -0.084, -0.308, 0.084 ], [ -0.72, -0.892, -0.492 ], [ 0.952, 0.612, 1.832 ], [ 0.572, 1.128, 0.024 ], [ 0.552, -1.356, 1.688 ], ... ],
...
]
[
[ [ -0.864, -1.508, -1.764 ], [ -1.16, -1.32, -0.408 ], [ -1.952, 1.476, -1.136 ], [ -1.176, -0.696, 0.82 ], [ 1.148, 0.18, 1.128 ], [ -1.484, -1.748, -0.916 ], [ 1.84, -0.072, -1.36 ], [ -0.14, 0.756, -0.26 ], ... ],
[ [ -1.544, -0.592, 1.412 ], [ -1.68, 1.008, 1.656 ], [ 1.144, 0.092, 1.96 ], [ -0.216, 1.076, -1.312 ], [ -1.552, 0.012, 1.072 ], [ 1.436, 0.624, 0.012 ], [ -0.236, -1.696, -1.216 ], [ 1.764, -0.812, 0.784 ], ... ],
[ [ 0.796, -1.812, -0.844 ], [ 1.008, 1.516, 1.82 ], [ 0.464, 1.004, 1.428 ], [ -0.768, 0.964, 1.184 ], [ 0.256, 2.0, -0.284 ], [ -1.692, -0.98, -1.608 ], [ -1.744, 0.248, -1.372 ], [ 0.284, 0.028, 0.588 ], ... ],
[ [ -0.104, 0.724, 1.088 ], [ -0.44, -0.684, -1.636 ], [ -1.312, -1.588, -1.456 ], [ -0.284, -1.34, -1.292 ], [ 1.972, -1.276, 0.684 ], [ 0.888, -0.588, 0.084 ], [ 0.376, -1.984, 0.696 ], [ -0.02, -1.46, -1.456 ], ... ],
[ [ -1.528, 1.472, -1.444 ], [ 0.504, -1.46, 0.388 ], [ -0.208, -1.796, 0.192 ], [ 1.044, 1.936, -1.548 ], [ -0.228, -0.276, -0.86 ], [ -1.468, 0.488, 0.596 ], [ 1.912, -1.656, 1.444 ], [ 0.644, 0.476, -0.208 ], ... ],
[ [ 0.872, -1.764, 0.548 ], [ 1.916, 0.868, -1.356 ], [ -0.548, 0.212, -0.352 ], [ -1.42, 1.176, 1.36 ], [ -0.268, 0.192, -1.956 ], [ 0.724, 0.992, 0.356 ], [ 0.432, 0.456, -1.76 ], [ -1.388, 1.864, 1.496 ], ... ],
[ [ -1.136, 1.696, -1.74 ], [ 0.284, -1.636, -1.856 ], [ 0.028, -0.568, -0.748 ], [ -1.868, -1.2, -1.384 ], [ 1.088, 1.488, 1.652 ], [ -0.876, -0.996, -0.804 ], [ -0.976, 1.78, 0.892 ], [ 0.412, -1.036, -1.82 ], ... ],
[ [ -0.448, -1.892, -1.136 ], [ -0.312, 1.576, -1.916 ], [ 1.12, -1.156, 1.84 ], [ 0.7, 1.58, -0.22 ], [ 1.276, -1.26, 0.296 ], [ 0.388, 0.48, 0.8 ], [ -0.524, 1.964, 0.948 ], [ -0.308, 0.916, 1.96 ], ... ],
...
]
[
[ [ -0.084, 0.324, 1.384 ], [ -1.584, 0.668, 0.756 ], [ -0.564, 1.94, -1.812 ], [ 1.196, -1.352, 1.384 ], [ -1.504, -1.428, -0.476 ], [ -1.396, -1.864, 0.636 ], [ -1.488, -1.212, 0.74 ], [ -1.388, 0.376, -1.66 ], ... ],
[ [ -1.824, 1.124, 1.276 ], [ 1.608, -0.688, 1.188 ], [ 0.532, -0.936, 1.18 ], [ 0.856, -0.852, 0.304 ], [ -1.24, 1.488, 1.888 ], [ -1.916, -1.128, 0.968 ], [ 1.432, -0.776, -0.476 ], [ -1.1, -1.924, 1.76 ], ... ],
[ [ 0.6, 0.244, 1.024 ], [ 1.848, -0.352, -0.832 ], [ -0.136, -1.852, -0.14 ], [ 1.128, 0.268, -1.38 ], [ -0.244, 0.928, 1.752 ], [ -0.82, 1.516, 1.916 ], [ -0.356, -0.748, 0.176 ], [ -0.128, 1.996, -1.092 ], ... ],
[ [ -0.328, -0.492, 0.668 ], [ 1.624, -0.52, -0.44 ], [ 1.156, 0.7, -1.656 ], [ -1.868, 0.196, 1.6 ], [ -1.464, -0.044, 0.728 ], [ 1.42, 1.056, 0.592 ], [ 0.092, -1.524, -0.684 ], [ -1.476, -1.12, 1.396 ], ... ],
[ [ -1.828, 0.568, -1.332 ], [ 0.868, 0.232, -1.892 ], [ -0.248, -1.664, 1.112 ], [ -0.512, 1.416, 1.476 ], [ 0.012, 1.788, 1.448 ], [ -0.048, -0.964, 1.336 ], [ 1.808, 0.972, -0.016 ], [ 1.48, 1.428, 1.612 ], ... ],
[ [ 0.376, -0.852, 1.712 ], [ 0.532, 0.276, 0.608 ], [ 0.912, 1.324, -1.52 ], [ 0.352, 0.404, -1.664 ], [ -1.004, 0.476, -0.648 ], [ 0.364, -1.764, 1.088 ], [ 0.74, -0.708, -0.6 ], [ -1.46, 1.156, -0.232 ], ... ],
[ [ 0.68, -1.864, 0.22 ], [ -1.584, 1.62, -1.996 ], [ -1.392, -0.476, 0.744 ], [ -1.596, -0.92, 1.732 ], [ 0.228, -0.172, 1.212 ], [ 0.064, -0.82, -0.336 ], [ 1.16, -1.06, -0.9 ], [ -1.972, -0.196, -1.416 ], ... ],
[ [ 0.508, -0.588, 0.98 ], [ -0.276, -0.848, -0.44 ], [ -1.652, 1.168, -0.444 ], [ 1.42, 0.308, -1.436 ], [ -1.752, 0.672, 1.124 ], [ 0.448, -0.868, 1.844 ], [ 0.42, -0.56, -0.04 ], [ -1.552, -0.16, -0.176 ], ... ],
...
]
[
[ [ -1.34, -0.144, 1.344 ], [ -1.32, -0.868, -0.78 ], [ 0.688, 0.548, -0.008 ], [ -1.228, 1.62, 0.756 ], [ -1.632, 0.548, -0.42 ], [ 1.2, -0.944, 0.968 ], [ 0.132, 0.808, 0.496 ], [ -1.672, -1.424, 0.608 ], ... ],
[ [ 1.772, 0.34, -0.8 ], [ 1.248, -0.192, -1.916 ], [ -1.704, 1.196, 0.816 ], [ 0.152, 0.388, 0.828 ], [ -1.396, 1.072, -1.396 ], [ 0.1, -1.564, 1.588 ], [ -0.36, 1.856, 0.116 ], [ -0.668, -1.176, -1.392 ], ... ],
[ [ 0.932, -0.152, 1.476 ], [ 0.516, -0.272, 1.14 ], [ 1.344, -0.192, 0.744 ], [ 0.38, -1.116, 1.172 ], [ -1.656, 1.272, 0.464 ], [ 1.12, 0.604, -1.048 ], [ -0.588, -0.2, -0.076 ], [ -1.492, -0.628, -1.872 ], ... ],
[ [ 1.044, 0.196, 1.484 ], [ 1.9, -1.868, -0.932 ], [ 0.644, -1.032, 0.272 ], [ 1.272, -1.428, 0.04 ], [ -1.332, -0.992, -1.068 ], [ -0.364, -1.116, 0.972 ], [ 0.156, -0.66, -1.852 ], [ 0.2, 0.672, 1.02 ], ... ],
[ [ -1.328, 0.308, 1.288 ], [ 0.756, 0.768, 1.108 ], [ 1.288, -0.664, -1.444 ], [ 1.664, 0.044, 0.364 ], [ -0.32, -0.976, 1.788 ], [ -1.296, 0.656, -0.332 ], [ 1.592, -0.272, 1.036 ], [ 1.46, -0.148, -0.776 ], ... ],
[ [ 0.3, -1.712, 1.232 ], [ -0.292, 1.128, -1.44 ], [ -1.636, -0.316, -1.952 ], [ -0.792, -0.204, -1.192 ], [ 1.424, -0.74, 0.224 ], [ 0.224, -1.912, 0.932 ], [ 0.58, -1.676, 0.644 ], [ 0.196, 0.976, 1.26 ], ... ],
[ [ -1.716, 1.396, -0.772 ], [ -1.108, 1.86, 0.136 ], [ 1.648, -1.972, 1.272 ], [ 0.872, 0.676, -0.944 ], [ 0.82, 1.228, 1.748 ], [ 0.548, -0.88, -1.476 ], [ -0.192, 0.16, 0.62 ], [ -1.7, -1.624, 1.004 ], ... ],
[ [ -1.212, 1.088, 1.852 ], [ -1.68, 1.616, -0.22 ], [ 0.084, 1.232, 0.572 ], [ -0.804, -0.576, -1.84 ], [ -1.328, 1.552, 0.248 ], [ 0.344, -1.764, 0.628 ], [ 0.932, -1.876, -1.62 ], [ 1.64, -0.224, 1.824 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.79 seconds (3.623 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 9840061190050
Reset training subject: 9841161483101
Low gradient: 4.53962236661526E-7
Constructing line search parameters: GD
th(0)=1.1128412464995708E-6;dx=-2.0608171231474478E-13
New Minimum: 1.1128412464995708E-6 > 1.1128408860764107E-6
WOLFE (weak): th(2.154434690031884)=1.1128408860764107E-6; dx=-2.0608167894220893E-13 evalInputDelta=3.604231601198629E-13
New Minimum: 1.1128408860764107E-6 > 1.1128402453447921E-6
WOLFE (weak): th(4.308869380063768)=1.1128402453447921E-6; dx=-2.0608161961521024E-13 evalInputDelta=1.0011547786793203E-12
New Minimum: 1.1128402453447921E-6 > 1.1128385402705345E-6
WOLFE (weak): th(12.926608140191302)=1.1128385402705345E-6; dx=-2.060814617378353E-13 evalInputDelta=2.706229036301749E-12
New Minimum: 1.1128385402705345E-6 > 1.1128305690038318E-6
WOLFE (weak): th(51.70643256076521)=1.1128305690038318E-6; dx=-2.0608072365535118E-13 evalInputDelta=1.0677495739034505E-11
New Minimum: 1.1128305690038318E-6 > 1.112788080138845E-6
WOLFE (weak): th(258.53216280382605)=1.112788080138845E-6; dx=-2.060767894447462E-13 evalInputDelta=5.3166360725868825E-11
New Minimum: 1.112788080138845E-6 > 1.1125215564080907E-6
WOLFE (weak): th(1551.1929768229563)=1.1125215564080907E-6; dx=-2.060521092541521E-13 evalInputDelta=3.196900914800672E-10
New Minimum: 1.1125215564080907E-6 > 1.1106047035109153E-6
WOLFE (weak): th(10858.350837760694)=1.1106047035109153E-6; dx=-2.058745208473663E-13 evalInputDelta=2.2365429886555342E-9
New Minimum: 1.1106047035109153E-6 > 1.095011289107596E-6
WOLFE (weak): th(86866.80670208555)=1.095011289107596E-6; dx=-2.0442412404899146E-13 evalInputDelta=1.782995739197487E-8
New Minimum: 1.095011289107596E-6 > 9.575576918517478E-7
WOLFE (weak): th(781801.26031877)=9.575576918517478E-7; dx=-1.9116365336041397E-13 evalInputDelta=1.5528355464782295E-7
New Minimum: 9.575576918517478E-7 > 8.483928777864716E-8
END: th(7818012.6031877)=8.483928777864716E-8; dx=-5.690119066803835E-14 evalInputDelta=1.0280019587209236E-6
Fitness changed from 1.1128412464995708E-6 to 8.483928777864716E-8
Iteration 1 complete. Error: 8.483928777864716E-8 Total: 16.8019; Orientation: 0.7335; Line Search: 14.0549
Low gradient: 1.2534344505504863E-7
th(0)=8.483928777864716E-8;dx=-1.5710979218268956E-14
New Minimum: 8.483928777864716E-8 > 2.6565098864036573E-8
WOLF (strong): th(1.684339755941405E7)=2.6565098864036573E-8; dx=8.791449088230101E-15 evalInputDelta=5.827418891461059E-8
New Minimum: 2.6565098864036573E-8 > 4.114195267613407E-9
END: th(8421698.779707026)=4.114195267613407E-9; dx=-3.459769751198657E-15 evalInputDelta=8.072509251103376E-8
Fitness changed from 8.483928777864716E-8 to 4.114195267613407E-9
Iteration 2 complete. Error: 4.114195267613407E-9 Total: 4.4576; Orientation: 0.7292; Line Search: 3.2768
Low gradient: 2.7602318969988763E-8
th(0)=4.114195267613407E-9;dx=-7.61888012520936E-16
New Minimum: 4.114195267613407E-9 > 1.9024162843502153E-9
WOLF (strong): th(1.8144E7)=1.9024162843502153E-9; dx=5.180855359496025E-16 evalInputDelta=2.211778983263192E-9
New Minimum: 1.9024162843502153E-9 > 1.0532257082081189E-10
END: th(9072000.0)=1.0532257082081189E-10; dx=-1.2190160279488427E-16 evalInputDelta=4.008872696792595E-9
Fitness changed from 4.114195267613407E-9 to 1.0532257082081189E-10
Iteration 3 complete. Error: 1.0532257082081189E-10 Total: 5.3385; Orientation: 0.7335; Line Search: 4.1610
Low gradient: 4.416353674880651E-9
th(0)=1.0532257082081189E-10;dx=-1.950417978163166E-17
New Minimum: 1.0532257082081189E-10 > 6.905589780368635E-11
WOLF (strong): th(1.954503150796925E7)=6.905589780368635E-11; dx=1.57930988473047E-17 evalInputDelta=3.626667301712554E-11
New Minimum: 6.905589780368635E-11 > 9.532719469351972E-13
END: th(9772515.753984625)=9.532719469351972E-13; dx=-1.8555615368178355E-18 evalInputDelta=1.043692988738767E-10
Fitness changed from 1.0532257082081189E-10 to 9.532719469351972E-13
Iteration 4 complete. Error: 9.532719469351972E-13 Total: 4.1783; Orientation: 0.7358; Line Search: 3.0002
Final threshold in iteration 4: 9.532719469351972E-13 (> 0.0) after 30.777s (< 30.000s)
Returns
9.532719469351972E-13
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 40.27 seconds (2.013 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 9870956796087
Reset training subject: 9871386892563
Low gradient: 4.53962236661526E-7
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=1.1128412464995708E-6}, derivative=-2.0608171231474478E-13}, evalInputDelta = 0.0
New Minimum: 1.1128412464995708E-6 > 1.1128410824683526E-6
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=1.1128410824683526E-6}, derivative=-2.0608169712666903E-13}, evalInputDelta = -1.6403121820593827E-13
New Minimum: 1.1128410824683526E-6 > 1.112839263676541E-6
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=1.112839263676541E-6}, derivative=-2.060815287199453E-13}, evalInputDelta = -1.9828230297175075E-12
New Minimum: 1.112839263676541E-6 > 1.112827196397415E-6
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=1.112827196397415E-6}, derivative=-2.06080411375247E-13}, evalInputDelta = -1.405010215570588E-11
New Minimum: 1.112827196397415E-6 > 1.1127432846035297E-6
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=1.1127432846035297E-6}, derivative=-2.06072641569171E-13}, evalInputDelta = -9.79618960411315E-11
New Minimum: 1.1127432846035297E-6 > 1.1121563150101574E-6
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=1.1121563150101574E-6}, derivative=-2.060182829710135E-13}, evalInputDelta = -6.84931489413437E-10
New Minimum: 1.1121563150101574E-6 > 1.1080522905105982E-6
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=1.1080522905105982E-6}, derivative=-2.0563781238375423E-13}, evalInputDelta = -4.788955988972587E-9
New Minimum: 1.1080522905105982E-6 > 1.0795356528504018E-6
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=1.0795356528504018E-6}, derivative=-2.029744354657353E-13}, evalInputDelta = -3.330559364916904E-8
New Minimum: 1.0795356528504018E-6 > 8.903278100961521E-7
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=8.903278100961521E-7}, derivative=-1.8433076129713734E-13}, evalInputDelta = -2.2251343640341873E-7
New Minimum: 8.903278100961521E-7 > 7.5914035194841E-8
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=7.5914035194841E-8}, derivative=-5.3824981054282084E-14}, evalInputDelta = -1.0369272113047297E-6
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=1.9367061383977856E-5}, derivative=8.597154107627019E-13}, evalInputDelta = 1.8254220137478286E-5
F(4296506.646794492) = LineSearchPoint{point=PointSample{avg=4.035329631247946E-7}, derivative=-1.2409730651580933E-13}, evalInputDelta = -7.093082833747762E-7
F(3.007554652756145E7) = LineSearchPoint{point=PointSample{avg=3.54486025144925E-6}, derivative=3.6780904630464784E-13}, evalInputDelta = 2.432019004949679E-6
F(2313503.579043188) = LineSearchPoint{point=PointSample{avg=6.871356618589201E-7}, derivative=-1.6193625505598092E-13}, evalInputDelta = -4.2570558464065066E-7
F(1.6194525053302318E7) = LineSearchPoint{point=PointSample{avg=2.776464960812844E-7}, derivative=1.0293639291888213E-13}, evalInputDelta = -8.351947504182863E-7
2.776464960812844E-7 <= 1.1128412464995708E-6
New Minimum: 7.5914035194841E-8 > 6.567328784378342E-20
F(1.079999973006567E7) = LineSearchPoint{point=PointSample{avg=6.567328784378342E-20}, derivative=1.2093886098251982E-20}, evalInputDelta = -1.1128412464995051E-6
Right bracket at 1.079999973006567E7
Converged to right
Fitness changed from 1.1128412464995708E-6 to 6.567328784378342E-20
Iteration 1 complete. Error: 6.567328784378342E-20 Total: 29.3337; Orientation: 0.7315; Line Search: 27.3059
Zero gradient: 1.1028018847948622E-13
F(0.0) = LineSearchPoint{point=PointSample{avg=6.567328784378342E-20}, derivative=-1.2161719971068174E-26}
New Minimum: 6.567328784378342E-20 > 2.6498014033272027E-20
F(1.079999973006567E7) = LineSearchPoint{point=PointSample{avg=2.6498014033272027E-20}, derivative=1.563350933512373E-27}, evalInputDelta = -3.917527381051139E-20
2.6498014033272027E-20 <= 6.567328784378342E-20
New Minimum: 2.6498014033272027E-20 > 1.2719611424602076E-20
F(9569828.332240928) = LineSearchPoint{point=PointSample{avg=1.2719611424602076E-20}, derivative=-1.934581422740853E-28}, evalInputDelta = -5.295367641918134E-20
Left bracket at 9569828.332240928
F(9705293.634977568) = LineSearchPoint{point=PointSample{avg=1.2719611424602076E-20}, derivative=-1.934581422740854E-28}, evalInputDelta = -5.295367641918134E-20
Left bracket at 9705293.634977568
F(9825841.627591424) = LineSearchPoint{point=PointSample{avg=1.2719611424602076E-20}, derivative=-1.934581422740854E-28}, evalInputDelta = -5.295367641918134E-20
Left bracket at 9825841.627591424
F(9933114.990066255) = LineSearchPoint{point=PointSample{avg=1.2719611424602076E-20}, derivative=-1.934581422740854E-28}, evalInputDelta = -5.295367641918134E-20
Left bracket at 9933114.990066255
Converged to left
Fitness changed from 6.567328784378342E-20 to 1.2719611424602076E-20
Iteration 2 complete. Error: 1.2719611424602076E-20 Total: 10.9392; Orientation: 0.7234; Line Search: 9.7833
Final threshold in iteration 2: 1.2719611424602076E-20 (> 0.0) after 40.274s (< 30.000s)
Returns
1.2719611424602076E-20
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 340.65 seconds (6.826 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 9911348606697
Reset training subject: 9911774580241
Adding measurement 6eb60866 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.1128412464995708E-6 < 1.1128412464995708E-6. Total: 1
th(0)=1.1128412464995708E-6;dx=-2.0608171231474478E-13
Adding measurement 2402e234 to history. Total: 1
New Minimum: 1.1128412464995708E-6 > 1.1128408860764107E-6
WOLFE (weak): th(2.154434690031884)=1.1128408860764107E-6; dx=-2.0608167894220893E-13 evalInputDelta=3.604231601198629E-13
Adding measurement 771ab093 to history. Total: 2
New Minimum: 1.1128408860764107E-6 > 1.1128402453447921E-6
WOLFE (weak): th(4.308869380063768)=1.1128402453447921E-6; dx=-2.0608161961521024E-13 evalInputDelta=1.0011547786793203E-12
Adding measurement 77d262d to history. Total: 3
New Minimum: 1.1128402453447921E-6 > 1.1128385402705345E-6
WOLFE (weak): th(12.926608140191302)=1.1128385402705345E-6; dx=-2.060814617378353E-13 evalInputDelta=2.706229036301749E-12
Adding measurement 64143006 to history. Total: 4
New Minimum: 1.1128385402705345E-6 > 1.1128305690038318E-6
WOLFE (weak): th(51.70643256076521)=1.1128305690038318E-6; dx=-2.0608072365535118E-13 evalInputDelta=1.0677495739034505E-11
Adding measurement 6957a39c to history. Total: 5
New Minimum: 1.1128305690038318E-6 > 1.112788080138845E-6
WOLFE (weak): th(258.53216280382605)=1.112788080138845E-6; dx=-2.060767894447462E-13 evalInputDelta=5.3166360725868825E-11
Adding measurement 23235efb to history. Total: 6
New Minimum: 1.112788080138845E-6 > 1.1125215564080907E-6
WOLFE (weak): th(1551.1929768229563)=1.1125215564080907E-6; dx=-2.060521092541521E-13 evalInputDelta=3.196900914800672E-10
Adding measurement 6c54cddf to history. Total: 7
New Minimum: 1.1125215564080907E-6 > 1.1106047035109153E-6
WOLFE (weak): th(10858.350837760694)=1.1106047035109153E-6; dx=-2.058745208473663E-13 evalInputDelta=2.2365429886555342E-9
Adding measurement 112ee458 to history. Total: 8
New Minimum: 1.1106047035109153E-6 > 1.095011289107596E-6
WOLFE (weak): th(86866.80670208555)=1.095011289107596E-6; dx=-2.0442412404899146E-13 evalInputDelta=1.782995739197487E-8
Adding measurement 2b363094 to history. Total: 9
New Minimum: 1.095011289107596E-6 > 9.575576918517478E-7
WOLFE (weak): th(781801.26031877)=9.575576918517478E-7; dx=-1.9116365336041397E-13 evalInputDelta=1.5528355464782295E-7
Adding measurement 33f9fc17 to history. Total: 10
New Minimum: 9.575576918517478E-7 > 8.483928777864716E-8
END: th(7818012.6031877)=8.483928777864716E-8; dx=-5.690119066803835E-14 evalInputDelta=1.0280019587209236E-6
Fitness changed from 1.1128412464995708E-6 to 8.483928777864716E-8
Iteration 1 complete. Error: 8.483928777864716E-8 Total: 19.2467; Orientation: 0.8604; Line Search: 17.0319
Non-optimal measurement 8.483928777864716E-8 < 8.483928777864716E-8. Total: 11
Rejected: LBFGS Orientation magnitude: 2.159e+00, gradient 1.253e-07, dot -0.946; [f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00, 243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, 8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6, 1.112788080138845E-6, 1.1128305690038318E-6, 1.1128385402705345E-6, 1.1128402453447921E-6, 1.1128408860764107E-6, 1.1128412464995708E-6
Rejected: LBFGS Orientation magnitude: 1.226e+00, gradient 1.253e-07, dot -0.946; [243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, 8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00, f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6, 1.112788080138845E-6, 1.1128305690038318E-6, 1.1128385402705345E-6, 1.1128402453447921E-6, 1.1128408860764107E-6
Rejected: LBFGS Orientation magnitude: 1.489e+00, gradient 1.253e-07, dot -0.983; [f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00, 243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, 8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6, 1.112788080138845E-6, 1.1128305690038318E-6, 1.1128385402705345E-6, 1.1128402453447921E-6
Rejected: LBFGS Orientation magnitude: 1.365e+00, gradient 1.253e-07, dot -0.998; [8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00, f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, 243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6, 1.112788080138845E-6, 1.1128305690038318E-6, 1.1128385402705345E-6
Rejected: LBFGS Orientation magnitude: 1.358e+00, gradient 1.253e-07, dot -1.000; [f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00, 243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, 8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6, 1.112788080138845E-6, 1.1128305690038318E-6
Rejected: LBFGS Orientation magnitude: 1.353e+00, gradient 1.253e-07, dot -1.000; [8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00, f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, 243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6, 1.112788080138845E-6
Rejected: LBFGS Orientation magnitude: 1.354e+00, gradient 1.253e-07, dot -1.000; [243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00, 8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6, 1.1125215564080907E-6
Rejected: LBFGS Orientation magnitude: 1.354e+00, gradient 1.253e-07, dot -1.000; [f39ea1bb-4c75-4606-bc7c-af158dbf6a11 = 1.000/1.000e+00, 243b3026-2ac0-4135-86c0-7d2fde9e549f = 1.000/1.000e+00, 8acf4631-b4e8-4413-a879-c8382e17e41f = 1.000/1.000e+00, e74cbd4e-a772-4f6d-a07c-e43e181a6bb2 = 1.000/1.000e+00, 1573e630-7ae4-4d70-9ddc-951a2c217d78 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.483928777864716E-8, 9.575576918517478E-7, 1.095011289107596E-6, 1.1106047035109153E-6
LBFGS Accumulation History: 3 points
Removed measurement 33f9fc17 to history. Total: 10
Removed measurement 2b363094 to history. Total: 9
Removed measurement 112ee458 to history. Total: 8
Removed measurement 6c54cddf to history. Total: 7
Removed measurement 23235efb to history. Total: 6
Removed measurement 6957a39c to history. Total: 5
Removed measurement 64143006 to history. Total: 4
Removed measurement 77d262d to history. Total: 3
Adding measurement 7ea58cf7 to history. Total: 3
th(0)=8.483928777864716E-8;dx=-1.5710979218268956E-14
Adding measurement 213b8558 to history. Total: 4
New Minimum: 8.483928777864716E-8 > 2.6565098864036573E-8
WOLF (strong): th(1.684339755941405E7)=2.6565098864036573E-8; dx=8.791449088230101E-15 evalInputDelta=5.827418891461059E-8
Adding measurement 742b42aa to history. Total: 5
New Minimum: 2.6565098864036573E-8 > 4.114195267613407E-9
END: th(8421698.779707026)=4.114195267613407E-9; dx=-3.4597697511986574E-15 evalInputDelta=8.072509251103376E-8
Fitness changed from 8.483928777864716E-8 to 4.114195267613407E-9
Iteration 2 complete. Error: 4.114195267613407E-9 Total: 321.4008; Orientation: 316.6338; Line Search: 4.3276
Final threshold in iteration 2: 4.114195267613407E-9 (> 0.0) after 340.648s (< 30.000s)
Returns
4.114195267613407E-9
TrainingTester.java:432 executed in 0.14 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -19.895526155883264], [4.0, -7.071402986019476]; valueStats=DoubleSummaryStatistics{count=8, sum=0.000000, min=0.000000, average=0.000000, max=0.000000}
Plotting 4 points for GD
Plotting 2 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -19.895526155883264], [321.401, -7.071402986019476]; valueStats=DoubleSummaryStatistics{count=8, sum=0.000000, min=0.000000, average=0.000000, max=0.000000}
Plotting 4 points for GD
Plotting 2 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 4.114195267613407E-9 }, "CjGD": { "type": "Converged", "value": 1.2719611424602076E-20 }, "GD": { "type": "Converged", "value": 9.532719469351972E-13 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 4.114195267613407E-9 }, "CjGD": { "type": "Converged", "value": 1.2719611424602076E-20 }, "GD": { "type": "Converged", "value": 9.532719469351972E-13 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "416.098",
"gc_time": "12.783"
},
"created_on": 1586744468496,
"file_name": "trainingTest",
"report": {
"simpleName": "Float",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandAvgReducerLayerTest.Float",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 4.114195267613407E-9
},
"CjGD": {
"type": "Converged",
"value": 1.2719611424602076E-20
},
"GD": {
"type": "Converged",
"value": 9.532719469351972E-13
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayer/Float/trainingTest/202004132108",
"id": "740eb0b5-6683-453d-a2dc-a36d25d32f05",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "BandAvgReducerLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandAvgReducerLayer",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayer.java",
"javaDoc": ""
}
}