Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 2823761349887525888
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.04 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 1.28, 0.432, 1.752 ], [ -1.46, -1.248, -0.744 ], [ 0.156, 1.02, 0.248 ], [ -0.792, -0.192, 1.056 ], [ -1.612, -0.36, 1.824 ], [ 1.7, 0.252, -1.444 ], [ 1.812, -1.48, 0.716 ], [ 1.624, 1.236, -0.288 ], ... ],
[ [ -0.136, 1.0, -1.636 ], [ -1.3, 0.796, -1.612 ], [ 0.44, 0.296, 1.308 ], [ -1.64, -0.152, 0.6 ], [ -0.984, -0.456, 0.2 ], [ -0.536, -0.936, 1.48 ], [ 0.836, 1.836, -1.98 ], [ -0.276, 0.408, -0.328 ], ... ],
[ [ 1.592, -1.252, -1.464 ], [ 0.008, 0.456, -1.284 ], [ 1.448, -1.976, -0.176 ], [ 1.528, 1.976, 0.044 ], [ -1.08, -0.352, 1.652 ], [ -0.98, 0.076, 0.816 ], [ 0.412, 1.812, -1.664 ], [ -0.724, 1.628, -1.256 ], ... ],
[ [ 0.2, 0.552, 0.008 ], [ 1.796, 0.052, -1.62 ], [ 1.324, -0.168, -1.536 ], [ -1.648, -1.768, -1.156 ], [ -0.968, -1.624, -1.2 ], [ 1.256, 0.58, -0.32 ], [ -0.56, 0.66, -1.784 ], [ 1.736, 1.064, -1.18 ], ... ],
[ [ 0.364, -1.64, 0.82 ], [ -0.376, -0.088, -0.924 ], [ 1.748, 0.18, -1.788 ], [ 0.72, 0.872, -1.596 ], [ -0.216, -0.632, 0.708 ], [ -0.852, -1.956, -1.204 ], [ 1.292, -0.876, -1.76 ], [ -0.936, -0.416, -1.9 ], ... ],
[ [ -0.84, -0.456, 1.216 ], [ -0.764, 1.376, -1.24 ], [ -0.96, -1.308, -1.536 ], [ -1.364, 1.604, 1.424 ], [ -1.404, 1.508, 0.544 ], [ 0.888, -0.76, 0.472 ], [ -0.26, -0.496, -1.704 ], [ -0.592, 1.4, 1.068 ], ... ],
[ [ -1.124, -1.724, -1.272 ], [ -0.812, -1.912, -1.408 ], [ -0.384, 1.368, -1.484 ], [ 0.164, -0.076, -1.328 ], [ 0.552, 1.748, -1.072 ], [ 0.84, -1.088, -1.708 ], [ -0.836, -1.56, 0.8 ], [ 1.092, 0.884, 0.804 ], ... ],
[ [ -1.724, 1.652, 0.576 ], [ -0.14, 0.64, -1.644 ], [ 0.172, -1.036, -1.232 ], [ 1.82, 1.964, 0.744 ], [ -1.636, -1.076, -1.38 ], [ 1.32, 1.272, 0.696 ], [ -0.864, -0.452, 0.524 ], [ 1.124, 0.972, -1.988 ], ... ],
...
]
[
[ [ -1.872, -0.156, -0.284 ], [ 0.964, -1.276, -1.452 ], [ 0.804, 0.836, 1.144 ], [ 0.2, -0.4, 0.836 ], [ -1.724, -0.232, -0.316 ], [ -0.328, 1.652, 1.412 ], [ -1.252, -1.336, -0.824 ], [ -0.456, -1.748, 0.868 ], ... ],
[ [ 1.668, -1.172, 0.576 ], [ 1.048, -1.628, 0.748 ], [ -0.288, 1.424, -0.852 ], [ -1.664, 1.648, -0.468 ], [ -1.136, 1.756, 0.312 ], [ 0.612, -1.2, 1.452 ], [ 0.224, -0.104, 1.256 ], [ 1.116, -1.18, -1.116 ], ... ],
[ [ -0.596, -0.96, 0.604 ], [ 1.688, -1.904, 1.0 ], [ 0.884, 1.216, 0.032 ], [ -1.224, 1.128, -0.356 ], [ 1.456, -0.468, 1.128 ], [ 1.348, 1.468, -1.272 ], [ 1.82, 1.732, 1.816 ], [ 1.848, 1.604, -0.748 ], ... ],
[ [ -1.572, -0.536, 1.224 ], [ -0.256, -0.844, -0.744 ], [ -1.2, -0.528, -1.232 ], [ 1.356, 1.732, 1.808 ], [ -0.556, 0.06, 1.768 ], [ 1.332, -1.1, 1.984 ], [ -1.7, -1.6, 1.036 ], [ -1.068, 1.576, -0.028 ], ... ],
[ [ -0.284, -1.616, -0.468 ], [ 1.544, -1.268, 0.316 ], [ 1.352, 0.248, -0.416 ], [ -0.644, 0.312, 0.912 ], [ -0.736, -1.036, -1.732 ], [ 1.62, -1.084, 1.508 ], [ -0.068, -0.696, 0.02 ], [ 0.928, -1.524, 1.58 ], ... ],
[ [ -0.392, 0.416, -0.192 ], [ 0.548, 0.024, -0.856 ], [ -1.156, -1.092, -1.464 ], [ -1.352, -1.696, -1.908 ], [ -0.744, -0.676, -1.724 ], [ -0.472, -0.508, 1.948 ], [ -1.568, 1.632, -0.392 ], [ 0.408, -0.964, -1.992 ], ... ],
[ [ -0.496, 1.96, -0.268 ], [ 1.524, -0.816, 1.824 ], [ 1.396, 0.036, -1.232 ], [ 0.56, -0.372, -1.356 ], [ 0.972, -1.952, -0.104 ], [ -1.388, -1.196, -1.98 ], [ -0.204, -1.264, 0.376 ], [ 1.596, 0.384, 1.956 ], ... ],
[ [ 1.004, 0.332, 0.192 ], [ 0.796, -1.44, 1.024 ], [ 0.348, -1.416, 0.992 ], [ 1.432, 0.772, -1.184 ], [ 0.384, -0.872, -1.768 ], [ 0.74, 0.232, 0.32 ], [ 0.556, -1.848, 0.776 ], [ 1.824, -0.752, 0.736 ], ... ],
...
]
[
[ [ -1.3, -1.22, -1.084 ], [ 0.12, 0.764, 1.656 ], [ 0.024, -1.376, 1.828 ], [ -1.928, 1.22, -0.832 ], [ -1.492, 0.556, 0.384 ], [ 0.828, 1.804, -0.756 ], [ 1.988, 1.0, -0.28 ], [ -1.956, -0.384, -0.16 ], ... ],
[ [ 0.576, -0.856, -1.332 ], [ -1.94, -1.3, 1.272 ], [ 0.392, 0.6, -0.932 ], [ 0.688, 1.704, 0.488 ], [ -0.064, 0.684, -1.096 ], [ -1.884, 1.012, -0.2 ], [ 0.3, 1.336, -0.84 ], [ -1.604, 0.728, -1.212 ], ... ],
[ [ 1.728, -0.292, 0.82 ], [ -0.728, -1.452, -1.32 ], [ 0.4, 0.852, 1.348 ], [ -1.056, 0.372, -0.264 ], [ 1.284, 1.904, 1.464 ], [ -1.844, 1.48, 0.98 ], [ -0.916, -0.624, -1.728 ], [ 1.624, 0.6, 0.932 ], ... ],
[ [ 0.42, -1.096, -0.376 ], [ 1.292, 1.884, 1.176 ], [ 1.892, 0.572, 1.256 ], [ 1.816, -1.392, 1.708 ], [ -0.424, -1.348, -0.048 ], [ -0.212, -1.548, -1.324 ], [ 1.1, 1.42, -0.524 ], [ -1.76, 1.384, -0.012 ], ... ],
[ [ 1.468, 0.14, 0.036 ], [ 0.932, -0.904, 0.504 ], [ -0.868, -1.08, -0.54 ], [ 0.516, 0.868, 0.16 ], [ 0.204, -1.54, 1.752 ], [ -0.936, -0.036, -0.028 ], [ 1.664, 0.672, -1.776 ], [ -0.032, -0.816, 1.376 ], ... ],
[ [ -1.76, 0.316, 1.224 ], [ -0.78, -1.616, -1.264 ], [ -1.72, -1.072, 0.508 ], [ -1.732, -0.092, 0.464 ], [ 0.14, -1.576, 1.104 ], [ -1.088, 0.452, 1.26 ], [ -1.996, 0.624, 1.328 ], [ 1.592, -0.8, -0.704 ], ... ],
[ [ 0.336, -1.896, 0.344 ], [ 0.392, -0.416, -0.52 ], [ -1.952, 1.668, -0.016 ], [ -1.796, 0.88, -0.188 ], [ 0.336, 0.76, -1.428 ], [ 1.576, 1.88, -0.312 ], [ -1.34, -0.208, -0.432 ], [ -1.444, -0.072, 0.4 ], ... ],
[ [ -0.632, 1.336, 1.86 ], [ -0.82, 0.204, -0.8 ], [ 1.116, -1.592, -1.236 ], [ 0.444, 0.164, 0.124 ], [ 1.78, -1.368, 0.244 ], [ -1.536, 1.724, 0.724 ], [ -0.748, 1.06, -1.492 ], [ -0.524, 1.532, -0.464 ], ... ],
...
]
[
[ [ -1.392, -0.704, -0.472 ], [ -1.196, -0.02, 1.716 ], [ -1.036, 1.468, 1.248 ], [ -1.372, -1.088, 0.284 ], [ -0.632, 0.652, 1.228 ], [ 0.9, 1.076, 0.176 ], [ 1.124, 0.576, 1.784 ], [ 0.404, -0.724, -1.76 ], ... ],
[ [ -1.22, -1.444, 0.42 ], [ -0.8, 0.336, -0.056 ], [ -0.152, 1.728, 1.108 ], [ 0.024, -0.696, -0.632 ], [ -0.76, -1.472, -0.284 ], [ -1.008, -0.552, -1.076 ], [ 0.452, 0.192, 1.34 ], [ -1.128, 0.824, -1.2 ], ... ],
[ [ 1.592, -0.92, -0.836 ], [ -0.412, 1.308, -1.004 ], [ -1.484, -0.12, -1.612 ], [ 1.676, -0.856, -1.82 ], [ -1.128, 1.824, 0.012 ], [ -0.576, -1.432, -0.608 ], [ -0.884, -1.024, 0.78 ], [ 1.78, 0.596, -1.76 ], ... ],
[ [ -0.388, -0.428, -0.052 ], [ -1.78, 0.312, 0.084 ], [ 1.012, 1.54, 0.924 ], [ -0.804, -0.956, 0.612 ], [ -0.052, -0.472, 0.952 ], [ 1.552, -0.236, 0.672 ], [ -1.624, 1.644, 0.816 ], [ 1.16, 1.232, 1.908 ], ... ],
[ [ 0.02, 0.432, 1.192 ], [ -1.196, -0.532, 0.168 ], [ 0.624, 0.508, 0.604 ], [ 1.852, -0.828, -0.444 ], [ -1.612, -0.708, -1.452 ], [ 0.564, 1.32, -0.204 ], [ -0.38, 1.684, -1.46 ], [ -0.964, -1.836, 0.532 ], ... ],
[ [ 1.156, -1.036, -1.632 ], [ 1.384, -0.152, -0.568 ], [ -1.564, -1.32, 0.792 ], [ 1.96, -0.052, 1.628 ], [ 0.288, 0.78, 0.708 ], [ -1.264, -0.9, 0.24 ], [ 0.868, 1.412, -0.56 ], [ 0.052, 0.66, 1.332 ], ... ],
[ [ 1.02, 0.088, 0.416 ], [ 1.324, 0.448, -0.444 ], [ -1.832, -0.148, 1.472 ], [ -1.228, -1.168, 1.62 ], [ -0.284, -1.976, 0.844 ], [ -0.624, -1.036, 0.404 ], [ 1.816, 0.932, -1.692 ], [ -1.016, -1.652, 1.34 ], ... ],
[ [ 1.692, -0.584, 0.02 ], [ 1.596, 1.524, 1.468 ], [ 1.092, 1.584, -0.096 ], [ 1.132, -1.456, 0.728 ], [ 0.012, -1.372, -1.044 ], [ 0.352, 0.528, 0.792 ], [ 0.944, -1.488, 1.0 ], [ 1.492, -1.584, 0.428 ], ... ],
...
]
[
[ [ -1.28, -0.576, 0.18 ], [ 0.44, 0.1, 0.256 ], [ -1.408, -1.48, -1.464 ], [ -0.984, -1.376, -1.444 ], [ 1.936, 1.632, -0.232 ], [ -0.476, -1.032, -1.276 ], [ 0.608, 1.388, 0.264 ], [ -1.556, -1.768, 0.532 ], ... ],
[ [ -0.94, 0.724, -0.596 ], [ -1.412, -1.884, 1.724 ], [ 0.996, -1.044, 1.54 ], [ -0.34, 0.824, -1.728 ], [ 1.448, -1.192, -0.168 ], [ -1.256, -0.676, 1.696 ], [ -0.06, -1.036, -0.204 ], [ 0.332, 1.276, 1.208 ], ... ],
[ [ 1.628, 0.28, -0.024 ], [ -0.36, 1.164, -1.128 ], [ -0.104, 1.824, -0.484 ], [ -1.404, 1.888, 1.48 ], [ 0.1, 0.556, -1.148 ], [ 1.648, -0.98, 0.404 ], [ -1.24, -0.72, -0.2 ], [ 1.572, 0.988, 0.24 ], ... ],
[ [ 0.844, -1.488, 0.992 ], [ 0.116, -1.98, 0.488 ], [ 1.444, -0.856, -0.544 ], [ 0.028, -1.124, -1.176 ], [ -0.472, -0.304, -1.908 ], [ 1.612, -0.184, -1.924 ], [ 0.792, -1.884, -1.272 ], [ -1.236, 0.78, -0.532 ], ... ],
[ [ 1.284, -0.788, -0.404 ], [ -0.976, -1.42, -0.452 ], [ -0.88, 0.712, 0.712 ], [ -0.024, -1.192, -0.32 ], [ 0.448, 0.456, 1.936 ], [ 0.924, -0.82, 1.14 ], [ 0.524, -1.832, -1.196 ], [ -0.956, 0.556, -0.744 ], ... ],
[ [ 1.64, -0.36, 0.244 ], [ -1.168, 0.88, 1.308 ], [ -0.188, -1.38, 1.076 ], [ -1.576, -1.964, -1.692 ], [ 1.956, -0.696, 1.704 ], [ -0.98, -1.832, 1.828 ], [ 1.732, 0.444, 0.58 ], [ 1.06, 0.156, 0.928 ], ... ],
[ [ 1.364, 0.856, -0.868 ], [ -1.656, 0.744, -1.096 ], [ -1.94, 0.892, -1.688 ], [ -0.516, -0.764, 0.832 ], [ 0.248, 0.396, -1.168 ], [ 0.488, 1.656, -1.664 ], [ 1.152, -0.66, 1.308 ], [ -1.588, -1.552, -0.54 ], ... ],
[ [ 1.68, 0.524, 0.172 ], [ -0.224, 0.264, 0.256 ], [ 1.724, 0.428, 0.548 ], [ 1.348, -0.064, 1.652 ], [ 0.956, -0.572, -0.824 ], [ 0.264, -1.068, -1.672 ], [ -1.34, -1.416, -0.588 ], [ -0.788, 0.808, -0.184 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.51 seconds (3.303 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 9368886293011
Reset training subject: 9370161941768
Low gradient: 5.302559822484755E-7
Constructing line search parameters: GD
th(0)=1.5183255962355965E-6;dx=-2.811714067102963E-13
New Minimum: 1.5183255962355965E-6 > 1.5183249904702451E-6
WOLFE (weak): th(2.154434690031884)=1.5183249904702451E-6; dx=-2.8117135062086966E-13 evalInputDelta=6.057653514137404E-13
New Minimum: 1.5183249904702451E-6 > 1.5183243847049646E-6
WOLFE (weak): th(4.308869380063768)=1.5183243847049646E-6; dx=-2.811712945314981E-13 evalInputDelta=1.2115306318884715E-12
New Minimum: 1.5183243847049646E-6 > 1.5183219616451855E-6
WOLFE (weak): th(12.926608140191302)=1.5183219616451855E-6; dx=-2.811710701739111E-13 evalInputDelta=3.6345904110284162E-12
New Minimum: 1.5183219616451855E-6 > 1.5183110578999896E-6
WOLFE (weak): th(51.70643256076521)=1.5183110578999896E-6; dx=-2.8117006056487355E-13 evalInputDelta=1.4538335606956141E-11
New Minimum: 1.5183110578999896E-6 > 1.518252905253729E-6
WOLFE (weak): th(258.53216280382605)=1.518252905253729E-6; dx=-2.8116467598327007E-13 evalInputDelta=7.269098186750696E-11
New Minimum: 1.518252905253729E-6 > 1.5178894764461986E-6
WOLFE (weak): th(1551.1929768229563)=1.5178894764461986E-6; dx=-2.8113102234812724E-13 evalInputDelta=4.3611978939793994E-10
New Minimum: 1.5178894764461986E-6 > 1.515274073232517E-6
WOLFE (weak): th(10858.350837760694)=1.515274073232517E-6; dx=-2.8088871617509873E-13 evalInputDelta=3.0515230030794246E-9
New Minimum: 1.515274073232517E-6 > 1.4939993596950692E-6
WOLFE (weak): th(86866.80670208555)=1.4939993596950692E-6; dx=-2.789098824291063E-13 evalInputDelta=2.432623654052731E-8
New Minimum: 1.4939993596950692E-6 > 1.3064617175036323E-6
WOLFE (weak): th(781801.26031877)=1.3064617175036323E-6; dx=-2.6081768817957297E-13 evalInputDelta=2.1186387873196424E-7
New Minimum: 1.3064617175036323E-6 > 1.1575213489278575E-7
END: th(7818012.6031877)=1.1575213489278575E-7; dx=-7.763422140316299E-14 evalInputDelta=1.4025734613428109E-6
Fitness changed from 1.5183255962355965E-6 to 1.1575213489278575E-7
Iteration 1 complete. Error: 1.1575213489278575E-7 Total: 16.4110; Orientation: 0.7502; Line Search: 13.5156
Low gradient: 1.4640894964345953E-7
th(0)=1.1575213489278575E-7;dx=-2.1435580535703752E-14
New Minimum: 1.1575213489278575E-7 > 3.624464317152488E-8
WOLF (strong): th(1.684339755941405E7)=3.624464317152488E-8; dx=1.1994790286487977E-14 evalInputDelta=7.950749172126087E-8
New Minimum: 3.624464317152488E-8 > 5.613260742556503E-9
END: th(8421698.779707026)=5.613260742556503E-9; dx=-4.720395124605644E-15 evalInputDelta=1.1013887415022924E-7
Fitness changed from 1.1575213489278575E-7 to 5.613260742556503E-9
Iteration 2 complete. Error: 5.613260742556503E-9 Total: 4.2485; Orientation: 0.7794; Line Search: 3.0373
Low gradient: 3.2241165148037934E-8
th(0)=5.613260742556503E-9;dx=-1.0394927301031062E-15
New Minimum: 5.613260742556503E-9 > 2.5955717673583993E-9
WOLF (strong): th(1.8144E7)=2.5955717673583993E-9; dx=7.068550564700722E-16 evalInputDelta=3.0176889751981034E-9
New Minimum: 2.5955717673583993E-9 > 1.43699475009475E-10
END: th(9072000.0)=1.43699475009475E-10; dx=-1.663188368165026E-16 evalInputDelta=5.469561267547027E-9
Fitness changed from 5.613260742556503E-9 to 1.43699475009475E-10
Iteration 3 complete. Error: 1.43699475009475E-10 Total: 5.5081; Orientation: 0.8284; Line Search: 4.2237
Low gradient: 5.158586423686582E-9
th(0)=1.43699475009475E-10;dx=-2.6611013890645966E-17
New Minimum: 1.43699475009475E-10 > 9.42172507980726E-11
WOLF (strong): th(1.954503150796925E7)=9.42172507980726E-11; dx=2.1547606938216903E-17 evalInputDelta=4.9482224211402405E-11
New Minimum: 9.42172507980726E-11 > 1.3006427187183845E-12
END: th(9772515.753984625)=1.3006427187183845E-12; dx=-2.5317034762364365E-18 evalInputDelta=1.4239883229075661E-10
Fitness changed from 1.43699475009475E-10 to 1.3006427187183845E-12
Iteration 4 complete. Error: 1.3006427187183845E-12 Total: 4.3319; Orientation: 0.8143; Line Search: 3.0522
Final threshold in iteration 4: 1.3006427187183845E-12 (> 0.0) after 30.500s (< 30.000s)
Returns
1.3006427187183845E-12
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 31.80 seconds (1.110 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 9399505679897
Reset training subject: 9399965563513
Low gradient: 5.302559822484755E-7
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5183255962355965E-6}, derivative=-2.811714067102963E-13}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.5183255962355965E-6}, derivative=-2.811714067102963E-13}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.5183255962355965E-6}, derivative=-2.811714067102963E-13}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.5183255962355965E-6}, derivative=-2.811714067102963E-13}, evalInputDelta = 0.0
New Minimum: 1.5183255962355965E-6 > 1.5183255962355963E-6
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.5183255962355963E-6}, derivative=-2.811714067102963E-13}, evalInputDelta = -2.1175823681357508E-22
New Minimum: 1.5183255962355963E-6 > 1.5183255962355698E-6
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.5183255962355698E-6}, derivative=-2.811714067102956E-13}, evalInputDelta = -2.668153783851046E-20
New Minimum: 1.5183255962355698E-6 > 1.518325596235126E-6
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.518325596235126E-6}, derivative=-2.811714067102765E-13}, evalInputDelta = -4.705268021997638E-19
New Minimum: 1.518325596235126E-6 > 1.5183255962322709E-6
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.5183255962322709E-6}, derivative=-2.811714067100217E-13}, evalInputDelta = -3.3256631091571967E-18
New Minimum: 1.5183255962322709E-6 > 1.518325596212403E-6
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.518325596212403E-6}, derivative=-2.81171406708155E-13}, evalInputDelta = -2.3193456161717252E-17
New Minimum: 1.518325596212403E-6 > 1.5183255960734988E-6
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.5183255960734988E-6}, derivative=-2.8117140669529217E-13}, evalInputDelta = -1.6209775390723952E-16
New Minimum: 1.5183255960734988E-6 > 1.518325595100989E-6
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.518325595100989E-6}, derivative=-2.811714066052655E-13}, evalInputDelta = -1.1346076208689502E-15
New Minimum: 1.518325595100989E-6 > 1.5183255882932127E-6
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.5183255882932127E-6}, derivative=-2.811714059748234E-13}, evalInputDelta = -7.942383789156528E-15
New Minimum: 1.5183255882932127E-6 > 1.5183255406388153E-6
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=1.5183255406388153E-6}, derivative=-2.8117140156240473E-13}, evalInputDelta = -5.5596781180027553E-14
New Minimum: 1.5183255406388153E-6 > 1.5183252070581825E-6
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=1.5183252070581825E-6}, derivative=-2.811713706753837E-13}, evalInputDelta = -3.8917741405008425E-13
New Minimum: 1.5183252070581825E-6 > 1.5183228719948885E-6
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=1.5183228719948885E-6}, derivative=-2.811711544656417E-13}, evalInputDelta = -2.7242407080575406E-12
New Minimum: 1.5183228719948885E-6 > 1.5183065266019148E-6
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=1.5183065266019148E-6}, derivative=-2.811696409979377E-13}, evalInputDelta = -1.906963368168684E-11
New Minimum: 1.5183065266019148E-6 > 1.5181921113147452E-6
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=1.5181921113147452E-6}, derivative=-2.8115904672371283E-13}, evalInputDelta = -1.3348492085136638E-10
New Minimum: 1.5181921113147452E-6 > 1.5173913250173365E-6
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=1.5173913250173365E-6}, derivative=-2.8108488680441744E-13}, evalInputDelta = -9.342712182600392E-10
New Minimum: 1.5173913250173365E-6 > 1.5117917358590084E-6
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=1.5117917358590084E-6}, derivative=-2.805657673689856E-13}, evalInputDelta = -6.5338603765881675E-9
New Minimum: 1.5117917358590084E-6 > 1.4728844430011116E-6
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=1.4728844430011116E-6}, derivative=-2.769319313211285E-13}, evalInputDelta = -4.544115323448489E-8
New Minimum: 1.4728844430011116E-6 > 1.2147351242749021E-6
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=1.2147351242749021E-6}, derivative=-2.5149507898605137E-13}, evalInputDelta = -3.035904719606944E-7
New Minimum: 1.2147351242749021E-6 > 1.0357472586223881E-7
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=1.0357472586223881E-7}, derivative=-7.34371126408416E-14}, evalInputDelta = -1.4147508703733578E-6
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=2.642380873784999E-5}, derivative=1.1729686517758358E-12}, evalInputDelta = 2.4905483141614396E-5
F(4296506.646794492) = LineSearchPoint{point=PointSample{avg=5.505677948260946E-7}, derivative=-1.693144791344172E-13}, evalInputDelta = -9.677578014095019E-7
F(3.007554652756145E7) = LineSearchPoint{point=PointSample{avg=4.836495675583217E-6}, derivative=5.018270863207101E-13}, evalInputDelta = 3.31817007934762E-6
F(2313503.579043188) = LineSearchPoint{point=PointSample{avg=9.375064564872028E-7}, derivative=-2.2094075340025654E-13}, evalInputDelta = -5.808191397483937E-7
F(1.6194525053302318E7) = LineSearchPoint{point=PointSample{avg=3.788120900175294E-7}, derivative=1.4044316646026904E-13}, evalInputDelta = -1.139513506218067E-6
3.788120900175294E-7 <= 1.5183255962355965E-6
New Minimum: 1.0357472586223881E-7 > 9.199160671045664E-34
F(1.0800000000000102E7) = LineSearchPoint{point=PointSample{avg=9.199160671045664E-34}, derivative=7.062513187991732E-28}, evalInputDelta = -1.5183255962355965E-6
Right bracket at 1.0800000000000102E7
Converged to right
Fitness changed from 1.5183255962355965E-6 to 9.199160671045664E-34
Iteration 1 complete. Error: 9.199160671045664E-34 Total: 31.8002; Orientation: 0.8070; Line Search: 29.6317
Final threshold in iteration 1: 9.199160671045664E-34 (> 0.0) after 31.801s (< 30.000s)
Returns
9.199160671045664E-34
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 359.47 seconds (7.558 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 9431421355821
Reset training subject: 9431874737304
Adding measurement 67538d7a to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.5183255962355965E-6 < 1.5183255962355965E-6. Total: 1
th(0)=1.5183255962355965E-6;dx=-2.811714067102963E-13
Adding measurement 371b6609 to history. Total: 1
New Minimum: 1.5183255962355965E-6 > 1.5183249904702451E-6
WOLFE (weak): th(2.154434690031884)=1.5183249904702451E-6; dx=-2.8117135062086966E-13 evalInputDelta=6.057653514137404E-13
Adding measurement af62f09 to history. Total: 2
New Minimum: 1.5183249904702451E-6 > 1.5183243847049646E-6
WOLFE (weak): th(4.308869380063768)=1.5183243847049646E-6; dx=-2.811712945314981E-13 evalInputDelta=1.2115306318884715E-12
Adding measurement 2843b2c5 to history. Total: 3
New Minimum: 1.5183243847049646E-6 > 1.5183219616451855E-6
WOLFE (weak): th(12.926608140191302)=1.5183219616451855E-6; dx=-2.811710701739111E-13 evalInputDelta=3.6345904110284162E-12
Adding measurement 6bfcd08b to history. Total: 4
New Minimum: 1.5183219616451855E-6 > 1.5183110578999896E-6
WOLFE (weak): th(51.70643256076521)=1.5183110578999896E-6; dx=-2.8117006056487355E-13 evalInputDelta=1.4538335606956141E-11
Adding measurement 4c5e9e04 to history. Total: 5
New Minimum: 1.5183110578999896E-6 > 1.518252905253729E-6
WOLFE (weak): th(258.53216280382605)=1.518252905253729E-6; dx=-2.8116467598327007E-13 evalInputDelta=7.269098186750696E-11
Adding measurement 3aeaa2a0 to history. Total: 6
New Minimum: 1.518252905253729E-6 > 1.5178894764461986E-6
WOLFE (weak): th(1551.1929768229563)=1.5178894764461986E-6; dx=-2.8113102234812724E-13 evalInputDelta=4.3611978939793994E-10
Adding measurement 23933ce8 to history. Total: 7
New Minimum: 1.5178894764461986E-6 > 1.515274073232517E-6
WOLFE (weak): th(10858.350837760694)=1.515274073232517E-6; dx=-2.8088871617509873E-13 evalInputDelta=3.0515230030794246E-9
Adding measurement 708a599c to history. Total: 8
New Minimum: 1.515274073232517E-6 > 1.4939993596950692E-6
WOLFE (weak): th(86866.80670208555)=1.4939993596950692E-6; dx=-2.7890988242910623E-13 evalInputDelta=2.432623654052731E-8
Adding measurement 1268b243 to history. Total: 9
New Minimum: 1.4939993596950692E-6 > 1.3064617175036323E-6
WOLFE (weak): th(781801.26031877)=1.3064617175036323E-6; dx=-2.6081768817957297E-13 evalInputDelta=2.1186387873196424E-7
Adding measurement 5b660a43 to history. Total: 10
New Minimum: 1.3064617175036323E-6 > 1.1575213489278575E-7
END: th(7818012.6031877)=1.1575213489278575E-7; dx=-7.763422140316299E-14 evalInputDelta=1.4025734613428109E-6
Fitness changed from 1.5183255962355965E-6 to 1.1575213489278575E-7
Iteration 1 complete. Error: 1.1575213489278575E-7 Total: 20.8653; Orientation: 1.1369; Line Search: 18.2399
Non-optimal measurement 1.1575213489278575E-7 < 1.1575213489278575E-7. Total: 11
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, 3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6, 1.518252905253729E-6, 1.5183110578999896E-6, 1.5183219616451855E-6, 1.5183243847049646E-6, 1.5183249904702451E-6, 1.5183255962355965E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00, acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6, 1.518252905253729E-6, 1.5183110578999896E-6, 1.5183219616451855E-6, 1.5183243847049646E-6, 1.5183249904702451E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00, acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6, 1.518252905253729E-6, 1.5183110578999896E-6, 1.5183219616451855E-6, 1.5183243847049646E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00, acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6, 1.518252905253729E-6, 1.5183110578999896E-6, 1.5183219616451855E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, 3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6, 1.518252905253729E-6, 1.5183110578999896E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00, 3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6, 1.518252905253729E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, 3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6, 1.5178894764461986E-6
Rejected: LBFGS Orientation magnitude: 1.581e+00, gradient 1.464e-07, dot -1.000; [3f855de1-227c-4b41-91cd-42e62be97c77 = 1.000/1.000e+00, acd31264-c663-41db-9e24-3b81b9b5e7b5 = 1.000/1.000e+00, dbeb2178-3463-431b-ae1c-57a6a92b8fac = 1.000/1.000e+00, f7109d36-6bc6-4a74-9524-04f202e7387c = 1.000/1.000e+00, c0a1a583-8255-4f8e-bb00-ff57e89d2a0e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.1575213489278575E-7, 1.3064617175036323E-6, 1.4939993596950692E-6, 1.515274073232517E-6
LBFGS Accumulation History: 3 points
Removed measurement 5b660a43 to history. Total: 10
Removed measurement 1268b243 to history. Total: 9
Removed measurement 708a599c to history. Total: 8
Removed measurement 23933ce8 to history. Total: 7
Removed measurement 3aeaa2a0 to history. Total: 6
Removed measurement 4c5e9e04 to history. Total: 5
Removed measurement 6bfcd08b to history. Total: 4
Removed measurement 2843b2c5 to history. Total: 3
Adding measurement 68d233dd to history. Total: 3
th(0)=1.1575213489278575E-7;dx=-2.1435580535703752E-14
Adding measurement 641defae to history. Total: 4
New Minimum: 1.1575213489278575E-7 > 3.624464317152488E-8
WOLF (strong): th(1.684339755941405E7)=3.624464317152488E-8; dx=1.1994790286487977E-14 evalInputDelta=7.950749172126087E-8
Adding measurement 181f42c4 to history. Total: 5
New Minimum: 3.624464317152488E-8 > 5.613260742556503E-9
END: th(8421698.779707026)=5.613260742556503E-9; dx=-4.720395124605644E-15 evalInputDelta=1.1013887415022924E-7
Fitness changed from 1.1575213489278575E-7 to 5.613260742556503E-9
Iteration 2 complete. Error: 5.613260742556503E-9 Total: 338.6027; Orientation: 333.8274; Line Search: 4.3261
Final threshold in iteration 2: 5.613260742556503E-9 (> 0.0) after 359.468s (< 30.000s)
Returns
5.613260742556503E-9
TrainingTester.java:432 executed in 0.17 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -33.036251795759014], [4.0, -6.936470990257257]; valueStats=DoubleSummaryStatistics{count=7, sum=0.000000, min=0.000000, average=0.000000, max=0.000000}
Plotting 4 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -33.036251795759014], [338.603, -6.936470990257257]; valueStats=DoubleSummaryStatistics{count=7, sum=0.000000, min=0.000000, average=0.000000, max=0.000000}
Plotting 4 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 5.613260742556503E-9 }, "CjGD": { "type": "Converged", "value": 9.199160671045664E-34 }, "GD": { "type": "Converged", "value": 1.3006427187183845E-12 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 5.613260742556503E-9 }, "CjGD": { "type": "Converged", "value": 9.199160671045664E-34 }, "GD": { "type": "Converged", "value": 1.3006427187183845E-12 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "426.175",
"gc_time": "12.289"
},
"created_on": 1586743997320,
"file_name": "trainingTest",
"report": {
"simpleName": "Double",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandAvgReducerLayerTest.Double",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 5.613260742556503E-9
},
"CjGD": {
"type": "Converged",
"value": 9.199160671045664E-34
},
"GD": {
"type": "Converged",
"value": 1.3006427187183845E-12
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayer/Double/trainingTest/202004131317",
"id": "03cc6039-bf6d-4144-b717-8ab5fe6500b8",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "BandAvgReducerLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.BandAvgReducerLayer",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/BandAvgReducerLayer.java",
"javaDoc": ""
}
}