Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 3660795533824300032
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.04 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.916 ], [ 1.004 ], [ -0.58 ], [ -0.176 ], [ -0.956 ], [ -1.4 ], [ -0.08 ], [ -1.02 ], ... ],
[ [ 0.904 ], [ -1.284 ], [ 1.704 ], [ 1.56 ], [ 1.46 ], [ 0.0 ], [ -0.936 ], [ -1.292 ], ... ],
[ [ 1.052 ], [ 1.156 ], [ 0.436 ], [ 1.732 ], [ -1.936 ], [ -0.64 ], [ 1.916 ], [ -0.888 ], ... ],
[ [ -0.288 ], [ 0.596 ], [ -0.98 ], [ 0.26 ], [ -1.74 ], [ 1.248 ], [ -1.456 ], [ 0.616 ], ... ],
[ [ 0.088 ], [ -1.784 ], [ 0.42 ], [ 0.884 ], [ 1.408 ], [ -0.164 ], [ -0.092 ], [ 0.192 ], ... ],
[ [ 1.136 ], [ 0.9 ], [ -1.088 ], [ 1.78 ], [ 0.632 ], [ 1.448 ], [ 0.52 ], [ -1.46 ], ... ],
[ [ -0.712 ], [ -1.248 ], [ 1.032 ], [ 1.708 ], [ -1.7 ], [ -1.736 ], [ 1.392 ], [ 1.124 ], ... ],
[ [ 1.896 ], [ -0.212 ], [ -1.852 ], [ 1.32 ], [ -1.444 ], [ 1.9 ], [ -0.544 ], [ 1.584 ], ... ],
...
]
[
[ [ 1.916 ], [ -1.24 ], [ 1.132 ], [ -1.552 ], [ 0.792 ], [ 1.236 ], [ -1.7 ], [ 1.788 ], ... ],
[ [ -0.436 ], [ 0.176 ], [ 0.724 ], [ -1.264 ], [ -1.276 ], [ 0.284 ], [ -1.736 ], [ -1.712 ], ... ],
[ [ 0.972 ], [ -0.392 ], [ -0.26 ], [ 1.392 ], [ 1.82 ], [ 0.028 ], [ -1.468 ], [ -1.128 ], ... ],
[ [ -1.672 ], [ -1.496 ], [ -0.112 ], [ 1.636 ], [ 0.512 ], [ -0.7 ], [ -1.952 ], [ -1.5 ], ... ],
[ [ -1.012 ], [ -1.192 ], [ -1.724 ], [ 0.44 ], [ 1.704 ], [ 0.496 ], [ 0.204 ], [ 1.776 ], ... ],
[ [ -0.816 ], [ 0.816 ], [ 1.88 ], [ 1.34 ], [ 0.308 ], [ 1.028 ], [ -1.076 ], [ -1.58 ], ... ],
[ [ 1.108 ], [ 1.664 ], [ 1.244 ], [ -1.76 ], [ 0.652 ], [ -1.12 ], [ 1.904 ], [ -1.004 ], ... ],
[ [ 1.312 ], [ 0.444 ], [ 0.728 ], [ 0.42 ], [ -1.944 ], [ -1.644 ], [ -1.808 ], [ 0.264 ], ... ],
...
]
[
[ [ -0.792 ], [ -0.988 ], [ -1.928 ], [ 0.468 ], [ 1.456 ], [ 1.14 ], [ 0.736 ], [ -0.392 ], ... ],
[ [ 1.14 ], [ 0.44 ], [ -1.484 ], [ -0.736 ], [ 1.252 ], [ -0.888 ], [ 1.588 ], [ 1.456 ], ... ],
[ [ 1.472 ], [ -0.468 ], [ -1.408 ], [ 1.932 ], [ -0.312 ], [ -1.356 ], [ -0.216 ], [ -0.444 ], ... ],
[ [ 0.408 ], [ -0.856 ], [ 1.848 ], [ 0.428 ], [ 0.104 ], [ 1.772 ], [ -1.428 ], [ 1.612 ], ... ],
[ [ 0.088 ], [ -1.356 ], [ -0.004 ], [ 1.496 ], [ -0.628 ], [ 1.028 ], [ -1.732 ], [ -0.892 ], ... ],
[ [ 1.492 ], [ -0.736 ], [ -0.544 ], [ 1.604 ], [ -0.468 ], [ 0.54 ], [ 1.608 ], [ -1.544 ], ... ],
[ [ -1.52 ], [ -1.02 ], [ 1.46 ], [ -1.284 ], [ -0.616 ], [ 1.24 ], [ -1.428 ], [ -1.912 ], ... ],
[ [ 0.736 ], [ 0.116 ], [ 0.856 ], [ 0.696 ], [ -1.328 ], [ -0.92 ], [ 0.02 ], [ -0.996 ], ... ],
...
]
[
[ [ -0.968 ], [ 0.312 ], [ -0.736 ], [ -1.612 ], [ 1.968 ], [ -0.2 ], [ -0.832 ], [ -1.156 ], ... ],
[ [ -0.528 ], [ 0.804 ], [ 1.584 ], [ -1.852 ], [ -0.68 ], [ 0.356 ], [ -1.704 ], [ 0.148 ], ... ],
[ [ -1.112 ], [ -0.532 ], [ -1.708 ], [ -1.796 ], [ 1.816 ], [ 1.152 ], [ 0.412 ], [ 1.764 ], ... ],
[ [ -1.192 ], [ 0.344 ], [ -0.812 ], [ 1.756 ], [ 0.336 ], [ -0.032 ], [ 1.144 ], [ 1.484 ], ... ],
[ [ -1.236 ], [ -0.008 ], [ 1.388 ], [ -1.292 ], [ 0.656 ], [ -0.192 ], [ 1.196 ], [ 1.124 ], ... ],
[ [ -0.2 ], [ 0.184 ], [ -1.144 ], [ 0.332 ], [ -0.204 ], [ 0.952 ], [ -0.26 ], [ -0.424 ], ... ],
[ [ -1.06 ], [ 1.232 ], [ -1.536 ], [ 1.048 ], [ 1.192 ], [ -1.796 ], [ -1.788 ], [ -1.568 ], ... ],
[ [ -0.02 ], [ -0.376 ], [ 0.256 ], [ -0.048 ], [ -1.384 ], [ 1.728 ], [ 1.784 ], [ 0.352 ], ... ],
...
]
[
[ [ -1.716 ], [ -1.552 ], [ -0.82 ], [ 0.976 ], [ 0.64 ], [ -1.388 ], [ 0.276 ], [ -1.796 ], ... ],
[ [ -1.58 ], [ 1.236 ], [ 0.608 ], [ 0.08 ], [ -0.504 ], [ 1.668 ], [ -0.912 ], [ -0.836 ], ... ],
[ [ 1.1 ], [ -0.56 ], [ -1.376 ], [ -0.2 ], [ 1.424 ], [ -1.572 ], [ 1.38 ], [ 1.616 ], ... ],
[ [ -1.056 ], [ -0.84 ], [ 0.352 ], [ 1.884 ], [ -1.944 ], [ 1.388 ], [ 1.804 ], [ -0.544 ], ... ],
[ [ 1.456 ], [ -1.464 ], [ 0.724 ], [ 1.44 ], [ 1.996 ], [ -0.084 ], [ -1.7 ], [ -0.564 ], ... ],
[ [ 1.8 ], [ 1.216 ], [ 1.764 ], [ 1.044 ], [ 0.468 ], [ -1.516 ], [ 1.752 ], [ 0.596 ], ... ],
[ [ -0.428 ], [ -0.06 ], [ 0.636 ], [ -1.656 ], [ -1.796 ], [ 0.856 ], [ -1.944 ], [ 0.288 ], ... ],
[ [ 1.9 ], [ -1.98 ], [ -0.536 ], [ -1.096 ], [ 0.404 ], [ -1.396 ], [ -0.828 ], [ 1.58 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.40 seconds (1.301 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 6129457191402
Reset training subject: 6129991692811
Constructing line search parameters: GD
th(0)=0.11919977647755502;dx=-2.103940862810305E-9
New Minimum: 0.11919977647755502 > 0.11919977180527175
WOLFE (weak): th(2.154434690031884)=0.11919977180527175; dx=-2.1039408538206603E-9 evalInputDelta=4.67228326861413E-9
New Minimum: 0.11919977180527175 > 0.11919976759861002
WOLFE (weak): th(4.308869380063768)=0.11919976759861002; dx=-2.1039408466024635E-9 evalInputDelta=8.878944995904803E-9
New Minimum: 0.11919976759861002 > 0.1191997495118567
WOLFE (weak): th(12.926608140191302)=0.1191997495118567; dx=-2.103940811386958E-9 evalInputDelta=2.696569831317497E-8
New Minimum: 0.1191997495118567 > 0.11919966776612181
WOLFE (weak): th(51.70643256076521)=0.11919966776612181; dx=-2.1039406697376447E-9 evalInputDelta=1.0871143320267951E-7
New Minimum: 0.11919966776612181 > 0.11919923248733107
WOLFE (weak): th(258.53216280382605)=0.11919923248733107; dx=-2.1039399138068397E-9 evalInputDelta=5.439902239501304E-7
New Minimum: 0.11919923248733107 > 0.11919651275260608
WOLFE (weak): th(1551.1929768229563)=0.11919651275260608; dx=-2.103935177631814E-9 evalInputDelta=3.263724948937563E-6
New Minimum: 0.11919651275260608 > 0.11917693131626858
WOLFE (weak): th(10858.350837760694)=0.11917693131626858; dx=-2.1039010374331897E-9 evalInputDelta=2.28451612864361E-5
New Minimum: 0.11917693131626858 > 0.11901702767599662
WOLFE (weak): th(86866.80670208555)=0.11901702767599662; dx=-2.1036197449105117E-9 evalInputDelta=1.8274880155839945E-4
New Minimum: 0.11901702767599662 > 0.11755609343512172
WOLFE (weak): th(781801.26031877)=0.11755609343512172; dx=-2.1008422453788594E-9 evalInputDelta=0.0016436830424332999
New Minimum: 0.11755609343512172 > 0.10292487533359262
WOLFE (weak): th(7818012.6031877)=0.10292487533359262; dx=-2.0515296358301212E-9 evalInputDelta=0.016274901143962392
New Minimum: 0.10292487533359262 > 0.00977722150018991
END: th(8.599813863506469E7)=0.00977722150018991; dx=-2.514904553990921E-10 evalInputDelta=0.10942255497736511
Fitness changed from 0.11919977647755502 to 0.00977722150018991
Iteration 1 complete. Error: 0.00977722150018991 Total: 8.5918; Orientation: 0.3617; Line Search: 6.9555
th(0)=0.00977722150018991;dx=-2.8187974165152507E-10
New Minimum: 0.00977722150018991 > 0.0013598417832073012
WOLF (strong): th(1.0E8)=0.0013598417832073012; dx=5.816776035863578E-11 evalInputDelta=0.008417379716982608
New Minimum: 0.0013598417832073012 > 0.0011320796543977363
END: th(5.0E7)=0.0011320796543977363; dx=-6.764032396257062E-11 evalInputDelta=0.008645141845792173
Fitness changed from 0.00977722150018991 to 0.0011320796543977363
Iteration 2 complete. Error: 0.0011320796543977363 Total: 2.8595; Orientation: 0.2551; Line Search: 2.1116
Low gradient: 4.669061366983103E-6
th(0)=0.0011320796543977363;dx=-2.1800134048654077E-11
New Minimum: 0.0011320796543977363 > 1.9209999898303772E-4
WOLF (strong): th(1.0E8)=1.9209999898303772E-4; dx=2.9989801673379212E-12 evalInputDelta=9.399796554146986E-4
END: th(5.0E7)=3.5114425505609637E-4; dx=-9.399769143259527E-12 evalInputDelta=7.8093539934164E-4
Fitness changed from 0.0011320796543977363 to 1.9209999898303772E-4
Iteration 3 complete. Error: 1.9209999898303772E-4 Total: 1.9211; Orientation: 0.2324; Line Search: 1.3898
Low gradient: 1.978944737308458E-6
th(0)=1.9209999898303772E-4;dx=-3.916222273320834E-12
New Minimum: 1.9209999898303772E-4 > 6.391997165365827E-5
WOLF (strong): th(1.0E8)=6.391997165365827E-5; dx=1.3776632695455731E-12 evalInputDelta=1.2818002732937946E-4
New Minimum: 6.391997165365827E-5 > 6.167322527393028E-5
END: th(5.0E7)=6.167322527393028E-5; dx=-1.2880891755265596E-12 evalInputDelta=1.3042677370910744E-4
Fitness changed from 1.9209999898303772E-4 to 6.167322527393028E-5
Iteration 4 complete. Error: 6.167322527393028E-5 Total: 2.0064; Orientation: 0.2351; Line Search: 1.4458
Low gradient: 8.122161608626515E-7
th(0)=6.167322527393028E-5;dx=-6.596950919664637E-13
New Minimum: 6.167322527393028E-5 > 1.897065567837083E-5
END: th(1.0E8)=1.897065567837083E-5; dx=-1.902693148237159E-13 evalInputDelta=4.270256959555945E-5
Fitness changed from 6.167322527393028E-5 to 1.897065567837083E-5
Iteration 5 complete. Error: 1.897065567837083E-5 Total: 1.6194; Orientation: 0.2331; Line Search: 1.0959
Low gradient: 4.299740857958605E-7
th(0)=1.897065567837083E-5;dx=-1.8487771445598587E-13
New Minimum: 1.897065567837083E-5 > 6.881117706255913E-6
END: th(1.0E8)=6.881117706255913E-6; dx=-5.6450784489880905E-14 evalInputDelta=1.2089537972114917E-5
Fitness changed from 1.897065567837083E-5 to 6.881117706255913E-6
Iteration 6 complete. Error: 6.881117706255913E-6 Total: 1.9956; Orientation: 0.2541; Line Search: 1.4275
Low gradient: 2.557093312905926E-7
th(0)=6.881117706255913E-6;dx=-6.538726210908203E-14
New Minimum: 6.881117706255913E-6 > 2.681467074836022E-6
END: th(1.0E8)=2.681467074836022E-6; dx=-1.8471228567861575E-14 evalInputDelta=4.199650631419891E-6
Fitness changed from 6.881117706255913E-6 to 2.681467074836022E-6
Iteration 7 complete. Error: 2.681467074836022E-6 Total: 1.4948; Orientation: 0.2352; Line Search: 0.9535
Low gradient: 1.597035905991878E-7
th(0)=2.681467074836022E-6;dx=-2.550523685027299E-14
New Minimum: 2.681467074836022E-6 > 1.0909258043941547E-6
END: th(1.0E8)=1.0909258043941547E-6; dx=-6.279686288572453E-15 evalInputDelta=1.590541270441867E-6
Fitness changed from 2.681467074836022E-6 to 1.0909258043941547E-6
Iteration 8 complete. Error: 1.0909258043941547E-6 Total: 1.4916; Orientation: 0.2372; Line Search: 0.9565
Low gradient: 1.0273125863059287E-7
th(0)=1.0909258043941547E-6;dx=-1.0553711499825763E-14
New Minimum: 1.0909258043941547E-6 > 4.5645193226942896E-7
END: th(1.0E8)=4.5645193226942896E-7; dx=-2.126343884262622E-15 evalInputDelta=6.344738721247257E-7
Fitness changed from 1.0909258043941547E-6 to 4.5645193226942896E-7
Iteration 9 complete. Error: 4.5645193226942896E-7 Total: 1.4893; Orientation: 0.2351; Line Search: 0.9529
Low gradient: 6.736155349290264E-8
th(0)=4.5645193226942896E-7;dx=-4.537578888977183E-15
New Minimum: 4.5645193226942896E-7 > 1.9508959234938421E-7
END: th(1.0E8)=1.9508959234938421E-7; dx=-6.877307232392241E-16 evalInputDelta=2.6136233992004477E-7
Fitness changed from 4.5645193226942896E-7 to 1.9508959234938421E-7
Iteration 10 complete. Error: 1.9508959234938421E-7 Total: 1.5673; Orientation: 0.2863; Line Search: 0.9801
Low gradient: 4.4816803352027796E-8
th(0)=1.9508959234938421E-7;dx=-2.00854586269433E-15
New Minimum: 1.9508959234938421E-7 > 8.478540135026513E-8
END: th(1.0E8)=8.478540135026513E-8; dx=-1.9675429657061524E-16 evalInputDelta=1.1030419099911908E-7
Fitness changed from 1.9508959234938421E-7 to 8.478540135026513E-8
Iteration 11 complete. Error: 8.478540135026513E-8 Total: 1.6085; Orientation: 0.2431; Line Search: 1.0596
Low gradient: 3.015744532854943E-8
th(0)=8.478540135026513E-8;dx=-9.094715087444477E-16
New Minimum: 8.478540135026513E-8 > 3.738477519260361E-8
END: th(1.0E8)=3.738477519260361E-8; dx=-3.83867073328389E-17 evalInputDelta=4.740062615766152E-8
Fitness changed from 8.478540135026513E-8 to 3.738477519260361E-8
Iteration 12 complete. Error: 3.738477519260361E-8 Total: 1.5288; Orientation: 0.2436; Line Search: 0.9788
Low gradient: 2.0490718797869597E-8
th(0)=3.738477519260361E-8;dx=-4.1986955685336666E-16
New Minimum: 3.738477519260361E-8 > 1.6693233246592527E-8
WOLF (strong): th(1.0E8)=1.6693233246592527E-8; dx=6.108581013191238E-18 evalInputDelta=2.0691541946011083E-8
END: th(5.0E7)=2.171427009708123E-8; dx=-2.0693287063386068E-16 evalInputDelta=1.567050509552238E-8
Fitness changed from 3.738477519260361E-8 to 1.6693233246592527E-8
Iteration 13 complete. Error: 1.6693233246592527E-8 Total: 2.2115; Orientation: 0.2435; Line Search: 1.6573
Final threshold in iteration 13: 1.6693233246592527E-8 (> 0.0) after 30.386s (< 30.000s)
Returns
1.6693233246592527E-8
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 34.15 seconds (1.005 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 6159908027513
Reset training subject: 6160285863257
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.11919977647755502}, derivative=-2.103940862810305E-9}, evalInputDelta = 0.0
New Minimum: 0.11919977647755502 > 0.11919977646512545
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.11919977646512545}, derivative=-2.103940863022209E-9}, evalInputDelta = -1.2429571261129979E-11
New Minimum: 0.11919977646512545 > 0.11919977598431226
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.11919977598431226}, derivative=-2.1039408618818907E-9}, evalInputDelta = -4.932427521975313E-10
New Minimum: 0.11919977598431226 > 0.11919977340325112
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.11919977340325112}, derivative=-2.1039408572651657E-9}, evalInputDelta = -3.0743038947900203E-9
New Minimum: 0.11919977340325112 > 0.11919975633946292
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.11919975633946292}, derivative=-2.103940823887541E-9}, evalInputDelta = -2.0138092096666504E-8
New Minimum: 0.11919975633946292 > 0.11919963374807212
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.11919963374807212}, derivative=-2.103940610842912E-9}, evalInputDelta = -1.4272948289928422E-7
New Minimum: 0.11919963374807212 > 0.11919877757791675
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.11919877757791675}, derivative=-2.1039391219503607E-9}, evalInputDelta = -9.988996382687132E-7
New Minimum: 0.11919877757791675 > 0.11919278446469597
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.11919278446469597}, derivative=-2.1039286826547635E-9}, evalInputDelta = -6.992012859044494E-6
New Minimum: 0.11919278446469597 > 0.1191508333221758
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.1191508333221758}, derivative=-2.1038554323827106E-9}, evalInputDelta = -4.8943155379219117E-5
New Minimum: 0.1191508333221758 > 0.1188572169560125
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.1188572169560125}, derivative=-2.103334152456588E-9}, evalInputDelta = -3.425595215425098E-4
New Minimum: 0.1188572169560125 > 0.11680408582810597
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.11680408582810597}, derivative=-2.0992658406699294E-9}, evalInputDelta = -0.002395690649449042
New Minimum: 0.11680408582810597 > 0.10259426745857178
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.10259426745857178}, derivative=-2.0499447618163143E-9}, evalInputDelta = -0.016605509018983236
New Minimum: 0.10259426745857178 > 0.027212376005571488
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.027212376005571488}, derivative=-9.540581460151465E-10}, evalInputDelta = -0.09198740047198353
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.07791374158279815}, derivative=1.5809377559853875E-10}, evalInputDelta = -0.041286034894756865
0.07791374158279815 <= 0.11919977647755502
F(3.6365633535019267E8) = LineSearchPoint{point=PointSample{avg=0.07337220111422696}, derivative=1.7470695778516007E-10}, evalInputDelta = -0.04582757536332806
Right bracket at 3.6365633535019267E8
F(3.357743206509096E8) = LineSearchPoint{point=PointSample{avg=0.06823403464651265}, derivative=1.9434492269753133E-10}, evalInputDelta = -0.05096574183104237
Right bracket at 3.357743206509096E8
F(3.073809699187255E8) = LineSearchPoint{point=PointSample{avg=0.06239466646272568}, derivative=2.175418865015227E-10}, evalInputDelta = -0.05680511001482934
Right bracket at 3.073809699187255E8
F(2.785768635386072E8) = LineSearchPoint{point=PointSample{avg=0.055747225372652344}, derivative=2.446092170449243E-10}, evalInputDelta = -0.06345255110490267
Right bracket at 2.785768635386072E8
F(2.4956216674269727E8) = LineSearchPoint{point=PointSample{avg=0.04821496564842022}, derivative=2.75000901339059E-10}, evalInputDelta = -0.0709848108291348
Right bracket at 2.4956216674269727E8
F(2.207132803056179E8) = LineSearchPoint{point=PointSample{avg=0.03983256096685052}, derivative=3.0581113761741007E-10}, evalInputDelta = -0.0793672155107045
Right bracket at 2.207132803056179E8
F(1.92703518585096E8) = LineSearchPoint{point=PointSample{avg=0.030911611083678448}, derivative=3.2904107983105434E-10}, evalInputDelta = -0.08828816539387657
Right bracket at 1.92703518585096E8
New Minimum: 0.027212376005571488 > 0.0222612043741093
F(1.666419302390495E8) = LineSearchPoint{point=PointSample{avg=0.0222612043741093}, derivative=3.294351525123661E-10}, evalInputDelta = -0.09693857210344572
Right bracket at 1.666419302390495E8
New Minimum: 0.0222612043741093 > 0.01517163493973063
F(1.4408162334131885E8) = LineSearchPoint{point=PointSample{avg=0.01517163493973063}, derivative=2.904782071977052E-10}, evalInputDelta = -0.10402814153782439
Right bracket at 1.4408162334131885E8
New Minimum: 0.01517163493973063 > 0.0106882852716809
F(1.2660240587159608E8) = LineSearchPoint{point=PointSample{avg=0.0106882852716809}, derivative=2.1383520896894681E-10}, evalInputDelta = -0.10851149120587411
Right bracket at 1.2660240587159608E8
New Minimum: 0.0106882852716809 > 0.008657254058122232
F(1.1492222147192618E8) = LineSearchPoint{point=PointSample{avg=0.008657254058122232}, derivative=1.2860534205259547E-10}, evalInputDelta = -0.11054252241943278
Right bracket at 1.1492222147192618E8
New Minimum: 0.008657254058122232 > 0.008010029230394379
F(1.0830215171943979E8) = LineSearchPoint{point=PointSample{avg=0.008010029230394379}, derivative=6.489796308752381E-11}, evalInputDelta = -0.11118974724716063
Loops = 12
Fitness changed from 0.11919977647755502 to 0.008010029230394379
Iteration 1 complete. Error: 0.008010029230394379 Total: 24.5626; Orientation: 0.2265; Line Search: 23.1936
F(0.0) = LineSearchPoint{point=PointSample{avg=0.008010029230394379}, derivative=-1.5352157163971655E-10}
New Minimum: 0.008010029230394379 > 0.0023958017677218175
F(1.0830215171943979E8) = LineSearchPoint{point=PointSample{avg=0.0023958017677218175}, derivative=3.869106649822698E-11}, evalInputDelta = -0.005614227462672561
0.0023958017677218175 <= 0.008010029230394379
New Minimum: 0.0023958017677218175 > 0.0018954884359287862
F(8.650168222548966E7) = LineSearchPoint{point=PointSample{avg=0.0018954884359287862}, derivative=6.542027445582814E-12}, evalInputDelta = -0.006114540794465593
Right bracket at 8.650168222548966E7
New Minimum: 0.0018954884359287862 > 0.0018822203749551537
F(8.296623517542887E7) = LineSearchPoint{point=PointSample{avg=0.0018822203749551537}, derivative=9.45537131184802E-13}, evalInputDelta = -0.006127808855439225
Right bracket at 8.296623517542887E7
Converged to right
Fitness changed from 0.008010029230394379 to 0.0018822203749551537
Iteration 2 complete. Error: 0.0018822203749551537 Total: 3.3097; Orientation: 0.2257; Line Search: 2.7923
Low gradient: 4.379558556462118E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0018822203749551537}, derivative=-1.918053314948051E-11}
New Minimum: 0.0018822203749551537 > 8.388379342122276E-4
F(8.296623517542887E7) = LineSearchPoint{point=PointSample{avg=8.388379342122276E-4}, derivative=-5.362465757307522E-12}, evalInputDelta = -0.0010433824407429261
F(5.807636462280021E8) = LineSearchPoint{point=PointSample{avg=0.026640031370530927}, derivative=1.0517485046749381E-10}, evalInputDelta = 0.024757810995575773
F(4.467412663292324E7) = LineSearchPoint{point=PointSample{avg=0.0011774295739140298}, derivative=-1.2194163284148796E-11}, evalInputDelta = -7.047908010411239E-4
F(3.1271888643046266E8) = LineSearchPoint{point=PointSample{avg=0.005435146747764826}, derivative=4.7999013530029136E-11}, evalInputDelta = 0.003552926372809672
F(2.4055298956189435E7) = LineSearchPoint{point=PointSample{avg=0.0014638588240729028}, derivative=-1.5551250914148187E-11}, evalInputDelta = -4.1836155088225096E-4
F(1.6838709269332606E8) = LineSearchPoint{point=PointSample{avg=0.001118722166570547}, derivative=1.2474130771535954E-11}, evalInputDelta = -7.634982083846067E-4
0.001118722166570547 <= 0.0018822203749551537
New Minimum: 8.388379342122276E-4 > 7.714223473363984E-4
F(1.0203091150826146E8) = LineSearchPoint{point=PointSample{avg=7.714223473363984E-4}, derivative=-1.679384406525924E-12}, evalInputDelta = -0.0011107980276187555
Left bracket at 1.0203091150826146E8
New Minimum: 7.714223473363984E-4 > 7.64375313950536E-4
F(1.0990439979416347E8) = LineSearchPoint{point=PointSample{avg=7.64375313950536E-4}, derivative=-1.0563692362778139E-13}, evalInputDelta = -0.0011178450610046177
Left bracket at 1.0990439979416347E8
Converged to left
Fitness changed from 0.0018822203749551537 to 7.64375313950536E-4
Iteration 3 complete. Error: 7.64375313950536E-4 Total: 6.2730; Orientation: 0.2257; Line Search: 5.7487
Final threshold in iteration 3: 7.64375313950536E-4 (> 0.0) after 34.145s (< 30.000s)
Returns
7.64375313950536E-4
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.01 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.9098646704015512 ], [ 0.995022800858907 ], [ -0.5102408826441172 ], [ -0.19638479552282767 ], [ -0.9412145841760526 ], [ -1.7287314996116223 ], [ -0.07817246552067537 ], [ -1.098944381675606 ], ... ],
[ [ 0.952565895578997 ], [ -1.2846861778904395 ], [ 1.7135712777829535 ], [ 1.6988086454161195 ], [ 1.5121000363473531 ], [ 0.10003392357389287 ], [ -0.9174217046880779 ], [ -1.3119263837075053 ], ... ],
[ [ 1.049784745947736 ], [ 1.5368318401661176 ], [ 0.47192783421280415 ], [ 1.7031038374101526 ], [ -1.9258306434162764 ], [ -0.6364620070789455 ], [ 2.0924321222091313 ], [ -0.9459795574486394 ], ... ],
[ [ -0.34826488793150817 ], [ 0.5987014584340304 ], [ -0.9710211110711314 ], [ 0.16863189942027818 ], [ -1.7097946313776453 ], [ 1.5901305354339335 ], [ -1.8831586293073987 ], [ 0.5280998156471154 ], ... ],
[ [ 0.20986992556685286 ], [ -2.296887546821706 ], [ 0.4219227767496918 ], [ 0.8734569129178203 ], [ 1.9010515469027907 ], [ -0.2689745313410179 ], [ -0.21699868073801448 ], [ 0.10310706685366955 ], ... ],
[ [ 1.4744169013394388 ], [ 0.8856319522523028 ], [ -1.0877552038158154 ], [ 1.926445754037975 ], [ 0.5726354046064288 ], [ 1.8921006447825557 ], [ 0.4595350983175932 ], [ -1.8835059980433735 ], ... ],
[ [ -0.6388562572044528 ], [ -1.2498108542409136 ], [ 1.0419554360100538 ], [ 1.7051799659396334 ], [ -2.497858613888216 ], [ -2.2065236899097784 ], [ 2.019525935204751 ], [ 1.1233335817393617 ], ... ],
[ [ 1.8040514590890329 ], [ -0.18402529870010345 ], [ -1.8380622960334072 ], [ 1.3217035414043221 ], [ -1.5038570920799865 ], [ 1.8434938893178372 ], [ -0.5560777036554762 ], [ 2.2704753570327565 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.07 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.7129724621772766 ], [ 0.7300788760185242 ], [ 0.375137060880661 ], [ 0.451060950756073 ], [ 0.2806550860404968 ], [ 0.15074990689754486 ], [ 0.4804667830467224 ], [ 0.2499377280473709 ], ... ],
[ [ 0.7216309309005737 ], [ 0.21675357222557068 ], [ 0.8472988605499268 ], [ 0.845379114151001 ], [ 0.8193721771240234 ], [ 0.5249876379966736 ], [ 0.28548353910446167 ], [ 0.21216468513011932 ], ... ],
[ [ 0.7407336235046387 ], [ 0.8230036497116089 ], [ 0.615839958190918 ], [ 0.8459396362304688 ], [ 0.127212792634964 ], [ 0.3460467457771301 ], [ 0.890165388584137 ], [ 0.2796940803527832 ], ... ],
[ [ 0.41380324959754944 ], [ 0.6453591585159302 ], [ 0.27467700839042664 ], [ 0.5420583486557007 ], [ 0.15319034457206726 ], [ 0.8306344747543335 ], [ 0.13202647864818573 ], [ 0.6290398240089417 ], ... ],
[ [ 0.5522757768630981 ], [ 0.09138105809688568 ], [ 0.6039432883262634 ], [ 0.7054645419120789 ], [ 0.870010495185852 ], [ 0.43315884470939636 ], [ 0.44596222043037415 ], [ 0.525753915309906 ], ... ],
[ [ 0.813727855682373 ], [ 0.7079879641532898 ], [ 0.252041220664978 ], [ 0.8728554844856262 ], [ 0.6393710970878601 ], [ 0.8689948320388794 ], [ 0.6129038333892822 ], [ 0.13198669254779816 ], ... ],
[ [ 0.34550514817237854 ], [ 0.22273288667201996 ], [ 0.7392271757125854 ], [ 0.8462100625038147 ], [ 0.07600844651460648 ], [ 0.09916619211435318 ], [ 0.8828319907188416 ], [ 0.7546065449714661 ], ... ],
[ [ 0.8586413860321045 ], [ 0.4541230797767639 ], [ 0.1372806280851364 ], [ 0.7894650101661682 ], [ 0.18185093998908997 ], [ 0.8633614182472229 ], [ 0.3644554913043976 ], [ 0.906402051448822 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 139.43 seconds (1.708 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 6194202724271
Reset training subject: 6194518334470
Adding measurement 1e2ffad5 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.11919977647755502 < 0.11919977647755502. Total: 1
th(0)=0.11919977647755502;dx=-2.103940862810305E-9
Adding measurement 65f21cdb to history. Total: 1
New Minimum: 0.11919977647755502 > 0.11919977180527175
WOLFE (weak): th(2.154434690031884)=0.11919977180527175; dx=-2.1039408538206603E-9 evalInputDelta=4.67228326861413E-9
Adding measurement 10ede8c2 to history. Total: 2
New Minimum: 0.11919977180527175 > 0.11919976759861002
WOLFE (weak): th(4.308869380063768)=0.11919976759861002; dx=-2.1039408466024635E-9 evalInputDelta=8.878944995904803E-9
Adding measurement 6edcf079 to history. Total: 3
New Minimum: 0.11919976759861002 > 0.1191997495118567
WOLFE (weak): th(12.926608140191302)=0.1191997495118567; dx=-2.103940811386958E-9 evalInputDelta=2.696569831317497E-8
Adding measurement 31bcda05 to history. Total: 4
New Minimum: 0.1191997495118567 > 0.11919966776612181
WOLFE (weak): th(51.70643256076521)=0.11919966776612181; dx=-2.1039406697376447E-9 evalInputDelta=1.0871143320267951E-7
Adding measurement 6e8e3e1a to history. Total: 5
New Minimum: 0.11919966776612181 > 0.11919923248733107
WOLFE (weak): th(258.53216280382605)=0.11919923248733107; dx=-2.1039399138068397E-9 evalInputDelta=5.439902239501304E-7
Adding measurement 54fca6cd to history. Total: 6
New Minimum: 0.11919923248733107 > 0.11919651275260608
WOLFE (weak): th(1551.1929768229563)=0.11919651275260608; dx=-2.103935177631814E-9 evalInputDelta=3.263724948937563E-6
Adding measurement 1985e547 to history. Total: 7
New Minimum: 0.11919651275260608 > 0.11917693131626858
WOLFE (weak): th(10858.350837760694)=0.11917693131626858; dx=-2.1039010374331897E-9 evalInputDelta=2.28451612864361E-5
Adding measurement 7e6afd58 to history. Total: 8
New Minimum: 0.11917693131626858 > 0.11901702767599662
WOLFE (weak): th(86866.80670208555)=0.11901702767599662; dx=-2.1036197449105117E-9 evalInputDelta=1.8274880155839945E-4
Adding measurement 7e9f8a3d to history. Total: 9
New Minimum: 0.11901702767599662 > 0.11755609343512172
WOLFE (weak): th(781801.26031877)=0.11755609343512172; dx=-2.1008422453788594E-9 evalInputDelta=0.0016436830424332999
Adding measurement 1c05340e to history. Total: 10
New Minimum: 0.11755609343512172 > 0.10292487533359262
WOLFE (weak): th(7818012.6031877)=0.10292487533359262; dx=-2.0515296358301212E-9 evalInputDelta=0.016274901143962392
Adding measurement 6b39ba3f to history. Total: 11
New Minimum: 0.10292487533359262 > 0.00977722150018991
END: th(8.599813863506469E7)=0.00977722150018991; dx=-2.514904553990921E-10 evalInputDelta=0.10942255497736511
Fitness changed from 0.11919977647755502 to 0.00977722150018991
Iteration 1 complete. Error: 0.00977722150018991 Total: 9.8439; Orientation: 0.2951; Line Search: 8.6204
Non-optimal measurement 0.00977722150018991 < 0.00977722150018991. Total: 12
Rejected: LBFGS Orientation magnitude: 5.631e+03, gradient 1.679e-05, dot -0.338; [14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608, 0.11919923248733107, 0.11919966776612181, 0.1191997495118567, 0.11919976759861002, 0.11919977180527175, 0.11919977647755502
Rejected: LBFGS Orientation magnitude: 6.992e+03, gradient 1.679e-05, dot -0.336; [b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608, 0.11919923248733107, 0.11919966776612181, 0.1191997495118567, 0.11919976759861002, 0.11919977180527175
Rejected: LBFGS Orientation magnitude: 6.032e+03, gradient 1.679e-05, dot -0.367; [b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608, 0.11919923248733107, 0.11919966776612181, 0.1191997495118567, 0.11919976759861002
Rejected: LBFGS Orientation magnitude: 7.214e+03, gradient 1.679e-05, dot -0.400; [00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608, 0.11919923248733107, 0.11919966776612181, 0.1191997495118567
Rejected: LBFGS Orientation magnitude: 7.291e+03, gradient 1.679e-05, dot -0.406; [b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608, 0.11919923248733107, 0.11919966776612181
Rejected: LBFGS Orientation magnitude: 7.278e+03, gradient 1.679e-05, dot -0.407; [b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608, 0.11919923248733107
Rejected: LBFGS Orientation magnitude: 7.271e+03, gradient 1.679e-05, dot -0.407; [78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858, 0.11919651275260608
Rejected: LBFGS Orientation magnitude: 7.215e+03, gradient 1.679e-05, dot -0.408; [14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00, 78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662, 0.11917693131626858
Rejected: LBFGS Orientation magnitude: 6.752e+03, gradient 1.679e-05, dot -0.416; [78c9b306-8000-4a44-a63f-0cf93676f0a0 = 1.000/1.000e+00, b17c0dec-a5e8-4173-9e28-a8932167f76b = 1.000/1.000e+00, 14d186f5-7e8f-4845-84a5-b1fa28595327 = 1.000/1.000e+00, b21441a7-88cf-4e20-8add-f420141d90b7 = 1.000/1.000e+00, 00249334-7bbb-4d9e-862c-9cce347829ba = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00977722150018991, 0.10292487533359262, 0.11755609343512172, 0.11901702767599662
LBFGS Accumulation History: 3 points
Removed measurement 6b39ba3f to history. Total: 11
Removed measurement 1c05340e to history. Total: 10
Removed measurement 7e9f8a3d to history. Total: 9
Removed measurement 7e6afd58 to history. Total: 8
Removed measurement 1985e547 to history. Total: 7
Removed measurement 54fca6cd to history. Total: 6
Removed measurement 6e8e3e1a to history. Total: 5
Removed measurement 31bcda05 to history. Total: 4
Removed measurement 6edcf079 to history. Total: 3
Adding measurement 3c8bd1d5 to history. Total: 3
th(0)=0.00977722150018991;dx=-2.8187974165152507E-10
Adding measurement 78afb081 to history. Total: 4
New Minimum: 0.00977722150018991 > 0.0013598417832073012
WOLF (strong): th(1.0E8)=0.0013598417832073012; dx=5.816776035863578E-11 evalInputDelta=0.008417379716982608
Adding measurement 763420c5 to history. Total: 5
New Minimum: 0.0013598417832073012 > 0.0011320796543977363
END: th(5.0E7)=0.0011320796543977363; dx=-6.764032396257062E-11 evalInputDelta=0.008645141845792173
Fitness changed from 0.00977722150018991 to 0.0011320796543977363
Iteration 2 complete. Error: 0.0011320796543977363 Total: 129.5819; Orientation: 127.4191; Line Search: 1.8682
Final threshold in iteration 2: 0.0011320796543977363 (> 0.0) after 139.426s (< 30.000s)
Returns
0.0011320796543977363
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 1.0563239445919765 ], [ 0.7983526207761912 ], [ -0.6603633082386683 ], [ -0.2395602224316712 ], [ -1.3366323223182675 ], [ -1.6639587917846184 ], [ -0.03295179032957085 ], [ -1.2672030361309383 ], ... ],
[ [ 1.1747828694989826 ], [ -1.061059724480021 ], [ 1.6015071193401882 ], [ 1.6260417076909923 ], [ 1.2514467635540656 ], [ -0.0025431771918691393 ], [ -1.260727378623498 ], [ -1.317709025799318 ], ... ],
[ [ 0.9072518978790023 ], [ 1.4922267358012022 ], [ 0.4727745733565375 ], [ 1.5999942396929439 ], [ -1.7535143780718276 ], [ -0.6716057079758506 ], [ 1.6250420681110402 ], [ -1.1236968414343371 ], ... ],
[ [ -0.3539430029486289 ], [ 0.6549329790245471 ], [ -0.8009402050973347 ], [ 0.2449247278244553 ], [ -1.326151925907539 ], [ 1.4525072696383452 ], [ -1.755064935501254 ], [ 0.7872563633934391 ], ... ],
[ [ 0.1066087279879464 ], [ -1.82158037092568 ], [ 0.7728229120011656 ], [ 0.7997757547919313 ], [ 1.6325466460187814 ], [ -0.2070509384557765 ], [ -0.1256377176305894 ], [ 0.17556408834339537 ], ... ],
[ [ 1.4389103990982233 ], [ 0.6515440407771522 ], [ -1.115294550947763 ], [ 1.7735762996111855 ], [ 1.075463161831142 ], [ 1.7590655435404994 ], [ 0.6530298190953505 ], [ -1.7556030939518428 ], ... ],
[ [ -0.46616940962359177 ], [ -1.0435025039319052 ], [ 1.0166068327746378 ], [ 1.6990206436815343 ], [ -2.030268192484726 ], [ -1.9563312311756977 ], [ 1.767737352150216 ], [ 1.2198873800288177 ], ... ],
[ [ 1.298120971269048 ], [ -0.20592047968889865 ], [ -1.6890269982798838 ], [ 1.2922054123355649 ], [ -1.252403309932751 ], [ 1.7485773942245038 ], [ -0.6328838976017356 ], [ 1.9643173485126177 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.06 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.7419874668121338 ], [ 0.6896219849586487 ], [ 0.3406580090522766 ], [ 0.44039472937583923 ], [ 0.20806442201137543 ], [ 0.1592312902212143 ], [ 0.49176281690597534 ], [ 0.21973641216754913 ], ... ],
[ [ 0.7640084624290466 ], [ 0.2571069896221161 ], [ 0.83222895860672 ], [ 0.8356266617774963 ], [ 0.7775501608848572 ], [ 0.4993642270565033 ], [ 0.2208487093448639 ], [ 0.21119971573352814 ], ... ],
[ [ 0.7124375104904175 ], [ 0.8164122104644775 ], [ 0.6160402894020081 ], [ 0.8320175409317017 ], [ 0.14760448038578033 ], [ 0.3381373882293701 ], [ 0.8354893326759338 ], [ 0.2453262060880661 ], ... ],
[ [ 0.41242656111717224 ], [ 0.6581212282180786 ], [ 0.3098244369029999 ], [ 0.5609269142150879 ], [ 0.20979660749435425 ], [ 0.8103839755058289 ], [ 0.14740949869155884 ], [ 0.6872419118881226 ], ... ],
[ [ 0.5266269445419312 ], [ 0.13924434781074524 ], [ 0.6841312646865845 ], [ 0.6899265050888062 ], [ 0.8365182280540466 ], [ 0.4484213888645172 ], [ 0.4686318337917328 ], [ 0.543778657913208 ], ... ],
[ [ 0.8082858324050903 ], [ 0.6573582887649536 ], [ 0.24688513576984406 ], [ 0.8549018502235413 ], [ 0.7456344962120056 ], [ 0.853092610836029 ], [ 0.6576929092407227 ], [ 0.14734187722206116 ], ... ],
[ [ 0.3855232894420624 ], [ 0.2604747712612152 ], [ 0.7343111038208008 ], [ 0.8454067707061768 ], [ 0.11606140434741974 ], [ 0.12386464327573776 ], [ 0.8541761040687561 ], [ 0.7720437049865723 ], ... ],
[ [ 0.7855185866355896 ], [ 0.44870105385780334 ], [ 0.15590384602546692 ], [ 0.7845202088356018 ], [ 0.22228440642356873 ], [ 0.8517732620239258 ], [ 0.34685689210891724 ], [ 0.8769993782043457 ], ... ],
...
]
TrainingTester.java:432 executed in 0.14 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -7.777459538386072], [13.0, -2.009784545884994]; valueStats=DoubleSummaryStatistics{count=18, sum=0.032759, min=0.000000, average=0.001820, max=0.009777}
Plotting 13 points for GD
Plotting 3 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -7.777459538386072], [129.582, -2.009784545884994]; valueStats=DoubleSummaryStatistics{count=18, sum=0.032759, min=0.000000, average=0.001820, max=0.009777}
Plotting 13 points for GD
Plotting 3 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.0011320796543977363 }, "CjGD": { "type": "NonConverged", "value": 7.64375313950536E-4 }, "GD": { "type": "NonConverged", "value": 1.6693233246592527E-8 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.0011320796543977363 }, "CjGD": { "type": "NonConverged", "value": 7.64375313950536E-4 }, "GD": { "type": "NonConverged", "value": 1.6693233246592527E-8 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "206.152",
"gc_time": "4.259"
},
"created_on": 1586740760130,
"file_name": "trainingTest",
"report": {
"simpleName": "Sigmoid_Float",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.ActivationLayerTest.Sigmoid_Float",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/ActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 0.0011320796543977363
},
"CjGD": {
"type": "NonConverged",
"value": 7.64375313950536E-4
},
"GD": {
"type": "NonConverged",
"value": 1.6693233246592527E-8
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/ActivationLayer/Sigmoid_Float/trainingTest/202004131920",
"id": "73ba27ac-16c8-4264-ba18-d0dad1ba56c9",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "ActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.ActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/ActivationLayer.java",
"javaDoc": ""
}
}