Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 1761696713753710592
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ -0.128, 0.08, 0.7 ]
[ -0.128, 0.7, 0.08 ]
[ -0.128, 0.7, 0.08 ]
[ -0.128, 0.08, 0.7 ]
[ 0.08, -0.128, 0.7 ]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.17 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2380094047824
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 2380119772346
Constructing line search parameters: GD
th(0)=0.356826553185465;dx=-0.27605626243498504
New Minimum: 0.356826553185465 > 0.07734440729790167
END: th(2.154434690031884)=0.07734440729790167; dx=-0.12557089933583648 evalInputDelta=0.27948214588756337
Fitness changed from 0.356826553185465 to 0.07734440729790167
Iteration 1 complete. Error: 0.07734440729790167 Total: 0.0518; Orientation: 0.0026; Line Search: 0.0154
th(0)=0.07734440729790167;dx=-0.06464833004593783
New Minimum: 0.07734440729790167 > 0.03488042699120847
WOLFE (weak): th(4.641588833612779)=0.03488042699120847; dx=-0.05834352355071595 evalInputDelta=0.0424639803066932
New Minimum: 0.03488042699120847 > 0.02429217822421248
END: th(9.283177667225559)=0.02429217822421248; dx=-0.057213835518036354 evalInputDelta=0.053052229073689186
Fitness changed from 0.07734440729790167 to 0.02429217822421248
Iteration 2 complete. Error: 0.02429217822421248 Total: 0.0175; Orientation: 0.0008; Line Search: 0.0130
th(0)=0.02429217822421248;dx=-0.05820181669606812
Armijo: th(20.000000000000004)=0.049383858861676885; dx=-0.056591549888063405 evalInputDelta=-0.025091680637464406
Armijo: th(10.000000000000002)=0.03341184105920876; dx=-0.05781479193427021 evalInputDelta=-0.009119662834996281
Armijo: th(3.333333333333334)=0.026929715159743982; dx=-0.058105985447699265 evalInputDelta=-0.0026375369355315023
Armijo: th(0.8333333333333335)=0.024922258721643154; dx=-0.05817996612294329 evalInputDelta=-6.300804974306747E-4
Armijo: th(0.16666666666666669)=0.02441674613415582; dx=-0.05819754673340123 evalInputDelta=-1.2456790994334194E-4
Armijo: th(0.02777777777777778)=0.024312890184526655; dx=-0.0582011084172811 evalInputDelta=-2.071196031417505E-5
Armijo: th(0.003968253968253969)=0.024295135871402675; dx=-0.05820171559573975 evalInputDelta=-2.957647190195145E-6
Armijo: th(4.960317460317461E-4)=0.02429254790817182; dx=-0.058201804060026945 evalInputDelta=-3.696839593406165E-7
Armijo: th(5.511463844797179E-5)=0.024292219299898436; dx=-0.05820181529208471 evalInputDelta=-4.1075685956737384E-8
Armijo: th(5.511463844797179E-6)=0.024292182331777594; dx=-0.05820181655567001 evalInputDelta=-4.107565114430667E-9
Armijo: th(5.010421677088345E-7)=0.024292178597627462; dx=-0.05820181668330466 evalInputDelta=-3.7341498296261655E-10
Armijo: th(4.175351397573621E-8)=0.02429217825533039; dx=-0.0582018166950045 evalInputDelta=-3.1117910331834864E-11
Armijo: th(3.211808767364324E-9)=0.024292178226606162; dx=-0.0582018166959863 evalInputDelta=-2.393682474455261E-12
Armijo: th(2.2941491195459456E-10)=0.024292178224383454; dx=-0.05820181669606228 evalInputDelta=-1.709743457922741E-13
Armijo: th(1.5294327463639638E-11)=0.024292178224223877; dx=-0.05820181669606773 evalInputDelta=-1.1397133237167623E-14
Armijo: th(9.558954664774774E-13)=0.02429217822421319; dx=-0.058201816696068095 evalInputDelta=-7.112366251504909E-16
Armijo: th(5.6229145086910434E-14)=0.024292178224212518; dx=-0.058201816696068116 evalInputDelta=-3.8163916471489756E-17
WOLFE (weak): th(3.1238413937172463E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
Armijo: th(2.967649324031384E-14)=0.0242921782242125; dx=-0.05820181669606812 evalInputDelta=-2.0816681711721685E-17
Armijo: th(1.6400167317015543E-14)=0.024292178224212486; dx=-0.05820181669606812 evalInputDelta=-6.938893903907228E-18
Armijo: th(9.762004355366395E-15)=0.024292178224212486; dx=-0.05820181669606812 evalInputDelta=-6.938893903907228E-18
WOLFE (weak): th(6.4429228745418206E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
WOLFE (weak): th(8.102463614954108E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
WOLFE (weak): th(8.932233985160251E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
mu >= nu (8.932233985160251E-15): th(0.0)=0.02429217822421248
Fitness changed from 0.02429217822421248 to 0.02429217822421248
Static Iteration Total: 0.0937; Orientation: 0.0009; Line Search: 0.0901
Iteration 3 failed. Error: 0.02429217822421248
Previous Error: 0.0 -> 0.02429217822421248
Optimization terminated 3
Final threshold in iteration 3: 0.02429217822421248 (> 0.0) after 0.164s (< 30.000s)
Returns
0.02429217822421248
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ -1.561710552906364, -0.11812229516748837, 2.048858951361061 ]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[ -0.3555070726747259, 2.9543825783514026, -0.9450422856421141 ]
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.32 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2380271985164
Reset training subject: 2380274087794
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.356826553185465}, derivative=-0.2760562624349851}
New Minimum: 0.356826553185465 > 0.35682655315785944
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.35682655315785944}, derivative=-0.27605626241278847}, evalInputDelta = -2.7605584485002055E-11
New Minimum: 0.35682655315785944 > 0.35682655299222565
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.35682655299222565}, derivative=-0.276056262279609}, evalInputDelta = -1.9323936895077054E-10
New Minimum: 0.35682655299222565 > 0.3568265518327894
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.3568265518327894}, derivative=-0.2760562613473529}, evalInputDelta = -1.352675638166545E-9
New Minimum: 0.3568265518327894 > 0.3568265437167354
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.3568265437167354}, derivative=-0.27605625482156004}, evalInputDelta = -9.468729633699269E-9
New Minimum: 0.3568265437167354 > 0.3568264869043629
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.3568264869043629}, derivative=-0.2760562091410185}, evalInputDelta = -6.628110210682436E-8
New Minimum: 0.3568264869043629 > 0.3568260892180183
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.3568260892180183}, derivative=-0.276055889377667}, evalInputDelta = -4.639674467399324E-7
New Minimum: 0.3568260892180183 > 0.35682330542650453
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.35682330542650453}, derivative=-0.2760536510557247}, evalInputDelta = -3.2477589604895662E-6
New Minimum: 0.35682330542650453 > 0.3568038195178887
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.3568038195178887}, derivative=-0.2760379838564175}, evalInputDelta = -2.2733667576324557E-5
New Minimum: 0.3568038195178887 > 0.35666744911473386
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.35666744911473386}, derivative=-0.2759283650960666}, evalInputDelta = -1.5910407073116417E-4
New Minimum: 0.35666744911473386 > 0.3557143698082079
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.3557143698082079}, derivative=-0.2751635552059352}, evalInputDelta = -0.0011121833772571033
New Minimum: 0.3557143698082079 > 0.34911583644815164
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.34911583644815164}, derivative=-0.2699305322372446}, evalInputDelta = -0.007710716737313383
New Minimum: 0.34911583644815164 > 0.3061565638436392
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.3061565638436392}, derivative=-0.23836019020580246}, evalInputDelta = -0.050669989341825816
New Minimum: 0.3061565638436392 > 0.09890339225860943
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.09890339225860943}, derivative=-0.13195660102287876}, evalInputDelta = -0.2579231609268556
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(0.7453000800538463) = LineSearchPoint{point=PointSample{avg=0.19528603435988184}, derivative=-0.1735692841721457}, evalInputDelta = -0.16154051882558318
New Minimum: 0.09890339225860943 > 0.07770295094959542
F(5.217100560376924) = LineSearchPoint{point=PointSample{avg=0.07770295094959542}, derivative=-0.11544038678312261}, evalInputDelta = -0.27912360223586963
F(36.51970392263847) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
New Minimum: 0.07770295094959542 > 0.07025887581046242
F(2.809207994049113) = LineSearchPoint{point=PointSample{avg=0.07025887581046242}, derivative=-0.12311432980169693}, evalInputDelta = -0.2865676773750026
F(19.66445595834379) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(1.5126504583341378) = LineSearchPoint{point=PointSample{avg=0.09442199917628465}, derivative=-0.13052900713070606}, evalInputDelta = -0.2624045540091804
F(10.588553208338965) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(0.8145040929491512) = LineSearchPoint{point=PointSample{avg=0.18347497058469991}, derivative=-0.16783589983504998}, evalInputDelta = -0.1733515826007651
F(5.701528650644058) = LineSearchPoint{point=PointSample{avg=0.08439868169424966}, derivative=-0.11116243278833016}, evalInputDelta = -0.2724278714912154
F(39.91070055450841) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
New Minimum: 0.07025887581046242 > 0.06995471597257726
F(3.070053888808339) = LineSearchPoint{point=PointSample{avg=0.06995471597257726}, derivative=-0.12262560091344166}, evalInputDelta = -0.2868718372128878
F(21.490377221658374) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(1.6531059401275672) = LineSearchPoint{point=PointSample{avg=0.09040660987536986}, derivative=-0.12932936777178364}, evalInputDelta = -0.26641994331009516
F(11.57174158089297) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(0.8901339677609977) = LineSearchPoint{point=PointSample{avg=0.17100521257206974}, derivative=-0.16199101184385295}, evalInputDelta = -0.18582134061339528
F(6.230937774326984) = LineSearchPoint{point=PointSample{avg=0.09527789775029682}, derivative=-0.10069465994004062}, evalInputDelta = -0.2615486554351682
F(43.61656442028889) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
New Minimum: 0.06995471597257726 > 0.0697742885576277
F(3.3551203400222223) = LineSearchPoint{point=PointSample{avg=0.0697742885576277}, derivative=-0.12204487730388597}, evalInputDelta = -0.28705226462783734
F(23.485842380155557) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(1.806603260011966) = LineSearchPoint{point=PointSample{avg=0.08620489334523573}, derivative=-0.12809934154046893}, evalInputDelta = -0.2706216598402293
F(12.64622282008376) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(0.9727863707756739) = LineSearchPoint{point=PointSample{avg=0.15786470216367107}, derivative=-0.15605348147803716}, evalInputDelta = -0.19896185102179395
F(6.809504595429717) = LineSearchPoint{point=PointSample{avg=0.12204933007849801}, derivative=-0.036375800186848065}, evalInputDelta = -0.23477722310696703
F(47.66653216800802) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(3.6666563206160014) = LineSearchPoint{point=PointSample{avg=0.06977693546018891}, derivative=-0.12133739958665682}, evalInputDelta = -0.2870496177252761
F(25.66659424431201) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(1.974353403408616) = LineSearchPoint{point=PointSample{avg=0.08182192195650065}, derivative=-0.12683931196110498}, evalInputDelta = -0.27500463122896435
F(13.820473823860313) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(1.063113371066178) = LineSearchPoint{point=PointSample{avg=0.1440438497948989}, derivative=-0.15004322637211148}, evalInputDelta = -0.21278270339056612
F(7.441793597463246) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=3.372185781767243E8}, evalInputDelta = 0.245646523429151
F(0.5724456613433266) = LineSearchPoint{point=PointSample{avg=0.22664750068361816}, derivative=-0.18978569252933775}, evalInputDelta = -0.13017905250184686
F(4.007119629403286) = LineSearchPoint{point=PointSample{avg=0.07005032932213967}, derivative=-0.12044646344064915}, evalInputDelta = -0.286776
...skipping 5186 bytes...
lta = 0.0
F(85.81263178881797) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.200920105512627E8}, evalInputDelta = 0.602473076614616
F(6.600971676062921) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(46.20680173244045) = LineSearchPoint{point=PointSample{avg=0.012429210178012002}, derivative=-0.05923957641609112}, evalInputDelta = 0.012429210178012002
F(3.5543693640338807) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(24.880585548237164) = LineSearchPoint{point=PointSample{avg=5.21037981877653E-4}, derivative=-0.05955218703242382}, evalInputDelta = 5.21037981877653E-4
F(1.9138911960182434) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(13.397238372127704) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(93.78066860489393) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.2009201055126274E8}, evalInputDelta = 0.602473076614616
F(7.213897584991841) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(50.49728309494289) = LineSearchPoint{point=PointSample{avg=0.01587491467704065}, derivative=-0.059112398134041434}, evalInputDelta = 0.01587491467704065
F(3.8844063919186835) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(27.190844743430784) = LineSearchPoint{point=PointSample{avg=0.001538400138743588}, derivative=-0.0595316362420031}, evalInputDelta = 0.001538400138743588
F(2.091603441802368) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(14.641224092616577) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(102.48856864831603) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.200920105512627E8}, evalInputDelta = 0.602473076614616
F(7.883736049870464) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(55.18615234909325) = LineSearchPoint{point=PointSample{avg=0.020391518625905054}, derivative=-0.05891215043216788}, evalInputDelta = 0.020391518625905054
F(4.245088642237942) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(29.715620495665593) = LineSearchPoint{point=PointSample{avg=0.0027073570766572687}, derivative=-0.059506808482837244}, evalInputDelta = 0.0027073570766572687
F(2.285816961205046) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(16.00071872843532) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(112.00503109904724) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.200920105512627E8}, evalInputDelta = 0.602473076614616
F(8.615771623003635) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(60.31040136102544) = LineSearchPoint{point=PointSample{avg=0.026708737324511334}, derivative=-0.05855247011957606}, evalInputDelta = 0.026708737324511334
F(4.639261643155804) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(32.47483150209062) = LineSearchPoint{point=PointSample{avg=0.004060875290438425}, derivative=-0.05947634763120656}, evalInputDelta = 0.004060875290438425
F(2.4980639616992786) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(17.48644773189495) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(122.40513412326465) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.200920105512627E8}, evalInputDelta = 0.602473076614616
F(9.415779547943433) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(65.91045683560404) = LineSearchPoint{point=PointSample{avg=0.03668856757638554}, derivative=-0.057722463029549455}, evalInputDelta = 0.03668856757638554
F(5.070035141200311) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(35.49024598840218) = LineSearchPoint{point=PointSample{avg=0.005642984421961339}, derivative=-0.05943826072612037}, evalInputDelta = 0.005642984421961339
F(2.730018922184783) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(19.11013245529348) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(133.77092718705438) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.200920105512627E8}, evalInputDelta = 0.602473076614616
F(10.290071322081106) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(72.03049925456774) = LineSearchPoint{point=PointSample{avg=0.05862433589715325}, derivative=-0.05380201513655802}, evalInputDelta = 0.05862433589715325
F(5.540807634966749) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(38.78565344476725) = LineSearchPoint{point=PointSample{avg=0.007514326928472641}, derivative=-0.0593894940338071}, evalInputDelta = 0.007514326928472641
F(2.9835118034436343) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(20.88458262410544) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
F(146.1920783687381) = LineSearchPoint{point=PointSample{avg=0.602473076614616}, derivative=4.200920105512627E8}, evalInputDelta = 0.602473076614616
F(11.24554448990293) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Loops = 52
F(5.622772244951465) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 5.622772244951465
F(2.8113861224757324) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 2.8113861224757324
F(1.4056930612378662) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 1.4056930612378662
F(0.7028465306189331) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.7028465306189331
F(0.35142326530946655) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.35142326530946655
F(0.17571163265473327) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.17571163265473327
F(0.08785581632736664) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.08785581632736664
F(0.04392790816368332) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.04392790816368332
F(0.02196395408184166) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.02196395408184166
F(0.01098197704092083) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.01098197704092083
F(0.005490988520460415) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Right bracket at 0.005490988520460415
F(0.0027454942602302074) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.059562346534441785}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.1477; Orientation: 0.0005; Line Search: 0.1457
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.325s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.09 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2380602997825
Reset training subject: 2380604290566
Adding measurement 25728172 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.356826553185465 < 0.356826553185465. Total: 1
th(0)=0.356826553185465;dx=-0.27605626243498504
Adding measurement 707833a2 to history. Total: 1
New Minimum: 0.356826553185465 > 0.07734440729790167
END: th(2.154434690031884)=0.07734440729790167; dx=-0.12557089933583648 evalInputDelta=0.27948214588756337
Fitness changed from 0.356826553185465 to 0.07734440729790167
Iteration 1 complete. Error: 0.07734440729790167 Total: 0.0113; Orientation: 0.0029; Line Search: 0.0049
Non-optimal measurement 0.07734440729790167 < 0.07734440729790167. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.07734440729790167 < 0.07734440729790167. Total: 2
th(0)=0.07734440729790167;dx=-0.06464833004593783
Adding measurement 545b19d0 to history. Total: 2
New Minimum: 0.07734440729790167 > 0.03488042699120847
WOLFE (weak): th(4.641588833612779)=0.03488042699120847; dx=-0.05834352355071595 evalInputDelta=0.0424639803066932
Adding measurement 7af8c8c7 to history. Total: 3
New Minimum: 0.03488042699120847 > 0.02429217822421248
END: th(9.283177667225559)=0.02429217822421248; dx=-0.057213835518036354 evalInputDelta=0.053052229073689186
Fitness changed from 0.07734440729790167 to 0.02429217822421248
Iteration 2 complete. Error: 0.02429217822421248 Total: 0.0131; Orientation: 0.0024; Line Search: 0.0092
Non-optimal measurement 0.02429217822421248 < 0.02429217822421248. Total: 4
Rejected: LBFGS Orientation magnitude: 1.157e+00, gradient 2.413e-01, dot -0.930; [2b1dc3fe-5ee4-414b-a768-b4c96008318d = 1.000/1.000e+00, a7117a5e-56f6-42e6-838e-c071e5d57031 = 1.000/1.000e+00, 1f118ac5-2832-47cb-80e6-f0c9d9ab8f4c = 1.000/1.000e+00, 502dd567-9bf6-4286-a049-cc25f11b54d0 = 1.000/1.000e+00, 078830db-8797-469c-9d1c-f311f3d92c8a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.02429217822421248, 0.03488042699120847, 0.07734440729790167, 0.356826553185465
LBFGS Accumulation History: 3 points
Removed measurement 7af8c8c7 to history. Total: 3
Adding measurement 46e7ed4d to history. Total: 3
th(0)=0.02429217822421248;dx=-0.058201816696068116
Non-optimal measurement 0.049383858861676885 < 0.02429217822421248. Total: 4
Armijo: th(20.000000000000004)=0.049383858861676885; dx=-0.056591549888063405 evalInputDelta=-0.025091680637464406
Non-optimal measurement 0.03341184105920876 < 0.02429217822421248. Total: 4
Armijo: th(10.000000000000002)=0.03341184105920876; dx=-0.05781479193427021 evalInputDelta=-0.009119662834996281
Non-optimal measurement 0.026929715159743982 < 0.02429217822421248. Total: 4
Armijo: th(3.333333333333334)=0.026929715159743982; dx=-0.058105985447699265 evalInputDelta=-0.0026375369355315023
Non-optimal measurement 0.024922258721643154 < 0.02429217822421248. Total: 4
Armijo: th(0.8333333333333335)=0.024922258721643154; dx=-0.05817996612294329 evalInputDelta=-6.300804974306747E-4
Non-optimal measurement 0.02441674613415582 < 0.02429217822421248. Total: 4
Armijo: th(0.16666666666666669)=0.02441674613415582; dx=-0.05819754673340123 evalInputDelta=-1.2456790994334194E-4
Non-optimal measurement 0.024312890184526655 < 0.02429217822421248. Total: 4
Armijo: th(0.02777777777777778)=0.024312890184526655; dx=-0.0582011084172811 evalInputDelta=-2.071196031417505E-5
Non-optimal measurement 0.024295135871402675 < 0.02429217822421248. Total: 4
Armijo: th(0.003968253968253969)=0.024295135871402675; dx=-0.05820171559573975 evalInputDelta=-2.957647190195145E-6
Non-optimal measurement 0.02429254790817182 < 0.02429217822421248. Total: 4
Armijo: th(4.960317460317461E-4)=0.02429254790817182; dx=-0.058201804060026945 evalInputDelta=-3.696839593406165E-7
Non-optimal measurement 0.024292219299898436 < 0.02429217822421248. Total: 4
Armijo: th(5.511463844797179E-5)=0.024292219299898436; dx=-0.05820181529208471 evalInputDelta=-4.1075685956737384E-8
Non-optimal measurement 0.024292182331777594 < 0.02429217822421248. Total: 4
Armijo: th(5.511463844797179E-6)=0.024292182331777594; dx=-0.05820181655567001 evalInputDelta=-4.107565114430667E-9
Non-optimal measurement 0.024292178597627462 < 0.02429217822421248. Total: 4
Armijo: th(5.010421677088345E-7)=0.024292178597627462; dx=-0.05820181668330466 evalInputDelta=-3.7341498296261655E-10
Non-optimal measurement 0.02429217825533039 < 0.02429217822421248. Total: 4
Armijo: th(4.175351397573621E-8)=0.02429217825533039; dx=-0.0582018166950045 evalInputDelta=-3.1117910331834864E-11
Non-optimal measurement 0.024292178226606162 < 0.02429217822421248. Total: 4
Armijo: th(3.211808767364324E-9)=0.024292178226606162; dx=-0.0582018166959863 evalInputDelta=-2.393682474455261E-12
Non-optimal measurement 0.024292178224383454 < 0.02429217822421248. Total: 4
Armijo: th(2.2941491195459456E-10)=0.024292178224383454; dx=-0.05820181669606228 evalInputDelta=-1.709743457922741E-13
Non-optimal measurement 0.024292178224223877 < 0.02429217822421248. Total: 4
Armijo: th(1.5294327463639638E-11)=0.024292178224223877; dx=-0.058201816696067735 evalInputDelta=-1.1397133237167623E-14
Non-optimal measurement 0.02429217822421319 < 0.02429217822421248. Total: 4
Armijo: th(9.558954664774774E-13)=0.02429217822421319; dx=-0.058201816696068095 evalInputDelta=-7.112366251504909E-16
Non-optimal measurement 0.024292178224212518 < 0.02429217822421248. Total: 4
Armijo: th(5.6229145086910434E-14)=0.024292178224212518; dx=-0.05820181669606812 evalInputDelta=-3.8163916471489756E-17
Non-optimal measurement 0.02429217822421248 < 0.02429217822421248. Total: 4
WOLFE (weak): th(3.1238413937172463E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
Non-optimal measurement 0.0242921782242125 < 0.02429217822421248. Total: 4
Armijo: th(2.967649324031384E-14)=0.0242921782242125; dx=-0.058201816696068116 evalInputDelta=-2.0816681711721685E-17
Non-optimal measurement 0.024292178224212486 < 0.02429217822421248. Total: 4
Armijo: th(1.6400167317015543E-14)=0.024292178224212486; dx=-0.05820181669606812 evalInputDelta=-6.938893903907228E-18
Non-optimal measurement 0.024292178224212486 < 0.02429217822421248. Total: 4
Armijo: th(9.762004355366395E-15)=0.024292178224212486; dx=-0.05820181669606812 evalInputDelta=-6.938893903907228E-18
Non-optimal measurement 0.02429217822421248 < 0.02429217822421248. Total: 4
WOLFE (weak): th(6.4429228745418206E-15)=0.02429217822421248; dx=-0.058201816696068116 evalInputDelta=0.0
Non-optimal measurement 0.02429217822421248 < 0.02429217822421248. Total: 4
WOLFE (weak): th(8.102463614954108E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
Non-optimal measurement 0.02429217822421248 < 0.02429217822421248. Total: 4
WOLFE (weak): th(8.932233985160251E-15)=0.02429217822421248; dx=-0.05820181669606812 evalInputDelta=0.0
Non-optimal measurement 0.02429217822421248 < 0.02429217822421248. Total: 4
mu >= nu (8.932233985160251E-15): th(0.0)=0.02429217822421248
Fitness changed from 0.02429217822421248 to 0.02429217822421248
Static Iteration Total: 0.0643; Orientation: 0.0199; Line Search: 0.0423
Iteration 3 failed. Error: 0.02429217822421248
Previous Error: 0.0 -> 0.02429217822421248
Optimization terminated 3
Final threshold in iteration 3: 0.02429217822421248 (> 0.0) after 0.089s (< 30.000s)
Returns
0.02429217822421248
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ -1.561710552906364, -0.11812229516748837, 2.048858951361061 ]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[ -0.3555070726747259, 2.9543825783514026, -0.9450422856421141 ]
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -1.6145335412532416], [2.0, -1.1115710842590303]; valueStats=DoubleSummaryStatistics{count=5, sum=0.273047, min=0.024292, average=0.054609, max=0.077344}
Plotting 2 points for GD
Plotting 2 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -1.6145335412532416], [0.034, -1.1115710842590303]; valueStats=DoubleSummaryStatistics{count=5, sum=0.273047, min=0.024292, average=0.054609, max=0.077344}
Plotting 2 points for GD
Plotting 2 points for CjGD
Plotting 2 points for LBFGS
Returns
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0042 = network_target.state();
assert temp_18_0042 != null;
String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0042.freeRef();
return temp_18_0041;
Returns
[0.3410053824151751, 0.7481782293830779, -0.49539870753101056, 0.07243099129442795, -0.6091525548308162, -0.24249229775602302, -0.9106156677046132, 0.005478915126374104, -0.8739821605502316]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.14 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2380995927524
Reset training subject: 2380998297889
Constructing line search parameters: GD
th(0)=11.196727398751673;dx=-5.09486895406369E22
New Minimum: 11.196727398751673 > 0.020913243463022566
Armijo: th(2.154434690031884)=0.020913243463022566; dx=-5.094868954064231E10 evalInputDelta=11.17581415528865
Armijo: th(1.077217345015942)=0.02804804100275779; dx=-5.09486895406451E10 evalInputDelta=11.168679357748914
Armijo: th(0.3590724483386473)=0.03527870402842889; dx=-5.0948689540649414E10 evalInputDelta=11.161448694723244
Armijo: th(0.08976811208466183)=0.03903472402093675; dx=-5.094868954065248E10 evalInputDelta=11.157692674730736
Armijo: th(0.017953622416932366)=0.04019278330820182; dx=-5.094868954065357E10 evalInputDelta=11.156534615443471
Armijo: th(0.002992270402822061)=0.04044421659768557; dx=-5.0948689540653824E10 evalInputDelta=11.156283182153986
Armijo: th(4.2746720040315154E-4)=0.040487692186557164; dx=-5.094868954065386E10 evalInputDelta=11.156239706565115
Armijo: th(5.343340005039394E-5)=0.04049404159902359; dx=-5.094868954065387E10 evalInputDelta=11.156233357152649
Armijo: th(5.9370444500437714E-6)=0.04049484804192981; dx=-5.094868954065387E10 evalInputDelta=11.156232550709744
Armijo: th(5.937044450043771E-7)=0.040494938769130906; dx=-5.094868954065387E10 evalInputDelta=11.156232459982542
Armijo: th(5.397313136403428E-8)=0.04049494793352136; dx=-5.094868954065387E10 evalInputDelta=11.156232450818152
Armijo: th(4.4977609470028565E-9)=0.04049494877359073; dx=-5.094868954065387E10 evalInputDelta=11.156232449978082
Armijo: th(3.4598161130791205E-10)=0.040494948844086054; dx=-5.094868954065387E10 evalInputDelta=11.156232449907586
Armijo: th(2.4712972236279432E-11)=0.405318413198418; dx=-8.939353730502051E19 evalInputDelta=10.791408985553254
Armijo: th(1.6475314824186289E-12)=11.196727398751644; dx=-5.09486895406369E22 evalInputDelta=2.8421709430404007E-14
Armijo: th(1.029707176511643E-13)=11.196727398751671; dx=-5.09486895406369E22 evalInputDelta=1.7763568394002505E-15
Armijo: th(6.057101038303783E-15)=11.196727398751673; dx=-5.09486895406369E22 evalInputDelta=0.0
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.020913243463022566
Fitness changed from 11.196727398751673 to 0.020913243463022566
Iteration 1 complete. Error: 0.020913243463022566 Total: 0.0468; Orientation: 0.0003; Line Search: 0.0410
th(0)=0.020913243463022566;dx=-0.05267744283570538
New Minimum: 0.020913243463022566 > 0.020913243463022563
WOLFE (weak): th(2.154434690031884E-15)=0.020913243463022563; dx=-0.05267744283570538 evalInputDelta=3.469446951953614E-18
New Minimum: 0.020913243463022563 > 0.020913243463022556
WOLFE (weak): th(4.308869380063768E-15)=0.020913243463022556; dx=-0.05267744283570538 evalInputDelta=1.0408340855860843E-17
New Minimum: 0.020913243463022556 > 0.020913243463022542
WOLFE (weak): th(1.2926608140191303E-14)=0.020913243463022542; dx=-0.05267744283570538 evalInputDelta=2.42861286636753E-17
New Minimum: 0.020913243463022542 > 0.020913243463022476
WOLFE (weak): th(5.1706432560765214E-14)=0.020913243463022476; dx=-0.05267744283570537 evalInputDelta=9.020562075079397E-17
New Minimum: 0.020913243463022476 > 0.020913243463022115
WOLFE (weak): th(2.5853216280382605E-13)=0.020913243463022115; dx=-0.052677442835705336 evalInputDelta=4.510281037539698E-16
New Minimum: 0.020913243463022115 > 0.020913243463019877
WOLFE (weak): th(1.5511929768229563E-12)=0.020913243463019877; dx=-0.05267744283570511 evalInputDelta=2.688821387764051E-15
New Minimum: 0.020913243463019877 > 0.020913243463003793
WOLFE (weak): th(1.0858350837760695E-11)=0.020913243463003793; dx=-0.05267744283570349 evalInputDelta=1.8773177457021006E-14
New Minimum: 0.020913243463003793 > 0.020913243462872395
WOLFE (weak): th(8.686680670208556E-11)=0.020913243462872395; dx=-0.05267744283569024 evalInputDelta=1.5017154186836024E-13
New Minimum: 0.020913243462872395 > 0.020913243461671026
WOLFE (weak): th(7.8180126031877E-10)=0.020913243461671026; dx=-0.052677442835569084 evalInputDelta=1.3515404073682902E-12
New Minimum: 0.020913243461671026 > 0.02091324344950715
WOLFE (weak): th(7.818012603187701E-9)=0.02091324344950715; dx=-0.052677442834342385 evalInputDelta=1.351541795147071E-11
New Minimum: 0.02091324344950715 > 0.020913243314353
WOLFE (weak): th(8.599813863506471E-8)=0.020913243314353; dx=-0.052677442820712454 evalInputDelta=1.4866956624115524E-10
New Minimum: 0.020913243314353 > 0.020913241678987872
WOLFE (weak): th(1.0319776636207765E-6)=0.020913241678987872; dx=-0.05267744265579026 evalInputDelta=1.7840346942799012E-9
New Minimum: 0.020913241678987872 > 0.020913220270586026
WOLFE (weak): th(1.3415709627070094E-5)=0.020913220270586026; dx=-0.052677440496811695 evalInputDelta=2.3192436540697692E-8
New Minimum: 0.020913220270586026 > 0.020912918771766438
WOLFE (weak): th(1.878199347789813E-4)=0.020912918771766438; dx=-0.05267741009176967 evalInputDelta=3.2469125612819005E-7
New Minimum: 0.020912918771766438 > 0.02090837373980865
WOLFE (weak): th(0.0028172990216847197)=0.02090837373980865; dx=-0.052676951806876246 evalInputDelta=4.869723213916932E-6
New Minimum: 0.02090837373980865 > 0.020835493410905723
WOLFE (weak): th(0.045076784346955515)=0.020835493410905723; dx=-0.052669619705244135 evalInputDelta=7.775005211684352E-5
New Minimum: 0.020835493410905723 > 0.01963718569611215
WOLFE (weak): th(0.7663053338982437)=0.01963718569611215; dx=-0.05255342883073551 evalInputDelta=0.0012760577669104177
New Minimum: 0.01963718569611215 > 0.005969849436955174
WOLFE (weak): th(13.793496010168386)=0.005969849436955174; dx=-0.051671703501592066 evalInputDelta=0.014943394026067394
New Minimum: 0.005969849436955174 > 0.0
WOLFE (weak): th(262.07642419319933)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(5241.528483863986)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(110072.09816114372)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
Armijo: th(2421586.1595451618)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
Armijo: th(1265829.1288531527)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
Armijo: th(687950.6135071482)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
Armijo: th(399011.355834146)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(254541.72699764484)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(326776.5414158954)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(362893.9486250207)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(380952.65222958336)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(389982.0040318647)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(394496.67993300536)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
WOLFE (weak): th(396754.01788357564)=0.0; dx=-0.05145907978610739 evalInputDelta=0.020913243463022566
mu ~= nu (396754.01788357564): th(262.07642419319933)=0.0
Fitness changed from 0.020913243463022566 to 0.0
Iteration 2 complete. Error: 0.0 Total: 0.0595; Orientation: 0.0003; Line Search: 0.0578
th(0)=0.0;dx=-0.05109937512626357
Armijo: th(857212.2631318228)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(428606.1315659114)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(142868.71052197047)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(35717.17763049262)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(7143.435526098523)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(1190.5725876830872)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(170.08179824044103)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(21.26022478005513)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(2.362247197783903)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(0.2362247197783903)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(0.021474974525308207)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(0.0017895812104423506)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(1.3766009311095006E-4)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(9.83286379363929E-6)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(6.55524252909286E-7)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(4.097026580683037E-8)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(2.410015635695904E-9)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(1.3388975753866133E-10)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(7.04682934414007E-12)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(3.523414672070035E-13)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Armijo: th(1.6778165105095405E-14)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
MIN ALPHA (7.626438684134275E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0355; Orientation: 0.0003; Line Search: 0.0337
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.142s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.09 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2381143113195
Reset training subject: 2381144412718
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=11.196727398751673}, derivative=-5.09486895406369E22}
New Minimum: 11.196727398751673 > 0.053920282748270876
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.053920282748270876}, derivative=-5.110641182056635E10}, evalInputDelta = -11.142807116003402
New Minimum: 0.053920282748270876 > 0.04049494883807498
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.04049494883807498}, derivative=-5.094868954065387E10}, evalInputDelta = -11.156232449913597
New Minimum: 0.04049494883807498 > 0.04049494876676089
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.04049494876676089}, derivative=-5.094868954065387E10}, evalInputDelta = -11.156232449984913
New Minimum: 0.04049494876676089 > 0.040494948267562204
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.040494948267562204}, derivative=-5.094868954065387E10}, evalInputDelta = -11.15623245048411
New Minimum: 0.040494948267562204 > 0.040494944773171844
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.040494944773171844}, derivative=-5.094868954065387E10}, evalInputDelta = -11.1562324539785
New Minimum: 0.040494944773171844 > 0.040494920312459215
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.040494920312459215}, derivative=-5.094868954065387E10}, evalInputDelta = -11.156232478439213
New Minimum: 0.040494920312459215 > 0.04049474908844814
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.04049474908844814}, derivative=-5.094868954065387E10}, evalInputDelta = -11.156232649663224
New Minimum: 0.04049474908844814 > 0.040493550568255335
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.040493550568255335}, derivative=-5.094868954065387E10}, evalInputDelta = -11.156233848183417
New Minimum: 0.040493550568255335 > 0.040485163272367314
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.040485163272367314}, derivative=-5.094868954065386E10}, evalInputDelta = -11.156242235479306
New Minimum: 0.040485163272367314 > 0.04042656682434535
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.04042656682434535}, derivative=-5.09486895406538E10}, evalInputDelta = -11.156300831927327
New Minimum: 0.04042656682434535 > 0.04002190594854728
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.04002190594854728}, derivative=-5.094868954065341E10}, evalInputDelta = -11.156705492803125
New Minimum: 0.04002190594854728 > 0.037428848036462604
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.037428848036462604}, derivative=-5.094868954065109E10}, evalInputDelta = -11.15929855071521
New Minimum: 0.037428848036462604 > 0.02569589446853036
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.02569589446853036}, derivative=-5.094868954064405E10}, evalInputDelta = -11.171031504283143
New Minimum: 0.02569589446853036 > 2.9501998084120073E-5
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=2.9501998084120073E-5}, derivative=-5.09486895406385E10}, evalInputDelta = -11.196697896753589
New Minimum: 2.9501998084120073E-5 > 0.0
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
0.0 <= 11.196727398751673
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.094868954063849E10}, evalInputDelta = -11.196727398751673
Right bracket at 1.0E10
Converged to right
Fitness changed from 11.196727398751673 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0476; Orientation: 0.0002; Line Search: 0.0435
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-0.05109937512626357}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0396; Orientation: 0.0003; Line Search: 0.0381
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.088s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.57 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2381234956300
Reset training subject: 2381236261953
Adding measurement 63ead833 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 11.196727398751673 < 11.196727398751673. Total: 1
th(0)=11.196727398751673;dx=-5.09486895406369E22
Adding measurement 49b1764c to history. Total: 1
New Minimum: 11.196727398751673 > 0.020913243463022566
Armijo: th(2.154434690031884)=0.020913243463022566; dx=-5.094868954064231E10 evalInputDelta=11.17581415528865
Non-optimal measurement 0.02804804100275779 < 0.020913243463022566. Total: 2
Armijo: th(1.077217345015942)=0.02804804100275779; dx=-5.09486895406451E10 evalInputDelta=11.168679357748914
Non-optimal measurement 0.03527870402842889 < 0.020913243463022566. Total: 2
Armijo: th(0.3590724483386473)=0.03527870402842889; dx=-5.0948689540649414E10 evalInputDelta=11.161448694723244
Non-optimal measurement 0.03903472402093675 < 0.020913243463022566. Total: 2
Armijo: th(0.08976811208466183)=0.03903472402093675; dx=-5.094868954065248E10 evalInputDelta=11.157692674730736
Non-optimal measurement 0.04019278330820182 < 0.020913243463022566. Total: 2
Armijo: th(0.017953622416932366)=0.04019278330820182; dx=-5.094868954065357E10 evalInputDelta=11.156534615443471
Non-optimal measurement 0.04044421659768557 < 0.020913243463022566. Total: 2
Armijo: th(0.002992270402822061)=0.04044421659768557; dx=-5.0948689540653824E10 evalInputDelta=11.156283182153986
Non-optimal measurement 0.040487692186557164 < 0.020913243463022566. Total: 2
Armijo: th(4.2746720040315154E-4)=0.040487692186557164; dx=-5.094868954065386E10 evalInputDelta=11.156239706565115
Non-optimal measurement 0.04049404159902359 < 0.020913243463022566. Total: 2
Armijo: th(5.343340005039394E-5)=0.04049404159902359; dx=-5.094868954065387E10 evalInputDelta=11.156233357152649
Non-optimal measurement 0.04049484804192981 < 0.020913243463022566. Total: 2
Armijo: th(5.9370444500437714E-6)=0.04049484804192981; dx=-5.094868954065387E10 evalInputDelta=11.156232550709744
Non-optimal measurement 0.040494938769130906 < 0.020913243463022566. Total: 2
Armijo: th(5.937044450043771E-7)=0.040494938769130906; dx=-5.094868954065387E10 evalInputDelta=11.156232459982542
Non-optimal measurement 0.04049494793352136 < 0.020913243463022566. Total: 2
Armijo: th(5.397313136403428E-8)=0.04049494793352136; dx=-5.094868954065387E10 evalInputDelta=11.156232450818152
Non-optimal measurement 0.04049494877359073 < 0.020913243463022566. Total: 2
Armijo: th(4.4977609470028565E-9)=0.04049494877359073; dx=-5.094868954065387E10 evalInputDelta=11.156232449978082
Non-optimal measurement 0.040494948844086054 < 0.020913243463022566. Total: 2
Armijo: th(3.4598161130791205E-10)=0.040494948844086054; dx=-5.094868954065387E10 evalInputDelta=11.156232449907586
Non-optimal measurement 0.405318413198418 < 0.020913243463022566. Total: 2
Armijo: th(2.4712972236279432E-11)=0.405318413198418; dx=-8.939353730502051E19 evalInputDelta=10.791408985553254
Non-optimal measurement 11.196727398751644 < 0.020913243463022566. Total: 2
Armijo: th(1.6475314824186289E-12)=11.196727398751644; dx=-5.09486895406369E22 evalInputDelta=2.8421709430404007E-14
Non-optimal measurement 11.196727398751671 < 0.020913243463022566. Total: 2
Armijo: th(1.029707176511643E-13)=11.196727398751671; dx=-5.09486895406369E22 evalInputDelta=1.7763568394002505E-15
Non-optimal measurement 11.196727398751673 < 0.020913243463022566. Total: 2
Armijo: th(6.057101038303783E-15)=11.196727398751673; dx=-5.09486895406369E22 evalInputDelta=0.0
Non-optimal measurement 0.020913243463022566 < 0.020913243463022566. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.020913243463022566
Fitness changed from 11.196727398751673 to 0.020913243463022566
Iteration 1 complete. Error: 0.020913243463022566 Total: 0.0305; Orientation: 0.0007; Line Search: 0.0264
Non-optimal measurement 0.020913243463022566 < 0.020913243463022566. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.020913243463022566 < 0.020913243463022566. Total: 2
th(0)=0.020913243463022566;dx=-0.05267744283570538
Adding measurement 54bd703d to history. Total: 2
New Minimum: 0.020913243463022566 > 0.020913243463022563
WOLFE (weak): th(2.154434690031884E-15)=0.020913243463022563; dx=-0.05267744283570538 evalInputDelta=3.469446951953614E-18
Adding measurement 5f75821b to history. Total: 3
New Minimum: 0.020913243463022563 > 0.020913243463022556
WOLFE (weak): th(4.308869380063768E-15)=0.020913243463022556; dx=-0.05267744283570538 evalInputDelta=1.0408340855860843E-17
Adding measurement 714fa41c to history. Total: 4
New Minimum: 0.020913243463022556 > 0.020913243463022542
WOLFE (weak): th(1.2926608140191303E-14)=0.020913243463022542; dx=-0.05267744283570538 evalInputDelta=2.42861286636753E-17
Adding measurement 42127453 to history. Total: 5
New Minimum: 0.020913243463022542 > 0.020913243463022476
WOLFE (weak): th(5.1706432560765214E-14)=0.020913243463022476; dx=-0.05267744283570537 evalInputDelta=9.020562075079397E-17
Adding measurement 27619187 to history. Total: 6
New Minimum: 0.020913243463022476 > 0.020913243463022115
WOLFE (weak): th(2.5853216280382605E-13)=0.020913243463022115; dx=-0.052677442835705336 evalInputDelta=4.510281037539698E-16
Adding measurement 25b5e978 to history. Total: 7
New Minimum: 0.020913243463022115 > 0.020913243463019877
WOLFE (weak): th(1.5511929768229563E-12)=0.020913243463019877; dx=-0.05267744283570511 evalInputDelta=2.688821387764051E-15
Adding measurement 320cb5cf to history. Total: 8
New Minimum: 0.020913243463019877 > 0.020913243463003793
WOLFE (weak): th(1.0858350837760695E-11)=0.020913243463003793; dx=-0.05267744283570349 evalInputDelta=1.8773177457021006E-14
Adding measurement 7585ba56 to history. Total: 9
New Minimum: 0.020913243463003793 > 0.020913243462872395
WOLFE (weak): th(8.686680670208556E-11)=0.020913243462872395; dx=-0.05267744283569024 evalInputDelta=1.5017154186836024E-13
Adding measurement 751db4da to history. Total: 10
New Minimum: 0.020913243462872395 > 0.020913243461671026
WOLFE (weak): th(7.8180126031877E-10)=0.020913243461671026; dx=-0.052677442835569084 evalInputDelta=1.3515404073682902E-12
Adding measurement b595ee0 to history. Total: 11
New Minimum: 0.020913243461671026 > 0.02091324344950715
WOLFE (weak): th(7.818012603187701E-9)=0.02091324344950715; dx=-0.052677442834342385 evalInputDelta=1.351541795147071E-11
Adding measurement 45397590 to history. Total: 12
New Minimum: 0.02091324344950715 > 0.020913243314353
WOLFE (weak): th(8.599813863506471E-8)=0.020913243314353; dx=-0.052677442820712454 evalInputDelta=1.4866956624115524E-10
Adding measurement 2636c718 to history. Total: 13
New Minimum: 0.020913243314353 > 0.020913241678987872
WOLFE (weak): th(1.0319776636207765E-6)=0.020913241678987872; dx=-0.05267744265579026 evalInputDelta=1.7840346942799012E-9
Adding measurement 48ec076f to history. Total: 14
New Minimum: 0.020913241678987872 > 0.020913220270586026
WOLFE (weak): th(1.3415709627070094E-5)=0.020913220270586026; dx=-0.052677440496811695 evalInputDelta=2.3192436540697692E-8
Adding measurement 1a495181 to history. Total: 15
New Minimum: 0.020913220270586026 > 0.020912918771766438
WOLFE (weak): th(1.878199347789813E-4)=0.020912918771766438; dx=-0.05267741009176967 evalInputDelta=3.2469125612819005E-7
Adding measurement 7df8af6 to history. Total: 16
New Minimum: 0.020912918771766438 > 0.02090837373980865
WOLFE (weak): th(0.0028172990216847197)=0.02090837373980865; dx=-0.052676951806876246 evalInputDelta=4.869723213916932E-6
Adding measurement 1a532f6a to history. Total: 17
New Minimum: 0.02090837373980865 > 0.020835493410905723
WOLFE (weak): th(0.045076784346955515)=0.020835493410905723; dx=-0.052669619705244135 evalInputDelta=7.775005211684352E-5
Adding measurement 546ac566 to history. Total: 18
New Minimum: 0.020835493410905723 > 0.01963718569611215
WOLFE (weak): th(0.7663053338982437)=0.01963718569611215; dx=-0.05255342883073551 evalInputDelta=0.0012760577669104177
Adding measurement 113a3010 to history. Total: 19
New Minimum: 0.01963718569611215 > 0.005969849436955174
WOLFE (weak): th(13.793496010168386)=0.00596984943695517
...skipping 6170 bytes...
20270586026, 0.020913241678987872, 0.020913243314353, 0.02091324344950715, 0.020913243461671026, 0.020913243462872395, 0.020913243463003793, 0.020913243463019877, 0.020913243463022115
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872, 0.020913243314353, 0.02091324344950715, 0.020913243461671026, 0.020913243462872395, 0.020913243463003793, 0.020913243463019877
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872, 0.020913243314353, 0.02091324344950715, 0.020913243461671026, 0.020913243462872395, 0.020913243463003793
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872, 0.020913243314353, 0.02091324344950715, 0.020913243461671026, 0.020913243462872395
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872, 0.020913243314353, 0.02091324344950715, 0.020913243461671026
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872, 0.020913243314353, 0.02091324344950715
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872, 0.020913243314353
Rejected: LBFGS Orientation magnitude: 3.244e+02, gradient 2.261e-01, dot -0.991; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026, 0.020913241678987872
Rejected: LBFGS Orientation magnitude: 3.263e+02, gradient 2.261e-01, dot -0.992; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438, 0.020913220270586026
Rejected: LBFGS Orientation magnitude: 1.244e+02, gradient 2.261e-01, dot -0.999; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865, 0.020912918771766438
Rejected: LBFGS Orientation magnitude: 1.368e+02, gradient 2.261e-01, dot -0.999; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723, 0.02090837373980865
Rejected: LBFGS Orientation magnitude: 1.465e+02, gradient 2.261e-01, dot -0.999; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.005969849436955174, 0.01963718569611215, 0.020835493410905723
LBFGS Accumulation History: 3 points
Removed measurement 1d542744 to history. Total: 20
Removed measurement 113a3010 to history. Total: 19
Removed measurement 546ac566 to history. Total: 18
Removed measurement 1a532f6a to history. Total: 17
Removed measurement 7df8af6 to history. Total: 16
Removed measurement 1a495181 to history. Total: 15
Removed measurement 48ec076f to history. Total: 14
Removed measurement 2636c718 to history. Total: 13
Removed measurement 45397590 to history. Total: 12
Removed measurement b595ee0 to history. Total: 11
Removed measurement 751db4da to history. Total: 10
Removed measurement 7585ba56 to history. Total: 9
Removed measurement 320cb5cf to history. Total: 8
Removed measurement 25b5e978 to history. Total: 7
Removed measurement 27619187 to history. Total: 6
Removed measurement 42127453 to history. Total: 5
Removed measurement 714fa41c to history. Total: 4
Removed measurement 5f75821b to history. Total: 3
Adding measurement 3da44543 to history. Total: 3
th(0)=0.0;dx=-0.05109937512626357
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(857212.2631318228)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(428606.1315659114)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(142868.71052197047)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(35717.17763049262)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7143.435526098523)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1190.5725876830872)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(170.08179824044103)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(21.26022478005513)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.362247197783903)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.2362247197783903)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.021474974525308207)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.0017895812104423506)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.3766009311095006E-4)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(9.83286379363929E-6)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.55524252909286E-7)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.097026580683037E-8)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.410015635695904E-9)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.3388975753866133E-10)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7.04682934414007E-12)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.523414672070035E-13)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.6778165105095405E-14)=0.0; dx=-0.05109937512626357 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
MIN ALPHA (7.626438684134275E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.4804; Orientation: 0.4531; Line Search: 0.0260
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.569s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -2.6795786066440654], [2.0, -0.6795786066440654]; valueStats=DoubleSummaryStatistics{count=2, sum=0.041826, min=0.020913, average=0.020913, max=0.020913}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -2.6795786066440654], [0.059, -0.6795786066440654]; valueStats=DoubleSummaryStatistics{count=2, sum=0.041826, min=0.020913, average=0.020913, max=0.020913}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0037 = temp_18_0035.state();
assert temp_18_0037 != null;
String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0037.freeRef();
return temp_18_0036;
Returns
[0.005478915126374104, 0.07243099129442795, 0.7481782293830779, -0.49539870753101056, -0.24249229775602302, -0.6091525548308162, 0.3410053824151751, -0.9106156677046132, -0.8739821605502316]
We simultaneously regress this target input:
TrainingTester.java:287 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
if (null != x)
RefUtil.freeRef(x);
return temp_18_0006;
}).map(x -> {
String temp_18_0007 = x.prettyPrint();
x.freeRef();
return temp_18_0007;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ -0.128, 0.7, 0.08 ]
[ 0.08, 0.7, -0.128 ]
[ -0.128, 0.08, 0.7 ]
[ 0.08, 0.7, -0.128 ]
[ 0.08, -0.128, 0.7 ]
Which produces the following output:
TrainingTester.java:308 executed in 0.00 seconds (0.000 gc):
return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
String temp_18_0008 = x.prettyPrint();
x.freeRef();
return temp_18_0008;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ 0.10985465112056991, -0.15506577825171206, -0.7509982291863901 ]
[ -0.044626806244824485, -0.13140497801335246, -0.49828082024958553 ]
[ 0.5288179387355328, -0.3823951376380838, -0.7282854547506735 ]
[ -0.044626806244824485, -0.13140497801335246, -0.49828082024958553 ]
[ 0.5148919068925777, -0.4349996708712812, -0.46794827632575753 ]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.10 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2381898891285
Reset training subject: 2381900230193
Constructing line search parameters: GD
th(0)=6.374832993649656;dx=-3.966220812162209E22
Armijo: th(2.154434690031884)=6.374832993649656; dx=5.448073859319616E33 evalInputDelta=0.0
Armijo: th(1.077217345015942)=6.374832993649656; dx=2.7240369296399764E33 evalInputDelta=0.0
Armijo: th(0.3590724483386473)=6.374832993649656; dx=9.08012309853551E32 evalInputDelta=0.0
Armijo: th(0.08976811208466183)=6.374832993649656; dx=2.270030774336411E32 evalInputDelta=0.0
Armijo: th(0.017953622416932366)=6.374832993649656; dx=4.540061545499844E31 evalInputDelta=0.0
Armijo: th(0.002992270402822061)=6.374832993649656; dx=7.5667692094479E30 evalInputDelta=0.0
Armijo: th(4.2746720040315154E-4)=6.374832993649656; dx=1.0809669959249501E30 evalInputDelta=0.0
Armijo: th(5.343340005039394E-5)=6.374832993649656; dx=1.3512083978618665E29 evalInputDelta=0.0
Armijo: th(5.9370444500437714E-6)=6.374832993649656; dx=1.501339138761352E28 evalInputDelta=0.0
Armijo: th(5.937044450043771E-7)=6.374832993649656; dx=1.5013034427740423E27 evalInputDelta=0.0
Armijo: th(5.397313136403428E-8)=6.374832993649656; dx=1.3644607460843875E26 evalInputDelta=0.0
Armijo: th(4.4977609470028565E-9)=6.374832993649656; dx=1.1334149193258407E25 evalInputDelta=0.0
Armijo: th(3.4598161130791205E-10)=6.374832993649656; dx=8.352463612153033E23 evalInputDelta=0.0
New Minimum: 6.374832993649656 > 0.19779507984871733
Armijo: th(2.4712972236279432E-11)=0.19779507984871733; dx=8.623010474629782E10 evalInputDelta=6.177037913800939
Armijo: th(1.6475314824186289E-12)=3.017809060661021; dx=-1.5846110869424282E22 evalInputDelta=3.3570239329886356
Armijo: th(1.029707176511643E-13)=3.3872944730170724; dx=-1.7705730211593838E22 evalInputDelta=2.987538520632584
Armijo: th(6.057101038303783E-15)=6.374832993649656; dx=-3.9646891094847984E22 evalInputDelta=0.0
MIN ALPHA (3.3650561323909904E-16): th(2.4712972236279432E-11)=0.19779507984871733
Fitness changed from 6.374832993649656 to 0.19779507984871733
Iteration 1 complete. Error: 0.19779507984871733 Total: 0.0297; Orientation: 0.0005; Line Search: 0.0259
th(0)=0.19779507984871733;dx=-6.38860370531756
New Minimum: 0.19779507984871733 > 0.1977950798487037
WOLFE (weak): th(2.154434690031884E-15)=0.1977950798487037; dx=-6.388603705316865 evalInputDelta=1.3627987627273797E-14
New Minimum: 0.1977950798487037 > 0.1977950798486899
WOLFE (weak): th(4.308869380063768E-15)=0.1977950798486899; dx=-6.388603705316163 evalInputDelta=2.7422508708241367E-14
New Minimum: 0.1977950798486899 > 0.19779507984863512
WOLFE (weak): th(1.2926608140191303E-14)=0.19779507984863512; dx=-6.388603705313379 evalInputDelta=8.221201497349284E-14
New Minimum: 0.19779507984863512 > 0.1977950798483883
WOLFE (weak): th(5.1706432560765214E-14)=0.1977950798483883; dx=-6.388603705300826 evalInputDelta=3.2904234892328077E-13
New Minimum: 0.1977950798483883 > 0.19779507984707254
WOLFE (weak): th(2.5853216280382605E-13)=0.19779507984707254; dx=-6.388603705233927 evalInputDelta=1.6447954109821694E-12
New Minimum: 0.19779507984707254 > 0.19779507983884884
WOLFE (weak): th(1.5511929768229563E-12)=0.19779507983884884; dx=-6.38860370481578 evalInputDelta=9.86849491013686E-12
New Minimum: 0.19779507983884884 > 0.19779507977963715
WOLFE (weak): th(1.0858350837760695E-11)=0.19779507977963715; dx=-6.3886037018051 evalInputDelta=6.908018601592403E-11
New Minimum: 0.19779507977963715 > 0.19779507929607668
WOLFE (weak): th(8.686680670208556E-11)=0.19779507929607668; dx=-6.38860367721793 evalInputDelta=5.526406554601238E-10
New Minimum: 0.19779507929607668 > 0.19779507487495274
WOLFE (weak): th(7.8180126031877E-10)=0.19779507487495274; dx=-6.388603452420989 evalInputDelta=4.97376459462906E-9
New Minimum: 0.19779507487495274 > 0.1977950301110773
WOLFE (weak): th(7.818012603187701E-9)=0.1977950301110773; dx=-6.388601176352678 evalInputDelta=4.973764003435299E-8
New Minimum: 0.1977950301110773 > 0.19779453273576478
WOLFE (weak): th(8.599813863506471E-8)=0.19779453273576478; dx=-6.388575886823915 evalInputDelta=5.471129525536078E-7
New Minimum: 0.19779453273576478 > 0.19778851465117736
WOLFE (weak): th(1.0319776636207765E-6)=0.19778851465117736; dx=-6.38826990081483 evalInputDelta=6.565197539970535E-6
New Minimum: 0.19778851465117736 > 0.19770975913731903
WOLFE (weak): th(1.3415709627070094E-5)=0.19770975913731903; dx=-6.384267209295036 evalInputDelta=8.532071139830122E-5
New Minimum: 0.19770975913731903 > 0.19660584902770786
WOLFE (weak): th(1.878199347789813E-4)=0.19660584902770786; dx=-6.328470858810967 evalInputDelta=0.0011892308210094682
New Minimum: 0.19660584902770786 > 0.18103626478617815
END: th(0.0028172990216847197)=0.18103626478617815; dx=-5.5997116695034705 evalInputDelta=0.01675881506253918
Fitness changed from 0.19779507984871733 to 0.18103626478617815
Iteration 2 complete. Error: 0.18103626478617815 Total: 0.0258; Orientation: 0.0003; Line Search: 0.0247
th(0)=0.18103626478617815;dx=-4.910649776878732
New Minimum: 0.18103626478617815 > 0.15430438835162769
END: th(0.0060696867445104484)=0.15430438835162769; dx=-4.013068098821553 evalInputDelta=0.026731876434550467
Fitness changed from 0.18103626478617815 to 0.15430438835162769
Iteration 3 complete. Error: 0.15430438835162769 Total: 0.0043; Orientation: 0.0004; Line Search: 0.0028
th(0)=0.15430438835162769;dx=-3.284324236930081
New Minimum: 0.15430438835162769 > 0.11851609478095074
END: th(0.013076743680000003)=0.11851609478095074; dx=-2.5319497411025953 evalInputDelta=0.03578829357067695
Fitness changed from 0.15430438835162769 to 0.11851609478095074
Iteration 4 complete. Error: 0.11851609478095074 Total: 0.0039; Orientation: 0.0003; Line Search: 0.0027
th(0)=0.11851609478095074;dx=-1.9589543761934252
New Minimum: 0.11851609478095074 > 0.0817044755637713
END: th(0.028172990216847202)=0.0817044755637713; dx=-1.4883048965554468 evalInputDelta=0.03681161921717943
Fitness changed from 0.11851609478095074 to 0.0817044755637713
Iteration 5 complete. Error: 0.0817044755637713 Total: 0.0040; Orientation: 0.0003; Line Search: 0.0027
th(0)=0.0817044755637713;dx=-1.1414728196596748
New Minimum: 0.0817044755637713 > 0.04145683316831847
END: th(0.060696867445104496)=0.04145683316831847; dx=-0.8807340178728184 evalInputDelta=0.04024764239545284
Fitness changed from 0.0817044755637713 to 0.04145683316831847
Iteration 6 complete. Error: 0.04145683316831847 Total: 0.0042; Orientation: 0.0004; Line Search: 0.0028
th(0)=0.04145683316831847;dx=-0.690096464863779
New Minimum: 0.04145683316831847 > 0.0
END: th(0.13076743680000005)=0.0; dx=-0.558746647920557 evalInputDelta=0.04145683316831847
Fitness changed from 0.04145683316831847 to 0.0
Iteration 7 complete. Error: 0.0 Total: 0.0040; Orientation: 0.0003; Line Search: 0.0027
th(0)=0.0;dx=-0.4610080154464221
Armijo: th(0.28172990216847205)=0.0; dx=-0.46500577171541424 evalInputDelta=0.0
Armijo: th(0.14086495108423602)=0.0; dx=-0.4630068935809182 evalInputDelta=0.0
Armijo: th(0.04695498369474534)=0.0; dx=-0.46167430815792077 evalInputDelta=0.0
Armijo: th(0.011738745923686336)=0.0; dx=-0.4611745886242968 evalInputDelta=0.0
Armijo: th(0.0023477491847372673)=0.0; dx=-0.46104133008199705 evalInputDelta=0.0
Armijo: th(3.912915307895445E-4)=0.0; dx=-0.4610135678856846 evalInputDelta=0.0
Armijo: th(5.589879011279207E-5)=0.0; dx=-0.46100880865203103 evalInputDelta=0.0
Armijo: th(6.987348764099009E-6)=0.0; dx=-0.4610081145971232 evalInputDelta=0.0
Armijo: th(7.763720848998899E-7)=0.0; dx=-0.46100802646316674 evalInputDelta=0.0
Armijo: th(7.763720848998898E-8)=0.0; dx=-0.4610080165480966 evalInputDelta=0.0
Armijo: th(7.057928044544453E-9)=0.0; dx=-0.4610080155465744 evalInputDelta=0.0
Armijo: th(5.881606703787044E-10)=0.0; dx=-0.4610080154547682 evalInputDelta=0.0
Armijo: th(4.5243128490669574E-11)=0.0; dx=-0.46100801544706416 evalInputDelta=0.0
Armijo: th(3.2316520350478265E-12)=0.0; dx=-0.4610080154464681 evalInputDelta=0.0
Armijo: th(2.1544346900318843E-13)=0.0; dx=-0.4610080154464252 evalInputDelta=0.0
Armijo: th(1.3465216812699277E-14)=0.0; dx=-0.46100801544642234 evalInputDelta=0.0
MIN ALPHA (7.920715772176045E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0265; Orientation: 0.0003; Line Search: 0.0251
Iteration 8 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 8
Final threshold in iteration 8: 0.0 (> 0.0) after 0.103s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.01 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2382005174152
Reset training subject: 2382006299452
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=6.374832993649656}, derivative=-3.966220812162209E22}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=6.374832993649656}, derivative=2.132149797813749E23}, evalInputDelta = 0.0
6.374832993649656 <= 6.374832993649656
Converged to right
Fitness changed from 6.374832993649656 to 6.374832993649656
Static Iteration Total: 0.0064; Orientation: 0.0003; Line Search: 0.0030
Iteration 1 failed. Error: 6.374832993649656
Previous Error: 0.0 -> 6.374832993649656
Optimization terminated 1
Final threshold in iteration 1: 6.374832993649656 (> 0.0) after 0.007s (< 30.000s)
Returns
6.374832993649656
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
[-0.49539870753101056, -0.8739821605502316, 0.005478915126374104, -0.6091525548308162, 0.07243099129442795, 0.7481782293830779, -0.24249229775602302, -0.9106156677046132, 0.3410053824151751]
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ -0.128, 0.08, 0.7 ]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[ -0.002672297691587313, 0.6074907668900532, 0.19689352838702445 ]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.98 seconds (0.124 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2382018173122
Reset training subject: 2382019374795
Adding measurement 747f0ede to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
th(0)=6.374832993649656;dx=-3.966220812162209E22
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(2.154434690031884)=6.374832993649656; dx=5.448073859319617E33 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(1.077217345015942)=6.374832993649656; dx=2.7240369296399764E33 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(0.3590724483386473)=6.374832993649656; dx=9.080123098535508E32 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(0.08976811208466183)=6.374832993649656; dx=2.2700307743364106E32 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(0.017953622416932366)=6.374832993649656; dx=4.540061545499844E31 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(0.002992270402822061)=6.374832993649656; dx=7.5667692094479E30 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(4.2746720040315154E-4)=6.374832993649656; dx=1.0809669959249501E30 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(5.343340005039394E-5)=6.374832993649656; dx=1.3512083978618665E29 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(5.9370444500437714E-6)=6.374832993649656; dx=1.501339138761352E28 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(5.937044450043771E-7)=6.374832993649656; dx=1.5013034427740423E27 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(5.397313136403428E-8)=6.374832993649656; dx=1.3644607460843875E26 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(4.4977609470028565E-9)=6.374832993649656; dx=1.1334149193258407E25 evalInputDelta=0.0
Non-optimal measurement 6.374832993649656 < 6.374832993649656. Total: 1
Armijo: th(3.4598161130791205E-10)=6.374832993649656; dx=8.352463612153033E23 evalInputDelta=0.0
Adding measurement 3faa9adc to history. Total: 1
New Minimum: 6.374832993649656 > 0.19779507984871733
Armijo: th(2.4712972236279432E-11)=0.19779507984871733; dx=8.623010474629782E10 evalInputDelta=6.177037913800939
Non-optimal measurement 3.017809060661021 < 0.19779507984871733. Total: 2
Armijo: th(1.6475314824186289E-12)=3.017809060661021; dx=-1.5846110869424282E22 evalInputDelta=3.3570239329886356
Non-optimal measurement 3.3872944730170724 < 0.19779507984871733. Total: 2
Armijo: th(1.029707176511643E-13)=3.3872944730170724; dx=-1.7705730211593838E22 evalInputDelta=2.987538520632584
Non-optimal measurement 6.374832993649656 < 0.19779507984871733. Total: 2
Armijo: th(6.057101038303783E-15)=6.374832993649656; dx=-3.9646891094847984E22 evalInputDelta=0.0
Non-optimal measurement 0.19779507984871733 < 0.19779507984871733. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.4712972236279432E-11)=0.19779507984871733
Fitness changed from 6.374832993649656 to 0.19779507984871733
Iteration 1 complete. Error: 0.19779507984871733 Total: 0.0344; Orientation: 0.0008; Line Search: 0.0305
Non-optimal measurement 0.19779507984871733 < 0.19779507984871733. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.19779507984871733 < 0.19779507984871733. Total: 2
th(0)=0.19779507984871733;dx=-6.38860370531756
Adding measurement 380c7969 to history. Total: 2
New Minimum: 0.19779507984871733 > 0.1977950798487037
WOLFE (weak): th(2.154434690031884E-15)=0.1977950798487037; dx=-6.388603705316865 evalInputDelta=1.3627987627273797E-14
Adding measurement 251e9325 to history. Total: 3
New Minimum: 0.1977950798487037 > 0.1977950798486899
WOLFE (weak): th(4.308869380063768E-15)=0.1977950798486899; dx=-6.388603705316163 evalInputDelta=2.7422508708241367E-14
Adding measurement 15475160 to history. Total: 4
New Minimum: 0.1977950798486899 > 0.19779507984863512
WOLFE (weak): th(1.2926608140191303E-14)=0.19779507984863512; dx=-6.388603705313379 evalInputDelta=8.221201497349284E-14
Adding measurement 43a914b9 to history. Total: 5
New Minimum: 0.19779507984863512 > 0.1977950798483883
WOLFE (weak): th(5.1706432560765214E-14)=0.1977950798483883; dx=-6.388603705300826 evalInputDelta=3.2904234892328077E-13
Adding measurement 78a5ec97 to history. Total: 6
New Minimum: 0.1977950798483883 > 0.19779507984707254
WOLFE (weak): th(2.5853216280382605E-13)=0.19779507984707254; dx=-6.388603705233927 evalInputDelta=1.6447954109821694E-12
Adding measurement 48533335 to history. Total: 7
New Minimum: 0.19779507984707254 > 0.19779507983884884
WOLFE (weak): th(1.5511929768229563E-12)=0.19779507983884884; dx=-6.38860370481578 evalInputDelta=9.86849491013686E-12
Adding measurement 5558b6db to history. Total: 8
New Minimum: 0.19779507983884884 > 0.19779507977963715
WOLFE (weak): th(1.0858350837760695E-11)=0.19779507977963715; dx=-6.3886037018051 evalInputDelta=6.908018601592403E-11
Adding measurement 6218e6fa to history. Total: 9
New Minimum: 0.19779507977963715 > 0.19779507929607668
WOLFE (weak): th(8.686680670208556E-11)=0.19779507929607668; dx=-6.38860367721793 evalInputDelta=5.526406554601238E-10
Adding measurement fc2ce3 to history. Total: 10
New Minimum: 0.19779507929607668 > 0.19779507487495274
WOLFE (weak): th(7.8180126031877E-10)=0.19779507487495274; dx=-6.38860345242099 evalInputDelta=4.97376459462906E-9
Adding measurement 33b6597c to history. Total: 11
New Minimum: 0.19779507487495274 > 0.1977950301110773
WOLFE (weak): th(7.818012603187701E-9)=0.1977950301110773; dx=-6.388601176352678 evalInputDelta=4.973764003435299E-8
Adding measurement 758f41c5 to history. Total: 12
New Minimum: 0.1977950301110773 > 0.19779453273576478
WOLFE (weak): th(8.599813863506471E-8)=0.19779453273576478; dx=-6.388575886823915 evalInputDelta=5.471129525536078E-7
Adding measurement 24e9e41 to history. Total: 13
New Minimum: 0.19779453273576478 > 0.19778851465117736
WOLFE (weak): th(1.0319776636207765E-6)=0.19778851465117736; dx=-6.38826990081483 evalInputDelta=6.565197539970535E-6
Adding measurement 1dd0d0e3 to history. Total: 14
New Minimum: 0.19778851465117736 > 0.19770975913731903
WOLFE (weak): th(1.3415709627070094E-5)=0.19770975913731903; dx=-6.384267209295036 evalInputDelta=8.532071139830122E-5
Adding measurement 4d645847 to history. Total: 15
New Minimum: 0.19770975913731903 > 0.19660584902770786
WOLFE (weak): th(1.878199347789813E-4)=0.19660584902770786; dx=-6.328470858810967 evalInputDelta=0.0011892308210094682
Adding measurement 6dc1b4e1 to history. Total: 16
New Minimum: 0.19660584902770786 > 0.18103626478617815
END: th(0.0028172990216847197)=0.18103626478617815; dx=-5.5997116695034705 evalInputDelta=0.01675881506253918
Fitness changed from 0.19779507984871733 to 0.18103626478617815
Iteration 2 complete. Error: 0.18103626478617815 Total: 0.0377; Orientation: 0.0008; Line Search: 0.0356
Non-optimal measurement 0.18103626478617815 < 0.18103626478617815. Total: 17
Rejected: LBFGS Orientation magnitude: 4.191e-02, gradient 2.216e+00, dot -0.984; [6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00, 7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, 63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.18103626478617815, 0.19660584902770786, 0.19770975913731903, 0.19778851465117736, 0.19779453273576478, 0.1977950301110773, 0.19779507487495274, 0.19779507929607668, 0.19779507977963715, 0.19779507983884884, 0.19779507984707254, 0.1977950798483883, 0.19779507984863512, 0.1977950798486899, 0.1977950798487037, 0.19779507984871733, 6.3
...skipping 12449 bytes...
6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, 7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00]
Orientation rejected. Popping history element from 0.11851609478095074, 0.15430438835162769, 0.1977950798487037, 0.19779507984871733
LBFGS Accumulation History: 3 points
Removed measurement dac111f to history. Total: 4
Removed measurement 31f1d7b to history. Total: 3
Adding measurement 1f8bc24 to history. Total: 3
th(0)=0.11851609478095074;dx=-1.9589543761934252
Adding measurement 3b97101d to history. Total: 4
New Minimum: 0.11851609478095074 > 0.0817044755637713
END: th(0.028172990216847202)=0.0817044755637713; dx=-1.4883048965554466 evalInputDelta=0.03681161921717943
Fitness changed from 0.11851609478095074 to 0.0817044755637713
Iteration 5 complete. Error: 0.0817044755637713 Total: 0.0328; Orientation: 0.0282; Line Search: 0.0032
Non-optimal measurement 0.0817044755637713 < 0.0817044755637713. Total: 5
Rejected: LBFGS Orientation magnitude: 1.982e-02, gradient 1.068e+00, dot -0.966; [63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00, 6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, 7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0817044755637713, 0.11851609478095074, 0.1977950798487037, 0.19779507984871733, 6.374832993649656
Rejected: LBFGS Orientation magnitude: 2.355e-02, gradient 1.068e+00, dot -0.988; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00, 6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0817044755637713, 0.11851609478095074, 0.1977950798487037, 0.19779507984871733
LBFGS Accumulation History: 3 points
Removed measurement 3b97101d to history. Total: 4
Removed measurement 1f8bc24 to history. Total: 3
Adding measurement 5f2a2e5a to history. Total: 3
th(0)=0.0817044755637713;dx=-1.1414728196596748
Adding measurement c92486c to history. Total: 4
New Minimum: 0.0817044755637713 > 0.04145683316831847
END: th(0.060696867445104496)=0.04145683316831847; dx=-0.8807340178728185 evalInputDelta=0.04024764239545284
Fitness changed from 0.0817044755637713 to 0.04145683316831847
Iteration 6 complete. Error: 0.04145683316831847 Total: 0.0277; Orientation: 0.0226; Line Search: 0.0039
Non-optimal measurement 0.04145683316831847 < 0.04145683316831847. Total: 5
Rejected: LBFGS Orientation magnitude: 1.540e-02, gradient 8.307e-01, dot -0.970; [1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, 7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, 63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00, 6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.04145683316831847, 0.0817044755637713, 0.1977950798487037, 0.19779507984871733, 6.374832993649656
Rejected: LBFGS Orientation magnitude: 2.052e-02, gradient 8.307e-01, dot -0.973; [7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, 6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.04145683316831847, 0.0817044755637713, 0.1977950798487037, 0.19779507984871733
LBFGS Accumulation History: 3 points
Removed measurement c92486c to history. Total: 4
Removed measurement 5f2a2e5a to history. Total: 3
Adding measurement 24e822af to history. Total: 3
th(0)=0.04145683316831847;dx=-0.6900964648637788
Adding measurement 2b00732 to history. Total: 4
New Minimum: 0.04145683316831847 > 0.0
END: th(0.13076743680000005)=0.0; dx=-0.558746647920557 evalInputDelta=0.04145683316831847
Fitness changed from 0.04145683316831847 to 0.0
Iteration 7 complete. Error: 0.0 Total: 0.0277; Orientation: 0.0223; Line Search: 0.0043
Non-optimal measurement 0.0 < 0.0. Total: 5
Rejected: LBFGS Orientation magnitude: 1.878e-02, gradient 6.790e-01, dot -0.955; [63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, 6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00, 7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00]
Orientation rejected. Popping history element from 0.0, 0.04145683316831847, 0.1977950798487037, 0.19779507984871733, 6.374832993649656
Rejected: LBFGS Orientation magnitude: 2.776e-02, gradient 6.790e-01, dot -0.879; [ccfc5d52-8aea-4867-9b51-33b5a7b42ed2 = 0.000e+00, 7c4d32f8-ff13-49ae-9628-19c3e11288e3 = 1.000/1.000e+00, 5ea79989-4f8d-44cc-8c09-e667c7acfc95 = 1.000/1.000e+00, 6a6f6dfa-536e-473f-b665-f0f6e2de902e = 1.000/1.000e+00, 63fb413c-0e73-4115-963a-bdaeffd809c3 = 1.000/1.000e+00, 1580d810-902f-4dbd-8112-59a79749b651 = 0.000e+00]
Orientation rejected. Popping history element from 0.0, 0.04145683316831847, 0.1977950798487037, 0.19779507984871733
LBFGS Accumulation History: 3 points
Removed measurement 2b00732 to history. Total: 4
Removed measurement 24e822af to history. Total: 3
Adding measurement 3627d4fc to history. Total: 3
th(0)=0.0;dx=-0.4610080154464221
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.28172990216847205)=0.0; dx=-0.46500577171541424 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.14086495108423602)=0.0; dx=-0.4630068935809182 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.04695498369474534)=0.0; dx=-0.46167430815792077 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.011738745923686336)=0.0; dx=-0.4611745886242968 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.0023477491847372673)=0.0; dx=-0.46104133008199705 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.912915307895445E-4)=0.0; dx=-0.4610135678856846 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(5.589879011279207E-5)=0.0; dx=-0.46100880865203103 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.987348764099009E-6)=0.0; dx=-0.46100811459712315 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7.763720848998899E-7)=0.0; dx=-0.46100802646316674 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7.763720848998898E-8)=0.0; dx=-0.4610080165480966 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7.057928044544453E-9)=0.0; dx=-0.4610080155465744 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(5.881606703787044E-10)=0.0; dx=-0.4610080154547682 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.5243128490669574E-11)=0.0; dx=-0.46100801544706416 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.2316520350478265E-12)=0.0; dx=-0.461008015446468 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.1544346900318843E-13)=0.0; dx=-0.4610080154464253 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.3465216812699277E-14)=0.0; dx=-0.46100801544642234 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
MIN ALPHA (7.920715772176045E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0662; Orientation: 0.0245; Line Search: 0.0406
Iteration 8 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 8
Final threshold in iteration 8: 0.0 (> 0.0) after 0.976s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -1.382403876145997], [7.0, -0.7037845156767987]; valueStats=DoubleSummaryStatistics{count=12, sum=1.549626, min=0.041457, average=0.129136, max=0.197795}
Plotting 7 points for GD
Plotting 7 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -1.382403876145997], [0.875, -0.7037845156767987]; valueStats=DoubleSummaryStatistics{count=12, sum=1.549626, min=0.041457, average=0.129136, max=0.197795}
Plotting 7 points for GD
Only 0 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.02429217822421248 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "NonConverged", "value": 0.02429217822421248 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "Converged", "value": 0.0 } }}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.02429217822421248 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "NonConverged", "value": 0.02429217822421248 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "Converged", "value": 0.0 } }} | OK |
{
"result": "OK",
"performance": {
"execution_time": "3.485",
"gc_time": "0.349"
},
"created_on": 1586736967900,
"file_name": "trainingTest",
"report": {
"simpleName": "Basic",
"canonicalName": "com.simiacryptus.mindseye.layers.java.FullyConnectedLayerTest.Basic",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/FullyConnectedLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 0.02429217822421248
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "NonConverged",
"value": 0.02429217822421248
}
},
"model": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"complete": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "NonConverged",
"value": "NaN"
},
"GD": {
"type": "Converged",
"value": 0.0
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/FullyConnectedLayer/Basic/trainingTest/202004131607",
"id": "28a3ddb3-ba51-4d13-9d13-19253f71d345",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "FullyConnectedLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.FullyConnectedLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/FullyConnectedLayer.java",
"javaDoc": ""
}
}