Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 5736978137337571328
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ 1.764, 1.208, -0.608, 0.048, 1.524 ]
[ 1.764, -0.608, 1.208, 0.048, 1.524 ]
[ -0.608, 1.524, 1.764, 0.048, 1.208 ]
[ 1.208, 1.764, -0.608, 0.048, 1.524 ]
[ 1.208, -0.608, 1.764, 1.524, 0.048 ]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.32 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 776605059774
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 776641031601
Constructing line search parameters: GD
th(0)=32.42178570862366;dx=-4.3659328E23
New Minimum: 32.42178570862366 > 0.0027671947991399753
Armijo: th(2.154434690031884)=0.0027671947991399753; dx=-4.365932800026625E11 evalInputDelta=32.41901851382452
Armijo: th(1.077217345015942)=0.029938429803399995; dx=-4.365932800026688E11 evalInputDelta=32.39184727882026
Armijo: th(0.3590724483386473)=0.3749259829030395; dx=-4.365932800029505E11 evalInputDelta=32.04685972572062
Armijo: th(0.08976811208466183)=0.8879710987390272; dx=-4.3659328000351575E11 evalInputDelta=31.533814609884633
Armijo: th(0.017953622416932366)=1.0548870652112325; dx=-4.3659328000374005E11 evalInputDelta=31.366898643412426
Armijo: th(0.002992270402822061)=1.091785040706696; dx=-4.365932800037927E11 evalInputDelta=31.330000667916963
Armijo: th(4.2746720040315154E-4)=1.0981903125413262; dx=-4.36593280003802E11 evalInputDelta=31.323595396082332
Armijo: th(5.343340005039394E-5)=1.0991264029668573; dx=-4.3659328000380334E11 evalInputDelta=31.322659305656803
Armijo: th(5.9370444500437714E-6)=1.0992453079130375; dx=-4.3659328000380347E11 evalInputDelta=31.322540400710622
Armijo: th(5.937044450043771E-7)=1.0992586852318968; dx=-4.365932800038035E11 evalInputDelta=31.322527023391764
Armijo: th(5.397313136403428E-8)=1.0992600364819893; dx=-4.365932800038035E11 evalInputDelta=31.32252567214167
Armijo: th(4.4977609470028565E-9)=1.0992601603466337; dx=-4.365932800038035E11 evalInputDelta=31.322525548277024
Armijo: th(3.4598161130791205E-10)=1.0992601707408702; dx=-4.365932800038035E11 evalInputDelta=31.32252553788279
Armijo: th(2.4712972236279432E-11)=1.099260171545186; dx=-4.365932800038035E11 evalInputDelta=31.322525537078473
Armijo: th(1.6475314824186289E-12)=1.8512313167384051; dx=-8.397132670926688E11 evalInputDelta=30.570554391885253
Armijo: th(1.029707176511643E-13)=32.4217857086234; dx=-4.3659328E23 evalInputDelta=2.5579538487363607E-13
Armijo: th(6.057101038303783E-15)=32.42178570862364; dx=-4.3659328E23 evalInputDelta=2.1316282072803006E-14
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0027671947991399753
Fitness changed from 32.42178570862366 to 0.0027671947991399753
Iteration 1 complete. Error: 0.0027671947991399753 Total: 0.1514; Orientation: 0.0034; Line Search: 0.1039
th(0)=0.0027671947991399753;dx=-2.478128119332879
New Minimum: 0.0027671947991399753 > 0.00276719479913995
WOLFE (weak): th(2.154434690031884E-15)=0.00276719479913995; dx=-2.478128119332879 evalInputDelta=2.5153490401663703E-17
New Minimum: 0.00276719479913995 > 0.0027671947991399137
WOLFE (weak): th(4.308869380063768E-15)=0.0027671947991399137; dx=-2.478128119332879 evalInputDelta=6.158268339717665E-17
New Minimum: 0.0027671947991399137 > 0.002767194799139802
WOLFE (weak): th(1.2926608140191303E-14)=0.002767194799139802; dx=-2.478128119332879 evalInputDelta=1.734723475976807E-16
New Minimum: 0.002767194799139802 > 0.002767194799139331
WOLFE (weak): th(5.1706432560765214E-14)=0.002767194799139331; dx=-2.478128119332879 evalInputDelta=6.444497713253838E-16
New Minimum: 0.002767194799139331 > 0.002767194799136754
WOLFE (weak): th(2.5853216280382605E-13)=0.002767194799136754; dx=-2.4781281193328786 evalInputDelta=3.2213814948889308E-15
New Minimum: 0.002767194799136754 > 0.002767194799120646
WOLFE (weak): th(1.5511929768229563E-12)=0.002767194799120646; dx=-2.478128119332877 evalInputDelta=1.9329156331071573E-14
New Minimum: 0.002767194799120646 > 0.002767194799004722
WOLFE (weak): th(1.0858350837760695E-11)=0.002767194799004722; dx=-2.4781281193328635 evalInputDelta=1.352533536558287E-13
New Minimum: 0.002767194799004722 > 0.002767194798058022
WOLFE (weak): th(8.686680670208556E-11)=0.002767194798058022; dx=-2.478128119332755 evalInputDelta=1.0819531034989005E-12
New Minimum: 0.002767194798058022 > 0.0027671947894025188
WOLFE (weak): th(7.8180126031877E-10)=0.0027671947894025188; dx=-2.4781281193317644 evalInputDelta=9.737456500846786E-12
New Minimum: 0.0027671947894025188 > 0.002767194701765385
WOLFE (weak): th(7.818012603187701E-9)=0.002767194701765385; dx=-2.478128119321732 evalInputDelta=9.737459016195826E-11
New Minimum: 0.002767194701765385 > 0.0027671937280194544
WOLFE (weak): th(8.599813863506471E-8)=0.0027671937280194544; dx=-2.4781281192102598 evalInputDelta=1.0711205208381591E-9
New Minimum: 0.0027671937280194544 > 0.002767181945694497
WOLFE (weak): th(1.0319776636207765E-6)=0.002767181945694497; dx=-2.478128117861448 evalInputDelta=1.2853445478105963E-8
New Minimum: 0.002767181945694497 > 0.002767027704467303
WOLFE (weak): th(1.3415709627070094E-5)=0.002767027704467303; dx=-2.478128100204302 evalInputDelta=1.6709467267218142E-7
New Minimum: 0.002767027704467303 > 0.0027648554970751784
WOLFE (weak): th(1.878199347789813E-4)=0.0027648554970751784; dx=-2.478127851538148 evalInputDelta=2.3393020647968603E-6
New Minimum: 0.0027648554970751784 > 0.0027321105482755762
WOLFE (weak): th(0.0028172990216847197)=0.0027321105482755762; dx=-2.4781241036206856 evalInputDelta=3.508425086439903E-5
New Minimum: 0.0027321105482755762 > 0.0022071998938567377
WOLFE (weak): th(0.045076784346955515)=0.0022071998938567377; dx=-2.4780641771750553 evalInputDelta=5.599949052832376E-4
New Minimum: 0.0022071998938567377 > 0.0
WOLFE (weak): th(0.7663053338982437)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(13.793496010168386)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(262.07642419319933)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Armijo: th(5241.528483863986)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Armijo: th(2751.802454028593)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Armijo: th(1506.939439110896)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(884.5079316520477)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Armijo: th(1195.7236853814718)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(1040.1158085167597)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Armijo: th(1117.9197469491157)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(1079.0177777329377)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(1098.4687623410268)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
WOLFE (weak): th(1108.1942546450714)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
mu ~= nu (1108.1942546450714): th(0.7663053338982437)=0.0
Fitness changed from 0.0027671947991399753 to 0.0
Iteration 2 complete. Error: 0.0 Total: 0.1124; Orientation: 0.0011; Line Search: 0.1079
th(0)=0.0;dx=-2.4775104
Armijo: th(2398.0086145001046)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(1199.0043072500523)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(399.6681024166841)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(99.91702560417103)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(19.983405120834206)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(3.3305675201390343)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(0.47579536001986206)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(0.05947442000248276)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(0.006608268889164751)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(6.608268889164751E-4)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(6.0075171719679556E-5)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(5.006264309973297E-6)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(3.850972546133305E-7)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(2.7506946758095036E-8)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(1.833796450539669E-9)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(1.1461227815872932E-10)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(6.741898715219372E-12)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(3.7454992862329843E-13)=0.0; dx=-2.4775104 evalInputDelta=0.0
Armijo: th(1.9713154138068337E-14)=0.0; dx=-2.4775104 evalInputDelta=0.0
MIN ALPHA (9.856577069034168E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0519; Orientation: 0.0008; Line Search: 0.0494
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.316s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.11 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 776926712304
Reset training subject: 776928148178
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=32.42178570862366}, derivative=-4.3659328E23}
New Minimum: 32.42178570862366 > 1.0992601713567005
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.0992601713567005}, derivative=-4.365932800038035E11}, evalInputDelta = -31.32252553726696
New Minimum: 1.0992601713567005 > 1.099260169854563
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.099260169854563}, derivative=-4.365932800038035E11}, evalInputDelta = -31.322525538769096
New Minimum: 1.099260169854563 > 1.0992601593396034
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.0992601593396034}, derivative=-4.365932800038035E11}, evalInputDelta = -31.322525549284055
New Minimum: 1.0992601593396034 > 1.0992600857348873
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.0992600857348873}, derivative=-4.365932800038035E11}, evalInputDelta = -31.322525622888772
New Minimum: 1.0992600857348873 > 1.0992595705019608
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.0992595705019608}, derivative=-4.365932800038035E11}, evalInputDelta = -31.3225261381217
New Minimum: 1.0992595705019608 > 1.09925596387578
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.09925596387578}, derivative=-4.365932800038035E11}, evalInputDelta = -31.32252974474788
New Minimum: 1.09925596387578 > 1.0992307177034477
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.0992307177034477}, derivative=-4.3659328000380347E11}, evalInputDelta = -31.32255499092021
New Minimum: 1.0992307177034477 > 1.0990540048319546
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.0990540048319546}, derivative=-4.365932800038032E11}, evalInputDelta = -31.322731703791703
New Minimum: 1.0990540048319546 > 1.0978175208520775
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.0978175208520775}, derivative=-4.3659328000380145E11}, evalInputDelta = -31.32396818777158
New Minimum: 1.0978175208520775 > 1.089186835170538
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.089186835170538}, derivative=-4.36593280003789E11}, evalInputDelta = -31.332598873453122
New Minimum: 1.089186835170538 > 1.0299499430124706
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.0299499430124706}, derivative=-4.365932800037051E11}, evalInputDelta = -31.391835765611187
New Minimum: 1.0299499430124706 > 0.6638507896865249
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.6638507896865249}, derivative=-4.365932800032477E11}, evalInputDelta = -31.757934918937135
New Minimum: 0.6638507896865249 > 0.021481955127564758
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.021481955127564758}, derivative=-4.365932800026666E11}, evalInputDelta = -32.4003037534961
New Minimum: 0.021481955127564758 > 0.0
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
0.0 <= 32.42178570862366
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.365932800026619E11}, evalInputDelta = -32.42178570862366
Right bracket at 1.0E10
Converged to right
Fitness changed from 32.42178570862366 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0635; Orientation: 0.0007; Line Search: 0.0585
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.4775104}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0445; Orientation: 0.0006; Line Search: 0.0428
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.108s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.94 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 777038696691
Reset training subject: 777039673649
Adding measurement 1ba7794 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 32.42178570862366 < 32.42178570862366. Total: 1
th(0)=32.42178570862366;dx=-4.3659328E23
Adding measurement f85ae60 to history. Total: 1
New Minimum: 32.42178570862366 > 0.0027671947991399753
Armijo: th(2.154434690031884)=0.0027671947991399753; dx=-4.365932800026625E11 evalInputDelta=32.41901851382452
Non-optimal measurement 0.029938429803399995 < 0.0027671947991399753. Total: 2
Armijo: th(1.077217345015942)=0.029938429803399995; dx=-4.365932800026688E11 evalInputDelta=32.39184727882026
Non-optimal measurement 0.3749259829030395 < 0.0027671947991399753. Total: 2
Armijo: th(0.3590724483386473)=0.3749259829030395; dx=-4.365932800029505E11 evalInputDelta=32.04685972572062
Non-optimal measurement 0.8879710987390272 < 0.0027671947991399753. Total: 2
Armijo: th(0.08976811208466183)=0.8879710987390272; dx=-4.3659328000351575E11 evalInputDelta=31.533814609884633
Non-optimal measurement 1.0548870652112325 < 0.0027671947991399753. Total: 2
Armijo: th(0.017953622416932366)=1.0548870652112325; dx=-4.3659328000374005E11 evalInputDelta=31.366898643412426
Non-optimal measurement 1.091785040706696 < 0.0027671947991399753. Total: 2
Armijo: th(0.002992270402822061)=1.091785040706696; dx=-4.365932800037927E11 evalInputDelta=31.330000667916963
Non-optimal measurement 1.0981903125413262 < 0.0027671947991399753. Total: 2
Armijo: th(4.2746720040315154E-4)=1.0981903125413262; dx=-4.36593280003802E11 evalInputDelta=31.323595396082332
Non-optimal measurement 1.0991264029668573 < 0.0027671947991399753. Total: 2
Armijo: th(5.343340005039394E-5)=1.0991264029668573; dx=-4.3659328000380334E11 evalInputDelta=31.322659305656803
Non-optimal measurement 1.0992453079130375 < 0.0027671947991399753. Total: 2
Armijo: th(5.9370444500437714E-6)=1.0992453079130375; dx=-4.365932800038035E11 evalInputDelta=31.322540400710622
Non-optimal measurement 1.0992586852318968 < 0.0027671947991399753. Total: 2
Armijo: th(5.937044450043771E-7)=1.0992586852318968; dx=-4.365932800038035E11 evalInputDelta=31.322527023391764
Non-optimal measurement 1.0992600364819893 < 0.0027671947991399753. Total: 2
Armijo: th(5.397313136403428E-8)=1.0992600364819893; dx=-4.365932800038035E11 evalInputDelta=31.32252567214167
Non-optimal measurement 1.0992601603466337 < 0.0027671947991399753. Total: 2
Armijo: th(4.4977609470028565E-9)=1.0992601603466337; dx=-4.365932800038035E11 evalInputDelta=31.322525548277024
Non-optimal measurement 1.0992601707408702 < 0.0027671947991399753. Total: 2
Armijo: th(3.4598161130791205E-10)=1.0992601707408702; dx=-4.365932800038035E11 evalInputDelta=31.32252553788279
Non-optimal measurement 1.099260171545186 < 0.0027671947991399753. Total: 2
Armijo: th(2.4712972236279432E-11)=1.099260171545186; dx=-4.365932800038035E11 evalInputDelta=31.322525537078473
Non-optimal measurement 1.8512313167384051 < 0.0027671947991399753. Total: 2
Armijo: th(1.6475314824186289E-12)=1.8512313167384051; dx=-8.39713267092669E11 evalInputDelta=30.570554391885253
Non-optimal measurement 32.4217857086234 < 0.0027671947991399753. Total: 2
Armijo: th(1.029707176511643E-13)=32.4217857086234; dx=-4.3659328E23 evalInputDelta=2.5579538487363607E-13
Non-optimal measurement 32.42178570862364 < 0.0027671947991399753. Total: 2
Armijo: th(6.057101038303783E-15)=32.42178570862364; dx=-4.3659328E23 evalInputDelta=2.1316282072803006E-14
Non-optimal measurement 0.0027671947991399753 < 0.0027671947991399753. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0027671947991399753
Fitness changed from 32.42178570862366 to 0.0027671947991399753
Iteration 1 complete. Error: 0.0027671947991399753 Total: 0.0469; Orientation: 0.0030; Line Search: 0.0410
Non-optimal measurement 0.0027671947991399753 < 0.0027671947991399753. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0027671947991399753 < 0.0027671947991399753. Total: 2
th(0)=0.0027671947991399753;dx=-2.478128119332879
Adding measurement 5e4c603b to history. Total: 2
New Minimum: 0.0027671947991399753 > 0.00276719479913995
WOLFE (weak): th(2.154434690031884E-15)=0.00276719479913995; dx=-2.478128119332879 evalInputDelta=2.5153490401663703E-17
Adding measurement 64ca6778 to history. Total: 3
New Minimum: 0.00276719479913995 > 0.0027671947991399137
WOLFE (weak): th(4.308869380063768E-15)=0.0027671947991399137; dx=-2.478128119332879 evalInputDelta=6.158268339717665E-17
Adding measurement 2e4f4c76 to history. Total: 4
New Minimum: 0.0027671947991399137 > 0.002767194799139802
WOLFE (weak): th(1.2926608140191303E-14)=0.002767194799139802; dx=-2.478128119332879 evalInputDelta=1.734723475976807E-16
Adding measurement 3914a566 to history. Total: 5
New Minimum: 0.002767194799139802 > 0.002767194799139331
WOLFE (weak): th(5.1706432560765214E-14)=0.002767194799139331; dx=-2.478128119332879 evalInputDelta=6.444497713253838E-16
Adding measurement 7d4f7c01 to history. Total: 6
New Minimum: 0.002767194799139331 > 0.002767194799136754
WOLFE (weak): th(2.5853216280382605E-13)=0.002767194799136754; dx=-2.4781281193328786 evalInputDelta=3.2213814948889308E-15
Adding measurement 1103c846 to history. Total: 7
New Minimum: 0.002767194799136754 > 0.002767194799120646
WOLFE (weak): th(1.5511929768229563E-12)=0.002767194799120646; dx=-2.478128119332877 evalInputDelta=1.9329156331071573E-14
Adding measurement 5349c65 to history. Total: 8
New Minimum: 0.002767194799120646 > 0.002767194799004722
WOLFE (weak): th(1.0858350837760695E-11)=0.002767194799004722; dx=-2.4781281193328635 evalInputDelta=1.352533536558287E-13
Adding measurement 17277339 to history. Total: 9
New Minimum: 0.002767194799004722 > 0.002767194798058022
WOLFE (weak): th(8.686680670208556E-11)=0.002767194798058022; dx=-2.478128119332755 evalInputDelta=1.0819531034989005E-12
Adding measurement 2a3a2ab9 to history. Total: 10
New Minimum: 0.002767194798058022 > 0.0027671947894025188
WOLFE (weak): th(7.8180126031877E-10)=0.0027671947894025188; dx=-2.4781281193317644 evalInputDelta=9.737456500846786E-12
Adding measurement e3f5286 to history. Total: 11
New Minimum: 0.0027671947894025188 > 0.002767194701765385
WOLFE (weak): th(7.818012603187701E-9)=0.002767194701765385; dx=-2.478128119321732 evalInputDelta=9.737459016195826E-11
Adding measurement 42d1d0b2 to history. Total: 12
New Minimum: 0.002767194701765385 > 0.0027671937280194544
WOLFE (weak): th(8.599813863506471E-8)=0.0027671937280194544; dx=-2.4781281192102598 evalInputDelta=1.0711205208381591E-9
Adding measurement 7aaea2f to history. Total: 13
New Minimum: 0.0027671937280194544 > 0.002767181945694497
WOLFE (weak): th(1.0319776636207765E-6)=0.002767181945694497; dx=-2.478128117861448 evalInputDelta=1.2853445478105963E-8
Adding measurement 1d41aa7d to history. Total: 14
New Minimum: 0.002767181945694497 > 0.002767027704467303
WOLFE (weak): th(1.3415709627070094E-5)=0.002767027704467303; dx=-2.478128100204302 evalInputDelta=1.6709467267218142E-7
Adding measurement 2e8f73b8 to history. Total: 15
New Minimum: 0.002767027704467303 > 0.0027648554970751784
WOLFE (weak): th(1.878199347789813E-4)=0.0027648554970751784; dx=-2.478127851538148 evalInputDelta=2.3393020647968603E-6
Adding measurement 76ffdca2 to history. Total: 16
New Minimum: 0.0027648554970751784 > 0.0027321105482755762
WOLFE (weak): th(0.0028172990216847197)=0.0027321105482755762; dx=-2.4781241036206856 evalInputDelta=3.508425086439903E-5
Adding measurement 7ab7d92f to history. Total: 17
New Minimum: 0.0027321105482755762 > 0.0022071998938567377
WOLFE (weak): th(0.045076784346955515)=0.0022071998938567377; dx=-2.4780641771750553 evalInputDelta=5.599949052832376E-4
Adding measurement 48575041 to history. Total: 18
New Minimum: 0.0022071998938567377 > 0.0
WOLFE (weak): th(0.7663053338982437)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Non-optimal measurement 0.0 < 0.0. Total: 19
WOLFE (weak): th(13.793496010168386)=0.0; dx=-2.477815332144412 evalInputDelta=0.0027671947991399753
Non-optimal measurement 0.0 < 0.0. Total: 19
WOLFE (weak): th(262.0764241931993
...skipping 7487 bytes...
7377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497, 0.0027671937280194544, 0.002767194701765385, 0.0027671947894025188, 0.002767194798058022, 0.002767194799004722, 0.002767194799120646
Rejected: LBFGS Orientation magnitude: 4.931e+03, gradient 1.574e+00, dot -0.999; [781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497, 0.0027671937280194544, 0.002767194701765385, 0.0027671947894025188, 0.002767194798058022, 0.002767194799004722
Rejected: LBFGS Orientation magnitude: 4.932e+03, gradient 1.574e+00, dot -0.999; [781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00, aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497, 0.0027671937280194544, 0.002767194701765385, 0.0027671947894025188, 0.002767194798058022
Rejected: LBFGS Orientation magnitude: 4.932e+03, gradient 1.574e+00, dot -0.999; [aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497, 0.0027671937280194544, 0.002767194701765385, 0.0027671947894025188
Rejected: LBFGS Orientation magnitude: 4.932e+03, gradient 1.574e+00, dot -0.999; [aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00, 781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497, 0.0027671937280194544, 0.002767194701765385
Rejected: LBFGS Orientation magnitude: 4.932e+03, gradient 1.574e+00, dot -0.999; [aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497, 0.0027671937280194544
Rejected: LBFGS Orientation magnitude: 4.932e+03, gradient 1.574e+00, dot -0.999; [242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303, 0.002767181945694497
Rejected: LBFGS Orientation magnitude: 6.480e+03, gradient 1.574e+00, dot -0.983; [aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784, 0.002767027704467303
Rejected: LBFGS Orientation magnitude: 5.406e+03, gradient 1.574e+00, dot -0.999; [aefdb3ec-3448-41fb-83c6-0d43694ce988 = 1.000/1.000e+00, 97968a4b-8c88-4a9e-a48b-e52c3a48e49e = 1.000/1.000e+00, 781cfbd5-fd4d-4a8a-9b5e-c26f5885cf25 = 1.000/1.000e+00, 242fe3cc-5f27-4ac0-b029-0e5ddcf79ba7 = 1.000/1.000e+00, 4c10d676-8643-49bf-b205-31972b1166b1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.0022071998938567377, 0.0027321105482755762, 0.0027648554970751784
LBFGS Accumulation History: 3 points
Removed measurement 48575041 to history. Total: 18
Removed measurement 7ab7d92f to history. Total: 17
Removed measurement 76ffdca2 to history. Total: 16
Removed measurement 2e8f73b8 to history. Total: 15
Removed measurement 1d41aa7d to history. Total: 14
Removed measurement 7aaea2f to history. Total: 13
Removed measurement 42d1d0b2 to history. Total: 12
Removed measurement e3f5286 to history. Total: 11
Removed measurement 2a3a2ab9 to history. Total: 10
Removed measurement 17277339 to history. Total: 9
Removed measurement 5349c65 to history. Total: 8
Removed measurement 1103c846 to history. Total: 7
Removed measurement 7d4f7c01 to history. Total: 6
Removed measurement 3914a566 to history. Total: 5
Removed measurement 2e4f4c76 to history. Total: 4
Removed measurement 64ca6778 to history. Total: 3
Adding measurement 1dcaad0c to history. Total: 3
th(0)=0.0;dx=-2.4775104
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2398.0086145001046)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1199.0043072500523)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(399.6681024166841)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(99.91702560417103)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(19.983405120834206)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.3305675201390343)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.47579536001986206)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.05947442000248276)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.006608268889164751)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.608268889164751E-4)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.0075171719679556E-5)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(5.006264309973297E-6)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.850972546133305E-7)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.7506946758095036E-8)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.833796450539669E-9)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.1461227815872932E-10)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.741898715219372E-12)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.7454992862329843E-13)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.9713154138068337E-14)=0.0; dx=-2.4775104 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
MIN ALPHA (9.856577069034168E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.8322; Orientation: 0.7995; Line Search: 0.0315
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.935s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -3.557960267236443], [2.0, -1.5579602672364432]; valueStats=DoubleSummaryStatistics{count=2, sum=0.005534, min=0.002767, average=0.002767, max=0.002767}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -3.557960267236443], [0.112, -1.5579602672364432]; valueStats=DoubleSummaryStatistics{count=2, sum=0.005534, min=0.002767, average=0.002767, max=0.002767}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0042 = network_target.state();
assert temp_18_0042 != null;
String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0042.freeRef();
return temp_18_0041;
Returns
[0.496]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.04 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 778252198713
Reset training subject: 778253544153
Constructing line search parameters: GD
th(0)=0.33119048147664976;dx=-48.77625600000001
New Minimum: 0.33119048147664976 > 0.0
WOLFE (weak): th(2.154434690031884)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
WOLFE (weak): th(4.308869380063768)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
WOLFE (weak): th(12.926608140191302)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
WOLFE (weak): th(51.70643256076521)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
WOLFE (weak): th(258.53216280382605)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
WOLFE (weak): th(1551.1929768229563)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(10858.350837760694)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
WOLFE (weak): th(6204.771907291825)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(8531.56137252626)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(7368.166639909043)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
WOLFE (weak): th(6786.4692736004345)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(7077.317956754739)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(6931.893615177587)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(6859.181444389011)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Armijo: th(6822.825358994723)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
mu ~= nu (6786.4692736004345): th(2.154434690031884)=0.0
Fitness changed from 0.33119048147664976 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0212; Orientation: 0.0003; Line Search: 0.0180
th(0)=0.0;dx=-42.61478400000001
Armijo: th(14660.168231663863)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(7330.084115831932)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(2443.3613719439772)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(610.8403429859943)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(122.16806859719887)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(20.36134476619981)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(2.9087635380285444)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(0.36359544225356805)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(0.04039949358372978)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(0.004039949358372978)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(3.6726812348845256E-4)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(3.060567695737105E-5)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(2.354282842874696E-6)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(1.6816306020533543E-7)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(1.1210870680355695E-8)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(7.006794175222309E-10)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(4.1216436324837114E-11)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(2.2898020180465063E-12)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(1.2051589568665822E-13)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Armijo: th(6.025794784332911E-15)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
MIN ALPHA (2.8694260877775766E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0207; Orientation: 0.0003; Line Search: 0.0193
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.042s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.05 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 778298172621
Reset training subject: 778299128590
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.33119048147664976}, derivative=-48.77625600000001}
New Minimum: 0.33119048147664976 > 0.33119048077824975
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.33119048077824975}, derivative=-48.77625599103378}, evalInputDelta = -6.984000044951699E-10
New Minimum: 0.33119048077824975 > 0.3311904765878499
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.3311904765878499}, derivative=-48.776255937236435}, evalInputDelta = -4.888799864932736E-9
New Minimum: 0.3311904765878499 > 0.33119044725505087
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.33119044725505087}, derivative=-48.77625556065508}, evalInputDelta = -3.42215988879957E-8
New Minimum: 0.33119044725505087 > 0.3311902419255025
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.3311902419255025}, derivative=-48.776252924586686}, evalInputDelta = -2.395511472519374E-7
New Minimum: 0.3311902419255025 > 0.33118880462083417
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.33118880462083417}, derivative=-48.77623447216367}, evalInputDelta = -1.6768558155910718E-6
New Minimum: 0.33118880462083417 > 0.33117874359448485
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.33117874359448485}, derivative=-48.77610530793265}, evalInputDelta = -1.1737882164908342E-5
New Minimum: 0.33117874359448485 > 0.33110832161962883
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.33110832161962883}, derivative=-48.77520129207167}, evalInputDelta = -8.215985702092476E-5
New Minimum: 0.33110832161962883 > 0.3306156228862884
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.3306156228862884}, derivative=-48.768879728189205}, evalInputDelta = -5.748585903613823E-4
New Minimum: 0.3306156228862884 > 0.32717917004805525
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.32717917004805525}, derivative=-48.72494724109758}, evalInputDelta = -0.004011311428594511
New Minimum: 0.32717917004805525 > 0.30371328519040325
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.30371328519040325}, derivative=-48.4322580122668}, evalInputDelta = -0.027477196286246508
New Minimum: 0.30371328519040325 > 0.1628604864710287
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.1628604864710287}, derivative=-46.91756943221259}, evalInputDelta = -0.16832999500562107
New Minimum: 0.1628604864710287 > 0.0
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
0.0 <= 0.33119048147664976
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.59155200000001}, evalInputDelta = -0.33119048147664976
Right bracket at 1.0E10
Converged to right
Fitness changed from 0.33119048147664976 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0273; Orientation: 0.0002; Line Search: 0.0245
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478400000001}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-42.61478399999999}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0213; Orientation: 0.0002; Line Search: 0.0203
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.049s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.04 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 778356485535
Reset training subject: 778357704212
Adding measurement 36976610 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.33119048147664976 < 0.33119048147664976. Total: 1
th(0)=0.33119048147664976;dx=-48.77625600000001
Adding measurement 75133ae8 to history. Total: 1
New Minimum: 0.33119048147664976 > 0.0
WOLFE (weak): th(2.154434690031884)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(4.308869380063768)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(12.926608140191302)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(51.70643256076521)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(258.53216280382605)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(1551.1929768229563)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(10858.350837760694)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(6204.771907291825)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(8531.56137252626)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(7368.166639909043)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
WOLFE (weak): th(6786.4692736004345)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(7077.317956754739)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(6931.893615177587)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(6859.181444389011)=0.0; dx=-45.591552 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(6822.825358994723)=0.0; dx=-45.59155200000001 evalInputDelta=0.33119048147664976
Non-optimal measurement 0.0 < 0.0. Total: 2
mu ~= nu (6786.4692736004345): th(2.154434690031884)=0.0
Fitness changed from 0.33119048147664976 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0190; Orientation: 0.0005; Line Search: 0.0158
Non-optimal measurement 0.0 < 0.0. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0 < 0.0. Total: 2
th(0)=0.0;dx=-42.61478400000001
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(14660.168231663863)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(7330.084115831932)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2443.3613719439772)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(610.8403429859943)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(122.16806859719887)=0.0; dx=-42.61478399999999 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(20.36134476619981)=0.0; dx=-42.61478399999999 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.9087635380285444)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.36359544225356805)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.04039949358372978)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.004039949358372978)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.6726812348845256E-4)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.060567695737105E-5)=0.0; dx=-42.61478399999999 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.354282842874696E-6)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.6816306020533543E-7)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.1210870680355695E-8)=0.0; dx=-42.61478399999999 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(7.006794175222309E-10)=0.0; dx=-42.61478399999999 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.1216436324837114E-11)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.2898020180465063E-12)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.2051589568665822E-13)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(6.025794784332911E-15)=0.0; dx=-42.61478400000001 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (2.8694260877775766E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0170; Orientation: 0.0003; Line Search: 0.0161
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.036s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[0.0, 0.0], [2.0, 1.0]; valueStats=DoubleSummaryStatistics{count=0, sum=0.000000, min=Infinity, average=0.000000, max=-Infinity}
Only 0 points for GD
Only 0 points for CjGD
Only 0 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
No Data
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0037 = temp_18_0035.state();
assert temp_18_0037 != null;
String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0037.freeRef();
return temp_18_0036;
Returns
[0.496]
We simultaneously regress this target input:
TrainingTester.java:287 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
if (null != x)
RefUtil.freeRef(x);
return temp_18_0006;
}).map(x -> {
String temp_18_0007 = x.prettyPrint();
x.freeRef();
return temp_18_0007;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ 1.208, -0.608, 0.048, 1.524, 1.764 ]
[ 1.764, 1.208, 0.048, -0.608, 1.524 ]
[ 0.048, -0.608, 1.524, 1.764, 1.208 ]
[ 0.048, 1.764, 1.208, 1.524, -0.608 ]
[ 1.764, 1.208, -0.608, 1.524, 0.048 ]
Which produces the following output:
TrainingTester.java:308 executed in 0.00 seconds (0.000 gc):
return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
String temp_18_0008 = x.prettyPrint();
x.freeRef();
return temp_18_0008;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ 1.704, -0.11199999999999999, 0.544, 2.02, 2.26 ]
[ 2.26, 1.704, 0.544, -0.11199999999999999, 2.02 ]
[ 0.544, -0.11199999999999999, 2.02, 2.26, 1.704 ]
[ 0.544, 2.26, 1.704, 2.02, -0.11199999999999999 ]
[ 2.26, 1.704, -0.11199999999999999, 2.02, 0.544 ]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.03 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 778446754275
Reset training subject: 778447738498
Constructing line search parameters: GD
th(0)=25.73620848320719;dx=-1.1451686400121916E24
New Minimum: 25.73620848320719 > 0.0
Armijo: th(2.154434690031884)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(1.077217345015942)=0.0; dx=-6.21793088004673E12 evalInputDelta=25.73620848320719
Armijo: th(0.3590724483386473)=0.0; dx=-6.21793088004673E12 evalInputDelta=25.73620848320719
Armijo: th(0.08976811208466183)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Armijo: th(0.017953622416932366)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Armijo: th(0.002992270402822061)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(4.2746720040315154E-4)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Armijo: th(5.343340005039394E-5)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(5.9370444500437714E-6)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(5.937044450043771E-7)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(5.397313136403428E-8)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(4.4977609470028565E-9)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Armijo: th(3.4598161130791205E-10)=0.0; dx=-6.21793088004673E12 evalInputDelta=25.73620848320719
Armijo: th(2.4712972236279432E-11)=0.0; dx=-6.217930880046731E12 evalInputDelta=25.73620848320719
Armijo: th(1.6475314824186289E-12)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Armijo: th(1.029707176511643E-13)=6.654044962051742; dx=-1.0993152005542722E23 evalInputDelta=19.082163521155447
Armijo: th(6.057101038303783E-15)=25.722687778030632; dx=-1.1451686400121695E24 evalInputDelta=0.013520705176556902
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0
Fitness changed from 25.73620848320719 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0246; Orientation: 0.0004; Line Search: 0.0216
th(0)=0.0;dx=-45.09229439999999
Armijo: th(2.154434690031884E-15)=0.0; dx=-45.09229440000001 evalInputDelta=0.0
Armijo: th(1.077217345015942E-15)=0.0; dx=-45.09229439999999 evalInputDelta=0.0
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0061; Orientation: 0.0004; Line Search: 0.0048
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.031s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.07 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 778480955538
Reset training subject: 778481818541
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=25.73620848320719}, derivative=-1.1451686400121919E24}
New Minimum: 25.73620848320719 > 0.0
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046733E12}, evalInputDelta = -25.73620848320719
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.217930880046732E12}, evalInputDelta = -25.73620848320719
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
0.0 <= 25.73620848320719
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.21793088004673E12}, evalInputDelta = -25.73620848320719
Right bracket at 1.0E10
Converged to right
Fitness changed from 25.73620848320719 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0338; Orientation: 0.0004; Line Search: 0.0310
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.092294399999986}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 1.0E10
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.092294399999986}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 1.0E10
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.092294399999986}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 1.0E10
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-45.09229439999999}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0319; Orientation: 0.0003; Line Search: 0.0308
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.066s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.04 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 778549978639
Reset training subject: 778550846275
Adding measurement 7e52d9b0 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 25.73620848320719 < 25.73620848320719. Total: 1
th(0)=25.73620848320719;dx=-1.1451686400121919E24
Adding measurement 2f30ac88 to history. Total: 1
New Minimum: 25.73620848320719 > 0.0
Armijo: th(2.154434690031884)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.077217345015942)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.3590724483386473)=0.0; dx=-6.217930880046731E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.08976811208466183)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.017953622416932366)=0.0; dx=-6.21793088004673E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.002992270402822061)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.2746720040315154E-4)=0.0; dx=-6.21793088004673E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.343340005039394E-5)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.9370444500437714E-6)=0.0; dx=-6.217930880046731E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.937044450043771E-7)=0.0; dx=-6.217930880046731E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.397313136403428E-8)=0.0; dx=-6.21793088004673E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.4977609470028565E-9)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.4598161130791205E-10)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.4712972236279432E-11)=0.0; dx=-6.217930880046733E12 evalInputDelta=25.73620848320719
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.6475314824186289E-12)=0.0; dx=-6.217930880046732E12 evalInputDelta=25.73620848320719
Non-optimal measurement 6.654044962051742 < 0.0. Total: 2
Armijo: th(1.029707176511643E-13)=6.654044962051742; dx=-1.099315200554272E23 evalInputDelta=19.082163521155447
Non-optimal measurement 25.722687778030632 < 0.0. Total: 2
Armijo: th(6.057101038303783E-15)=25.722687778030632; dx=-1.1451686400121695E24 evalInputDelta=0.013520705176556902
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0
Fitness changed from 25.73620848320719 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0303; Orientation: 0.0008; Line Search: 0.0273
Non-optimal measurement 0.0 < 0.0. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0 < 0.0. Total: 2
th(0)=0.0;dx=-45.092294399999986
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.154434690031884E-15)=0.0; dx=-45.092294399999986 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.077217345015942E-15)=0.0; dx=-45.092294399999986 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0067; Orientation: 0.0008; Line Search: 0.0051
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.037s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[0.0, 0.0], [2.0, 1.0]; valueStats=DoubleSummaryStatistics{count=0, sum=0.000000, min=Infinity, average=0.000000, max=-Infinity}
Only 0 points for GD
Only 0 points for CjGD
Only 0 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
No Data
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }} | OK |
{
"result": "OK",
"performance": {
"execution_time": "2.671",
"gc_time": "0.452"
},
"created_on": 1586735364407,
"file_name": "trainingTest",
"report": {
"simpleName": "Reducing",
"canonicalName": "com.simiacryptus.mindseye.layers.java.BiasLayerTest.Reducing",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/BiasLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"model": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"complete": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/BiasLayer/Reducing/trainingTest/202004124924",
"id": "362211ac-d1ac-4b73-9a2f-e48993fa5eb4",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "BiasLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.BiasLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/BiasLayer.java",
"javaDoc": ""
}
}