Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 8434018969874044928
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ -1.616, 1.356, 1.032, 1.556, 1.512 ]
[ -1.616, 1.356, 1.032, 1.512, 1.556 ]
[ 1.556, 1.356, -1.616, 1.032, 1.512 ]
[ -1.616, 1.356, 1.556, 1.512, 1.032 ]
[ 1.512, 1.556, -1.616, 1.032, 1.356 ]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.32 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 743297497511
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 743325492977
Constructing line search parameters: GD
th(0)=14.485537739630754;dx=-7.037760000000001E22
New Minimum: 14.485537739630754 > 0.01737882160084258
Armijo: th(2.154434690031884)=0.01737882160084258; dx=-7.037760000232262E10 evalInputDelta=14.46815891802991
Armijo: th(1.077217345015942)=0.10167577537500996; dx=-7.037760000235352E10 evalInputDelta=14.383861964255743
Armijo: th(0.3590724483386473)=0.36275696020500897; dx=-7.037760000254753E10 evalInputDelta=14.122780779425744
Armijo: th(0.08976811208466183)=0.5152966622260962; dx=-7.037760000274858E10 evalInputDelta=13.970241077404658
Armijo: th(0.017953622416932366)=0.5674208400801727; dx=-7.037760000284375E10 evalInputDelta=13.918116899550581
Armijo: th(0.002992270402822061)=0.5792067847778842; dx=-7.037760000286781E10 evalInputDelta=13.90633095485287
Armijo: th(4.2746720040315154E-4)=0.5812639726613756; dx=-7.037760000287212E10 evalInputDelta=13.904273766969379
Armijo: th(5.343340005039394E-5)=0.5815649045084212; dx=-7.037760000287276E10 evalInputDelta=13.903972835122332
Armijo: th(5.9370444500437714E-6)=0.5816031350137707; dx=-7.037760000287283E10 evalInputDelta=13.903934604616984
Armijo: th(5.937044450043771E-7)=0.581607436184648; dx=-7.037760000287283E10 evalInputDelta=13.903930303446106
Armijo: th(5.397313136403428E-8)=0.5816078706490412; dx=-7.037760000287285E10 evalInputDelta=13.903929868981713
Armijo: th(4.4977609470028565E-9)=0.5816079104749685; dx=-7.037760000287285E10 evalInputDelta=13.903929829155786
Armijo: th(3.4598161130791205E-10)=0.5816079138170045; dx=-7.037760000287285E10 evalInputDelta=13.90392982581375
Armijo: th(2.4712972236279432E-11)=2.4798946067934646; dx=-4.0960000000936274E21 evalInputDelta=12.005643132837289
Armijo: th(1.6475314824186289E-12)=14.485537739629427; dx=-7.037760000000001E22 evalInputDelta=1.326938559031987E-12
Armijo: th(1.029707176511643E-13)=14.485537739630672; dx=-7.037760000000001E22 evalInputDelta=8.171241461241152E-14
Armijo: th(6.057101038303783E-15)=14.485537739630749; dx=-7.037760000000001E22 evalInputDelta=5.329070518200751E-15
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.01737882160084258
Fitness changed from 14.485537739630754 to 0.01737882160084258
Iteration 1 complete. Error: 0.01737882160084258 Total: 0.1442; Orientation: 0.0032; Line Search: 0.1053
th(0)=0.01737882160084258;dx=-2.2350156442069267
New Minimum: 0.01737882160084258 > 0.017378821600842552
WOLFE (weak): th(2.154434690031884E-15)=0.017378821600842552; dx=-2.2350156442069267 evalInputDelta=2.7755575615628914E-17
WOLFE (weak): th(4.308869380063768E-15)=0.017378821600842552; dx=-2.2350156442069267 evalInputDelta=2.7755575615628914E-17
New Minimum: 0.017378821600842552 > 0.01737882160084248
WOLFE (weak): th(1.2926608140191303E-14)=0.01737882160084248; dx=-2.2350156442069267 evalInputDelta=1.0061396160665481E-16
New Minimum: 0.01737882160084248 > 0.017378821600842143
WOLFE (weak): th(5.1706432560765214E-14)=0.017378821600842143; dx=-2.2350156442069267 evalInputDelta=4.371503159461554E-16
New Minimum: 0.017378821600842143 > 0.017378821600840387
WOLFE (weak): th(2.5853216280382605E-13)=0.017378821600840387; dx=-2.2350156442069267 evalInputDelta=2.192690473634684E-15
New Minimum: 0.017378821600840387 > 0.017378821600829458
WOLFE (weak): th(1.5511929768229563E-12)=0.017378821600829458; dx=-2.2350156442069258 evalInputDelta=1.3121448372288569E-14
New Minimum: 0.017378821600829458 > 0.0173788216007507
WOLFE (weak): th(1.0858350837760695E-11)=0.0173788216007507; dx=-2.2350156442069196 evalInputDelta=9.188136362858756E-14
New Minimum: 0.0173788216007507 > 0.017378821600107446
WOLFE (weak): th(8.686680670208556E-11)=0.017378821600107446; dx=-2.235015644206869 evalInputDelta=7.351341757555474E-13
New Minimum: 0.017378821600107446 > 0.01737882159422645
WOLFE (weak): th(7.8180126031877E-10)=0.01737882159422645; dx=-2.2350156442064044 evalInputDelta=6.616131253966984E-12
New Minimum: 0.01737882159422645 > 0.017378821534681205
WOLFE (weak): th(7.818012603187701E-9)=0.017378821534681205; dx=-2.235015644201705 evalInputDelta=6.616137498971497E-11
New Minimum: 0.017378821534681205 > 0.017378820873067496
WOLFE (weak): th(8.599813863506471E-8)=0.017378820873067496; dx=-2.235015644149487 evalInputDelta=7.277750832535013E-10
New Minimum: 0.017378820873067496 > 0.017378812867541865
WOLFE (weak): th(1.0319776636207765E-6)=0.017378812867541865; dx=-2.2350156435176505 evalInputDelta=8.733300714547365E-9
New Minimum: 0.017378812867541865 > 0.0173787080679887
WOLFE (weak): th(1.3415709627070094E-5)=0.0173787080679887; dx=-2.2350156352463433 evalInputDelta=1.1353285387857848E-7
New Minimum: 0.0173787080679887 > 0.01737723215182751
WOLFE (weak): th(1.878199347789813E-4)=0.01737723215182751; dx=-2.235015518760517 evalInputDelta=1.5894490150714091E-6
New Minimum: 0.01737723215182751 > 0.017354982339195406
WOLFE (weak): th(0.0028172990216847197)=0.017354982339195406; dx=-2.235013762908878 evalInputDelta=2.383926164717387E-5
New Minimum: 0.017354982339195406 > 0.016998027950037752
WOLFE (weak): th(0.045076784346955515)=0.016998027950037752; dx=-2.23498564543159 evalInputDelta=3.807936508048275E-4
New Minimum: 0.016998027950037752 > 0.01108222844325175
WOLFE (weak): th(0.7663053338982437)=0.01108222844325175; dx=-2.2345335185287634 evalInputDelta=0.006296593157590829
New Minimum: 0.01108222844325175 > 0.0
WOLFE (weak): th(13.793496010168386)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
WOLFE (weak): th(262.07642419319933)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
WOLFE (weak): th(5241.528483863986)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(110072.09816114372)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(57656.813322503855)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(31449.17090318392)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(18345.349693523953)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(11793.43908869397)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(8517.483786278977)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
WOLFE (weak): th(6879.506135071482)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
WOLFE (weak): th(7698.49496067523)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(8107.989373477103)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(7903.242167076167)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Armijo: th(7800.868563875698)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
WOLFE (weak): th(7749.681762275464)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
mu ~= nu (7749.681762275464): th(13.793496010168386)=0.0
Fitness changed from 0.01737882160084258 to 0.0
Iteration 2 complete. Error: 0.0 Total: 0.1104; Orientation: 0.0011; Line Search: 0.1063
th(0)=0.0;dx=-2.2327283200000005
Armijo: th(16751.322535873343)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(8375.661267936672)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(2791.887089312224)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(697.971772328056)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(139.5943544656112)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(23.265725744268533)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(3.323675106324076)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(0.4154593882905095)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(0.046162154254501056)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(0.004616215425450106)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(4.196559477681914E-4)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(3.4971328980682615E-5)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(2.690102229283278E-6)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(1.9215015923451987E-7)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(1.2810010615634659E-8)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(8.006256634771662E-10)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(4.7095627263362714E-11)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(2.616423736853484E-12)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(1.3770651246597284E-13)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Armijo: th(6.885325623298642E-15)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
MIN ALPHA (3.2787264872850675E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0600; Orientation: 0.0009; Line Search: 0.0573
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.315s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.11 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 743620228567
Reset training subject: 743621568473
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=14.485537739630754}, derivative=-7.037760000000001E22}
New Minimum: 14.485537739630754 > 0.5816079140150111
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.5816079140150111}, derivative=-7.037760000287285E10}, evalInputDelta = -13.903929825615743
New Minimum: 0.5816079140150111 > 0.5816079135320323
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.5816079135320323}, derivative=-7.037760000287283E10}, evalInputDelta = -13.903929826098722
New Minimum: 0.5816079135320323 > 0.5816079101511804
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.5816079101511804}, derivative=-7.037760000287285E10}, evalInputDelta = -13.903929829479573
New Minimum: 0.5816079101511804 > 0.5816078864852173
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.5816078864852173}, derivative=-7.037760000287283E10}, evalInputDelta = -13.903929853145536
New Minimum: 0.5816078864852173 > 0.5816077208235169
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.5816077208235169}, derivative=-7.037760000287283E10}, evalInputDelta = -13.903930018807237
New Minimum: 0.5816077208235169 > 0.5816065611936215
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.5816065611936215}, derivative=-7.037760000287283E10}, evalInputDelta = -13.903931178437132
New Minimum: 0.5816065611936215 > 0.5815984438827515
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.5815984438827515}, derivative=-7.037760000287282E10}, evalInputDelta = -13.903939295748003
New Minimum: 0.5815984438827515 > 0.5815416275274262
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.5815416275274262}, derivative=-7.03776000028727E10}, evalInputDelta = -13.903996112103327
New Minimum: 0.5815416275274262 > 0.5811441490296
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.5811441490296}, derivative=-7.037760000287186E10}, evalInputDelta = -13.904393590601154
New Minimum: 0.5811441490296 > 0.5783732854545126
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.5783732854545126}, derivative=-7.037760000286607E10}, evalInputDelta = -13.907164454176241
New Minimum: 0.5783732854545126 > 0.5595148688776608
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.5595148688776608}, derivative=-7.037760000282817E10}, evalInputDelta = -13.926022870753092
New Minimum: 0.5595148688776608 > 0.44760925521776007
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.44760925521776007}, derivative=-7.037760000264746E10}, evalInputDelta = -14.037928484412994
New Minimum: 0.44760925521776007 > 0.04209034946507249
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.04209034946507249}, derivative=-7.037760000232771E10}, evalInputDelta = -14.443447390165682
New Minimum: 0.04209034946507249 > 0.0
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
0.0 <= 14.485537739630754
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.037760000232103E10}, evalInputDelta = -14.485537739630754
Right bracket at 1.0E10
Converged to right
Fitness changed from 14.485537739630754 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0616; Orientation: 0.0007; Line Search: 0.0568
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.2327283200000005}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0428; Orientation: 0.0005; Line Search: 0.0409
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.105s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.82 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 743729876391
Reset training subject: 743730948371
Adding measurement 1d47e50e to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 14.485537739630754 < 14.485537739630754. Total: 1
th(0)=14.485537739630754;dx=-7.037760000000001E22
Adding measurement 2db23da2 to history. Total: 1
New Minimum: 14.485537739630754 > 0.01737882160084258
Armijo: th(2.154434690031884)=0.01737882160084258; dx=-7.037760000232262E10 evalInputDelta=14.46815891802991
Non-optimal measurement 0.10167577537500996 < 0.01737882160084258. Total: 2
Armijo: th(1.077217345015942)=0.10167577537500996; dx=-7.037760000235352E10 evalInputDelta=14.383861964255743
Non-optimal measurement 0.36275696020500897 < 0.01737882160084258. Total: 2
Armijo: th(0.3590724483386473)=0.36275696020500897; dx=-7.037760000254753E10 evalInputDelta=14.122780779425744
Non-optimal measurement 0.5152966622260962 < 0.01737882160084258. Total: 2
Armijo: th(0.08976811208466183)=0.5152966622260962; dx=-7.037760000274858E10 evalInputDelta=13.970241077404658
Non-optimal measurement 0.5674208400801727 < 0.01737882160084258. Total: 2
Armijo: th(0.017953622416932366)=0.5674208400801727; dx=-7.037760000284375E10 evalInputDelta=13.918116899550581
Non-optimal measurement 0.5792067847778842 < 0.01737882160084258. Total: 2
Armijo: th(0.002992270402822061)=0.5792067847778842; dx=-7.037760000286781E10 evalInputDelta=13.90633095485287
Non-optimal measurement 0.5812639726613756 < 0.01737882160084258. Total: 2
Armijo: th(4.2746720040315154E-4)=0.5812639726613756; dx=-7.037760000287212E10 evalInputDelta=13.904273766969379
Non-optimal measurement 0.5815649045084212 < 0.01737882160084258. Total: 2
Armijo: th(5.343340005039394E-5)=0.5815649045084212; dx=-7.037760000287276E10 evalInputDelta=13.903972835122332
Non-optimal measurement 0.5816031350137707 < 0.01737882160084258. Total: 2
Armijo: th(5.9370444500437714E-6)=0.5816031350137707; dx=-7.037760000287283E10 evalInputDelta=13.903934604616984
Non-optimal measurement 0.581607436184648 < 0.01737882160084258. Total: 2
Armijo: th(5.937044450043771E-7)=0.581607436184648; dx=-7.037760000287283E10 evalInputDelta=13.903930303446106
Non-optimal measurement 0.5816078706490412 < 0.01737882160084258. Total: 2
Armijo: th(5.397313136403428E-8)=0.5816078706490412; dx=-7.037760000287285E10 evalInputDelta=13.903929868981713
Non-optimal measurement 0.5816079104749685 < 0.01737882160084258. Total: 2
Armijo: th(4.4977609470028565E-9)=0.5816079104749685; dx=-7.037760000287285E10 evalInputDelta=13.903929829155786
Non-optimal measurement 0.5816079138170045 < 0.01737882160084258. Total: 2
Armijo: th(3.4598161130791205E-10)=0.5816079138170045; dx=-7.037760000287285E10 evalInputDelta=13.90392982581375
Non-optimal measurement 2.4798946067934646 < 0.01737882160084258. Total: 2
Armijo: th(2.4712972236279432E-11)=2.4798946067934646; dx=-4.0960000000936274E21 evalInputDelta=12.005643132837289
Non-optimal measurement 14.485537739629427 < 0.01737882160084258. Total: 2
Armijo: th(1.6475314824186289E-12)=14.485537739629427; dx=-7.037760000000001E22 evalInputDelta=1.326938559031987E-12
Non-optimal measurement 14.485537739630672 < 0.01737882160084258. Total: 2
Armijo: th(1.029707176511643E-13)=14.485537739630672; dx=-7.037760000000001E22 evalInputDelta=8.171241461241152E-14
Non-optimal measurement 14.485537739630749 < 0.01737882160084258. Total: 2
Armijo: th(6.057101038303783E-15)=14.485537739630749; dx=-7.037760000000001E22 evalInputDelta=5.329070518200751E-15
Non-optimal measurement 0.01737882160084258 < 0.01737882160084258. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.01737882160084258
Fitness changed from 14.485537739630754 to 0.01737882160084258
Iteration 1 complete. Error: 0.01737882160084258 Total: 0.0544; Orientation: 0.0025; Line Search: 0.0490
Non-optimal measurement 0.01737882160084258 < 0.01737882160084258. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.01737882160084258 < 0.01737882160084258. Total: 2
th(0)=0.01737882160084258;dx=-2.2350156442069267
Adding measurement 51558762 to history. Total: 2
New Minimum: 0.01737882160084258 > 0.017378821600842552
WOLFE (weak): th(2.154434690031884E-15)=0.017378821600842552; dx=-2.2350156442069267 evalInputDelta=2.7755575615628914E-17
Non-optimal measurement 0.017378821600842552 < 0.017378821600842552. Total: 3
WOLFE (weak): th(4.308869380063768E-15)=0.017378821600842552; dx=-2.2350156442069267 evalInputDelta=2.7755575615628914E-17
Adding measurement 6a0f03c to history. Total: 3
New Minimum: 0.017378821600842552 > 0.01737882160084248
WOLFE (weak): th(1.2926608140191303E-14)=0.01737882160084248; dx=-2.2350156442069267 evalInputDelta=1.0061396160665481E-16
Adding measurement 3c3452d5 to history. Total: 4
New Minimum: 0.01737882160084248 > 0.017378821600842143
WOLFE (weak): th(5.1706432560765214E-14)=0.017378821600842143; dx=-2.2350156442069267 evalInputDelta=4.371503159461554E-16
Adding measurement 62986275 to history. Total: 5
New Minimum: 0.017378821600842143 > 0.017378821600840387
WOLFE (weak): th(2.5853216280382605E-13)=0.017378821600840387; dx=-2.2350156442069267 evalInputDelta=2.192690473634684E-15
Adding measurement 6f53aa5b to history. Total: 6
New Minimum: 0.017378821600840387 > 0.017378821600829458
WOLFE (weak): th(1.5511929768229563E-12)=0.017378821600829458; dx=-2.2350156442069258 evalInputDelta=1.3121448372288569E-14
Adding measurement 7f1dd5cd to history. Total: 7
New Minimum: 0.017378821600829458 > 0.0173788216007507
WOLFE (weak): th(1.0858350837760695E-11)=0.0173788216007507; dx=-2.2350156442069196 evalInputDelta=9.188136362858756E-14
Adding measurement 708483dc to history. Total: 8
New Minimum: 0.0173788216007507 > 0.017378821600107446
WOLFE (weak): th(8.686680670208556E-11)=0.017378821600107446; dx=-2.2350156442068685 evalInputDelta=7.351341757555474E-13
Adding measurement 616d7f1e to history. Total: 9
New Minimum: 0.017378821600107446 > 0.01737882159422645
WOLFE (weak): th(7.8180126031877E-10)=0.01737882159422645; dx=-2.2350156442064044 evalInputDelta=6.616131253966984E-12
Adding measurement 289599ff to history. Total: 10
New Minimum: 0.01737882159422645 > 0.017378821534681205
WOLFE (weak): th(7.818012603187701E-9)=0.017378821534681205; dx=-2.235015644201705 evalInputDelta=6.616137498971497E-11
Adding measurement 3dd53e40 to history. Total: 11
New Minimum: 0.017378821534681205 > 0.017378820873067496
WOLFE (weak): th(8.599813863506471E-8)=0.017378820873067496; dx=-2.2350156441494873 evalInputDelta=7.277750832535013E-10
Adding measurement 64bc7160 to history. Total: 12
New Minimum: 0.017378820873067496 > 0.017378812867541865
WOLFE (weak): th(1.0319776636207765E-6)=0.017378812867541865; dx=-2.2350156435176505 evalInputDelta=8.733300714547365E-9
Adding measurement 73f678bb to history. Total: 13
New Minimum: 0.017378812867541865 > 0.0173787080679887
WOLFE (weak): th(1.3415709627070094E-5)=0.0173787080679887; dx=-2.2350156352463433 evalInputDelta=1.1353285387857848E-7
Adding measurement 596ed766 to history. Total: 14
New Minimum: 0.0173787080679887 > 0.01737723215182751
WOLFE (weak): th(1.878199347789813E-4)=0.01737723215182751; dx=-2.235015518760517 evalInputDelta=1.5894490150714091E-6
Adding measurement 7ad91133 to history. Total: 15
New Minimum: 0.01737723215182751 > 0.017354982339195406
WOLFE (weak): th(0.0028172990216847197)=0.017354982339195406; dx=-2.235013762908878 evalInputDelta=2.383926164717387E-5
Adding measurement 2b208c18 to history. Total: 16
New Minimum: 0.017354982339195406 > 0.016998027950037752
WOLFE (weak): th(0.045076784346955515)=0.016998027950037752; dx=-2.23498564543159 evalInputDelta=3.807936508048275E-4
Adding measurement 3b500a49 to history. Total: 17
New Minimum: 0.016998027950037752 > 0.01108222844325175
WOLFE (weak): th(0.7663053338982437)=0.01108222844325175; dx=-2.2345335185287634 evalInputDelta=0.006296593157590829
Adding measurement 3e994fd0 to history. Total: 18
New Minimum: 0.01108222844325175 > 0.0
WOLFE (weak): th(13.793496010168386)=0.0; dx=-2.2337819326222133 evalInputDelta=0.01737882160084258
Non-optimal measurement 0.0 < 0.0. Total: 19
WOLFE (weak): th(262.076424193199
...skipping 8015 bytes...
FGS Orientation magnitude: 9.719e+03, gradient 1.494e+00, dot -0.996; [8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, 7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751, 0.0173787080679887, 0.017378812867541865, 0.017378820873067496, 0.017378821534681205, 0.01737882159422645, 0.017378821600107446
Rejected: LBFGS Orientation magnitude: 9.719e+03, gradient 1.494e+00, dot -0.996; [30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00, 7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751, 0.0173787080679887, 0.017378812867541865, 0.017378820873067496, 0.017378821534681205, 0.01737882159422645
Rejected: LBFGS Orientation magnitude: 9.719e+03, gradient 1.494e+00, dot -0.996; [ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00, 7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751, 0.0173787080679887, 0.017378812867541865, 0.017378820873067496, 0.017378821534681205
Rejected: LBFGS Orientation magnitude: 9.719e+03, gradient 1.494e+00, dot -0.996; [a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00, ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, 7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751, 0.0173787080679887, 0.017378812867541865, 0.017378820873067496
Rejected: LBFGS Orientation magnitude: 9.719e+03, gradient 1.494e+00, dot -0.996; [7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751, 0.0173787080679887, 0.017378812867541865
Rejected: LBFGS Orientation magnitude: 1.006e+04, gradient 1.494e+00, dot -0.996; [7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751, 0.0173787080679887
Rejected: LBFGS Orientation magnitude: 9.987e+03, gradient 1.494e+00, dot -1.000; [7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406, 0.01737723215182751
Rejected: LBFGS Orientation magnitude: 1.003e+04, gradient 1.494e+00, dot -1.000; [ceb18b8b-df33-4612-a94d-5c9801904df5 = 1.000/1.000e+00, 8797a338-1e60-44c6-a45d-5d2753acf561 = 1.000/1.000e+00, 7ae1720f-52d7-4b7b-a8d8-b52dca8bea39 = 1.000/1.000e+00, a53a7e1c-cd88-46fa-ba5c-feb4c0797532 = 1.000/1.000e+00, 30323be7-5d03-4284-9ef6-d9677bfb5692 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.01108222844325175, 0.016998027950037752, 0.017354982339195406
LBFGS Accumulation History: 3 points
Removed measurement 3e994fd0 to history. Total: 18
Removed measurement 3b500a49 to history. Total: 17
Removed measurement 2b208c18 to history. Total: 16
Removed measurement 7ad91133 to history. Total: 15
Removed measurement 596ed766 to history. Total: 14
Removed measurement 73f678bb to history. Total: 13
Removed measurement 64bc7160 to history. Total: 12
Removed measurement 3dd53e40 to history. Total: 11
Removed measurement 289599ff to history. Total: 10
Removed measurement 616d7f1e to history. Total: 9
Removed measurement 708483dc to history. Total: 8
Removed measurement 7f1dd5cd to history. Total: 7
Removed measurement 6f53aa5b to history. Total: 6
Removed measurement 62986275 to history. Total: 5
Removed measurement 3c3452d5 to history. Total: 4
Removed measurement 6a0f03c to history. Total: 3
Adding measurement 631a28c6 to history. Total: 3
th(0)=0.0;dx=-2.2327283200000005
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(16751.322535873343)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(8375.661267936672)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2791.887089312224)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(697.971772328056)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(139.5943544656112)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(23.265725744268533)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.323675106324076)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.4154593882905095)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.046162154254501056)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.004616215425450106)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.196559477681914E-4)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.4971328980682615E-5)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.690102229283278E-6)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.9215015923451987E-7)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.2810010615634659E-8)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(8.006256634771662E-10)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.7095627263362714E-11)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.616423736853484E-12)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.3770651246597284E-13)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.885325623298642E-15)=0.0; dx=-2.2327283200000005 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
MIN ALPHA (3.2787264872850675E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.6909; Orientation: 0.6497; Line Search: 0.0401
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.819s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -2.759979674930041], [2.0, -0.7599796749300409]; valueStats=DoubleSummaryStatistics{count=2, sum=0.034758, min=0.017379, average=0.017379, max=0.017379}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -2.759979674930041], [0.11, -0.7599796749300409]; valueStats=DoubleSummaryStatistics{count=2, sum=0.034758, min=0.017379, average=0.017379, max=0.017379}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0042 = network_target.state();
assert temp_18_0042 != null;
String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0042.freeRef();
return temp_18_0041;
Returns
[1.108, -0.712, 1.048, -0.804, 0.028]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.03 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 744833853176
Reset training subject: 744835231679
Constructing line search parameters: GD
th(0)=1.7382913758153855;dx=-28.28936675599244
New Minimum: 1.7382913758153855 > 0.0
END: th(2.154434690031884)=0.0; dx=-15.951545001739134 evalInputDelta=1.7382913758153855
Fitness changed from 1.7382913758153855 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0067; Orientation: 0.0004; Line Search: 0.0024
th(0)=0.0;dx=-9.908042880000004
Armijo: th(4.641588833612779)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(2.3207944168063896)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(0.7735981389354633)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(0.1933995347338658)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Armijo: th(0.03867990694677316)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(0.006446651157795527)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(9.20950165399361E-4)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(1.1511877067492013E-4)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(1.279097451943557E-5)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(1.279097451943557E-6)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(1.1628158654032335E-7)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(9.690132211693613E-9)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(7.453947855148933E-10)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(5.324248467963524E-11)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(3.5494989786423493E-12)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(2.2184368616514683E-13)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
Armijo: th(1.3049628597949813E-14)=0.0; dx=-9.908042880000004 evalInputDelta=0.0
MIN ALPHA (7.249793665527674E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0199; Orientation: 0.0003; Line Search: 0.0188
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.027s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.04 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 744863400485
Reset training subject: 744868402854
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.7382913758153855}, derivative=-28.289366755992432}
New Minimum: 1.7382913758153855 > 1.738291373389003
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.738291373389003}, derivative=-28.289366728618678}, evalInputDelta = -2.426382428311058E-9
New Minimum: 1.738291373389003 > 1.7382913588307072
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.7382913588307072}, derivative=-28.28936656437614}, evalInputDelta = -1.6984678330445035E-8
New Minimum: 1.7382913588307072 > 1.7382912569226405
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.7382912569226405}, derivative=-28.28936541467845}, evalInputDelta = -1.1889274498244617E-7
New Minimum: 1.7382912569226405 > 1.7382905435663054
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.7382905435663054}, derivative=-28.289357366798104}, evalInputDelta = -8.32249080096048E-7
New Minimum: 1.7382905435663054 > 1.738285550078587
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.738285550078587}, derivative=-28.289301031807128}, evalInputDelta = -5.825736798525938E-6
New Minimum: 1.738285550078587 > 1.7382505959891787
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.7382505959891787}, derivative=-28.288906695270256}, evalInputDelta = -4.077982620676046E-5
New Minimum: 1.7382505959891787 > 1.7380059332681106
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.7380059332681106}, derivative=-28.286146751041088}, evalInputDelta = -2.854425472749167E-4
New Minimum: 1.7380059332681106 > 1.7362940730039411
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.7362940730039411}, derivative=-28.266847283255526}, evalInputDelta = -0.0019973028114443903
New Minimum: 1.7362940730039411 > 1.7243490235090426
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.7243490235090426}, derivative=-28.132730099955907}, evalInputDelta = -0.013942352306342887
New Minimum: 1.7243490235090426 > 1.6425324302465314
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.6425324302465314}, derivative=-27.239339356386992}, evalInputDelta = -0.09575894556885411
New Minimum: 1.6425324302465314 > 1.1414497304992788
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.1414497304992788}, derivative=-22.59119855066287}, evalInputDelta = -0.5968416453161067
New Minimum: 1.1414497304992788 > 0.0988068264959967
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.0988068264959967}, derivative=-16.04008737617162}, evalInputDelta = -1.6394845493193888
New Minimum: 0.0988068264959967 > 0.0
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.95154500173913}, evalInputDelta = -1.7382913758153855
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.95154500173913}, evalInputDelta = -1.7382913758153855
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.95154500173913}, evalInputDelta = -1.7382913758153855
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.95154500173913}, evalInputDelta = -1.7382913758153855
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.95154500173913}, evalInputDelta = -1.7382913758153855
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.95154500173913}, evalInputDelta = -1.7382913758153855
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
0.0 <= 1.7382913758153855
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-15.951545001739131}, evalInputDelta = -1.7382913758153855
Right bracket at 1.0E10
Converged to right
Fitness changed from 1.7382913758153855 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0340; Orientation: 0.0002; Line Search: 0.0270
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-9.908042880000004}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-9.908042880000002}, evalInputDelta = 0.0
0.0 <= 0.0
Converged to right
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0027; Orientation: 0.0002; Line Search: 0.0017
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.036s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.03 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 744904038589
Reset training subject: 744905208551
Adding measurement 158e992f to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.7382913758153855 < 1.7382913758153855. Total: 1
th(0)=1.7382913758153855;dx=-28.289366755992432
Adding measurement 1e879823 to history. Total: 1
New Minimum: 1.7382913758153855 > 0.0
END: th(2.154434690031884)=0.0; dx=-15.951545001739131 evalInputDelta=1.7382913758153855
Fitness changed from 1.7382913758153855 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0054; Orientation: 0.0006; Line Search: 0.0022
Non-optimal measurement 0.0 < 0.0. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0 < 0.0. Total: 2
th(0)=0.0;dx=-9.908042880000002
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.641588833612779)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.3207944168063896)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.7735981389354633)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.1933995347338658)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.03867990694677316)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.006446651157795527)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(9.20950165399361E-4)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.1511877067492013E-4)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.279097451943557E-5)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.279097451943557E-6)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.1628158654032335E-7)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(9.690132211693613E-9)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(7.453947855148933E-10)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.324248467963524E-11)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.5494989786423493E-12)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.2184368616514683E-13)=0.0; dx=-9.90804288 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.3049628597949813E-14)=0.0; dx=-9.908042880000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (7.249793665527674E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0196; Orientation: 0.0004; Line Search: 0.0185
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.025s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[0.0, 0.0], [2.0, 1.0]; valueStats=DoubleSummaryStatistics{count=0, sum=0.000000, min=Infinity, average=0.000000, max=-Infinity}
Only 0 points for GD
Only 0 points for CjGD
Only 0 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
No Data
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0037 = temp_18_0035.state();
assert temp_18_0037 != null;
String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0037.freeRef();
return temp_18_0036;
Returns
[-0.712, 0.028, 1.108, -0.804, 1.048]
We simultaneously regress this target input:
TrainingTester.java:287 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
if (null != x)
RefUtil.freeRef(x);
return temp_18_0006;
}).map(x -> {
String temp_18_0007 = x.prettyPrint();
x.freeRef();
return temp_18_0007;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ 1.512, -1.616, 1.556, 1.356, 1.032 ]
[ -1.616, 1.356, 1.512, 1.556, 1.032 ]
[ 1.512, 1.032, -1.616, 1.356, 1.556 ]
[ 1.032, 1.556, 1.356, 1.512, -1.616 ]
[ 1.356, -1.616, 1.032, 1.512, 1.556 ]
Which produces the following output:
TrainingTester.java:308 executed in 0.00 seconds (0.000 gc):
return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
String temp_18_0008 = x.prettyPrint();
x.freeRef();
return temp_18_0008;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[ 0.8, -1.588, 2.664, 0.552, 2.08 ]
[ -2.3280000000000003, 1.3840000000000001, 2.62, 0.752, 2.08 ]
[ 0.8, 1.06, -0.508, 0.552, 2.604 ]
[ 0.32000000000000006, 1.584, 2.4640000000000004, 0.708, -0.5680000000000001 ]
[ 0.6440000000000001, -1.588, 2.14, 0.708, 2.604 ]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.04 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 744994336994
Reset training subject: 744995549218
Constructing line search parameters: GD
th(0)=43.11251288026838;dx=-1.3441203200062158E24
New Minimum: 43.11251288026838 > 0.0
Armijo: th(2.154434690031884)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(1.077217345015942)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(0.3590724483386473)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(0.08976811208466183)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(0.017953622416932366)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(0.002992270402822061)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(4.2746720040315154E-4)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(5.343340005039394E-5)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(5.9370444500437714E-6)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(5.937044450043771E-7)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(5.397313136403428E-8)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(4.4977609470028565E-9)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(3.4598161130791205E-10)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Armijo: th(2.4712972236279432E-11)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Armijo: th(1.6475314824186289E-12)=41.424426857000086; dx=-1.3441203200047866E24 evalInputDelta=1.6880860232682977
Armijo: th(1.029707176511643E-13)=42.830549618851805; dx=-1.344120320005899E24 evalInputDelta=0.28196326141657835
Armijo: th(6.057101038303783E-15)=43.094989682616585; dx=-1.3441203200061946E24 evalInputDelta=0.017523197651797773
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0
Fitness changed from 43.11251288026838 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0328; Orientation: 0.0005; Line Search: 0.0293
th(0)=0.0;dx=-11.01544576
Armijo: th(2.154434690031884E-15)=0.0; dx=-11.015445760000002 evalInputDelta=0.0
Armijo: th(1.077217345015942E-15)=0.0; dx=-11.015445760000002 evalInputDelta=0.0
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0057; Orientation: 0.0004; Line Search: 0.0046
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.038s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.10 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 745035966494
Reset training subject: 745036892017
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=43.11251288026838}, derivative=-1.3441203200062158E24}
New Minimum: 43.11251288026838 > 0.0
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144507E12}, evalInputDelta = -43.11251288026838
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014451E12}, evalInputDelta = -43.11251288026838
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.0227590400144517E12}, evalInputDelta = -43.11251288026838
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014452E12}, evalInputDelta = -43.11251288026838
0.0 <= 43.11251288026838
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.022759040014451E12}, evalInputDelta = -43.11251288026838
Right bracket at 1.0E10
Converged to right
Fitness changed from 43.11251288026838 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0384; Orientation: 0.0003; Line Search: 0.0358
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000004}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000004}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000002}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000004}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.01544576}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-11.015445760000004}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0578; Orientation: 0.0003; Line Search: 0.0567
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.097s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.07 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 745136266252
Reset training subject: 745137165321
Adding measurement 21fb98b6 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 43.11251288026838 < 43.11251288026838. Total: 1
th(0)=43.11251288026838;dx=-1.3441203200062158E24
Adding measurement 785cab20 to history. Total: 1
New Minimum: 43.11251288026838 > 0.0
Armijo: th(2.154434690031884)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.077217345015942)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.3590724483386473)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.08976811208466183)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.017953622416932366)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.002992270402822061)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.2746720040315154E-4)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.343340005039394E-5)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.9370444500437714E-6)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.937044450043771E-7)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.397313136403428E-8)=0.0; dx=-3.0227590400144517E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.4977609470028565E-9)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.4598161130791205E-10)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.4712972236279432E-11)=0.0; dx=-3.022759040014452E12 evalInputDelta=43.11251288026838
Non-optimal measurement 41.424426857000086 < 0.0. Total: 2
Armijo: th(1.6475314824186289E-12)=41.424426857000086; dx=-1.3441203200047866E24 evalInputDelta=1.6880860232682977
Non-optimal measurement 42.830549618851805 < 0.0. Total: 2
Armijo: th(1.029707176511643E-13)=42.830549618851805; dx=-1.344120320005899E24 evalInputDelta=0.28196326141657835
Non-optimal measurement 43.094989682616585 < 0.0. Total: 2
Armijo: th(6.057101038303783E-15)=43.094989682616585; dx=-1.3441203200061948E24 evalInputDelta=0.017523197651797773
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0
Fitness changed from 43.11251288026838 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0320; Orientation: 0.0008; Line Search: 0.0290
Non-optimal measurement 0.0 < 0.0. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0 < 0.0. Total: 2
th(0)=0.0;dx=-11.015445760000004
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.154434690031884E-15)=0.0; dx=-11.01544576 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.077217345015942E-15)=0.0; dx=-11.015445760000002 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0349; Orientation: 0.0006; Line Search: 0.0335
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.068s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[0.0, 0.0], [2.0, 1.0]; valueStats=DoubleSummaryStatistics{count=0, sum=0.000000, min=Infinity, average=0.000000, max=-Infinity}
Only 0 points for GD
Only 0 points for CjGD
Only 0 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
No Data
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }} | OK |
{
"result": "OK",
"performance": {
"execution_time": "2.548",
"gc_time": "0.394"
},
"created_on": 1586735331094,
"file_name": "trainingTest",
"report": {
"simpleName": "Basic",
"canonicalName": "com.simiacryptus.mindseye.layers.java.BiasLayerTest.Basic",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/BiasLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"model": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"complete": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/BiasLayer/Basic/trainingTest/202004124851",
"id": "141f2953-f714-4eb3-926c-ecbf9279848e",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "BiasLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.BiasLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/BiasLayer.java",
"javaDoc": ""
}
}