Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 2803102653318710272
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 1.556, 1.356, 1.048 ], [ 1.512, 1.108, -0.804 ] ],
[ [ -0.712, -1.616, -1.688 ], [ -0.852, 0.028, 1.912 ] ]
]
[
[ [ -0.804, -0.852, 1.912 ], [ -1.616, 1.108, 1.556 ] ],
[ [ 1.512, -0.712, 0.028 ], [ 1.356, 1.048, -1.688 ] ]
]
[
[ [ -0.804, 1.048, 1.356 ], [ -1.688, 1.912, 1.512 ] ],
[ [ -0.712, 1.556, 0.028 ], [ -0.852, 1.108, -1.616 ] ]
]
[
[ [ 1.556, -1.688, 1.912 ], [ 1.356, -1.616, -0.852 ] ],
[ [ -0.804, 1.048, 1.108 ], [ 1.512, -0.712, 0.028 ] ]
]
[
[ [ -1.688, 1.048, 1.356 ], [ 1.512, -0.804, -0.852 ] ],
[ [ 0.028, -0.712, -1.616 ], [ 1.556, 1.108, 1.912 ] ]
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.46 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4765545430116
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 4765573101574
Constructing line search parameters: GD
th(0)=96.40497059202553;dx=-3.605078692607362E24
New Minimum: 96.40497059202553 > 0.2172805043380504
Armijo: th(2.154434690031884)=0.2172805043380504; dx=-3.605078692608996E12 evalInputDelta=96.18769008768749
Armijo: th(1.077217345015942)=0.2514571633688658; dx=-3.605078692609E12 evalInputDelta=96.15351342865667
Armijo: th(0.3590724483386473)=0.3340071310953624; dx=-3.605078692609027E12 evalInputDelta=96.07096346093017
Armijo: th(0.08976811208466183)=0.45963927639458957; dx=-3.6050786926090845E12 evalInputDelta=95.94533131563095
Armijo: th(0.017953622416932366)=0.4959638587169496; dx=-3.605078692609103E12 evalInputDelta=95.90900673330859
Armijo: th(0.002992270402822061)=0.50370136782179; dx=-3.6050786926091074E12 evalInputDelta=95.90126922420374
Armijo: th(4.2746720040315154E-4)=0.5050338853239461; dx=-3.605078692609108E12 evalInputDelta=95.8999367067016
Armijo: th(5.343340005039394E-5)=0.5052283606997089; dx=-3.605078692609108E12 evalInputDelta=95.89974223132583
Armijo: th(5.9370444500437714E-6)=0.5052530587191134; dx=-3.605078692609108E12 evalInputDelta=95.89971753330641
Armijo: th(5.937044450043771E-7)=0.5052558372848523; dx=-3.605078692609108E12 evalInputDelta=95.89971475474069
Armijo: th(5.397313136403428E-8)=0.505256117948492; dx=-3.605078692609108E12 evalInputDelta=95.89971447407704
Armijo: th(4.4977609470028565E-9)=0.5052561436759962; dx=-3.605078692609108E12 evalInputDelta=95.89971444834954
Armijo: th(3.4598161130791205E-10)=0.5052561458349476; dx=-3.605078692609108E12 evalInputDelta=95.89971444619059
Armijo: th(2.4712972236279432E-11)=7.11633933215622; dx=-1.8849225083165879E21 evalInputDelta=89.28863125986932
Armijo: th(1.6475314824186289E-12)=52.340418492519085; dx=-6.958200822273676E23 evalInputDelta=44.06455209950645
Armijo: th(1.029707176511643E-13)=96.40497059202548; dx=-3.605078692607362E24 evalInputDelta=5.6843418860808015E-14
Armijo: th(6.057101038303783E-15)=96.40497059202552; dx=-3.605078692607362E24 evalInputDelta=1.4210854715202004E-14
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.2172805043380504
Fitness changed from 96.40497059202553 to 0.2172805043380504
Iteration 1 complete. Error: 0.2172805043380504 Total: 0.2332; Orientation: 0.0031; Line Search: 0.1930
th(0)=0.2172805043380504;dx=-5.152574465202571
New Minimum: 0.2172805043380504 > 0.21728050433805035
WOLFE (weak): th(2.154434690031884E-15)=0.21728050433805035; dx=-5.152574465202571 evalInputDelta=5.551115123125783E-17
New Minimum: 0.21728050433805035 > 0.2172805043380503
WOLFE (weak): th(4.308869380063768E-15)=0.2172805043380503; dx=-5.152574465202571 evalInputDelta=1.1102230246251565E-16
New Minimum: 0.2172805043380503 > 0.21728050433805007
WOLFE (weak): th(1.2926608140191303E-14)=0.21728050433805007; dx=-5.152574465202571 evalInputDelta=3.3306690738754696E-16
New Minimum: 0.21728050433805007 > 0.21728050433804924
WOLFE (weak): th(5.1706432560765214E-14)=0.21728050433804924; dx=-5.15257446520257 evalInputDelta=1.1657341758564144E-15
New Minimum: 0.21728050433804924 > 0.21728050433804463
WOLFE (weak): th(2.5853216280382605E-13)=0.21728050433804463; dx=-5.15257446520257 evalInputDelta=5.773159728050814E-15
New Minimum: 0.21728050433804463 > 0.21728050433801566
WOLFE (weak): th(1.5511929768229563E-12)=0.21728050433801566; dx=-5.152574465202568 evalInputDelta=3.47499806707674E-14
New Minimum: 0.21728050433801566 > 0.21728050433780757
WOLFE (weak): th(1.0858350837760695E-11)=0.21728050433780757; dx=-5.152574465202551 evalInputDelta=2.4283353106113736E-13
New Minimum: 0.21728050433780757 > 0.2172805043361076
WOLFE (weak): th(8.686680670208556E-11)=0.2172805043361076; dx=-5.152574465202413 evalInputDelta=1.942807026367177E-12
New Minimum: 0.2172805043361076 > 0.21728050432056484
WOLFE (weak): th(7.8180126031877E-10)=0.21728050432056484; dx=-5.152574465201154 evalInputDelta=1.7485568548636365E-11
New Minimum: 0.21728050432056484 > 0.21728050416319475
WOLFE (weak): th(7.818012603187701E-9)=0.21728050416319475; dx=-5.1525744651883985 evalInputDelta=1.7485565773078804E-10
New Minimum: 0.21728050416319475 > 0.2172805024146383
WOLFE (weak): th(8.599813863506471E-8)=0.2172805024146383; dx=-5.152574465046679 evalInputDelta=1.9234120962607903E-9
New Minimum: 0.2172805024146383 > 0.2172804812571064
WOLFE (weak): th(1.0319776636207765E-6)=0.2172804812571064; dx=-5.15257446333187 evalInputDelta=2.3080944017150884E-8
New Minimum: 0.2172804812571064 > 0.2172802042859286
WOLFE (weak): th(1.3415709627070094E-5)=0.2172802042859286; dx=-5.1525744408834955 evalInputDelta=3.0005212181549723E-7
New Minimum: 0.2172802042859286 > 0.21727630363803402
WOLFE (weak): th(1.878199347789813E-4)=0.21727630363803402; dx=-5.1525741247407435 evalInputDelta=4.200700016387948E-6
New Minimum: 0.21727630363803402 > 0.21721750055101124
WOLFE (weak): th(0.0028172990216847197)=0.21721750055101124; dx=-5.152569359456423 evalInputDelta=6.300378703916198E-5
New Minimum: 0.21721750055101124 > 0.21627416546733547
WOLFE (weak): th(0.045076784346955515)=0.21627416546733547; dx=-5.152493075771811 evalInputDelta=0.00100633887071494
New Minimum: 0.21627416546733547 > 0.20065102572076512
WOLFE (weak): th(0.7663053338982437)=0.20065102572076512; dx=-5.151272879990985 evalInputDelta=0.01662947861728528
New Minimum: 0.20065102572076512 > 0.020565784925124904
WOLFE (weak): th(13.793496010168386)=0.020565784925124904; dx=-5.14149110560229 evalInputDelta=0.1967147194129255
New Minimum: 0.020565784925124904 > 0.0
WOLFE (weak): th(262.07642419319933)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
WOLFE (weak): th(5241.528483863986)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
Armijo: th(110072.09816114372)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
Armijo: th(57656.813322503855)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
WOLFE (weak): th(31449.17090318392)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
Armijo: th(44552.99211284389)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
WOLFE (weak): th(38001.0815080139)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
WOLFE (weak): th(41277.036810428894)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
Armijo: th(42915.01446163639)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
WOLFE (weak): th(42096.025636032646)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
Armijo: th(42505.520048834514)=0.0; dx=-5.140823878272537 evalInputDelta=0.2172805043380504
mu ~= nu (42096.025636032646): th(262.07642419319933)=0.0
Fitness changed from 0.2172805043380504 to 0.0
Iteration 2 complete. Error: 0.0 Total: 0.1324; Orientation: 0.0012; Line Search: 0.1271
th(0)=0.0;dx=-5.135589491842355
Armijo: th(91134.25242689752)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(45567.12621344876)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(15189.042071149588)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(3797.260517787397)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(759.4521035574794)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(126.57535059291324)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(18.08219294184475)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(2.2602741177305936)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(0.25114156863673265)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(0.025114156863673263)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(0.0022831051694248423)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(1.9025876411873685E-4)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(1.4635289547595142E-5)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(1.0453778248282244E-6)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(6.969185498854829E-8)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(4.355740936784268E-9)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(2.562200551049569E-10)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(1.423444750583094E-11)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(7.491814476753127E-13)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(3.7459072383765636E-14)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Armijo: th(1.7837653516078873E-15)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
MIN ALPHA (8.108024325490396E-17): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0867; Orientation: 0.0008; Line Search: 0.0837
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 0.453s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.22 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4766004238079
Reset training subject: 4766005824794
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=96.40497059202553}, derivative=-3.605078692607362E24}
New Minimum: 96.40497059202553 > 0.7439777327457794
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.7439777327457794}, derivative=-3.6087728818655576E12}, evalInputDelta = -95.66099285927976
New Minimum: 0.7439777327457794 > 0.5052561456508557
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.5052561456508557}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89971444637467
New Minimum: 0.5052561456508557 > 0.5052561434668295
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.5052561434668295}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89971444855871
New Minimum: 0.5052561434668295 > 0.5052561281786441
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.5052561281786441}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89971446384689
New Minimum: 0.5052561281786441 > 0.5052560211613537
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.5052560211613537}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89971457086418
New Minimum: 0.5052560211613537 > 0.5052552720406454
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.5052552720406454}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89971531998489
New Minimum: 0.5052552720406454 > 0.505250028211558
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.505250028211558}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89972056381397
New Minimum: 0.505250028211558 > 0.505213322185593
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.505213322185593}, derivative=-3.605078692609108E12}, evalInputDelta = -95.89975726983994
New Minimum: 0.505213322185593 > 0.5049564180999428
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.5049564180999428}, derivative=-3.605078692609108E12}, evalInputDelta = -95.9000141739256
New Minimum: 0.5049564180999428 > 0.5031599532575367
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.5031599532575367}, derivative=-3.605078692609107E12}, evalInputDelta = -95.901810638768
New Minimum: 0.5031599532575367 > 0.4906750241038348
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.4906750241038348}, derivative=-3.6050786926091E12}, evalInputDelta = -95.9142955679217
New Minimum: 0.4906750241038348 > 0.407392366701887
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.407392366701887}, derivative=-3.6050786926090596E12}, evalInputDelta = -95.99757822532365
New Minimum: 0.407392366701887 > 0.2412443098343589
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.2412443098343589}, derivative=-3.6050786926089985E12}, evalInputDelta = -96.16372628219118
New Minimum: 0.2412443098343589 > 0.05430791983184609
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.05430791983184609}, derivative=-3.6050786926089824E12}, evalInputDelta = -96.35066267219369
New Minimum: 0.05430791983184609 > 0.0
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.6050786926089795E12}, evalInputDelta = -96.40497059202553
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.6050786926089795E12}, evalInputDelta = -96.40497059202553
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.6050786926089795E12}, evalInputDelta = -96.40497059202553
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.6050786926089795E12}, evalInputDelta = -96.40497059202553
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.6050786926089795E12}, evalInputDelta = -96.40497059202553
0.0 <= 96.40497059202553
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.60507869260898E12}, evalInputDelta = -96.40497059202553
Right bracket at 1.0E10
Converged to right
Fitness changed from 96.40497059202553 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.1005; Orientation: 0.0007; Line Search: 0.0953
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.135589491842355}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.1162; Orientation: 0.0007; Line Search: 0.1136
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.217s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 2.64 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4766226617362
Reset training subject: 4766228177632
Adding measurement 5ce03960 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 96.40497059202553 < 96.40497059202553. Total: 1
th(0)=96.40497059202553;dx=-3.605078692607362E24
Adding measurement 6dcb678d to history. Total: 1
New Minimum: 96.40497059202553 > 0.2172805043380504
Armijo: th(2.154434690031884)=0.2172805043380504; dx=-3.6050786926089956E12 evalInputDelta=96.18769008768749
Non-optimal measurement 0.2514571633688658 < 0.2172805043380504. Total: 2
Armijo: th(1.077217345015942)=0.2514571633688658; dx=-3.605078692609E12 evalInputDelta=96.15351342865667
Non-optimal measurement 0.3340071310953624 < 0.2172805043380504. Total: 2
Armijo: th(0.3590724483386473)=0.3340071310953624; dx=-3.605078692609027E12 evalInputDelta=96.07096346093017
Non-optimal measurement 0.45963927639458957 < 0.2172805043380504. Total: 2
Armijo: th(0.08976811208466183)=0.45963927639458957; dx=-3.6050786926090845E12 evalInputDelta=95.94533131563095
Non-optimal measurement 0.4959638587169496 < 0.2172805043380504. Total: 2
Armijo: th(0.017953622416932366)=0.4959638587169496; dx=-3.605078692609103E12 evalInputDelta=95.90900673330859
Non-optimal measurement 0.50370136782179 < 0.2172805043380504. Total: 2
Armijo: th(0.002992270402822061)=0.50370136782179; dx=-3.6050786926091074E12 evalInputDelta=95.90126922420374
Non-optimal measurement 0.5050338853239461 < 0.2172805043380504. Total: 2
Armijo: th(4.2746720040315154E-4)=0.5050338853239461; dx=-3.605078692609108E12 evalInputDelta=95.8999367067016
Non-optimal measurement 0.5052283606997089 < 0.2172805043380504. Total: 2
Armijo: th(5.343340005039394E-5)=0.5052283606997089; dx=-3.605078692609108E12 evalInputDelta=95.89974223132583
Non-optimal measurement 0.5052530587191134 < 0.2172805043380504. Total: 2
Armijo: th(5.9370444500437714E-6)=0.5052530587191134; dx=-3.605078692609108E12 evalInputDelta=95.89971753330641
Non-optimal measurement 0.5052558372848523 < 0.2172805043380504. Total: 2
Armijo: th(5.937044450043771E-7)=0.5052558372848523; dx=-3.605078692609108E12 evalInputDelta=95.89971475474069
Non-optimal measurement 0.505256117948492 < 0.2172805043380504. Total: 2
Armijo: th(5.397313136403428E-8)=0.505256117948492; dx=-3.605078692609108E12 evalInputDelta=95.89971447407704
Non-optimal measurement 0.5052561436759962 < 0.2172805043380504. Total: 2
Armijo: th(4.4977609470028565E-9)=0.5052561436759962; dx=-3.605078692609108E12 evalInputDelta=95.89971444834954
Non-optimal measurement 0.5052561458349476 < 0.2172805043380504. Total: 2
Armijo: th(3.4598161130791205E-10)=0.5052561458349476; dx=-3.605078692609108E12 evalInputDelta=95.89971444619059
Non-optimal measurement 7.11633933215622 < 0.2172805043380504. Total: 2
Armijo: th(2.4712972236279432E-11)=7.11633933215622; dx=-1.8849225083165879E21 evalInputDelta=89.28863125986932
Non-optimal measurement 52.340418492519085 < 0.2172805043380504. Total: 2
Armijo: th(1.6475314824186289E-12)=52.340418492519085; dx=-6.958200822273676E23 evalInputDelta=44.06455209950645
Non-optimal measurement 96.40497059202548 < 0.2172805043380504. Total: 2
Armijo: th(1.029707176511643E-13)=96.40497059202548; dx=-3.605078692607362E24 evalInputDelta=5.6843418860808015E-14
Non-optimal measurement 96.40497059202552 < 0.2172805043380504. Total: 2
Armijo: th(6.057101038303783E-15)=96.40497059202552; dx=-3.605078692607362E24 evalInputDelta=1.4210854715202004E-14
Non-optimal measurement 0.2172805043380504 < 0.2172805043380504. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.2172805043380504
Fitness changed from 96.40497059202553 to 0.2172805043380504
Iteration 1 complete. Error: 0.2172805043380504 Total: 0.1443; Orientation: 0.0035; Line Search: 0.1361
Non-optimal measurement 0.2172805043380504 < 0.2172805043380504. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.2172805043380504 < 0.2172805043380504. Total: 2
th(0)=0.2172805043380504;dx=-5.152574465202571
Adding measurement 3e5d25bf to history. Total: 2
New Minimum: 0.2172805043380504 > 0.21728050433805035
WOLFE (weak): th(2.154434690031884E-15)=0.21728050433805035; dx=-5.152574465202571 evalInputDelta=5.551115123125783E-17
Adding measurement 3de16532 to history. Total: 3
New Minimum: 0.21728050433805035 > 0.2172805043380503
WOLFE (weak): th(4.308869380063768E-15)=0.2172805043380503; dx=-5.152574465202571 evalInputDelta=1.1102230246251565E-16
Adding measurement 123a3741 to history. Total: 4
New Minimum: 0.2172805043380503 > 0.21728050433805007
WOLFE (weak): th(1.2926608140191303E-14)=0.21728050433805007; dx=-5.152574465202571 evalInputDelta=3.3306690738754696E-16
Adding measurement 2bb765d2 to history. Total: 5
New Minimum: 0.21728050433805007 > 0.21728050433804924
WOLFE (weak): th(5.1706432560765214E-14)=0.21728050433804924; dx=-5.15257446520257 evalInputDelta=1.1657341758564144E-15
Adding measurement 3942cf5 to history. Total: 6
New Minimum: 0.21728050433804924 > 0.21728050433804463
WOLFE (weak): th(2.5853216280382605E-13)=0.21728050433804463; dx=-5.15257446520257 evalInputDelta=5.773159728050814E-15
Adding measurement 28810c25 to history. Total: 7
New Minimum: 0.21728050433804463 > 0.21728050433801566
WOLFE (weak): th(1.5511929768229563E-12)=0.21728050433801566; dx=-5.152574465202568 evalInputDelta=3.47499806707674E-14
Adding measurement 62265e83 to history. Total: 8
New Minimum: 0.21728050433801566 > 0.21728050433780757
WOLFE (weak): th(1.0858350837760695E-11)=0.21728050433780757; dx=-5.152574465202551 evalInputDelta=2.4283353106113736E-13
Adding measurement 409c51aa to history. Total: 9
New Minimum: 0.21728050433780757 > 0.2172805043361076
WOLFE (weak): th(8.686680670208556E-11)=0.2172805043361076; dx=-5.152574465202413 evalInputDelta=1.942807026367177E-12
Adding measurement 3aac5060 to history. Total: 10
New Minimum: 0.2172805043361076 > 0.21728050432056484
WOLFE (weak): th(7.8180126031877E-10)=0.21728050432056484; dx=-5.152574465201154 evalInputDelta=1.7485568548636365E-11
Adding measurement 6a81b3d7 to history. Total: 11
New Minimum: 0.21728050432056484 > 0.21728050416319475
WOLFE (weak): th(7.818012603187701E-9)=0.21728050416319475; dx=-5.1525744651883985 evalInputDelta=1.7485565773078804E-10
Adding measurement 320d9810 to history. Total: 12
New Minimum: 0.21728050416319475 > 0.2172805024146383
WOLFE (weak): th(8.599813863506471E-8)=0.2172805024146383; dx=-5.152574465046679 evalInputDelta=1.9234120962607903E-9
Adding measurement 3035c94d to history. Total: 13
New Minimum: 0.2172805024146383 > 0.2172804812571064
WOLFE (weak): th(1.0319776636207765E-6)=0.2172804812571064; dx=-5.15257446333187 evalInputDelta=2.3080944017150884E-8
Adding measurement 618d5284 to history. Total: 14
New Minimum: 0.2172804812571064 > 0.2172802042859286
WOLFE (weak): th(1.3415709627070094E-5)=0.2172802042859286; dx=-5.1525744408834955 evalInputDelta=3.0005212181549723E-7
Adding measurement 64c54206 to history. Total: 15
New Minimum: 0.2172802042859286 > 0.21727630363803402
WOLFE (weak): th(1.878199347789813E-4)=0.21727630363803402; dx=-5.1525741247407435 evalInputDelta=4.200700016387948E-6
Adding measurement 28347e0d to history. Total: 16
New Minimum: 0.21727630363803402 > 0.21721750055101124
WOLFE (weak): th(0.0028172990216847197)=0.21721750055101124; dx=-5.152569359456423 evalInputDelta=6.300378703916198E-5
Adding measurement 39e09215 to history. Total: 17
New Minimum: 0.21721750055101124 > 0.21627416546733547
WOLFE (weak): th(0.045076784346955515)=0.21627416546733547; dx=-5.152493075771811 evalInputDelta=0.00100633887071494
Adding measurement 2b8040ec to history. Total: 18
New Minimum: 0.21627416546733547 > 0.20065102572076512
WOLFE (weak): th(0.7663053338982437)=0.20065102572076512; dx=-5.151272879990985 evalInputDelta=0.01662947861728528
Adding measurement 48f115be to history. Total: 19
New Minimum: 0.20065102572076512 > 0.020565784925124904
WOLFE (weak): th(13.793496010168386)=0.020565784925124904; dx=-5.14149110560229 evalInputDelta=0.1967147194129255
Adding measurement 4d53672f to history. Total: 20
New Minimum: 0.020565784925124904 > 0.0
WOLFE (weak): th(262.07642419319933)=0.0; dx
...skipping 9318 bytes...
e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124, 0.21727630363803402, 0.2172802042859286, 0.2172804812571064, 0.2172805024146383, 0.21728050416319475, 0.21728050432056484
Rejected: LBFGS Orientation magnitude: 1.477e+04, gradient 2.266e+00, dot -0.962; [c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, 144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, 43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124, 0.21727630363803402, 0.2172802042859286, 0.2172804812571064, 0.2172805024146383, 0.21728050416319475
Rejected: LBFGS Orientation magnitude: 1.477e+04, gradient 2.266e+00, dot -0.962; [8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00, 144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, 43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124, 0.21727630363803402, 0.2172802042859286, 0.2172804812571064, 0.2172805024146383
Rejected: LBFGS Orientation magnitude: 1.477e+04, gradient 2.266e+00, dot -0.962; [43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00, c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, 144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124, 0.21727630363803402, 0.2172802042859286, 0.2172804812571064
Rejected: LBFGS Orientation magnitude: 1.829e+04, gradient 2.266e+00, dot -0.990; [c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, 144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, 43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124, 0.21727630363803402, 0.2172802042859286
Rejected: LBFGS Orientation magnitude: 1.288e+04, gradient 2.266e+00, dot -1.000; [43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00, c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, 144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124, 0.21727630363803402
Rejected: LBFGS Orientation magnitude: 1.291e+04, gradient 2.266e+00, dot -1.000; [144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, 43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547, 0.21721750055101124
Rejected: LBFGS Orientation magnitude: 1.379e+04, gradient 2.266e+00, dot -1.000; [c3f90eb2-fd63-43cc-bcb4-8856e8efd59a = 1.000/1.000e+00, 144ece20-5513-4d6f-97e2-5aa69f623a10 = 1.000/1.000e+00, b14db6fe-264c-4190-86f1-b39f618be134 = 1.000/1.000e+00, 8b040e01-1f0c-4664-aa11-5f4a0293c9e9 = 1.000/1.000e+00, 43091419-bafc-491a-b049-6cb07974f5f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 0.020565784925124904, 0.20065102572076512, 0.21627416546733547
LBFGS Accumulation History: 3 points
Removed measurement 4d53672f to history. Total: 20
Removed measurement 48f115be to history. Total: 19
Removed measurement 2b8040ec to history. Total: 18
Removed measurement 39e09215 to history. Total: 17
Removed measurement 28347e0d to history. Total: 16
Removed measurement 64c54206 to history. Total: 15
Removed measurement 618d5284 to history. Total: 14
Removed measurement 3035c94d to history. Total: 13
Removed measurement 320d9810 to history. Total: 12
Removed measurement 6a81b3d7 to history. Total: 11
Removed measurement 3aac5060 to history. Total: 10
Removed measurement 409c51aa to history. Total: 9
Removed measurement 62265e83 to history. Total: 8
Removed measurement 28810c25 to history. Total: 7
Removed measurement 3942cf5 to history. Total: 6
Removed measurement 2bb765d2 to history. Total: 5
Removed measurement 123a3741 to history. Total: 4
Removed measurement 3de16532 to history. Total: 3
Adding measurement 7817b53c to history. Total: 3
th(0)=0.0;dx=-5.135589491842355
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(91134.25242689752)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(45567.12621344876)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(15189.042071149588)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3797.260517787397)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(759.4521035574794)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(126.57535059291324)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(18.08219294184475)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.2602741177305936)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.25114156863673265)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.025114156863673263)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.0022831051694248423)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.9025876411873685E-4)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.4635289547595142E-5)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.0453778248282244E-6)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(6.969185498854829E-8)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.355740936784268E-9)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.562200551049569E-10)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.423444750583094E-11)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7.491814476753127E-13)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.7459072383765636E-14)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.7837653516078873E-15)=0.0; dx=-5.135589491842355 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
MIN ALPHA (8.108024325490396E-17): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 2.4158; Orientation: 2.3692; Line Search: 0.0452
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 2.635s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -1.662979239340967], [2.0, 0.33702076065903297]; valueStats=DoubleSummaryStatistics{count=2, sum=0.434561, min=0.217281, average=0.217281, max=0.217281}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -1.662979239340967], [0.132, 0.33702076065903297]; valueStats=DoubleSummaryStatistics{count=2, sum=0.434561, min=0.217281, average=0.217281, max=0.217281}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS
Returns
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0042 = network_target.state();
assert temp_18_0042 != null;
String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0042.freeRef();
return temp_18_0041;
Returns
[-1.028, -1.72, -0.384]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.02 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4769173609251
Reset training subject: 4769174675467
Constructing line search parameters: GD
th(0)=2.9902625796248836;dx=-146.21208670747342
New Minimum: 2.9902625796248836 > 0.0
END: th(2.154434690031884)=0.0; dx=-84.1898694628723 evalInputDelta=2.9902625796248836
Fitness changed from 2.9902625796248836 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0054; Orientation: 0.0003; Line Search: 0.0021
th(0)=0.0;dx=-49.38272398702641
Armijo: th(4.641588833612779)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(2.3207944168063896)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(0.7735981389354633)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(0.1933995347338658)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(0.03867990694677316)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(0.006446651157795527)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(9.20950165399361E-4)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(1.1511877067492013E-4)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(1.279097451943557E-5)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(1.279097451943557E-6)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(1.1628158654032335E-7)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(9.690132211693613E-9)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(7.453947855148933E-10)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(5.324248467963524E-11)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(3.5494989786423493E-12)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(2.2184368616514683E-13)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Armijo: th(1.3049628597949813E-14)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
MIN ALPHA (7.249793665527674E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0178; Orientation: 0.0002; Line Search: 0.0169
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.023s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.05 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4769200631592
Reset training subject: 4769201379558
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=2.9902625796248836}, derivative=-146.21208670747342}
New Minimum: 2.9902625796248836 > 2.990262565435006
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=2.990262565435006}, derivative=-146.21208627600814}, evalInputDelta = -1.4189877717285526E-8
New Minimum: 2.990262565435006 > 2.9902624802957405
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=2.9902624802957405}, derivative=-146.21208368721653}, evalInputDelta = -9.932914313282026E-8
New Minimum: 2.9902624802957405 > 2.990261884320925
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=2.990261884320925}, derivative=-146.21206556567836}, evalInputDelta = -6.953039584089993E-7
New Minimum: 2.990261884320925 > 2.9902577124993464
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=2.9902577124993464}, derivative=-146.21193871504187}, evalInputDelta = -4.867125537266759E-6
New Minimum: 2.9902577124993464 > 2.990228509852716
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=2.990228509852716}, derivative=-146.21105076701318}, evalInputDelta = -3.406977216746654E-5
New Minimum: 2.990228509852716 > 2.9900240964428586
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=2.9900240964428586}, derivative=-146.20483544570212}, evalInputDelta = -2.3848318202501773E-4
New Minimum: 2.9900240964428586 > 2.988593453224473
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=2.988593453224473}, derivative=-146.1613436205445}, evalInputDelta = -0.0016691264004107254
New Minimum: 2.988593453224473 > 2.978591211926497
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=2.978591211926497}, derivative=-145.85765471473886}, evalInputDelta = -0.011671367698386614
New Minimum: 2.978591211926497 > 2.9091693332924082
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=2.9091693332924082}, derivative=-143.7681294320035}, evalInputDelta = -0.08109324633247539
New Minimum: 2.9091693332924082 > 2.4501184478356137
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=2.4501184478356137}, derivative=-130.72395910782245}, evalInputDelta = -0.5401441317892699
New Minimum: 2.4501184478356137 > 0.7050853686412986
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.7050853686412986}, derivative=-92.04445398977047}, evalInputDelta = -2.2851772109835853
New Minimum: 0.7050853686412986 > 0.0
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
0.0 <= 2.9902625796248836
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-84.1898694628723}, evalInputDelta = -2.9902625796248836
Right bracket at 1.0E10
Converged to right
Fitness changed from 2.9902625796248836 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0261; Orientation: 0.0002; Line Search: 0.0240
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-49.38272398702641}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0197; Orientation: 0.0002; Line Search: 0.0189
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.046s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.02 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4769250581063
Reset training subject: 4769251854223
Adding measurement 791f245 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 2.9902625796248836 < 2.9902625796248836. Total: 1
th(0)=2.9902625796248836;dx=-146.21208670747342
Adding measurement 61c18854 to history. Total: 1
New Minimum: 2.9902625796248836 > 0.0
END: th(2.154434690031884)=0.0; dx=-84.1898694628723 evalInputDelta=2.9902625796248836
Fitness changed from 2.9902625796248836 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.0049; Orientation: 0.0005; Line Search: 0.0018
Non-optimal measurement 0.0 < 0.0. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0 < 0.0. Total: 2
th(0)=0.0;dx=-49.38272398702641
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.641588833612779)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.3207944168063896)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.7735981389354633)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.1933995347338658)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.03867990694677316)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.006446651157795527)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(9.20950165399361E-4)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.1511877067492013E-4)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.279097451943557E-5)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.279097451943557E-6)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.1628158654032335E-7)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(9.690132211693613E-9)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(7.453947855148933E-10)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.324248467963524E-11)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.5494989786423493E-12)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.2184368616514683E-13)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.3049628597949813E-14)=0.0; dx=-49.38272398702641 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (7.249793665527674E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.0172; Orientation: 0.0003; Line Search: 0.0162
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 0.022s (< 30.000s)
Returns
0.0
TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[0.0, 0.0], [2.0, 1.0]; valueStats=DoubleSummaryStatistics{count=0, sum=0.000000, min=Infinity, average=0.000000, max=-Infinity}
Only 0 points for GD
Only 0 points for CjGD
Only 0 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
No Data
In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:
TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):
RefList<double[]> temp_18_0037 = temp_18_0035.state();
assert temp_18_0037 != null;
String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
temp_18_0037.freeRef();
return temp_18_0036;
Returns
[-0.384, -1.72, -1.028]
We simultaneously regress this target input:
TrainingTester.java:287 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
if (null != x)
RefUtil.freeRef(x);
return temp_18_0006;
}).map(x -> {
String temp_18_0007 = x.prettyPrint();
x.freeRef();
return temp_18_0007;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ -1.616, -1.688, -0.712 ], [ 1.108, 1.048, 1.556 ] ],
[ [ 1.912, 1.356, -0.804 ], [ -0.852, 1.512, 0.028 ] ]
]
[
[ [ 1.556, 1.048, -1.688 ], [ 1.912, -1.616, 1.108 ] ],
[ [ -0.804, -0.712, 1.512 ], [ 1.356, -0.852, 0.028 ] ]
]
[
[ [ 1.912, 1.556, -0.712 ], [ 0.028, 1.512, 1.356 ] ],
[ [ 1.048, -0.804, -1.688 ], [ -1.616, 1.108, -0.852 ] ]
]
[
[ [ -1.688, 0.028, 1.512 ], [ -1.616, 1.108, 1.048 ] ],
[ [ -0.852, -0.804, -0.712 ], [ 1.556, 1.356, 1.912 ] ]
]
[
[ [ -1.688, 1.048, -0.804 ], [ 0.028, -1.616, 1.912 ] ],
[ [ 1.356, -0.852, 1.512 ], [ 1.556, 1.108, -0.712 ] ]
]
Which produces the following output:
TrainingTester.java:308 executed in 0.00 seconds (0.000 gc):
return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
String temp_18_0008 = x.prettyPrint();
x.freeRef();
return temp_18_0008;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.6205440000000001, 2.9033599999999997, 0.731936 ], [ -0.4254720000000001, -1.80256, -1.599568 ] ],
[ [ -0.734208, -2.33232, 0.826512 ], [ 0.327168, -2.60064, -0.028784 ] ]
]
[
[ [ -0.597504, -1.80256, 1.735264 ], [ -0.734208, 2.77952, -1.139024 ] ],
[ [ 0.308736, 1.22464, -1.554336 ], [ -0.5207040000000001, 1.4654399999999999, -0.028784 ] ]
]
[
[ [ -0.734208, -2.67632, 0.731936 ], [ -0.010752000000000001, -2.60064, -1.393968 ] ],
[ [ -0.402432, 1.38288, 1.735264 ], [ 0.6205440000000001, -1.9057600000000001, 0.875856 ] ]
]
[
[ [ 0.648192, -0.04816, -1.554336 ], [ 0.6205440000000001, -1.9057600000000001, -1.077344 ] ],
[ [ 0.327168, 1.38288, 0.731936 ], [ -0.597504, -2.33232, -1.965536 ] ]
]
[
[ [ 0.648192, -1.80256, 0.826512 ], [ -0.010752000000000001, 2.77952, -1.965536 ] ],
[ [ -0.5207040000000001, 1.4654399999999999, -1.554336 ], [ -0.597504, -1.9057600000000001, 0.731936 ] ]
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 0.05 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4769330475205
Reset training subject: 4769331718263
Constructing line search parameters: GD
th(0)=92.027884477252;dx=-1.205069834027516E25
Armijo: th(2.154434690031884)=157.12059637727825; dx=2.1416645137770505E37 evalInputDelta=-65.09271190002625
Armijo: th(1.077217345015942)=157.12059637727825; dx=1.0708322568881165E37 evalInputDelta=-65.09271190002625
Armijo: th(0.3590724483386473)=157.12059637727825; dx=3.56944085628827E36 evalInputDelta=-65.09271190002625
Armijo: th(0.08976811208466183)=157.12059637727825; dx=8.923602140659356E35 evalInputDelta=-65.09271190002625
Armijo: th(0.017953622416932366)=157.12059637727825; dx=1.78472042806646E35 evalInputDelta=-65.09271190002625
Armijo: th(0.002992270402822061)=157.12059637727825; dx=2.974534046096068E34 evalInputDelta=-65.09271190002625
Armijo: th(4.2746720040315154E-4)=157.12059637727825; dx=4.2493343445574875E33 evalInputDelta=-65.09271190002625
Armijo: th(5.343340005039394E-5)=157.12059637727825; dx=5.311667859153552E32 evalInputDelta=-65.09271190002625
Armijo: th(5.9370444500437714E-6)=157.12059637727825; dx=5.9018524500481305E31 evalInputDelta=-65.09271190002625
Armijo: th(5.937044450043771E-7)=157.12059637727825; dx=5.901845091307996E30 evalInputDelta=-65.09271190002625
Armijo: th(5.397313136403428E-8)=157.12059637727825; dx=5.365239388662467E29 evalInputDelta=-65.09271190002625
Armijo: th(4.4977609470028565E-9)=157.12059637727825; dx=4.470283322575301E28 evalInputDelta=-65.09271190002625
Armijo: th(3.4598161130791205E-10)=157.12059637727825; dx=3.4311320531241816E27 evalInputDelta=-65.09271190002625
Armijo: th(2.4712972236279432E-11)=149.03094074867494; dx=2.37218495524783E26 evalInputDelta=-57.003056271422935
Armijo: th(1.6475314824186289E-12)=122.06233666681817; dx=9.009220577330164E24 evalInputDelta=-30.034452189566167
Armijo: th(1.029707176511643E-13)=92.37853571469857; dx=-1.1027094640556364E25 evalInputDelta=-0.35065123744657
Armijo: th(6.057101038303783E-15)=92.04558426583131; dx=-1.199048635793878E25 evalInputDelta=-0.017699788579307096
MIN ALPHA (3.3650561323909904E-16): th(0.0)=92.027884477252
Fitness changed from 92.027884477252 to 92.027884477252
Static Iteration Total: 0.0496; Orientation: 0.0004; Line Search: 0.0461
Iteration 1 failed. Error: 92.027884477252
Previous Error: 0.0 -> 92.027884477252
Optimization terminated 1
Final threshold in iteration 1: 92.027884477252 (> 0.0) after 0.050s (< 30.000s)
Returns
92.027884477252
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
[-1.028, -1.72, -0.384]
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 1.356, 1.048, -0.804 ], [ 1.512, 0.028, -1.616 ] ],
[ [ -0.712, -1.688, -0.852 ], [ 1.108, 1.912, 1.556 ] ]
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ -1.393968, -1.80256, 0.308736 ], [ -1.554336, -0.04816, 0.6205440000000001 ] ],
[ [ 0.731936, 2.9033599999999997, 0.327168 ], [ -1.139024, -3.28864, -0.597504 ] ]
]
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 0.01 seconds (0.000 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4769393405802
Reset training subject: 4769394602705
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=92.027884477252}, derivative=-1.205069834027516E25}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=157.12059637727825}, derivative=9.858962313213884E26}, evalInputDelta = 65.09271190002625
157.12059637727825 <= 92.027884477252
Converged to right
Fitness changed from 92.027884477252 to 92.027884477252
Static Iteration Total: 0.0077; Orientation: 0.0006; Line Search: 0.0037
Iteration 1 failed. Error: 92.027884477252
Previous Error: 0.0 -> 92.027884477252
Optimization terminated 1
Final threshold in iteration 1: 92.027884477252 (> 0.0) after 0.007s (< 30.000s)
Returns
92.027884477252
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
[-1.028, -1.72, -0.384]
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 1.356, 1.048, -0.804 ], [ 1.512, 0.028, -1.616 ] ],
[ [ -0.712, -1.688, -0.852 ], [ 1.108, 1.912, 1.556 ] ]
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ -1.393968, -1.80256, 0.308736 ], [ -1.554336, -0.04816, 0.6205440000000001 ] ],
[ [ 0.731936, 2.9033599999999997, 0.327168 ], [ -1.139024, -3.28864, -0.597504 ] ]
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 0.17 seconds (0.103 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4769408379992
Reset training subject: 4769409412319
Adding measurement 41f11372 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 92.027884477252 < 92.027884477252. Total: 1
th(0)=92.027884477252;dx=-1.205069834027516E25
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(2.154434690031884)=157.12059637727825; dx=2.1416645137770505E37 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(1.077217345015942)=157.12059637727825; dx=1.0708322568881165E37 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(0.3590724483386473)=157.12059637727825; dx=3.56944085628827E36 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(0.08976811208466183)=157.12059637727825; dx=8.923602140659356E35 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(0.017953622416932366)=157.12059637727825; dx=1.78472042806646E35 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(0.002992270402822061)=157.12059637727825; dx=2.974534046096068E34 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(4.2746720040315154E-4)=157.12059637727825; dx=4.2493343445574875E33 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(5.343340005039394E-5)=157.12059637727825; dx=5.311667859153552E32 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(5.9370444500437714E-6)=157.12059637727825; dx=5.9018524500481305E31 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(5.937044450043771E-7)=157.12059637727825; dx=5.901845091307996E30 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(5.397313136403428E-8)=157.12059637727825; dx=5.365239388662467E29 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(4.4977609470028565E-9)=157.12059637727825; dx=4.470283322575301E28 evalInputDelta=-65.09271190002625
Non-optimal measurement 157.12059637727825 < 92.027884477252. Total: 1
Armijo: th(3.4598161130791205E-10)=157.12059637727825; dx=3.4311320531241816E27 evalInputDelta=-65.09271190002625
Non-optimal measurement 149.03094074867494 < 92.027884477252. Total: 1
Armijo: th(2.4712972236279432E-11)=149.03094074867494; dx=2.37218495524783E26 evalInputDelta=-57.003056271422935
Non-optimal measurement 122.06233666681817 < 92.027884477252. Total: 1
Armijo: th(1.6475314824186289E-12)=122.06233666681817; dx=9.009220577330163E24 evalInputDelta=-30.034452189566167
Non-optimal measurement 92.37853571469857 < 92.027884477252. Total: 1
Armijo: th(1.029707176511643E-13)=92.37853571469857; dx=-1.1027094640556364E25 evalInputDelta=-0.35065123744657
Non-optimal measurement 92.04558426583131 < 92.027884477252. Total: 1
Armijo: th(6.057101038303783E-15)=92.04558426583131; dx=-1.199048635793878E25 evalInputDelta=-0.017699788579307096
Non-optimal measurement 92.027884477252 < 92.027884477252. Total: 1
MIN ALPHA (3.3650561323909904E-16): th(0.0)=92.027884477252
Fitness changed from 92.027884477252 to 92.027884477252
Static Iteration Total: 0.1674; Orientation: 0.1047; Line Search: 0.0597
Iteration 1 failed. Error: 92.027884477252
Previous Error: 0.0 -> 92.027884477252
Optimization terminated 1
Final threshold in iteration 1: 92.027884477252 (> 0.0) after 0.167s (< 30.000s)
Returns
92.027884477252
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
[-1.028, -1.72, -0.384]
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 1.356, 1.048, -0.804 ], [ 1.512, 0.028, -1.616 ] ],
[ [ -0.712, -1.688, -0.852 ], [ 1.108, 1.912, 1.556 ] ]
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ -1.393968, -1.80256, 0.308736 ], [ -1.554336, -0.04816, 0.6205440000000001 ] ],
[ [ 0.731936, 2.9033599999999997, 0.327168 ], [ -1.139024, -3.28864, -0.597504 ] ]
]
TrainingTester.java:432 executed in 0.00 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
No Data
TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
No Data
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": NaN }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "NonConverged", "value": NaN } }}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": NaN }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "NonConverged", "value": NaN } }} | OK |
{
"result": "OK",
"performance": {
"execution_time": "4.434",
"gc_time": "0.318"
},
"created_on": 1586739353350,
"file_name": "trainingTest",
"report": {
"simpleName": "Basic",
"canonicalName": "com.simiacryptus.mindseye.layers.java.ImgBandScaleLayerTest.Basic",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/ImgBandScaleLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"model": {
"LBFGS": {
"type": "Converged",
"value": 0.0
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
},
"complete": {
"LBFGS": {
"type": "NonConverged",
"value": "NaN"
},
"CjGD": {
"type": "NonConverged",
"value": "NaN"
},
"GD": {
"type": "NonConverged",
"value": "NaN"
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/ImgBandScaleLayer/Basic/trainingTest/202004135553",
"id": "289b77eb-7387-4cf8-85c1-98c628d65d0a",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "ImgBandScaleLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.ImgBandScaleLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/ImgBandScaleLayer.java",
"javaDoc": ""
}
}