Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 6583348929311750144
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.04 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.816 ], [ -0.192 ], [ 0.22 ], [ 1.48 ], [ 0.148 ], [ -1.076 ], [ 0.168 ], [ -0.308 ], ... ],
[ [ 1.888 ], [ 0.404 ], [ 1.492 ], [ -0.328 ], [ 0.136 ], [ -0.84 ], [ 1.684 ], [ 1.248 ], ... ],
[ [ -0.992 ], [ 0.348 ], [ 0.308 ], [ -0.18 ], [ 0.244 ], [ -0.752 ], [ -1.584 ], [ -0.424 ], ... ],
[ [ -0.928 ], [ -1.252 ], [ -2.0 ], [ -0.844 ], [ -1.208 ], [ -0.956 ], [ 0.316 ], [ -0.612 ], ... ],
[ [ -1.976 ], [ -1.512 ], [ -0.964 ], [ 1.948 ], [ 1.024 ], [ -0.684 ], [ -0.732 ], [ 0.028 ], ... ],
[ [ 0.116 ], [ 1.556 ], [ -0.34 ], [ 0.856 ], [ 0.992 ], [ 1.724 ], [ -1.62 ], [ -0.552 ], ... ],
[ [ 0.072 ], [ 1.316 ], [ 1.312 ], [ 1.896 ], [ -0.812 ], [ -1.496 ], [ 1.42 ], [ -0.456 ], ... ],
[ [ 0.156 ], [ -1.86 ], [ -1.136 ], [ -0.228 ], [ 1.064 ], [ -0.356 ], [ -1.248 ], [ -1.456 ], ... ],
...
]
[
[ [ -0.684 ], [ -1.472 ], [ -0.1 ], [ -1.18 ], [ 0.52 ], [ -0.032 ], [ 1.072 ], [ -0.568 ], ... ],
[ [ 0.096 ], [ -0.088 ], [ 1.556 ], [ -1.824 ], [ 1.456 ], [ -1.596 ], [ 1.644 ], [ 1.716 ], ... ],
[ [ 0.496 ], [ -0.536 ], [ -1.26 ], [ -1.34 ], [ 1.384 ], [ -0.092 ], [ -0.428 ], [ -0.944 ], ... ],
[ [ -0.964 ], [ -0.488 ], [ 1.416 ], [ -0.396 ], [ 0.236 ], [ 1.232 ], [ -1.824 ], [ 0.836 ], ... ],
[ [ 1.244 ], [ -0.624 ], [ -0.328 ], [ 1.728 ], [ 0.544 ], [ 0.476 ], [ 0.98 ], [ -0.152 ], ... ],
[ [ -1.428 ], [ 0.004 ], [ -0.58 ], [ -0.728 ], [ 1.972 ], [ 0.656 ], [ -1.972 ], [ 1.004 ], ... ],
[ [ -0.252 ], [ 1.996 ], [ 1.772 ], [ -1.112 ], [ -0.62 ], [ -1.848 ], [ -0.536 ], [ -0.22 ], ... ],
[ [ 1.52 ], [ 0.86 ], [ 0.044 ], [ 1.532 ], [ 1.084 ], [ -1.932 ], [ -0.148 ], [ -0.312 ], ... ],
...
]
[
[ [ -1.344 ], [ -0.036 ], [ 0.732 ], [ 0.824 ], [ -0.788 ], [ -0.472 ], [ -1.688 ], [ -0.988 ], ... ],
[ [ -1.524 ], [ 0.828 ], [ -0.284 ], [ 1.876 ], [ -0.528 ], [ 1.556 ], [ -1.564 ], [ 1.356 ], ... ],
[ [ 0.064 ], [ -0.772 ], [ 0.448 ], [ -0.02 ], [ 1.24 ], [ -0.1 ], [ 1.456 ], [ -1.656 ], ... ],
[ [ -0.624 ], [ 0.32 ], [ 0.944 ], [ 1.008 ], [ 1.396 ], [ 1.66 ], [ 0.832 ], [ 0.62 ], ... ],
[ [ -0.152 ], [ 1.636 ], [ 1.084 ], [ 1.26 ], [ 1.024 ], [ -1.988 ], [ 1.156 ], [ -0.54 ], ... ],
[ [ 1.8 ], [ 0.728 ], [ -1.148 ], [ -1.628 ], [ -1.412 ], [ 0.476 ], [ 0.952 ], [ 1.664 ], ... ],
[ [ -1.74 ], [ -1.676 ], [ -0.576 ], [ -0.644 ], [ 1.828 ], [ -1.492 ], [ -0.472 ], [ 0.796 ], ... ],
[ [ 0.572 ], [ -1.392 ], [ -0.536 ], [ 0.14 ], [ 0.792 ], [ -0.788 ], [ -0.348 ], [ 1.62 ], ... ],
...
]
[
[ [ -1.508 ], [ -1.844 ], [ -1.472 ], [ 1.328 ], [ -1.048 ], [ -0.268 ], [ -0.592 ], [ -0.948 ], ... ],
[ [ -1.044 ], [ -0.156 ], [ -0.92 ], [ -0.98 ], [ -1.608 ], [ 1.56 ], [ -0.408 ], [ -1.064 ], ... ],
[ [ 1.776 ], [ 0.844 ], [ 0.052 ], [ 1.1 ], [ 0.296 ], [ -1.252 ], [ 1.216 ], [ -0.976 ], ... ],
[ [ 0.936 ], [ 1.256 ], [ 1.808 ], [ 0.632 ], [ -1.724 ], [ -1.04 ], [ 0.18 ], [ 0.9 ], ... ],
[ [ -1.924 ], [ 0.888 ], [ 0.832 ], [ -0.86 ], [ 0.484 ], [ -0.444 ], [ 1.78 ], [ -1.348 ], ... ],
[ [ -0.756 ], [ 1.768 ], [ 1.388 ], [ 1.892 ], [ -1.328 ], [ -1.28 ], [ -1.216 ], [ 1.124 ], ... ],
[ [ 1.384 ], [ -0.584 ], [ 1.984 ], [ -0.916 ], [ 0.032 ], [ -0.556 ], [ -1.04 ], [ -0.504 ], ... ],
[ [ 0.736 ], [ 1.052 ], [ -0.572 ], [ 1.144 ], [ 1.3 ], [ 1.384 ], [ -1.936 ], [ -1.948 ], ... ],
...
]
[
[ [ -0.656 ], [ 1.792 ], [ 0.272 ], [ 0.716 ], [ 0.212 ], [ 1.5 ], [ 1.964 ], [ -1.504 ], ... ],
[ [ 0.296 ], [ -0.548 ], [ 0.584 ], [ 0.84 ], [ -1.864 ], [ 1.432 ], [ 1.248 ], [ 0.816 ], ... ],
[ [ -0.524 ], [ -1.584 ], [ 1.876 ], [ -1.924 ], [ 0.788 ], [ -1.048 ], [ 1.556 ], [ 0.564 ], ... ],
[ [ -1.604 ], [ -0.924 ], [ -1.236 ], [ 0.008 ], [ -0.132 ], [ 0.98 ], [ 0.956 ], [ -0.212 ], ... ],
[ [ 0.504 ], [ 1.916 ], [ 1.888 ], [ 1.244 ], [ 0.64 ], [ -0.052 ], [ -1.064 ], [ -0.804 ], ... ],
[ [ -1.5 ], [ 0.376 ], [ 1.5 ], [ 0.76 ], [ 1.32 ], [ 1.356 ], [ 1.596 ], [ -0.452 ], ... ],
[ [ -0.416 ], [ -1.792 ], [ 1.26 ], [ 0.792 ], [ 0.588 ], [ 0.716 ], [ 1.548 ], [ -1.38 ], ... ],
[ [ 1.196 ], [ 0.812 ], [ 1.528 ], [ -1.104 ], [ 1.776 ], [ 1.076 ], [ 1.332 ], [ 0.132 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 31.41 seconds (1.493 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 5865924287723
Reset training subject: 5866220245583
Constructing line search parameters: GD
th(0)=0.11922722039197835;dx=-2.1045719379244403E-9
New Minimum: 0.11922722039197835 > 0.11922721585781557
WOLFE (weak): th(2.154434690031884)=0.11922721585781557; dx=-2.104571930020167E-9 evalInputDelta=4.534162781344442E-9
New Minimum: 0.11922721585781557 > 0.11922721132365281
WOLFE (weak): th(4.308869380063768)=0.11922721132365281; dx=-2.10457192211589E-9 evalInputDelta=9.068325534933308E-9
New Minimum: 0.11922721132365281 > 0.11922719318700192
WOLFE (weak): th(12.926608140191302)=0.11922719318700192; dx=-2.104571890498746E-9 evalInputDelta=2.7204976424388683E-8
New Minimum: 0.11922719318700192 > 0.11922711157207629
WOLFE (weak): th(51.70643256076521)=0.11922711157207629; dx=-2.1045717482208954E-9 evalInputDelta=1.0881990206157433E-7
New Minimum: 0.11922711157207629 > 0.11922667629256609
WOLFE (weak): th(258.53216280382605)=0.11922667629256609; dx=-2.1045709893862307E-9 evalInputDelta=5.440994122613008E-7
New Minimum: 0.11922667629256609 > 0.11922395579918348
WOLFE (weak): th(1551.1929768229563)=0.11922395579918348; dx=-2.104566245927073E-9 evalInputDelta=3.264592794871568E-6
New Minimum: 0.11922395579918348 > 0.11920436842796148
WOLFE (weak): th(10858.350837760694)=0.11920436842796148; dx=-2.1045320552289807E-9 evalInputDelta=2.285196401687062E-5
New Minimum: 0.11920436842796148 > 0.11904441687420504
WOLFE (weak): th(86866.80670208555)=0.11904441687420504; dx=-2.104250346419807E-9 evalInputDelta=1.828035177733056E-4
New Minimum: 0.11904441687420504 > 0.11758304573761787
WOLFE (weak): th(781801.26031877)=0.11758304573761787; dx=-2.101468979832483E-9 evalInputDelta=0.0016441746543604807
New Minimum: 0.11758304573761787 > 0.10294757011127327
WOLFE (weak): th(7818012.6031877)=0.10294757011127327; dx=-2.052111452474942E-9 evalInputDelta=0.016279650280705077
New Minimum: 0.10294757011127327 > 0.009778844249577878
END: th(8.599813863506469E7)=0.009778844249577878; dx=-2.5146981427515033E-10 evalInputDelta=0.10944837614240047
Fitness changed from 0.11922722039197835 to 0.009778844249577878
Iteration 1 complete. Error: 0.009778844249577878 Total: 7.1573; Orientation: 0.2108; Line Search: 6.2452
th(0)=0.009778844249577878;dx=-2.819206555762429E-10
New Minimum: 0.009778844249577878 > 0.0013594212079240677
WOLF (strong): th(1.0E8)=0.0013594212079240677; dx=5.81374214842216E-11 evalInputDelta=0.00841942304165381
New Minimum: 0.0013594212079240677 > 0.0011327208457505501
END: th(5.0E7)=0.0011327208457505501; dx=-6.765055025272928E-11 evalInputDelta=0.008646123403827328
Fitness changed from 0.009778844249577878 to 0.0011327208457505501
Iteration 2 complete. Error: 0.0011327208457505501 Total: 1.6719; Orientation: 0.2504; Line Search: 1.2030
Low gradient: 4.669989430655837E-6
th(0)=0.0011327208457505501;dx=-2.1808801282437223E-11
New Minimum: 0.0011327208457505501 > 1.9219287801333336E-4
WOLF (strong): th(1.0E8)=1.9219287801333336E-4; dx=2.9982503762332034E-12 evalInputDelta=9.405279677372168E-4
END: th(5.0E7)=3.5141026838328003E-4; dx=-9.40565311652976E-12 evalInputDelta=7.813105773672701E-4
Fitness changed from 0.0011327208457505501 to 1.9219287801333336E-4
Iteration 3 complete. Error: 1.9219287801333336E-4 Total: 1.6734; Orientation: 0.2521; Line Search: 1.1667
Low gradient: 1.9795243260963345E-6
th(0)=1.9219287801333336E-4;dx=-3.918516557607147E-12
New Minimum: 1.9219287801333336E-4 > 6.395782180636513E-5
WOLF (strong): th(1.0E8)=6.395782180636513E-5; dx=1.3785879887046687E-12 evalInputDelta=1.2823505620696823E-4
New Minimum: 6.395782180636513E-5 > 6.169811589539494E-5
END: th(5.0E7)=6.169811589539494E-5; dx=-1.288571603861713E-12 evalInputDelta=1.304947621179384E-4
Fitness changed from 1.9219287801333336E-4 to 6.169811589539494E-5
Iteration 4 complete. Error: 6.169811589539494E-5 Total: 1.9101; Orientation: 0.2526; Line Search: 1.4377
Low gradient: 8.124357349934255E-7
th(0)=6.169811589539494E-5;dx=-6.600518234943075E-13
New Minimum: 6.169811589539494E-5 > 1.8966948671255522E-5
END: th(1.0E8)=1.8966948671255522E-5; dx=-1.9047851097009345E-13 evalInputDelta=4.2731167224139424E-5
Fitness changed from 6.169811589539494E-5 to 1.8966948671255522E-5
Iteration 5 complete. Error: 1.8966948671255522E-5 Total: 1.2589; Orientation: 0.2540; Line Search: 0.7891
Low gradient: 4.299770861848994E-7
th(0)=1.8966948671255522E-5;dx=-1.8488029464405646E-13
New Minimum: 1.8966948671255522E-5 > 6.87631197987714E-6
END: th(1.0E8)=6.87631197987714E-6; dx=-5.646908347096325E-14 evalInputDelta=1.2090636691378382E-5
Fitness changed from 1.8966948671255522E-5 to 6.87631197987714E-6
Iteration 6 complete. Error: 6.87631197987714E-6 Total: 1.2664; Orientation: 0.2533; Line Search: 0.7924
Low gradient: 2.556506807963402E-7
th(0)=6.87631197987714E-6;dx=-6.535727059163223E-14
New Minimum: 6.87631197987714E-6 > 2.6784458685023015E-6
END: th(1.0E8)=2.6784458685023015E-6; dx=-1.8465703777171444E-14 evalInputDelta=4.197866111374839E-6
Fitness changed from 6.87631197987714E-6 to 2.6784458685023015E-6
Iteration 7 complete. Error: 2.6784458685023015E-6 Total: 1.2511; Orientation: 0.2525; Line Search: 0.7842
Low gradient: 1.5963847094941004E-7
th(0)=2.6784458685023015E-6;dx=-2.5484441407065626E-14
New Minimum: 2.6784458685023015E-6 > 1.0892883815811563E-6
END: th(1.0E8)=1.0892883815811563E-6; dx=-6.272768683879297E-15 evalInputDelta=1.5891574869211453E-6
Fitness changed from 2.6784458685023015E-6 to 1.0892883815811563E-6
Iteration 8 complete. Error: 1.0892883815811563E-6 Total: 1.2658; Orientation: 0.2535; Line Search: 0.7947
Low gradient: 1.0267452517701626E-7
th(0)=1.0892883815811563E-6;dx=-1.054205812032574E-14
New Minimum: 1.0892883815811563E-6 > 4.5563610017665307E-7
END: th(1.0E8)=4.5563610017665307E-7; dx=-2.12160347062007E-15 evalInputDelta=6.336522814045032E-7
Fitness changed from 1.0892883815811563E-6 to 4.5563610017665307E-7
Iteration 9 complete. Error: 4.5563610017665307E-7 Total: 1.7659; Orientation: 0.2580; Line Search: 1.2903
Low gradient: 6.731830614857461E-8
th(0)=4.5563610017665307E-7;dx=-4.5317543427132164E-15
New Minimum: 4.5563610017665307E-7 > 1.9470195664423872E-7
END: th(1.0E8)=1.9470195664423872E-7; dx=-6.849899649375421E-16 evalInputDelta=2.6093414353241435E-7
Fitness changed from 4.5563610017665307E-7 to 1.9470195664423872E-7
Iteration 10 complete. Error: 1.9470195664423872E-7 Total: 1.2511; Orientation: 0.2527; Line Search: 0.7754
Low gradient: 4.4786398552535125E-8
th(0)=1.9470195664423872E-7;dx=-2.005821495306519E-15
New Minimum: 1.9470195664423872E-7 > 8.460755810672694E-8
END: th(1.0E8)=8.460755810672694E-8; dx=-1.9528843412261224E-16 evalInputDelta=1.1009439853751178E-7
Fitness changed from 1.9470195664423872E-7 to 8.460755810672694E-8
Iteration 11 complete. Error: 8.460755810672694E-8 Total: 1.2492; Orientation: 0.2517; Line Search: 0.7820
Low gradient: 3.0137516531993667E-8
th(0)=8.460755810672694E-8;dx=-9.082699027161908E-16
New Minimum: 8.460755810672694E-8 > 3.7304880061385765E-8
END: th(1.0E8)=3.7304880061385765E-8; dx=-3.762903960839749E-17 evalInputDelta=4.730267804534118E-8
Fitness changed from 8.460755810672694E-8 to 3.7304880061385765E-8
Iteration 12 complete. Error: 3.7304880061385765E-8 Total: 1.2511; Orientation: 0.2538; Line Search: 0.7836
Low gradient: 2.0478372054825325E-8
th(0)=3.7304880061385765E-8;dx=-4.193637220158503E-16
New Minimum: 3.7304880061385765E-8 > 1.6657971837731195E-8
WOLF (strong): th(1.0E8)=1.6657971837731195E-8; dx=6.494929407167307E-18 evalInputDelta=2.064690822365457E-8
END: th(5.0E7)=2.1658190816790667E-8; dx=-2.064864252055174E-16 evalInputDelta=1.5646689244595098E-8
Fitness changed from 3.7304880061385765E-8 to 1.6657971837731195E-8
Iteration 13 complete. Error: 1.6657971837731195E-8 Total: 1.6225; Orientation: 0.2534; Line Search: 1.1535
Low gradient: 1.4030658715997582E-8
th(0)=1.6657971837731195E-8;dx=-1.9685938400479862E-16
New Minimum: 1.6657971837731195E-8 > 7.525676101107196E-9
WOLF (strong): th(1.0E8)=7.525676101107196E-9; dx=1.422591228274998E-17 evalInputDelta=9.132295736624E-9
END: th(5.0E7)=9.453257182062193E-9; dx=-9.132606811771776E-17 evalInputDelta=7.204714655669003E-9
Fitness changed from 1.6657971837731195E-8 to 7.525676101107196E-9
Iteration 14 complete. Error: 7.525676101107196E-9 Total: 1.8885; Orientation: 0.2613; Line Search: 1.4070
Low gradient: 9.684028547266404E-9
th(0)=7.525676101107196E-9;dx=-9.378040890427052E-17
New Minimum: 7.525676101107196E-9 > 3.4364192588937824E-9
WOLF (strong): th(1.0E8)=3.4364192588937824E-9; dx=1.2001792554835487E-17 evalInputDelta=4.089256842213414E-9
END: th(5.0E7)=4.15876999237491E-9; dx=-4.0894198545961693E-17 evalInputDelta=3.3669061087322863E-9
Fitness changed from 7.525676101107196E-9 to 3.4364192588937824E-9
Iteration 15 complete. Error: 3.4364192588937824E-9 Total: 1.6314; Orientation: 0.2573; Line Search: 1.1496
Low gradient: 6.7272027442593365E-9
th(0)=3.4364192588937824E-9;dx=-4.5255256762370284E-17
New Minimum: 3.4364192588937824E-9 > 1.5850899755974204E-9
WOLF (strong): th(1.0E8)=1.5850899755974204E-9; dx=8.229652387007995E-18 evalInputDelta=1.851329283296362E-9
END: th(5.0E7)=1.842193204080008E-9; dx=-1.8513538155568957E-17 evalInputDelta=1.5942260548137744E-9
Fitness changed from 3.4364192588937824E-9 to 1.5850899755974204E-9
Iteration 16 complete. Error: 1.5850899755974204E-9 Total: 1.6401; Orientation: 0.2625; Line Search: 1.1587
Low gradient: 4.700255811831539E-9
th(0)=1.5850899755974204E-9;dx=-2.209240469665614E-17
New Minimum: 1.5850899755974204E-9 > 7.38112909685413E-10
WOLF (strong): th(1.0E8)=7.38112909685413E-10; dx=5.153507216008776E-18 evalInputDelta=8.469770659120074E-10
END: th(5.0E7)=8.210275297327133E-10; dx=-8.469931618523079E-18 evalInputDelta=7.640624458647071E-10
Fitness changed from 1.5850899755974204E-9 to 7.38112909685413E-10
Iteration 17 complete. Error: 7.38112909685413E-10 Total: 1.6442; Orientation: 0.2581; Line Search: 1.1667
Final threshold in iteration 17: 7.38112909685413E-10 (> 0.0) after 31.400s (< 30.000s)
Returns
7.38112909685413E-10
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 34.45 seconds (0.946 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 5897391898897
Reset training subject: 5897603627505
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.11922722039197835}, derivative=-2.1045719379244403E-9}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.11922722039197835}, derivative=-2.1045719379244403E-9}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.11922722039197835}, derivative=-2.1045719379244403E-9}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.11922722039197835}, derivative=-2.1045719379244403E-9}, evalInputDelta = 0.0
New Minimum: 0.11922722039197835 > 0.11922722039197828
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.11922722039197828}, derivative=-2.1045719379244403E-9}, evalInputDelta = -6.938893903907228E-17
New Minimum: 0.11922722039197828 > 0.11922722039197783
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.11922722039197783}, derivative=-2.10457193792444E-9}, evalInputDelta = -5.134781488891349E-16
New Minimum: 0.11922722039197783 > 0.11922722039197482
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.11922722039197482}, derivative=-2.1045719379244345E-9}, evalInputDelta = -3.524958103184872E-15
New Minimum: 0.11922722039197482 > 0.11922722039195359
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.11922722039195359}, derivative=-2.1045719379243973E-9}, evalInputDelta = -2.475797344914099E-14
New Minimum: 0.11922722039195359 > 0.11922722039180504
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.11922722039180504}, derivative=-2.104571937924139E-9}, evalInputDelta = -1.7330581414398694E-13
New Minimum: 0.11922722039180504 > 0.1192272203907651
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.1192272203907651}, derivative=-2.1045719379223256E-9}, evalInputDelta = -1.213251721310371E-12
New Minimum: 0.1192272203907651 > 0.11922722038348565
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.11922722038348565}, derivative=-2.104571937909636E-9}, evalInputDelta = -8.492692660233558E-12
New Minimum: 0.11922722038348565 > 0.11922722033252942
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.11922722033252942}, derivative=-2.104571937820805E-9}, evalInputDelta = -5.944893188836176E-11
New Minimum: 0.11922722033252942 > 0.11922721997583571
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.11922721997583571}, derivative=-2.104571937198991E-9}, evalInputDelta = -4.1614263424083475E-10
New Minimum: 0.11922721997583571 > 0.11922721747897991
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.11922721747897991}, derivative=-2.104571932846296E-9}, evalInputDelta = -2.9129984396858433E-9
New Minimum: 0.11922721747897991 > 0.1192272000009893
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.1192272000009893}, derivative=-2.1045719023773966E-9}, evalInputDelta = -2.0390989050045327E-8
New Minimum: 0.1192272000009893 > 0.11922707765506213
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.11922707765506213}, derivative=-2.104571689093625E-9}, evalInputDelta = -1.4273691621713436E-7
New Minimum: 0.11922707765506213 > 0.1192262212339191
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.1192262212339191}, derivative=-2.1045701960347244E-9}, evalInputDelta = -9.991580592477733E-7
New Minimum: 0.1192262212339191 > 0.11922022630293369
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.11922022630293369}, derivative=-2.104559741070111E-9}, evalInputDelta = -6.994089044656815E-6
New Minimum: 0.11922022630293369 > 0.11917826262115339
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.11917826262115339}, derivative=-2.104486382242874E-9}, evalInputDelta = -4.8957770824961666E-5
New Minimum: 0.11917826262115339 > 0.11888455823124944
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.11888455823124944}, derivative=-2.1039643367269486E-9}, evalInputDelta = -3.4266216072890987E-4
New Minimum: 0.11888455823124944 > 0.11683081402696342
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.11683081402696342}, derivative=-2.099890539286975E-9}, evalInputDelta = -0.0023964063650149248
New Minimum: 0.11683081402696342 > 0.1026168685311549
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.1026168685311549}, derivative=-2.05052544052787E-9}, evalInputDelta = -0.01661035186082345
New Minimum: 0.1026168685311549 > 0.027216457315859553
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.027216457315859553}, derivative=-9.542506367419234E-10}, evalInputDelta = -0.0920107630761188
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.07791818128851899}, derivative=1.5804138273390616E-10}, evalInputDelta = -0.04130903910345936
0.07791818128851899 <= 0.11922722039197835
F(3.6367237768934554E8) = LineSearchPoint{point=PointSample{avg=0.07338084538572084}, derivative=1.7464604466044473E-10}, evalInputDelta = -0.0458463750062575
Right bracket at 3.6367237768934554E8
F(3.358058274949355E8) = LineSearchPoint{point=PointSample{avg=0.06824738750008727}, derivative=1.9427168917257307E-10}, evalInputDelta = -0.05097983289189108
Right bracket at 3.358058274949355E8
F(3.07427400805767E8) = LineSearchPoint{point=PointSample{avg=0.0624133873257553}, derivative=2.174524925823893E-10}, evalInputDelta = -0.05681383306622305
Right bracket at 3.07427400805767E8
F(2.786374993236591E8) = LineSearchPoint{point=PointSample{avg=0.055772050982101695}, derivative=2.445023598040854E-10}, evalInputDelta = -0.06345516940987665
Right bracket at 2.786374993236591E8
F(2.4963563838617373E8) = LineSearchPoint{point=PointSample{avg=0.04824639978085303}, derivative=2.748839582974499E-10}, evalInputDelta = -0.07098082061112532
Right bracket at 2.4963563838617373E8
F(2.207967628597735E8) = LineSearchPoint{point=PointSample{avg=0.03987011739164059}, derivative=3.057116540018883E-10}, evalInputDelta = -0.07935710300033776
Right bracket at 2.207967628597735E8
F(1.9279169997081396E8) = LineSearchPoint{point=PointSample{avg=0.030952534897316475}, derivative=3.2901975365083643E-10}, evalInputDelta = -0.08827468549466187
Right bracket at 1.9279169997081396E8
New Minimum: 0.027216457315859553 > 0.022299549002826262
F(1.6672640814313552E8) = LineSearchPoint{point=PointSample{avg=0.022299549002826262}, derivative=3.2957602363956483E-10}, evalInputDelta = -0.09692767138915209
Right bracket at 1.6672640814313552E8
New Minimum: 0.022299549002826262 > 0.01520026688228988
F(1.4415217375016314E8) = LineSearchPoint{point=PointSample{avg=0.01520026688228988}, derivative=2.9081579038824414E-10}, evalInputDelta = -0.10402695350968846
Right bracket at 1.4415217375016314E8
New Minimum: 0.01520026688228988 > 0.010704466075479712
F(1.2665115383182745E8) = LineSearchPoint{point=PointSample{avg=0.010704466075479712}, derivative=2.1426752016762636E-10}, evalInputDelta = -0.10852275431649863
Right bracket at 1.2665115383182745E8
New Minimum: 0.010704466075479712 > 0.008664735889233159
F(1.149482183124178E8) = LineSearchPoint{point=PointSample{avg=0.008664735889233159}, derivative=1.2896369226897874E-10}, evalInputDelta = -0.1105624845027452
Right bracket at 1.149482183124178E8
New Minimum: 0.008664735889233159 > 0.008013917191547036
F(1.08311141897377E8) = LineSearchPoint{point=PointSample{avg=0.008013917191547036}, derivative=6.51105816051869E-11}, evalInputDelta = -0.11121330320043131
Loops = 12
Fitness changed from 0.11922722039197835 to 0.008013917191547036
Iteration 1 complete. Error: 0.008013917191547036 Total: 20.3165; Orientation: 0.2468; Line Search: 19.4324
F(0.0) = LineSearchPoint{point=PointSample{avg=0.008013917191547036}, derivative=-1.5351082285879496E-10}
New Minimum: 0.008013917191547036 > 0.0023975706010640843
F(1.08311141897377E8) = LineSearchPoint{point=PointSample{avg=0.0023975706010640843}, derivative=3.8658245968600076E-11}, evalInputDelta = -0.005616346590482952
0.0023975706010640843 <= 0.008013917191547036
New Minimum: 0.0023975706010640843 > 0.0018980199879516722
F(8.652241809204085E7) = LineSearchPoint{point=PointSample{avg=0.0018980199879516722}, derivative=6.531634778642427E-12}, evalInputDelta = -0.006115897203595363
Right bracket at 8.652241809204085E7
New Minimum: 0.0018980199879516722 > 0.001884789663111119
F(8.299127489738637E7) = LineSearchPoint{point=PointSample{avg=0.001884789663111119}, derivative=9.437855483137445E-13}, evalInputDelta = -0.006129127528435916
Right bracket at 8.299127489738637E7
Converged to right
Fitness changed from 0.008013917191547036 to 0.001884789663111119
Iteration 2 complete. Error: 0.001884789663111119 Total: 2.7545; Orientation: 0.2533; Line Search: 2.2897
Low gradient: 4.3819870079811036E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=0.001884789663111119}, derivative=-1.9201810138115187E-11}
New Minimum: 0.001884789663111119 > 8.400533452622898E-4
F(8.299127489738637E7) = LineSearchPoint{point=PointSample{avg=8.400533452622898E-4}, derivative=-5.365329671787081E-12}, evalInputDelta = -0.0010447363178488292
F(5.809389242817047E8) = LineSearchPoint{point=PointSample{avg=0.026680968045453807}, derivative=1.0529059990151968E-10}, evalInputDelta = 0.024796178382342687
F(4.4687609560131125E7) = LineSearchPoint{point=PointSample{avg=0.0011790358234399002}, derivative=-1.2206193262280177E-11}, evalInputDelta = -7.057538396712188E-4
F(3.1281326692091787E8) = LineSearchPoint{point=PointSample{avg=0.00544457983566472}, derivative=4.8062299358232254E-11}, evalInputDelta = 0.0035597901725536014
F(2.4062558993916757E7) = LineSearchPoint{point=PointSample{avg=0.0014658463457958638}, derivative=-1.5567752927816403E-11}, evalInputDelta = -4.189433173152552E-4
F(1.684379129574173E8) = LineSearchPoint{point=PointSample{avg=0.0011207603291576462}, derivative=1.249486101581438E-11}, evalInputDelta = -7.640293339534728E-4
0.0011207603291576462 <= 0.001884789663111119
New Minimum: 8.400533452622898E-4 > 7.726467934537568E-4
F(1.0203951099349837E8) = LineSearchPoint{point=PointSample{avg=7.726467934537568E-4}, derivative=-1.6817079140388578E-12}, evalInputDelta = -0.0011121428696573622
Left bracket at 1.0203951099349837E8
New Minimum: 7.726467934537568E-4 > 7.655871782883009E-4
F(1.0991607961985439E8) = LineSearchPoint{point=PointSample{avg=7.655871782883009E-4}, derivative=-1.058000285130041E-13}, evalInputDelta = -0.001119202484822818
Left bracket at 1.0991607961985439E8
Converged to left
Fitness changed from 0.001884789663111119 to 7.655871782883009E-4
Iteration 3 complete. Error: 7.655871782883009E-4 Total: 4.8266; Orientation: 0.2501; Line Search: 4.3669
Low gradient: 2.767260446344938E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=7.655871782883009E-4}, derivative=-7.657730377905186E-12}
New Minimum: 7.655871782883009E-4 > 3.916654786152666E-4
F(1.0991607961985439E8) = LineSearchPoint{point=PointSample{avg=3.916654786152666E-4}, derivative=9.398188294151723E-13}, evalInputDelta = -3.739216996730343E-4
3.916654786152666E-4 <= 7.655871782883009E-4
New Minimum: 3.916654786152666E-4 > 3.8615635643128197E-4
F(9.79008879889323E7) = LineSearchPoint{point=PointSample{avg=3.8615635643128197E-4}, derivative=-2.2167390987201465E-14}, evalInputDelta = -3.794308218570189E-4
Left bracket at 9.79008879889323E7
Converged to left
Fitness changed from 7.655871782883009E-4 to 3.8615635643128197E-4
Iteration 4 complete. Error: 3.8615635643128197E-4 Total: 1.9969; Orientation: 0.2542; Line Search: 1.5290
Low gradient: 1.83638909719451E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=3.8615635643128197E-4}, derivative=-3.372324916294867E-12}
New Minimum: 3.8615635643128197E-4 > 2.0606662448405876E-4
F(9.79008879889323E7) = LineSearchPoint{point=PointSample{avg=2.0606662448405876E-4}, derivative=-2.6791426563001206E-13}, evalInputDelta = -1.800897319472232E-4
F(6.853062159225261E8) = LineSearchPoint{point=PointSample{avg=0.00616597265754678}, derivative=2.090837750373774E-11}, evalInputDelta = 0.005779816301115497
F(5.271586276327124E7) = LineSearchPoint{point=PointSample{avg=2.513695662982281E-4}, derivative=-1.729552182598405E-12}, evalInputDelta = -1.347867901330539E-4
F(3.690110393428987E8) = LineSearchPoint{point=PointSample{avg=0.0013989491176348192}, derivative=9.235195441549839E-12}, evalInputDelta = 0.0010127927612035373
F(2.8385464564838365E7) = LineSearchPoint{point=PointSample{avg=3.028108305709083E-4}, derivative=-2.4965638416207664E-12}, evalInputDelta = -8.334552586037366E-5
F(1.9869825195386857E8) = LineSearchPoint{point=PointSample{avg=3.493671441873539E-4}, derivative=3.142589663576246E-12}, evalInputDelta = -3.678921224392807E-5
3.493671441873539E-4 <= 3.8615635643128197E-4
New Minimum: 2.0606662448405876E-4 > 2.0514320211532092E-4
F(1.028524714596523E8) = LineSearchPoint{point=PointSample{avg=2.0514320211532092E-4}, derivative=-1.0497921899504057E-13}, evalInputDelta = -1.8101315431596105E-4
Left bracket at 1.028524714596523E8
New Minimum: 2.0514320211532092E-4 > 2.0497624099226024E-4
F(1.0595073220301995E8) = LineSearchPoint{point=PointSample{avg=2.0497624099226024E-4}, derivative=-2.7643607155728358E-15}, evalInputDelta = -1.8118011543902173E-4
Left bracket at 1.0595073220301995E8
Converged to left
Fitness changed from 3.8615635643128197E-4 to 2.0497624099226024E-4
Iteration 5 complete. Error: 2.0497624099226024E-4 Total: 4.5525; Orientation: 0.2472; Line Search: 4.0934
Final threshold in iteration 5: 2.0497624099226024E-4 (> 0.0) after 34.448s (< 30.000s)
Returns
2.0497624099226024E-4
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.01 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.8159827756095881 ], [ -0.2432662331255025 ], [ 0.2620694814544178 ], [ 1.5274000311944058 ], [ 0.20651442245980672 ], [ -1.0759656194712033 ], [ 0.15858680018108487 ], [ -0.2727565338504776 ], ... ],
[ [ 1.8746548057932289 ], [ 0.3831790007139919 ], [ 1.4953686598846339 ], [ -0.3126428957687196 ], [ 0.16853605824829992 ], [ -0.8378744854127842 ], [ 1.6795321027187327 ], [ 1.248248816719912 ], ... ],
[ [ -0.9919106773952229 ], [ 0.31288348074745564 ], [ 0.33229845769788774 ], [ -0.08608685664325616 ], [ 0.1999225051059602 ], [ -0.7516673280830577 ], [ -1.5820003186611835 ], [ -0.4264813643397689 ], ... ],
[ [ -0.9280038484959258 ], [ -1.2521145728242276 ], [ -1.9613672359796328 ], [ -0.8442441966537437 ], [ -1.207994988910089 ], [ -0.9558216040196789 ], [ 0.28104666404766887 ], [ -0.5924739238921523 ], ... ],
[ [ -2.4585616457131465 ], [ -1.6691506831110305 ], [ -0.9637540927380643 ], [ 1.8951098193523888 ], [ 1.0239892421941768 ], [ -0.6828753619874105 ], [ -0.7322883139098646 ], [ 0.06751870810218388 ], ... ],
[ [ 0.13115211651743258 ], [ 1.5663059871611107 ], [ -0.31219253182875406 ], [ 0.8559934373617129 ], [ 0.9919994995214012 ], [ 1.7365305953459194 ], [ -1.6289224249823404 ], [ -0.5486764027479359 ], ... ],
[ [ -0.0011448636458939454 ], [ 1.315979013954028 ], [ 1.3124217844230928 ], [ 2.186744364861732 ], [ -0.8111391737991274 ], [ -1.5020285774591806 ], [ 1.4247397956718917 ], [ -0.4369807389637942 ], ... ],
[ [ 0.09134067421034126 ], [ -2.0516487360462192 ], [ -1.1359999031050072 ], [ -0.2720818777575612 ], [ 1.062911864937522 ], [ -0.36708731784832976 ], [ -1.2480524400013593 ], [ -1.4718823359954194 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.04 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.6933829301108894 ], [ 0.43948159659755864 ], [ 0.565144947961868 ], [ 0.8216255892195693 ], [ 0.5514458958462168 ], [ 0.2542702446733293 ], [ 0.5395638163634504 ], [ 0.4322304955543442 ], ... ],
[ [ 0.8669959591316927 ], [ 0.5946396098628762 ], [ 0.8168827120400514 ], [ 0.4224697694232168 ], [ 0.5420345644075292 ], [ 0.3019826306876645 ], [ 0.8428425638612218 ], [ 0.7769965757672448 ], ... ],
[ [ 0.27053484832291746 ], [ 0.5775889295852276 ], [ 0.5823185216849402 ], [ 0.47849156735312637 ], [ 0.5498148159678361 ], [ 0.3204581069705362 ], [ 0.1705123746460472 ], [ 0.3949668615301435 ], ... ],
[ [ 0.2833298657951573 ], [ 0.22233431079416222 ], [ 0.12331915743194022 ], [ 0.30064166076865384 ], [ 0.23005600681457464 ], [ 0.27771555881205373 ], [ 0.5698028088743088 ], [ 0.3560674228309381 ], ... ],
[ [ 0.07881470268413275 ], [ 0.1585374476863395 ], [ 0.27612719290220816 ], [ 0.8693370518761243 ], [ 0.7357489267566396 ], [ 0.3356198540328366 ], [ 0.3246927736865836 ], [ 0.5168732673920748 ], ... ],
[ [ 0.5327411112573971 ], [ 0.8272563597180939 ], [ 0.4225796571341663 ], [ 0.701822889463345 ], [ 0.7294826799884053 ], [ 0.8502458502360946 ], [ 0.1639780316760108 ], [ 0.36617154815333647 ], ... ],
[ [ 0.49971378411978873 ], [ 0.7885119413310283 ], [ 0.7879181258677295 ], [ 0.8990528188829953 ], [ 0.30764779780661927 ], [ 0.18212316356710823 ], [ 0.8060803916296022 ], [ 0.39246063340399584 ], ... ],
[ [ 0.5228193054103237 ], [ 0.11388589165588978 ], [ 0.24305554040202906 ], [ 0.4323960686445728 ], [ 0.7432466131193285 ], [ 0.4092450153022835 ], [ 0.22303745284658488 ], [ 0.18665667675219372 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 145.06 seconds (1.922 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 5931955963805
Reset training subject: 5932166751216
Adding measurement 4125e18c to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.11922722039197835 < 0.11922722039197835. Total: 1
th(0)=0.11922722039197835;dx=-2.1045719379244403E-9
Adding measurement 5fa3f940 to history. Total: 1
New Minimum: 0.11922722039197835 > 0.11922721585781557
WOLFE (weak): th(2.154434690031884)=0.11922721585781557; dx=-2.104571930020167E-9 evalInputDelta=4.534162781344442E-9
Adding measurement 54cb8cb4 to history. Total: 2
New Minimum: 0.11922721585781557 > 0.11922721132365281
WOLFE (weak): th(4.308869380063768)=0.11922721132365281; dx=-2.10457192211589E-9 evalInputDelta=9.068325534933308E-9
Adding measurement 4162878a to history. Total: 3
New Minimum: 0.11922721132365281 > 0.11922719318700192
WOLFE (weak): th(12.926608140191302)=0.11922719318700192; dx=-2.104571890498746E-9 evalInputDelta=2.7204976424388683E-8
Adding measurement 4d432808 to history. Total: 4
New Minimum: 0.11922719318700192 > 0.11922711157207629
WOLFE (weak): th(51.70643256076521)=0.11922711157207629; dx=-2.1045717482208954E-9 evalInputDelta=1.0881990206157433E-7
Adding measurement 351e1dc6 to history. Total: 5
New Minimum: 0.11922711157207629 > 0.11922667629256609
WOLFE (weak): th(258.53216280382605)=0.11922667629256609; dx=-2.1045709893862307E-9 evalInputDelta=5.440994122613008E-7
Adding measurement 5c531c6f to history. Total: 6
New Minimum: 0.11922667629256609 > 0.11922395579918348
WOLFE (weak): th(1551.1929768229563)=0.11922395579918348; dx=-2.104566245927073E-9 evalInputDelta=3.264592794871568E-6
Adding measurement 82cb0d8 to history. Total: 7
New Minimum: 0.11922395579918348 > 0.11920436842796148
WOLFE (weak): th(10858.350837760694)=0.11920436842796148; dx=-2.1045320552289807E-9 evalInputDelta=2.285196401687062E-5
Adding measurement 229188da to history. Total: 8
New Minimum: 0.11920436842796148 > 0.11904441687420504
WOLFE (weak): th(86866.80670208555)=0.11904441687420504; dx=-2.104250346419807E-9 evalInputDelta=1.828035177733056E-4
Adding measurement 1a82382b to history. Total: 9
New Minimum: 0.11904441687420504 > 0.11758304573761787
WOLFE (weak): th(781801.26031877)=0.11758304573761787; dx=-2.101468979832483E-9 evalInputDelta=0.0016441746543604807
Adding measurement 7f597e1 to history. Total: 10
New Minimum: 0.11758304573761787 > 0.10294757011127327
WOLFE (weak): th(7818012.6031877)=0.10294757011127327; dx=-2.052111452474942E-9 evalInputDelta=0.016279650280705077
Adding measurement 2efb2b77 to history. Total: 11
New Minimum: 0.10294757011127327 > 0.009778844249577878
END: th(8.599813863506469E7)=0.009778844249577878; dx=-2.5146981427515033E-10 evalInputDelta=0.10944837614240047
Fitness changed from 0.11922722039197835 to 0.009778844249577878
Iteration 1 complete. Error: 0.009778844249577878 Total: 8.5059; Orientation: 0.3015; Line Search: 7.5759
Non-optimal measurement 0.009778844249577878 < 0.009778844249577878. Total: 12
Rejected: LBFGS Orientation magnitude: 7.270e+03, gradient 1.679e-05, dot -0.407; [851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348, 0.11922667629256609, 0.11922711157207629, 0.11922719318700192, 0.11922721132365281, 0.11922721585781557, 0.11922722039197835
Rejected: LBFGS Orientation magnitude: 7.270e+03, gradient 1.679e-05, dot -0.407; [851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348, 0.11922667629256609, 0.11922711157207629, 0.11922719318700192, 0.11922721132365281, 0.11922721585781557
Rejected: LBFGS Orientation magnitude: 7.270e+03, gradient 1.679e-05, dot -0.407; [7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00, 851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348, 0.11922667629256609, 0.11922711157207629, 0.11922719318700192, 0.11922721132365281
Rejected: LBFGS Orientation magnitude: 7.270e+03, gradient 1.679e-05, dot -0.407; [0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00, 7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348, 0.11922667629256609, 0.11922711157207629, 0.11922719318700192
Rejected: LBFGS Orientation magnitude: 7.270e+03, gradient 1.679e-05, dot -0.407; [57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00, 7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348, 0.11922667629256609, 0.11922711157207629
Rejected: LBFGS Orientation magnitude: 7.269e+03, gradient 1.679e-05, dot -0.407; [7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348, 0.11922667629256609
Rejected: LBFGS Orientation magnitude: 7.262e+03, gradient 1.679e-05, dot -0.407; [851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00, 7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148, 0.11922395579918348
Rejected: LBFGS Orientation magnitude: 7.207e+03, gradient 1.679e-05, dot -0.408; [0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00, 7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504, 0.11920436842796148
Rejected: LBFGS Orientation magnitude: 6.744e+03, gradient 1.679e-05, dot -0.416; [7665075d-840e-49aa-806d-bf35b6149f6a = 1.000/1.000e+00, 57429a7d-7c46-4cb9-a156-9058e4143b36 = 1.000/1.000e+00, 835c5b18-e493-4fe5-a931-acd7eadec621 = 1.000/1.000e+00, 851d7bd4-969b-4113-8dc4-c53145edba3b = 1.000/1.000e+00, 0b445180-2190-4092-8430-e550637d872e = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.009778844249577878, 0.10294757011127327, 0.11758304573761787, 0.11904441687420504
LBFGS Accumulation History: 3 points
Removed measurement 2efb2b77 to history. Total: 11
Removed measurement 7f597e1 to history. Total: 10
Removed measurement 1a82382b to history. Total: 9
Removed measurement 229188da to history. Total: 8
Removed measurement 82cb0d8 to history. Total: 7
Removed measurement 5c531c6f to history. Total: 6
Removed measurement 351e1dc6 to history. Total: 5
Removed measurement 4d432808 to history. Total: 4
Removed measurement 4162878a to history. Total: 3
Adding measurement 34cfd139 to history. Total: 3
th(0)=0.009778844249577878;dx=-2.819206555762429E-10
Adding measurement 474169a3 to history. Total: 4
New Minimum: 0.009778844249577878 > 0.0013594212079240677
WOLF (strong): th(1.0E8)=0.0013594212079240677; dx=5.81374214842216E-11 evalInputDelta=0.00841942304165381
Adding measurement 25c701b8 to history. Total: 5
New Minimum: 0.0013594212079240677 > 0.0011327208457505501
END: th(5.0E7)=0.0011327208457505501; dx=-6.765055025272928E-11 evalInputDelta=0.008646123403827328
Fitness changed from 0.009778844249577878 to 0.0011327208457505501
Iteration 2 complete. Error: 0.0011327208457505501 Total: 136.5578; Orientation: 134.6940; Line Search: 1.6419
Final threshold in iteration 2: 0.0011327208457505501 (> 0.0) after 145.065s (< 30.000s)
Returns
0.0011327208457505501
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.9002738456971314 ], [ -0.23797489475480607 ], [ 0.26198931633922007 ], [ 1.6497667699283733 ], [ 0.18713857264183606 ], [ -0.8168247323880327 ], [ 0.20550372394675764 ], [ -0.2932942571340935 ], ... ],
[ [ 1.802415637666678 ], [ 0.391883376100015 ], [ 1.233834049490408 ], [ -0.3190160456826388 ], [ 0.11368791518087612 ], [ -0.5915897895708455 ], [ 1.6131243222893068 ], [ 1.268263605251664 ], ... ],
[ [ -1.0473814181867906 ], [ 0.25409941959494575 ], [ 0.4373947281903637 ], [ 0.060443907684479936 ], [ 0.23896339598474917 ], [ -0.7514485272659834 ], [ -1.5223104116285904 ], [ -0.484431078891571 ], ... ],
[ [ -0.9302845756446613 ], [ -1.2496647452367162 ], [ -1.7498656720763799 ], [ -1.0352020103665125 ], [ -0.963545093097107 ], [ -1.002521809714055 ], [ 0.3017328881545732 ], [ -0.777545178436197 ], ... ],
[ [ -2.0679777812281754 ], [ -1.6551543031857703 ], [ -1.0564027085963712 ], [ 1.3220465670790433 ], [ 0.9720923160222731 ], [ -0.6811246926214707 ], [ -0.6476064317839172 ], [ 0.010884489349436671 ], ... ],
[ [ 0.18137902864382976 ], [ 1.5434957778716867 ], [ -0.3380302945065984 ], [ 0.877750348712142 ], [ 0.9915632891720796 ], [ 1.8308910513905625 ], [ -1.5715856311882914 ], [ -0.545040616018102 ], ... ],
[ [ 0.05682269958588866 ], [ 1.297744484492857 ], [ 1.4209460992675271 ], [ 2.0642116713851095 ], [ -1.0079776177264899 ], [ -1.257671437811833 ], [ 1.6345334768485287 ], [ -0.5539385087790096 ], ... ],
[ [ 0.1376047142525179 ], [ -1.9690572852483756 ], [ -0.9133413084186959 ], [ -0.2457965715483454 ], [ 1.3532211055588015 ], [ -0.43810065082908023 ], [ -1.236007883690581 ], [ -1.3139469859146466 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.04 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.7110057747488708 ], [ 0.44078546627169785 ], [ 0.5651252467900587 ], [ 0.8388595262216257 ], [ 0.5466485831547571 ], [ 0.3064380964675307 ], [ 0.5511958832352509 ], [ 0.42719757034116523 ], ... ],
[ [ 0.8584427347530444 ], [ 0.5967360014165214 ], [ 0.7744889180316837 ], [ 0.42091556375708244 ], [ 0.528391405607853 ], [ 0.35627016595289807 ], [ 0.8338447057468974 ], [ 0.7804453608696553 ], ... ],
[ [ 0.2597282573476482 ], [ 0.5631852492374566 ], [ 0.6076380718392951 ], [ 0.5151063779813156 ], [ 0.5594581787123549 ], [ 0.32050575593836605 ], [ 0.17912155147570696 ], [ 0.3812063378267616 ], ... ],
[ [ 0.28286698384486697 ], [ 0.2227581784248669 ], [ 0.14806414150962946 ], [ 0.2620768296277816 ], [ 0.2761689699069532 ], [ 0.26844589248708606 ], [ 0.5748660809782686 ], [ 0.3148491973129637 ], ... ],
[ [ 0.11224839205594442 ], [ 0.160413547012682 ], [ 0.2579975017009711 ], [ 0.7895220004370809 ], [ 0.7255363442591051 ], [ 0.33601032910016276 ], [ 0.3435291249814165 ], [ 0.5027210954729191 ], ... ],
[ [ 0.545220850794535 ], [ 0.8239723339470614 ], [ 0.41628802128096387 ], [ 0.7063558212665321 ], [ 0.7293965906251206 ], [ 0.8618678421880837 ], [ 0.17199046494315562 ], [ 0.3670157875502376 ], ... ],
[ [ 0.5142018538339121 ], [ 0.7854551387496115 ], [ 0.8054866920805214 ], [ 0.887375771634546 ], [ 0.2673758206963778 ], [ 0.22137500166498614 ], [ 0.8367897335549004 ], [ 0.36495112851090133 ], ... ],
[ [ 0.5343469987930074 ], [ 0.12249017972117932 ], [ 0.2863165890992073 ], [ 0.4388583750662101 ], [ 0.7946557406317433 ], [ 0.39219363908289645 ], [ 0.225131635705877 ], [ 0.21182712007471943 ], ... ],
...
]
TrainingTester.java:432 executed in 0.17 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -9.131877198742547], [17.0, -2.009712470947699]; valueStats=DoubleSummaryStatistics{count=24, sum=0.033363, min=0.000000, average=0.001390, max=0.009779}
Plotting 17 points for GD
Plotting 5 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -9.131877198742547], [136.557, -2.009712470947699]; valueStats=DoubleSummaryStatistics{count=24, sum=0.033363, min=0.000000, average=0.001390, max=0.009779}
Plotting 17 points for GD
Plotting 5 points for CjGD
Plotting 2 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.0011327208457505501 }, "CjGD": { "type": "NonConverged", "value": 2.0497624099226024E-4 }, "GD": { "type": "Converged", "value": 7.38112909685413E-10 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.0011327208457505501 }, "CjGD": { "type": "NonConverged", "value": 2.0497624099226024E-4 }, "GD": { "type": "Converged", "value": 7.38112909685413E-10 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "213.028",
"gc_time": "4.643"
},
"created_on": 1586740496654,
"file_name": "trainingTest",
"report": {
"simpleName": "Sigmoid_Double",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.ActivationLayerTest.Sigmoid_Double",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/ActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 0.0011327208457505501
},
"CjGD": {
"type": "NonConverged",
"value": 2.0497624099226024E-4
},
"GD": {
"type": "Converged",
"value": 7.38112909685413E-10
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/ActivationLayer/Sigmoid_Double/trainingTest/202004131456",
"id": "8dc6645c-d382-43c5-a507-d7e5423c3158",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "ActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.ActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/ActivationLayer.java",
"javaDoc": ""
}
}