Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 6658524679634642944
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ -0.010833392093138272 ], [ 0.050572164654282936 ], [ 0.30370969847147866 ], [ -0.5483246357195068 ], [ -0.25337157256356324 ], [ 0.9947456528651064 ], [ 0.5326133067339842 ], [ 0.16437512552407996 ], ... ],
[ [ -0.5097883626281223 ], [ 0.8513107377562366 ], [ -0.11248606127867089 ], [ -0.3430053189783524 ], [ -0.26706554581772646 ], [ -0.21800728032277017 ], [ -0.9764079143349043 ], [ 0.10474837986347274 ], ... ],
[ [ -0.445823756583023 ], [ 0.326309882190601 ], [ -0.29925392830807107 ], [ 0.8828209120413082 ], [ -0.9450511611517212 ], [ -0.6263259294426735 ], [ -0.659954779677242 ], [ 0.5961089794995793 ], ... ],
[ [ 0.06870759668913795 ], [ -0.151379045907732 ], [ -0.491407821924827 ], [ -0.10539660084471647 ], [ 0.21581782107494885 ], [ 0.5612905095775209 ], [ -0.6763517081712684 ], [ -0.7704932920049281 ], ... ],
[ [ -0.557424110336709 ], [ -0.20094256384762277 ], [ 0.5363774602951819 ], [ 0.797937160570912 ], [ -0.22741791472549822 ], [ -0.8960681460293751 ], [ 0.3722822218721802 ], [ -0.4690231047808209 ], ... ],
[ [ 0.9481486069222222 ], [ 0.14509296556482343 ], [ -0.6923476532697812 ], [ -0.8626759419837757 ], [ -0.9181088734703211 ], [ 0.573470401867048 ], [ 0.6728277178529183 ], [ -0.6246073604071847 ], ... ],
[ [ 0.8015172905415655 ], [ 0.6173214684319941 ], [ 0.45811762509681575 ], [ -0.8131072054489883 ], [ 0.3733177450939187 ], [ -0.9329701259518748 ], [ -0.7270857583931369 ], [ -0.14499446915599146 ], ... ],
[ [ 0.37763456412824525 ], [ 0.5239439817948077 ], [ 0.14196481343786616 ], [ 0.6355474041653915 ], [ -0.45526565239230377 ], [ -0.2798729504593942 ], [ -0.6787253893079139 ], [ 0.09080615429570749 ], ... ],
...
]
[
[ [ -0.19198956847881 ], [ 0.6024634765293766 ], [ -0.40856725866346744 ], [ -0.919153507567009 ], [ -0.9007214342445988 ], [ -0.6922437002081174 ], [ -0.8941400174522918 ], [ 0.7437442561764802 ], ... ],
[ [ 0.6889260621141371 ], [ -0.4616294641040397 ], [ -0.8084975089269686 ], [ 0.6760393696002032 ], [ -0.36842258774372194 ], [ 0.32349386277788206 ], [ 0.7722838763323772 ], [ 0.6658736154604736 ], ... ],
[ [ -0.616638101413584 ], [ -0.8231699186349875 ], [ 0.38929727524794355 ], [ -0.6421041736110755 ], [ 0.057147597858625554 ], [ 0.5234517996280894 ], [ -0.08709339125734683 ], [ 0.19057145253572583 ], ... ],
[ [ -0.8277349821388786 ], [ 0.7907973617646984 ], [ 0.45488619542476405 ], [ 0.4195490715646044 ], [ -0.4682439106020766 ], [ 0.8662889294008433 ], [ -0.8054056805059076 ], [ -0.7557873114151319 ], ... ],
[ [ 0.32102144027070967 ], [ 0.897538308735087 ], [ -0.050840717015510295 ], [ 0.7456393006535744 ], [ -0.22191292250785088 ], [ -0.30160747874116156 ], [ -0.8111070508782281 ], [ -0.28551264401472487 ], ... ],
[ [ -0.8136086417451882 ], [ -0.8197944037641636 ], [ -0.637147861943244 ], [ -0.9093990000429941 ], [ -0.23575882403923742 ], [ 0.7781958383879659 ], [ 0.5558365132341754 ], [ -0.6907592090348371 ], ... ],
[ [ -0.4318057407363669 ], [ 0.3861474263460649 ], [ -0.5827155052061797 ], [ 0.2905710407847897 ], [ 0.6333226632920765 ], [ -0.1614249882020109 ], [ -0.4477255828103648 ], [ -0.1977760000349169 ], ... ],
[ [ -0.012138854503320973 ], [ 0.7275654869998749 ], [ 0.18684304363619053 ], [ -0.22581811100342986 ], [ 0.8248700187882574 ], [ -0.9821988415980571 ], [ -0.8873589891047924 ], [ 0.9668867715316263 ], ... ],
...
]
[
[ [ 0.8164702037234912 ], [ -0.7449149307256604 ], [ -0.7536933536526085 ], [ 0.7798289276786317 ], [ 0.033108079999545126 ], [ -0.7817488754510884 ], [ 0.38129891378820435 ], [ 0.7838476804989212 ], ... ],
[ [ -0.35129660473265556 ], [ -0.7627037132331143 ], [ -0.16337468986504455 ], [ 0.13191822839813194 ], [ 0.45639192692381214 ], [ -0.8756242351938628 ], [ -0.10426042256064671 ], [ 0.37197844296429383 ], ... ],
[ [ 0.063028385972979 ], [ 0.008225043466830284 ], [ 0.9865167195868993 ], [ 0.8971461978962825 ], [ 0.010329297077855015 ], [ 0.11703142040641934 ], [ 0.6033370817136992 ], [ -0.6071672054434343 ], ... ],
[ [ 0.4011559169214962 ], [ 0.08284128120537337 ], [ -0.7671157350308011 ], [ 0.3023185309737859 ], [ 0.8346607173577283 ], [ -0.004668145913101496 ], [ -0.8586514139386039 ], [ -0.3533227184710981 ], ... ],
[ [ -0.95499121139756 ], [ 0.30865108338181413 ], [ 0.5404654265588908 ], [ 0.43993671670772705 ], [ -0.6583635000108783 ], [ -0.501775298893939 ], [ -0.3173935022575365 ], [ -0.8682342618283623 ], ... ],
[ [ -0.6566028513423401 ], [ 0.7476391470783468 ], [ 0.5595875200477771 ], [ 0.28785218029559334 ], [ 0.5833861932052866 ], [ -0.18529244184663218 ], [ 0.19050107849392228 ], [ 0.4506580323110019 ], ... ],
[ [ 0.5557477258162704 ], [ -0.5645767010793803 ], [ -0.1707637558472661 ], [ -0.8513430887485549 ], [ 0.005358609738266296 ], [ 0.5517767139050784 ], [ 0.5905598874358087 ], [ 0.5701624899296414 ], ... ],
[ [ 0.2869025856755094 ], [ 0.7952962679176037 ], [ 0.9680952389124273 ], [ -0.7391925508218544 ], [ 0.4940842086577162 ], [ -0.18137086590396745 ], [ 0.7780556082404442 ], [ 0.9370379779345785 ], ... ],
...
]
[
[ [ -0.6101951228104996 ], [ 0.25436171090227755 ], [ -0.6099168152126131 ], [ -0.6487107157871423 ], [ 0.5468696932529848 ], [ 0.18147924326619957 ], [ 0.43648137739579496 ], [ -0.6591036760651263 ], ... ],
[ [ -0.3540831999846741 ], [ -0.7015296877726662 ], [ -0.9979006744681587 ], [ 0.592318051314467 ], [ -0.9056144955538832 ], [ -0.6155601240914867 ], [ 0.7790557623480052 ], [ 0.48639076479214793 ], ... ],
[ [ 0.27622128784214794 ], [ -0.5123124067459861 ], [ 0.7539472948283321 ], [ -0.21582753883813255 ], [ 0.9380208302313107 ], [ 0.5235524708441321 ], [ -0.6577513125559757 ], [ -0.6301327672448684 ], ... ],
[ [ 0.6578407826056951 ], [ -0.8705226508053212 ], [ -0.05999131908862876 ], [ 0.17397790528749058 ], [ 0.4096068126525232 ], [ 0.04119375455204577 ], [ 0.1303066295415154 ], [ 0.25897652161672946 ], ... ],
[ [ 0.19510022074756694 ], [ -0.10310336198754433 ], [ 0.9640186592595925 ], [ -0.554609553865276 ], [ 0.7117352020983949 ], [ -0.9793717581912831 ], [ -0.6791293217006211 ], [ 0.38901863591241637 ], ... ],
[ [ 0.5788639139850786 ], [ 0.4532662834955765 ], [ 0.5300488931150742 ], [ 0.49519445116700034 ], [ 0.7658505082311469 ], [ -0.2683035374442384 ], [ 0.7861225450802574 ], [ -0.44890952942418116 ], ... ],
[ [ 0.2576615818837258 ], [ 0.06431717329204534 ], [ 0.8798531666040588 ], [ 0.6111819539077066 ], [ 0.075115037615757 ], [ 0.6162123020716659 ], [ -0.4261539998209849 ], [ -0.08895957441700508 ], ... ],
[ [ -0.25477680718849105 ], [ 0.09601721210268277 ], [ -0.3074021263505544 ], [ 0.015544220477108128 ], [ 0.7505638404572452 ], [ -0.26783079859921144 ], [ -0.9558597784674077 ], [ 0.25193149310950624 ], ... ],
...
]
[
[ [ 0.1163477968155513 ], [ 0.6034970663118975 ], [ -0.7878891109338604 ], [ 0.5075474957514068 ], [ -0.8116110897925795 ], [ 0.40417029874302957 ], [ 0.13146035332929173 ], [ -0.12113766007184523 ], ... ],
[ [ 0.7997020005406659 ], [ 0.22059454519378807 ], [ 0.01291295976032325 ], [ -0.14933167423771443 ], [ 0.7114243718726767 ], [ -7.684261103364509E-4 ], [ 0.17435304986508537 ], [ -0.044238313044372424 ], ... ],
[ [ -0.3320305827975718 ], [ -0.44553240673427363 ], [ 0.43418842377424416 ], [ -0.09281791036725817 ], [ -0.9779175880679083 ], [ 0.2700147589611348 ], [ -0.8206335321337317 ], [ 0.345465422153042 ], ... ],
[ [ -0.4699815364719184 ], [ 0.3674854655806401 ], [ 0.6897610572329933 ], [ -0.05647128424874359 ], [ -0.5000819836727919 ], [ 0.794292749540952 ], [ -0.23539021077501165 ], [ 0.3193666880973256 ], ... ],
[ [ -0.48790874243180093 ], [ 0.9543582191766025 ], [ -0.2747883501930961 ], [ 0.024682162243582217 ], [ 0.8694478681401505 ], [ 0.003415131054600451 ], [ -0.5009640533631479 ], [ -0.7874907811885854 ], ... ],
[ [ -0.9265100817444063 ], [ -0.5903068658134716 ], [ 0.6064126512946566 ], [ 0.9701278234384945 ], [ 0.80970809041532 ], [ 0.5089642623355328 ], [ 0.4126936470988536 ], [ -0.6870991542227667 ], ... ],
[ [ -0.8665535078603943 ], [ -0.9834451461514098 ], [ -0.7823451274077147 ], [ -0.9359254397662582 ], [ 0.10823629013289815 ], [ 0.8518981088886974 ], [ 0.43118182810616457 ], [ -0.22973840545830404 ], ... ],
[ [ -0.705336169338683 ], [ -0.2552087698578023 ], [ 0.501046797288115 ], [ -0.8765740298622688 ], [ -0.12741419402893994 ], [ 0.18795303584349154 ], [ -0.3372942697895951 ], [ 0.9986004970208997 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 5.44 seconds (0.020 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 285860822118
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 285947773829
Constructing line search parameters: GD
th(0)=0.6658476244979714;dx=-5.3267809959837715E-5
New Minimum: 0.6658476244979714 > 0.6657328674252766
WOLFE (weak): th(2.154434690031884)=0.6657328674252766; dx=-5.326321947913213E-5 evalInputDelta=1.147570726948155E-4
New Minimum: 0.6657328674252766 > 0.6656181202424736
WOLFE (weak): th(4.308869380063768)=0.6656181202424736; dx=-5.3258628998426556E-5 evalInputDelta=2.295042554978366E-4
New Minimum: 0.6656181202424736 > 0.665159230410171
WOLFE (weak): th(12.926608140191302)=0.665159230410171; dx=-5.324026707560423E-5 evalInputDelta=6.883940878004147E-4
New Minimum: 0.665159230410171 > 0.6630961843631951
WOLFE (weak): th(51.70643256076521)=0.6630961843631951; dx=-5.3157638422903796E-5 evalInputDelta=0.0027514401347763506
New Minimum: 0.6630961843631951 > 0.6521473895955381
WOLFE (weak): th(258.53216280382605)=0.6521473895955381; dx=-5.271695227516811E-5 evalInputDelta=0.013700234902433328
New Minimum: 0.6521473895955381 > 0.5857824315125962
WOLFE (weak): th(1551.1929768229563)=0.5857824315125962; dx=-4.996266385182011E-5 evalInputDelta=0.08006519298537518
New Minimum: 0.5857824315125962 > 0.21305658163309077
END: th(10858.350837760694)=0.21305658163309077; dx=-3.013178720371446E-5 evalInputDelta=0.45279104286488064
Fitness changed from 0.6658476244979714 to 0.21305658163309077
Iteration 1 complete. Error: 0.21305658163309077 Total: 0.4455; Orientation: 0.0107; Line Search: 0.3122
th(0)=0.21305658163309077;dx=-1.7044526530647263E-5
New Minimum: 0.21305658163309077 > 8.796667027449357E-4
END: th(23393.607721408407)=8.796667027449357E-4; dx=-1.095207832443251E-6 evalInputDelta=0.21217691493034582
Fitness changed from 0.21305658163309077 to 8.796667027449357E-4
Iteration 2 complete. Error: 8.796667027449357E-4 Total: 0.2643; Orientation: 0.0041; Line Search: 0.2468
th(0)=8.796667027449357E-4;dx=-7.037333621959483E-8
Armijo: th(50400.00000000001)=9.080412319086766E-4; dx=7.14993095991084E-8 evalInputDelta=-2.8374529163740884E-5
New Minimum: 8.796667027449357E-4 > 5.6298668975679306E-8
WOLF (strong): th(25200.000000000004)=5.6298668975679306E-8; dx=5.629866897567758E-10 evalInputDelta=8.7961040407596E-4
END: th(8400.000000000002)=3.878415305734307E-4; dx=-4.672789524981097E-8 evalInputDelta=4.918251721715049E-4
Fitness changed from 8.796667027449357E-4 to 5.6298668975679306E-8
Iteration 3 complete. Error: 5.6298668975679306E-8 Total: 0.3707; Orientation: 0.0041; Line Search: 0.3548
Low gradient: 2.1222378561448624E-6
th(0)=5.6298668975679306E-8;dx=-4.503893518054342E-12
New Minimum: 5.6298668975679306E-8 > 4.29202480792879E-9
END: th(18097.251396267828)=4.29202480792879E-9; dx=-1.243569787724321E-12 evalInputDelta=5.200664416775052E-8
Fitness changed from 5.6298668975679306E-8 to 4.29202480792879E-9
Iteration 4 complete. Error: 4.29202480792879E-9 Total: 0.0642; Orientation: 0.0033; Line Search: 0.0514
Low gradient: 5.859709759316609E-7
th(0)=4.29202480792879E-9;dx=-3.4336198463430315E-13
New Minimum: 4.29202480792879E-9 > 1.3439312181222332E-9
WOLF (strong): th(38989.346202347355)=1.3439312181222332E-9; dx=1.9213638703097288E-13 evalInputDelta=2.9480935898065567E-9
New Minimum: 1.3439312181222332E-9 > 2.0813658756917315E-10
END: th(19494.673101173677)=2.0813658756917315E-10; dx=-7.561279880166467E-14 evalInputDelta=4.083888220359617E-9
Fitness changed from 4.29202480792879E-9 to 2.0813658756917315E-10
Iteration 5 complete. Error: 2.0813658756917315E-10 Total: 0.0645; Orientation: 0.0045; Line Search: 0.0488
Low gradient: 1.290384710291232E-7
th(0)=2.0813658756917315E-10;dx=-1.6650927005533866E-14
New Minimum: 2.0813658756917315E-10 > 9.624235809198708E-11
WOLF (strong): th(42000.000000000015)=9.624235809198708E-11; dx=1.132263036376311E-14 evalInputDelta=1.1189422947718606E-10
New Minimum: 9.624235809198708E-11 > 5.328296641770246E-12
END: th(21000.000000000007)=5.328296641770246E-12; dx=-2.6641483208852708E-15 evalInputDelta=2.028082909274029E-10
Fitness changed from 2.0813658756917315E-10 to 5.328296641770246E-12
Iteration 6 complete. Error: 5.328296641770246E-12 Total: 0.0495; Orientation: 0.0027; Line Search: 0.0377
Low gradient: 2.064615536465856E-8
th(0)=5.328296641770246E-12;dx=-4.262637313416195E-16
New Minimum: 5.328296641770246E-12 > 3.493523278291704E-12
WOLF (strong): th(45243.12849066957)=3.493523278291704E-12; dx=3.4515645937845967E-16 evalInputDelta=1.834773363478542E-12
New Minimum: 3.493523278291704E-12 > 4.82271089000708E-14
END: th(22621.564245334786)=4.82271089000708E-14; dx=-4.05536359815618E-17 evalInputDelta=5.280069532870175E-12
Fitness changed from 5.328296641770246E-12 to 4.82271089000708E-14
Iteration 7 complete. Error: 4.82271089000708E-14 Total: 0.4351; Orientation: 0.0034; Line Search: 0.2308
Low gradient: 1.9642221646253923E-9
th(0)=4.82271089000708E-14;dx=-3.8581687120056626E-18
New Minimum: 4.82271089000708E-14 > 4.3476168290532224E-14
WOLF (strong): th(48736.6827529342)=4.3476168290532224E-14; dx=3.663205068968148E-18 evalInputDelta=4.750940609538578E-15
New Minimum: 4.3476168290532224E-14 > 3.0787616601238224E-17
END: th(24368.3413764671)=3.0787616601238224E-17; dx=-9.748182152098823E-20 evalInputDelta=4.819632128346956E-14
Fitness changed from 4.82271089000708E-14 to 3.0787616601238224E-17
Iteration 8 complete. Error: 3.0787616601238224E-17 Total: 0.2052; Orientation: 0.0031; Line Search: 0.1928
Zero gradient: 4.962871475364882E-11
th(0)=3.0787616601238224E-17;dx=-2.46300932809904E-21
Armijo: th(52500.00000000002)=3.7253016126995366E-17; dx=2.7093102623452084E-21 evalInputDelta=-6.465399525757143E-18
New Minimum: 3.0787616601238224E-17 > 7.696904235338266E-20
WOLF (strong): th(26250.00000000001)=7.696904235338266E-20; dx=1.2315046708518386E-22 evalInputDelta=3.071064755888484E-17
END: th(8750.000000000004)=1.3007768001987265E-17; dx=-1.6009560625237068E-21 evalInputDelta=1.777984859925096E-17
Fitness changed from 3.0787616601238224E-17 to 7.696904235338266E-20
Iteration 9 complete. Error: 7.696904235338266E-20 Total: 0.4054; Orientation: 0.0035; Line Search: 0.3866
Zero gradient: 2.4814357513888673E-12
th(0)=7.696904235338266E-20;dx=-6.157523388270833E-24
New Minimum: 7.696904235338266E-20 > 4.655884248386066E-21
END: th(18851.303537778993)=4.655884248386066E-21; dx=-1.5144296920627565E-24 evalInputDelta=7.23131581049966E-20
Fitness changed from 7.696904235338266E-20 to 4.655884248386066E-21
Iteration 10 complete. Error: 4.655884248386066E-21 Total: 0.3004; Orientation: 0.0037; Line Search: 0.2714
Zero gradient: 6.103038094841748E-13
th(0)=4.655884248386066E-21;dx=-3.72470739870896E-25
New Minimum: 4.655884248386066E-21 > 1.8161222222632825E-21
WOLF (strong): th(40613.90229411184)=1.8161222222632825E-21; dx=2.326288697169242E-25 evalInputDelta=2.8397620261227836E-21
New Minimum: 1.8161222222632825E-21 > 1.6407118417936316E-22
END: th(20306.95114705592)=1.6407118417936316E-22; dx=-6.99209355558479E-26 evalInputDelta=4.491813064206703E-21
Fitness changed from 4.655884248386066E-21 to 1.6407118417936316E-22
Iteration 11 complete. Error: 1.6407118417936316E-22 Total: 0.2893; Orientation: 0.0027; Line Search: 0.2784
Zero gradient: 1.1456742440305226E-13
th(0)=1.6407118417936316E-22;dx=-1.3125694734349093E-26
New Minimum: 1.6407118417936316E-22 > 9.229019929949452E-23
WOLF (strong): th(43750.00000000003)=9.229019929949452E-23; dx=9.844279487985698E-27 evalInputDelta=7.178098487986864E-23
New Minimum: 9.229019929949452E-23 > 2.563598017061746E-24
END: th(21875.000000000015)=2.563598017061746E-24; dx=-1.640707286161122E-27 evalInputDelta=1.6150758616230142E-22
Fitness changed from 1.6407118417936316E-22 to 2.563598017061746E-24
Iteration 12 complete. Error: 2.563598017061746E-24 Total: 0.1676; Orientation: 0.0027; Line Search: 0.1576
Zero gradient: 1.4320888288264098E-14
th(0)=2.563598017061746E-24;dx=-2.050878413649398E-28
New Minimum: 2.563598017061746E-24 > 2.0084652156389776E-24
WOLF (strong): th(47128.25884444749)=2.0084652156389776E-24; dx=1.815294566536094E-28 evalInputDelta=5.551328014227686E-25
New Minimum: 2.0084652156389776E-24 > 8.456693473800735E-27
END: th(23564.129422223745)=8.456693473800735E-27; dx=-1.1779184657435164E-29 evalInputDelta=2.5551413235879455E-24
Fitness changed from 2.563598017061746E-24 to 8.456693473800735E-27
Iteration 13 complete. Error: 8.456693473800735E-27 Total: 0.3337; Orientation: 0.0279; Line Search: 0.2986
Zero gradient: 8.225177675309263E-16
th(0)=8.456693473800735E-27;dx=-6.76535477904059E-31
Armijo: th(50767.37786763981)=8.983818543295871E-27; dx=6.973017467535978E-31 evalInputDelta=-5.271250694951357E-28
New Minimum: 8.456693473800735E-27 > 1.992976010273115E-30
WOLF (strong): th(25383.688933819903)=1.992976010273115E-30; dx=1.0384308079785242E-32 evalInputDelta=8.454700497790463E-27
END: th(8461.229644606634)=3.701056044348532E-27; dx=-4.475618679030435E-31 evalInputDelta=4.755637429452203E-27
Fitness changed from 8.456693473800735E-27 to 1.992976010273115E-30
Iteration 14 complete. Error: 1.992976010273115E-30 Total: 0.2334; Orientation: 0.0031; Line Search: 0.2219
Zero gradient: 1.2626879298617265E-17
th(0)=1.992976010273115E-30;dx=-1.5943808082184924E-34
New Minimum: 1.992976010273115E-30 > 1.4591540952079677E-31
END: th(18229.16666666668)=1.4591540952079677E-31; dx=-4.3053738053547824E-35 evalInputDelta=1.8470606007523183E-30
Fitness changed from 1.992976010273115E-30 to 1.4591540952079677E-31
Iteration 15 complete. Error: 1.4591540952079677E-31 Total: 0.1445; Orientation: 0.0168; Line Search: 0.1199
Zero gradient: 3.4166112980062193E-18
th(0)=1.4591540952079677E-31;dx=-1.1673232761663745E-35
New Minimum: 1.4591540952079677E-31 > 4.72007269986907E-32
WOLF (strong): th(39273.54903703958)=4.72007269986907E-32; dx=6.59600036802883E-36 evalInputDelta=9.871468252210606E-32
New Minimum: 4.72007269986907E-32 > 7.424970544711892E-33
END: th(19636.77451851979)=7.424970544711892E-33; dx=-2.5332485083591113E-36 evalInputDelta=1.3849043897608488E-31
Fitness changed from 1.4591540952079677E-31 to 7.424970544711892E-33
Iteration 16 complete. Error: 7.424970544711892E-33 Total: 0.3464; Orientation: 0.0050; Line Search: 0.1838
Zero gradient: 7.707124259910122E-19
th(0)=7.424970544711892E-33;dx=-5.939976435769516E-37
New Minimum: 7.424970544711892E-33 > 4.5017419078121515E-33
WOLF (strong): th(42306.148223033175)=4.5017419078121515E-33; dx=4.408223375313835E-37 evalInputDelta=2.9232286368997403E-33
New Minimum: 4.5017419078121515E-33 > 2.1922867134218138E-35
END: th(21153.074111516587)=2.1922867134218138E-35; dx=-8.747395383160077E-39 evalInputDelta=7.403047677577673E-33
Fitness changed from 7.424970544711892E-33 to 2.1922867134218138E-35
Iteration 17 complete. Error: 2.1922867134218138E-35 Total: 0.2276; Orientation: 0.0035; Line Search: 0.2137
Zero gradient: 4.187874605020369E-20
th(0)=2.1922867134218138E-35;dx=-1.7538293707374517E-39
New Minimum: 2.1922867134218138E-35 > 2.1024985535562E-35
WOLF (strong): th(45572.91666666671)=2.1024985535562E-35; dx=1.712881489401927E-39 evalInputDelta=8.97881598656138E-37
New Minimum: 2.1024985535562E-35 > 1.3174338759736067E-38
END: th(22786.458333333354)=1.3174338759736067E-38; dx=-8.944931076776013E-42 evalInputDelta=2.1909692795458403E-35
Fitness changed from 2.1922867134218138E-35 to 1.3174338759736067E-38
Iteration 18 complete. Error: 1.3174338759736067E-38 Total: 0.2434; Orientation: 0.0035; Line Search: 0.2283
Zero gradient: 1.026619257942732E-21
th(0)=1.3174338759736067E-38;dx=-1.0539471007788856E-42
New Minimum: 1.3174338759736067E-38 > 1.315918085191771E-38
WOLF (strong): th(49091.93629629948)=1.315918085191771E-38; dx=1.0533270458759262E-42 evalInputDelta=1.5157907818356807E-41
New Minimum: 1.315918085191771E-38 > 4.155556201149102E-44
END: th(24545.96814814974)=4.155556201149102E-44; dx=-1.6151108709305918E-46 evalInputDelta=1.3174297204174056E-38
Fitness changed from 1.3174338759736067E-38 to 4.155556201149102E-44
Iteration 19 complete. Error: 4.155556201149102E-44 Total: 0.2275; Orientation: 0.0030; Line Search: 0.2176
Zero gradient: 1.8233060524550678E-24
th(0)=4.155556201149102E-44;dx=-3.3244449609192825E-48
Armijo: th(52882.685278791476)=4.155556201149102E-44; dx=3.3244449609192825E-48 evalInputDelta=0.0
New Minimum: 4.155556201149102E-44 > 0.0
END: th(26441.342639395738)=0.0; dx=0.0 evalInputDelta=4.155556201149102E-44
Fitness changed from 4.155556201149102E-44 to 0.0
Iteration 20 complete. Error: 0.0 Total: 0.4038; Orientation: 0.1099; Line Search: 0.2855
Zero gradient: 0.0
th(0)=0.0;dx=0.0 (ERROR: Starting derivative negative)
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.2089; Orientation: 0.0028; Line Search: 0.0963
Iteration 21 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 21
Final threshold in iteration 21: 0.0 (> 0.0) after 5.432s (< 30.000s)
Returns
0.0
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 1.61 seconds (0.020 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 291298308800
Reset training subject: 291307606157
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.6658476244979714}, derivative=-5.3267809959837715E-5}
New Minimum: 0.6658476244979714 > 0.6658476244979662
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.6658476244979662}, derivative=-5.32678099598375E-5}, evalInputDelta = -5.218048215738236E-15
New Minimum: 0.6658476244979662 > 0.6658476244979344
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.6658476244979344}, derivative=-5.326780995983622E-5}, evalInputDelta = -3.697042672001771E-14
New Minimum: 0.6658476244979344 > 0.6658476244977111
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.6658476244977111}, derivative=-5.326780995982727E-5}, evalInputDelta = -2.603472992745992E-13
New Minimum: 0.6658476244977111 > 0.6658476244961435
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.6658476244961435}, derivative=-5.3267809959764626E-5}, evalInputDelta = -1.8278711877428577E-12
New Minimum: 0.6658476244961435 > 0.6658476244851823
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.6658476244851823}, derivative=-5.326780995932613E-5}, evalInputDelta = -1.2789103109867028E-11
New Minimum: 0.6658476244851823 > 0.6658476244084452
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.6658476244084452}, derivative=-5.326780995625662E-5}, evalInputDelta = -8.952616425972337E-11
New Minimum: 0.6658476244084452 > 0.6658476238712806
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.6658476238712806}, derivative=-5.3267809934770094E-5}, evalInputDelta = -6.266908103569335E-10
New Minimum: 0.6658476238712806 > 0.6658476201111377
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.6658476201111377}, derivative=-5.326780978436438E-5}, evalInputDelta = -4.38683367409709E-9
New Minimum: 0.6658476201111377 > 0.6658475937901404
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.6658475937901404}, derivative=-5.326780873152442E-5}, evalInputDelta = -3.070783105574293E-8
New Minimum: 0.6658475937901404 > 0.6658474095431612
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.6658474095431612}, derivative=-5.3267801361644635E-5}, evalInputDelta = -2.1495481017375084E-7
New Minimum: 0.6658474095431612 > 0.6658461198150324
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.6658461198150324}, derivative=-5.326774977248618E-5}, evalInputDelta = -1.5046829390241712E-6
New Minimum: 0.6658461198150324 > 0.6658370917531057
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.6658370917531057}, derivative=-5.3267388648377014E-5}, evalInputDelta = -1.0532744865732191E-5
New Minimum: 0.6658370917531057 > 0.6657738970333711
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.6657738970333711}, derivative=-5.326486077961282E-5}, evalInputDelta = -7.372746460032609E-5
New Minimum: 0.6657738970333711 > 0.6653316179687195
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.6653316179687195}, derivative=-5.324716569826348E-5}, evalInputDelta = -5.160065292518778E-4
New Minimum: 0.6653316179687195 > 0.6622397792175632
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.6622397792175632}, derivative=-5.312330012881812E-5}, evalInputDelta = -0.0036078452804082506
New Minimum: 0.6622397792175632 > 0.6407985283285887
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.6407985283285887}, derivative=-5.225624114270058E-5}, evalInputDelta = -0.025049096169382667
New Minimum: 0.6407985283285887 > 0.500589170192173
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.500589170192173}, derivative=-4.618682823987776E-5}, evalInputDelta = -0.16525845430579844
New Minimum: 0.500589170192173 > 0.0032141694718852898
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0032141694718852898}, derivative=-3.700937920118059E-6}, evalInputDelta = -0.6626334550260862
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=20.242023969925377}, derivative=2.9370029431819984E-4}, evalInputDelta = 19.576176345427406
F(12526.258445464993) = LineSearchPoint{point=PointSample{avg=0.16576327575250466}, derivative=-2.6577955784604053E-5}, evalInputDelta = -0.5000843487454667
F(87683.80911825495) = LineSearchPoint{point=PointSample{avg=4.186061419965441}, derivative=1.335611692667979E-4}, evalInputDelta = 3.5202137954674693
F(6744.908393711919) = LineSearchPoint{point=PointSample{avg=0.3550282163765325}, derivative=-3.889635001932728E-5}, evalInputDelta = -0.31081940812143893
F(47214.358755983434) = LineSearchPoint{point=PointSample{avg=0.5257295640834099}, derivative=4.733240962373531E-5}, evalInputDelta = -0.14011806041456154
0.5257295640834099 <= 0.6658476244979714
New Minimum: 0.0032141694718852898 > 5.471056660754185E-33
F(24999.999999999996) = LineSearchPoint{point=PointSample{avg=5.471056660754185E-33}, derivative=-2.3774055883215367E-21}, evalInputDelta = -0.6658476244979714
Left bracket at 24999.999999999996
Converged to left
Fitness changed from 0.6658476244979714 to 5.471056660754185E-33
Iteration 1 complete. Error: 5.471056660754185E-33 Total: 1.4337; Orientation: 0.0031; Line Search: 1.2802
Zero gradient: 6.615773067906235E-19
F(0.0) = LineSearchPoint{point=PointSample{avg=5.471056660754185E-33}, derivative=-4.376845328603349E-37}
New Minimum: 5.471056660754185E-33 > 0.0
F(24999.999999999996) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=0.0}, evalInputDelta = -5.471056660754185E-33
0.0 <= 5.471056660754185E-33
Converged to right
Fitness changed from 5.471056660754185E-33 to 0.0
Iteration 2 complete. Error: 0.0 Total: 0.0350; Orientation: 0.0029; Line Search: 0.0245
Zero gradient: 0.0
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=0.0}
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.1445; Orientation: 0.0030; Line Search: 0.1340
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 1.613s (< 30.000s)
Returns
0.0
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 30.10 seconds (0.058 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 292916295842
Reset training subject: 292923772934
Adding measurement 14d5e0f5 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.6658476244979714 < 0.6658476244979714. Total: 1
th(0)=0.6658476244979714;dx=-5.3267809959837715E-5
Adding measurement 159c821f to history. Total: 1
New Minimum: 0.6658476244979714 > 0.6657328674252766
WOLFE (weak): th(2.154434690031884)=0.6657328674252766; dx=-5.326321947913213E-5 evalInputDelta=1.147570726948155E-4
Adding measurement 1ce771c5 to history. Total: 2
New Minimum: 0.6657328674252766 > 0.6656181202424736
WOLFE (weak): th(4.308869380063768)=0.6656181202424736; dx=-5.3258628998426556E-5 evalInputDelta=2.295042554978366E-4
Adding measurement 5accc7ac to history. Total: 3
New Minimum: 0.6656181202424736 > 0.665159230410171
WOLFE (weak): th(12.926608140191302)=0.665159230410171; dx=-5.324026707560423E-5 evalInputDelta=6.883940878004147E-4
Adding measurement 6c859ca5 to history. Total: 4
New Minimum: 0.665159230410171 > 0.6630961843631951
WOLFE (weak): th(51.70643256076521)=0.6630961843631951; dx=-5.3157638422903796E-5 evalInputDelta=0.0027514401347763506
Adding measurement 30bd0069 to history. Total: 5
New Minimum: 0.6630961843631951 > 0.6521473895955381
WOLFE (weak): th(258.53216280382605)=0.6521473895955381; dx=-5.271695227516811E-5 evalInputDelta=0.013700234902433328
Adding measurement 47b76a37 to history. Total: 6
New Minimum: 0.6521473895955381 > 0.5857824315125962
WOLFE (weak): th(1551.1929768229563)=0.5857824315125962; dx=-4.996266385182011E-5 evalInputDelta=0.08006519298537518
Adding measurement 2dfe61b5 to history. Total: 7
New Minimum: 0.5857824315125962 > 0.21305658163309077
END: th(10858.350837760694)=0.21305658163309077; dx=-3.013178720371446E-5 evalInputDelta=0.45279104286488064
Fitness changed from 0.6658476244979714 to 0.21305658163309077
Iteration 1 complete. Error: 0.21305658163309077 Total: 0.6549; Orientation: 0.0129; Line Search: 0.4973
Non-optimal measurement 0.21305658163309077 < 0.21305658163309077. Total: 8
Rejected: LBFGS Orientation magnitude: 1.032e+02, gradient 4.129e-03, dot -1.000; [5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.21305658163309077, 0.5857824315125962, 0.6521473895955381, 0.6630961843631951, 0.665159230410171, 0.6656181202424736, 0.6657328674252766, 0.6658476244979714
Rejected: LBFGS Orientation magnitude: 1.032e+02, gradient 4.129e-03, dot -1.000; [a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.21305658163309077, 0.5857824315125962, 0.6521473895955381, 0.6630961843631951, 0.665159230410171, 0.6656181202424736, 0.6657328674252766
Rejected: LBFGS Orientation magnitude: 1.032e+02, gradient 4.129e-03, dot -1.000; [b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.21305658163309077, 0.5857824315125962, 0.6521473895955381, 0.6630961843631951, 0.665159230410171, 0.6656181202424736
Rejected: LBFGS Orientation magnitude: 1.032e+02, gradient 4.129e-03, dot -1.000; [a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.21305658163309077, 0.5857824315125962, 0.6521473895955381, 0.6630961843631951, 0.665159230410171
Rejected: LBFGS Orientation magnitude: 1.032e+02, gradient 4.129e-03, dot -1.000; [5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.21305658163309077, 0.5857824315125962, 0.6521473895955381, 0.6630961843631951
LBFGS Accumulation History: 3 points
Removed measurement 2dfe61b5 to history. Total: 7
Removed measurement 47b76a37 to history. Total: 6
Removed measurement 30bd0069 to history. Total: 5
Removed measurement 6c859ca5 to history. Total: 4
Removed measurement 5accc7ac to history. Total: 3
Adding measurement 5f938bd1 to history. Total: 3
th(0)=0.21305658163309077;dx=-1.7044526530647263E-5
Adding measurement 73eb2c65 to history. Total: 4
New Minimum: 0.21305658163309077 > 8.796667027449357E-4
END: th(23393.607721408407)=8.796667027449357E-4; dx=-1.095207832443251E-6 evalInputDelta=0.21217691493034582
Fitness changed from 0.21305658163309077 to 8.796667027449357E-4
Iteration 2 complete. Error: 8.796667027449357E-4 Total: 5.7829; Orientation: 5.7194; Line Search: 0.0561
Non-optimal measurement 8.796667027449357E-4 < 8.796667027449357E-4. Total: 5
Rejected: LBFGS Orientation magnitude: 6.632e+00, gradient 2.653e-04, dot -1.000; [6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.796667027449357E-4, 0.21305658163309077, 0.6656181202424736, 0.6657328674252766, 0.6658476244979714
Rejected: LBFGS Orientation magnitude: 6.632e+00, gradient 2.653e-04, dot -1.000; [ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.796667027449357E-4, 0.21305658163309077, 0.6656181202424736, 0.6657328674252766
LBFGS Accumulation History: 3 points
Removed measurement 73eb2c65 to history. Total: 4
Removed measurement 5f938bd1 to history. Total: 3
Adding measurement 3cf5d703 to history. Total: 3
th(0)=8.796667027449357E-4;dx=-7.037333621959483E-8
Non-optimal measurement 9.080412319086766E-4 < 8.796667027449357E-4. Total: 4
Armijo: th(50400.00000000001)=9.080412319086766E-4; dx=7.14993095991084E-8 evalInputDelta=-2.8374529163740884E-5
Adding measurement 1ea72f2c to history. Total: 4
New Minimum: 8.796667027449357E-4 > 5.6298668975679306E-8
WOLF (strong): th(25200.000000000004)=5.6298668975679306E-8; dx=5.629866897567758E-10 evalInputDelta=8.7961040407596E-4
Non-optimal measurement 3.878415305734307E-4 < 5.6298668975679306E-8. Total: 5
END: th(8400.000000000002)=3.878415305734307E-4; dx=-4.672789524981097E-8 evalInputDelta=4.918251721715049E-4
Fitness changed from 8.796667027449357E-4 to 5.6298668975679306E-8
Iteration 3 complete. Error: 5.6298668975679306E-8 Total: 2.7731; Orientation: 2.5594; Line Search: 0.1002
Non-optimal measurement 5.6298668975679306E-8 < 5.6298668975679306E-8. Total: 5
Rejected: LBFGS Orientation magnitude: 5.306e-02, gradient 2.122e-06, dot -1.000; [6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 5.6298668975679306E-8, 8.796667027449357E-4, 0.66561
...skipping 9174 bytes...
.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.82271089000708E-14, 3.493523278291704E-12, 5.328296641770246E-12, 0.6656181202424736, 0.6657328674252766
Rejected: LBFGS Orientation magnitude: 4.911e-05, gradient 1.964e-09, dot -1.000; [6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.82271089000708E-14, 3.493523278291704E-12, 5.328296641770246E-12, 0.6656181202424736
LBFGS Accumulation History: 3 points
Removed measurement 4a0b25bf to history. Total: 5
Removed measurement 2cc4ffb2 to history. Total: 4
Removed measurement 13bacd2e to history. Total: 3
Adding measurement 62c783d2 to history. Total: 3
th(0)=4.82271089000708E-14;dx=-3.8581687120056626E-18
Adding measurement 353be7ef to history. Total: 4
New Minimum: 4.82271089000708E-14 > 4.3476168290532224E-14
WOLF (strong): th(48736.6827529342)=4.3476168290532224E-14; dx=3.663205068968148E-18 evalInputDelta=4.750940609538578E-15
Adding measurement 2739d18e to history. Total: 5
New Minimum: 4.3476168290532224E-14 > 3.0787616601238224E-17
END: th(24368.3413764671)=3.0787616601238224E-17; dx=-9.748182152098823E-20 evalInputDelta=4.819632128346956E-14
Fitness changed from 4.82271089000708E-14 to 3.0787616601238224E-17
Iteration 8 complete. Error: 3.0787616601238224E-17 Total: 3.3536; Orientation: 3.0388; Line Search: 0.2028
Non-optimal measurement 3.0787616601238224E-17 < 3.0787616601238224E-17. Total: 6
Rejected: LBFGS Orientation magnitude: 1.241e-06, gradient 4.963e-11, dot -1.000; [6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3.0787616601238224E-17, 4.3476168290532224E-14, 4.82271089000708E-14, 0.6656181202424736, 0.6657328674252766, 0.6658476244979714
Rejected: LBFGS Orientation magnitude: 1.241e-06, gradient 4.963e-11, dot -1.000; [b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3.0787616601238224E-17, 4.3476168290532224E-14, 4.82271089000708E-14, 0.6656181202424736, 0.6657328674252766
Rejected: LBFGS Orientation magnitude: 1.241e-06, gradient 4.963e-11, dot -1.000; [6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3.0787616601238224E-17, 4.3476168290532224E-14, 4.82271089000708E-14, 0.6656181202424736
LBFGS Accumulation History: 3 points
Removed measurement 2739d18e to history. Total: 5
Removed measurement 353be7ef to history. Total: 4
Removed measurement 62c783d2 to history. Total: 3
Adding measurement 448e0dc8 to history. Total: 3
th(0)=3.0787616601238224E-17;dx=-2.46300932809904E-21
Non-optimal measurement 3.7253016126995366E-17 < 3.0787616601238224E-17. Total: 4
Armijo: th(52500.00000000002)=3.7253016126995366E-17; dx=2.7093102623452084E-21 evalInputDelta=-6.465399525757143E-18
Adding measurement 60515f93 to history. Total: 4
New Minimum: 3.0787616601238224E-17 > 7.696904235338266E-20
WOLF (strong): th(26250.00000000001)=7.696904235338266E-20; dx=1.2315046708518386E-22 evalInputDelta=3.071064755888484E-17
Non-optimal measurement 1.3007768001987265E-17 < 7.696904235338266E-20. Total: 5
END: th(8750.000000000004)=1.3007768001987265E-17; dx=-1.6009560625237068E-21 evalInputDelta=1.777984859925096E-17
Fitness changed from 3.0787616601238224E-17 to 7.696904235338266E-20
Iteration 9 complete. Error: 7.696904235338266E-20 Total: 2.9643; Orientation: 2.7550; Line Search: 0.2028
Non-optimal measurement 7.696904235338266E-20 < 7.696904235338266E-20. Total: 5
Rejected: LBFGS Orientation magnitude: 6.204e-08, gradient 2.481e-12, dot -1.000; [ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 7.696904235338266E-20, 3.0787616601238224E-17, 0.6656181202424736, 0.6657328674252766, 0.6658476244979714
Rejected: LBFGS Orientation magnitude: 6.204e-08, gradient 2.481e-12, dot -1.000; [5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 7.696904235338266E-20, 3.0787616601238224E-17, 0.6656181202424736, 0.6657328674252766
LBFGS Accumulation History: 3 points
Removed measurement 60515f93 to history. Total: 4
Removed measurement 448e0dc8 to history. Total: 3
Adding measurement 9b3377c to history. Total: 3
th(0)=7.696904235338266E-20;dx=-6.157523388270833E-24
Adding measurement 25650bb3 to history. Total: 4
New Minimum: 7.696904235338266E-20 > 4.655884248386066E-21
END: th(18851.303537778993)=4.655884248386066E-21; dx=-1.5144296920627565E-24 evalInputDelta=7.23131581049966E-20
Fitness changed from 7.696904235338266E-20 to 4.655884248386066E-21
Iteration 10 complete. Error: 4.655884248386066E-21 Total: 1.7794; Orientation: 1.7156; Line Search: 0.0247
Non-optimal measurement 4.655884248386066E-21 < 4.655884248386066E-21. Total: 5
Rejected: LBFGS Orientation magnitude: 1.526e-08, gradient 6.103e-13, dot -1.000; [b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, 5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.655884248386066E-21, 7.696904235338266E-20, 0.6656181202424736, 0.6657328674252766, 0.6658476244979714
Rejected: LBFGS Orientation magnitude: 1.526e-08, gradient 6.103e-13, dot -1.000; [5311a59f-812a-4efd-ab4b-c11b86df1b35 = 1.000/1.000e+00, ee7e7e0e-03c0-4249-b5ba-2fd11367a230 = 1.000/1.000e+00, a15fbcce-00ea-47ca-9da3-ea95048517c3 = 1.000/1.000e+00, 6e78c811-b16a-4f96-9057-6d13dca6c806 = 1.000/1.000e+00, b1ebb0af-ef50-48f8-9b98-982beba597e5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 4.655884248386066E-21, 7.696904235338266E-20, 0.6656181202424736, 0.6657328674252766
LBFGS Accumulation History: 3 points
Removed measurement 25650bb3 to history. Total: 4
Removed measurement 9b3377c to history. Total: 3
Adding measurement 27226145 to history. Total: 3
th(0)=4.655884248386066E-21;dx=-3.72470739870896E-25
Adding measurement 4e7c5b40 to history. Total: 4
New Minimum: 4.655884248386066E-21 > 1.8161222222632825E-21
WOLF (strong): th(40613.90229411184)=1.8161222222632825E-21; dx=2.326288697169242E-25 evalInputDelta=2.8397620261227836E-21
Adding measurement 2e70f6e8 to history. Total: 5
New Minimum: 1.8161222222632825E-21 > 1.6407118417936316E-22
END: th(20306.95114705592)=1.6407118417936316E-22; dx=-6.99209355558479E-26 evalInputDelta=4.491813064206703E-21
Fitness changed from 4.655884248386066E-21 to 1.6407118417936316E-22
Iteration 11 complete. Error: 1.6407118417936316E-22 Total: 2.4969; Orientation: 2.0226; Line Search: 0.4627
Final threshold in iteration 11: 1.6407118417936316E-22 (> 0.0) after 30.103s (< 30.000s)
Returns
1.6407118417936316E-22
TrainingTester.java:432 executed in 0.11 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -43.38137083976828], [20.0, -0.6715050452570825]; valueStats=DoubleSummaryStatistics{count=31, sum=0.427873, min=0.000000, average=0.013802, max=0.213057}
Plotting 20 points for GD
Plotting 2 points for CjGD
Plotting 11 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -43.38137083976828], [29.448, -0.6715050452570825]; valueStats=DoubleSummaryStatistics{count=31, sum=0.427873, min=0.000000, average=0.013802, max=0.213057}
Plotting 20 points for GD
Plotting 2 points for CjGD
Plotting 11 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "Converged", "value": 1.6407118417936316E-22 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "Converged", "value": 1.6407118417936316E-22 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "37.822",
"gc_time": "0.299"
},
"created_on": 1586734873626,
"file_name": "trainingTest",
"report": {
"simpleName": "Basic",
"canonicalName": "com.simiacryptus.mindseye.layers.java.BoundedActivationLayerTest.Basic",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/BoundedActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "Converged",
"value": 1.6407118417936316E-22
},
"CjGD": {
"type": "Converged",
"value": 0.0
},
"GD": {
"type": "Converged",
"value": 0.0
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/BoundedActivationLayer/Basic/trainingTest/202004124113",
"id": "da8e94c4-8dbd-4bde-a4a8-aaa27ddea52d",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "BoundedActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.BoundedActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/BoundedActivationLayer.java",
"javaDoc": ""
}
}