Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 6159481755037988864
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ -0.08146086037346212 ], [ 0.35317561592737334 ], [ -0.16816631277237337 ], [ -0.6594909300029155 ], [ 0.5887227436916922 ], [ 0.13566916394591622 ], [ 0.852822828466834 ], [ -0.7788890598027733 ], ... ],
[ [ -0.09182264472682045 ], [ -0.9561130367974842 ], [ 0.12914551004939656 ], [ -0.3602933669944861 ], [ -0.8046453184816706 ], [ 0.12008028498022072 ], [ -0.33662486773148226 ], [ 0.34022711396669036 ], ... ],
[ [ -0.6337307812545718 ], [ -0.890217159513703 ], [ -0.5628663424611302 ], [ 0.8795880103623206 ], [ 0.47829432586722875 ], [ 0.026933452818830084 ], [ -0.8022677820278079 ], [ 0.003050739308732453 ], ... ],
[ [ -0.728577800261027 ], [ -0.7753697945274831 ], [ -0.14078142051272824 ], [ -0.5408592789247528 ], [ 0.28116790426327665 ], [ -0.4076165799120126 ], [ 0.872000805731147 ], [ 0.31575013246541594 ], ... ],
[ [ -0.05666976341189134 ], [ 0.9197683434254695 ], [ 0.21872928771350653 ], [ 0.7453156088951773 ], [ 0.31968186181331015 ], [ -0.025809521373501676 ], [ 0.17351182039176005 ], [ 0.37960592249787806 ], ... ],
[ [ -0.4591031743814862 ], [ -0.5845878326095819 ], [ -0.3834365692241697 ], [ -0.5423891260370208 ], [ -0.8098343244099748 ], [ 0.2984371841925726 ], [ 0.4419652171448267 ], [ 0.5063713155679466 ], ... ],
[ [ 0.13049729769841845 ], [ 0.5803775148819643 ], [ -0.028851622663005783 ], [ 0.6760271575099701 ], [ -0.18364402175477745 ], [ 0.8054165949621275 ], [ -0.8947402937382627 ], [ -0.1461635238765595 ], ... ],
[ [ -0.23412627735513247 ], [ 0.068169624310358 ], [ -0.3330231153743577 ], [ -0.2489479362593605 ], [ -0.22627277642734334 ], [ 0.09413534464332463 ], [ 0.3023518054247465 ], [ -0.10010032142868397 ], ... ],
...
]
[
[ [ -0.3237423042004146 ], [ 0.6537371620987691 ], [ -0.6764154666385573 ], [ -0.12223423807984912 ], [ 0.6001212687241375 ], [ 0.0303028322482779 ], [ 0.7554281058130456 ], [ -0.7271263358840387 ], ... ],
[ [ -0.670616149442109 ], [ 0.7234767717666615 ], [ -0.23313890928711922 ], [ 0.017842167136439047 ], [ 0.32967714022080763 ], [ 0.11681293104507229 ], [ 0.4895185937155373 ], [ 0.9723061443577248 ], ... ],
[ [ 0.888681424506172 ], [ 0.48308382199514943 ], [ -0.7970456363243041 ], [ -0.37321798406897644 ], [ 0.5476863666089766 ], [ 0.8721421294067737 ], [ 0.7909836559987851 ], [ -0.25478898794405147 ], ... ],
[ [ 0.3518244887272546 ], [ -0.9802272093361442 ], [ 0.008379991680457222 ], [ -0.6280797287429418 ], [ -0.2080493574699771 ], [ -0.9465886440484432 ], [ -0.8537679599720909 ], [ -0.4864998933835569 ], ... ],
[ [ -0.5025409200998923 ], [ -0.5964954952900305 ], [ 0.3108434148512089 ], [ 0.8263335619441978 ], [ 0.12100342031207201 ], [ 0.11289675457261472 ], [ 0.6219337278143173 ], [ -0.8859614858555818 ], ... ],
[ [ -0.17813972770060515 ], [ 0.25341395967386715 ], [ -0.16084923697949294 ], [ -0.25809767911286685 ], [ 0.4640848515792486 ], [ -0.49744989785758387 ], [ -0.5704912374057332 ], [ 0.3668733382636691 ], ... ],
[ [ 0.035471413907606886 ], [ -0.961598743241785 ], [ 0.04558159177025001 ], [ -0.9155528048149808 ], [ -0.2810680970685695 ], [ 0.6011645707950306 ], [ 0.5079079891363065 ], [ -0.16065876793852274 ], ... ],
[ [ 0.6094078863597097 ], [ -0.7696549099761285 ], [ -0.008585741195057839 ], [ -0.006985852298002193 ], [ -0.882676447560538 ], [ -0.32716971797188577 ], [ 0.8792701322941916 ], [ -0.5027632746485955 ], ... ],
...
]
[
[ [ 0.5955231950138502 ], [ -0.8242014651979723 ], [ -0.7697795369905691 ], [ 0.7561065798333233 ], [ 0.006927368907788445 ], [ -0.8831173224179523 ], [ -0.3242937154355463 ], [ -0.357569161272413 ], ... ],
[ [ -0.04691659322575803 ], [ 0.8544906683517448 ], [ -0.11017397024607267 ], [ -0.9484976370739111 ], [ 0.9284708275167743 ], [ 0.5779349613491798 ], [ 0.7539851453482098 ], [ -0.5336411631789639 ], ... ],
[ [ -0.08978581968624377 ], [ 0.5906419917826746 ], [ 0.3688658564283338 ], [ -0.2331308395752134 ], [ -0.2510060548966224 ], [ -0.7847832791061915 ], [ 0.6868698825294717 ], [ -0.22235659749143344 ], ... ],
[ [ -0.4005926421768595 ], [ -0.810428204935066 ], [ 0.7346628739021901 ], [ -0.376628961095431 ], [ 0.2671318370620783 ], [ 0.05487173271330614 ], [ 0.1756909842734926 ], [ 0.21335339592958524 ], ... ],
[ [ 0.7036993665142375 ], [ -0.8596953647719925 ], [ 0.5906561612283616 ], [ -0.3221333442027503 ], [ 0.9961815371068923 ], [ -0.1163868561794823 ], [ 0.2469152767597611 ], [ 0.2514574372923639 ], ... ],
[ [ -0.13133618552240356 ], [ -0.4421639488032587 ], [ -0.3209913655395966 ], [ -0.6133019116065708 ], [ 0.22788137505920747 ], [ -0.41188592437287763 ], [ 0.6225590881732865 ], [ 0.14708531040844375 ], ... ],
[ [ 0.7027938042679205 ], [ -0.6734219628801559 ], [ 0.6749354458627757 ], [ -0.837509484128264 ], [ 0.1542597515769224 ], [ -0.7550445491117319 ], [ -0.17398927521836316 ], [ 0.2382391131983428 ], ... ],
[ [ -0.8252175804840391 ], [ -0.930187833683018 ], [ 0.9422266133557726 ], [ -0.8251666959542405 ], [ 0.45702340607713765 ], [ -0.4961072302089702 ], [ -0.4464817156854046 ], [ 0.5794605416991905 ], ... ],
...
]
[
[ [ 0.6276693613788888 ], [ -0.017702745466326775 ], [ 0.891661445813958 ], [ 0.590942500427919 ], [ 0.5043373538911005 ], [ -0.12187255887158743 ], [ -0.2822657694393753 ], [ 0.36280784162253976 ], ... ],
[ [ -0.33038235589983533 ], [ -0.49845324472170516 ], [ 0.4708254073632834 ], [ 0.6466676173175521 ], [ -0.724016497535207 ], [ -0.7150160290833534 ], [ -0.09251905715850617 ], [ 0.6709013556144718 ], ... ],
[ [ 0.7147842284623915 ], [ 0.9131696431508025 ], [ 0.4306477068576533 ], [ -0.48907416915641355 ], [ -0.8740656692177227 ], [ -0.8728704025400471 ], [ 0.04065953374703235 ], [ -0.8304628963757983 ], ... ],
[ [ 0.6132346085874425 ], [ -0.45668084971661305 ], [ 0.751499796962539 ], [ -0.056930499691481895 ], [ -0.8260677099939417 ], [ 0.4490332866064455 ], [ 0.21240766948633794 ], [ 0.015530581167025881 ], ... ],
[ [ -0.034438015163876345 ], [ -0.45847247840230754 ], [ 0.7542028859334007 ], [ -0.23050806098077659 ], [ 0.986892009638572 ], [ 0.2310204021703497 ], [ 0.10705270159860558 ], [ 0.5279464323360039 ], ... ],
[ [ -0.6262010192198875 ], [ -0.3141627896275032 ], [ 0.906624091070759 ], [ 0.40076858001700955 ], [ 0.8837641509972105 ], [ 0.21503800307688836 ], [ 0.015999472881426335 ], [ -0.9039417617544685 ], ... ],
[ [ -0.10415697274624502 ], [ -0.7034957761085863 ], [ 0.9806991461487069 ], [ -0.19342163870166562 ], [ -0.7177473918115955 ], [ -0.5825695347447271 ], [ -0.8029932102902149 ], [ 0.29482557239618545 ], ... ],
[ [ 0.5767291781275714 ], [ 0.5518849637270369 ], [ 0.4990714240294647 ], [ 0.2200186836939415 ], [ 0.6718488704376815 ], [ 0.9570792440382774 ], [ 0.8107738211543323 ], [ 0.1547288983048012 ], ... ],
...
]
[
[ [ 0.6142968341003998 ], [ -0.8021601488798018 ], [ -0.657675856807246 ], [ -0.19794045218923317 ], [ 0.6944771788438671 ], [ 0.8145405559760732 ], [ -0.36184580704293645 ], [ 0.29392727235356086 ], ... ],
[ [ -0.06939585108101554 ], [ 0.0768091873812935 ], [ 0.3161676230860304 ], [ 0.3691507486911114 ], [ -0.37643836949739073 ], [ 0.4154444581698382 ], [ -0.5122856023271998 ], [ 0.11533955152069364 ], ... ],
[ [ -0.6132609291752429 ], [ -0.8582626954853818 ], [ -0.7763967583061497 ], [ -0.4764188033281138 ], [ -0.9597054657570401 ], [ 0.8775535649616188 ], [ 0.8686905821618851 ], [ 0.5608826464730219 ], ... ],
[ [ 0.1403845970371611 ], [ -0.505242519575962 ], [ 0.6032295695138219 ], [ -0.2214013487592843 ], [ -0.36405094848625663 ], [ 0.6248054317626102 ], [ -0.7421633386615933 ], [ -0.2943950864860583 ], ... ],
[ [ 0.14532141344279936 ], [ 0.8796545904587896 ], [ 0.8630515812049644 ], [ -0.9415955930899129 ], [ -0.6221848793287548 ], [ -0.7528844312114593 ], [ 0.4771216460251293 ], [ 0.6448072714271647 ], ... ],
[ [ -0.3652503213489402 ], [ 0.8863270906195928 ], [ 0.43632548163112506 ], [ 0.2947729407056374 ], [ 0.4404602230247163 ], [ -0.3584143600018803 ], [ -0.277263372130107 ], [ 0.7689920116993176 ], ... ],
[ [ -0.31728391767679454 ], [ -0.6843084404229864 ], [ -0.5786625825640535 ], [ -0.5533439542733698 ], [ -0.36599088340403596 ], [ -0.5851833347303017 ], [ 0.29068421652385856 ], [ 0.14049661838395489 ], ... ],
[ [ -0.995232379639232 ], [ 0.5564021221290736 ], [ 0.6815121973440867 ], [ -0.10636368579586697 ], [ 0.8509247701336937 ], [ -0.06168626184749826 ], [ -0.1529084690106055 ], [ 0.3545557597924003 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.14 seconds (0.211 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2251975760897
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 2252049339231
Constructing line search parameters: GD
th(0)=0.17726094106397533;dx=-2.4378975770333734E-5
New Minimum: 0.17726094106397533 > 0.17720842718171878
WOLFE (weak): th(2.154434690031884)=0.17720842718171878; dx=-2.437059447699295E-5 evalInputDelta=5.2513882256549627E-5
New Minimum: 0.17720842718171878 > 0.1771559313541557
WOLFE (weak): th(4.308869380063768)=0.1771559313541557; dx=-2.4362215277266792E-5 evalInputDelta=1.0500970981963675E-4
New Minimum: 0.1771559313541557 > 0.1769461285006471
WOLFE (weak): th(12.926608140191302)=0.1769461285006471; dx=-2.4328719408740067E-5 evalInputDelta=3.148125633282428E-4
New Minimum: 0.1769461285006471 > 0.17600558186316975
WOLFE (weak): th(51.70643256076521)=0.17600558186316975; dx=-2.417840198403685E-5 evalInputDelta=0.001255359200805578
New Minimum: 0.17600558186316975 > 0.17108692415388357
WOLFE (weak): th(258.53216280382605)=0.17108692415388357; dx=-2.3388087979078938E-5 evalInputDelta=0.006174016910091756
New Minimum: 0.17108692415388357 > 0.14385171737406613
END: th(1551.1929768229563)=0.14385171737406613; dx=-1.8868488674881193E-5 evalInputDelta=0.0334092236899092
Fitness changed from 0.17726094106397533 to 0.14385171737406613
Iteration 1 complete. Error: 0.14385171737406613 Total: 1.0170; Orientation: 0.0064; Line Search: 0.9019
th(0)=0.14385171737406613;dx=-1.4837876634737385E-5
New Minimum: 0.14385171737406613 > 0.1030957617931636
END: th(3341.943960201201)=0.1030957617931636; dx=-9.801173154234549E-6 evalInputDelta=0.04075595558090253
Fitness changed from 0.14385171737406613 to 0.1030957617931636
Iteration 2 complete. Error: 0.1030957617931636 Total: 0.0635; Orientation: 0.0096; Line Search: 0.0393
th(0)=0.1030957617931636;dx=-6.943877812472072E-6
New Minimum: 0.1030957617931636 > 0.06367898243254494
END: th(7200.000000000001)=0.06367898243254494; dx=-4.060728971028692E-6 evalInputDelta=0.03941677936061866
Fitness changed from 0.1030957617931636 to 0.06367898243254494
Iteration 3 complete. Error: 0.06367898243254494 Total: 0.0522; Orientation: 0.0040; Line Search: 0.0358
th(0)=0.06367898243254494;dx=-2.98934935755973E-6
New Minimum: 0.06367898243254494 > 0.032121464852960024
END: th(15511.929768229566)=0.032121464852960024; dx=-5.866196439732105E-7 evalInputDelta=0.031557517579584915
Fitness changed from 0.06367898243254494 to 0.032121464852960024
Iteration 4 complete. Error: 0.032121464852960024 Total: 0.0492; Orientation: 0.0051; Line Search: 0.0286
th(0)=0.032121464852960024;dx=-1.6821716072383375E-6
Armijo: th(33419.43960201202)=0.05137946126280359; dx=3.563900983766489E-6 evalInputDelta=-0.019257996409843568
New Minimum: 0.032121464852960024 > 0.020728924889044507
WOLF (strong): th(16709.71980100601)=0.020728924889044507; dx=3.939446459903857E-7 evalInputDelta=0.011392539963915517
END: th(5569.906600335336)=0.024560122737580917; dx=-1.0324475054627855E-6 evalInputDelta=0.007561342115379108
Fitness changed from 0.032121464852960024 to 0.020728924889044507
Iteration 5 complete. Error: 0.020728924889044507 Total: 0.0785; Orientation: 0.0031; Line Search: 0.0659
th(0)=0.020728924889044507;dx=-1.53258798862395E-6
New Minimum: 0.020728924889044507 > 0.01157430297724502
WOLF (strong): th(12000.000000000004)=0.01157430297724502; dx=1.2245400378173124E-7 evalInputDelta=0.009154621911799488
END: th(6000.000000000002)=0.013685721670338134; dx=-7.91794242764381E-7 evalInputDelta=0.007043203218706374
Fitness changed from 0.020728924889044507 to 0.01157430297724502
Iteration 6 complete. Error: 0.01157430297724502 Total: 0.1564; Orientation: 0.0043; Line Search: 0.1431
th(0)=0.01157430297724502;dx=-6.35601024961388E-7
New Minimum: 0.01157430297724502 > 0.007150251495423714
WOLF (strong): th(12926.608140191307)=0.007150251495423714; dx=1.6918235601959183E-8 evalInputDelta=0.004424051481821305
END: th(6463.3040700956535)=0.008313640265527306; dx=-3.586950015938886E-7 evalInputDelta=0.0032606627117177135
Fitness changed from 0.01157430297724502 to 0.007150251495423714
Iteration 7 complete. Error: 0.007150251495423714 Total: 0.1131; Orientation: 0.0037; Line Search: 0.0958
th(0)=0.007150251495423714;dx=-3.940771703292385E-7
New Minimum: 0.007150251495423714 > 0.0048530268111762935
WOLF (strong): th(13924.766500838343)=0.0048530268111762935; dx=8.995199762010804E-8 evalInputDelta=0.0022972246842474205
END: th(6962.383250419171)=0.00516255395528827; dx=-1.7142973434972723E-7 evalInputDelta=0.0019876975401354443
Fitness changed from 0.007150251495423714 to 0.0048530268111762935
Iteration 8 complete. Error: 0.0048530268111762935 Total: 0.2609; Orientation: 0.0039; Line Search: 0.2480
th(0)=0.0048530268111762935;dx=-3.283662662760575E-7
New Minimum: 0.0048530268111762935 > 0.003746467208605833
WOLF (strong): th(15000.000000000007)=0.003746467208605833; dx=1.9391987776583701E-7 evalInputDelta=0.0011065596025704606
New Minimum: 0.003746467208605833 > 0.003323798792179633
END: th(7500.000000000004)=0.003323798792179633; dx=-7.704436312949067E-8 evalInputDelta=0.0015292280189966607
Fitness changed from 0.0048530268111762935 to 0.003323798792179633
Iteration 9 complete. Error: 0.003323798792179633 Total: 0.2143; Orientation: 0.0031; Line Search: 0.2023
th(0)=0.003323798792179633;dx=-9.946943204976875E-8
New Minimum: 0.003323798792179633 > 0.00217748775628272
END: th(16158.260175239137)=0.00217748775628272; dx=-2.8242825489106173E-8 evalInputDelta=0.0011463110358969127
Fitness changed from 0.003323798792179633 to 0.00217748775628272
Iteration 10 complete. Error: 0.00217748775628272 Total: 0.1357; Orientation: 0.0036; Line Search: 0.0328
th(0)=0.00217748775628272;dx=-7.578799155036619E-8
Armijo: th(34811.91625209586)=0.0027690719528446817; dx=1.2270157043214643E-7 evalInputDelta=-5.915841965619616E-4
New Minimum: 0.00217748775628272 > 0.0016308665010736898
WOLF (strong): th(17405.95812604793)=0.0016308665010736898; dx=1.3762195582509528E-8 evalInputDelta=5.466212552090303E-4
END: th(5801.986042015978)=0.0018236480617418566; dx=-4.6278353823498616E-8 evalInputDelta=3.538396945408635E-4
Fitness changed from 0.00217748775628272 to 0.0016308665010736898
Iteration 11 complete. Error: 0.0016308665010736898 Total: 0.2175; Orientation: 0.0045; Line Search: 0.2037
th(0)=0.0016308665010736898;dx=-8.850134991249E-8
New Minimum: 0.0016308665010736898 > 0.0011672176436779722
WOLF (strong): th(12500.00000000001)=0.0011672176436779722; dx=2.096121842384057E-8 evalInputDelta=4.636488573957177E-4
END: th(6250.000000000005)=0.0012285035704977139; dx=-3.875283001532425E-8 evalInputDelta=4.0236293057597594E-4
Fitness changed from 0.0016308665010736898 to 0.0011672176436779722
Iteration 12 complete. Error: 0.0011672176436779722 Total: 0.2937; Orientation: 0.1300; Line Search: 0.1552
th(0)=0.0011672176436779722;dx=-5.875757650471927E-8
New Minimum: 0.0011672176436779722 > 9.166616821343402E-4
WOLF (strong): th(13465.216812699284)=9.166616821343402E-4; dx=2.0230111478134265E-8 evalInputDelta=2.50555961543632E-4
New Minimum: 9.166616821343402E-4 > 9.092186690004627E-4
END: th(6732.608406349642)=9.092186690004627E-4; dx=-1.827959982245862E-8 evalInputDelta=2.5799897467750944E-4
Fitness changed from 0.0011672176436779722 to 9.092186690004627E-4
Iteration 13 complete. Error: 9.092186690004627E-4 Total: 0.2970; Orientation: 0.0036; Line Search: 0.2529
th(0)=9.092186690004627E-4;dx=-2.0678925477216505E-8
New Minimum: 9.092186690004627E-4 > 6.563172159083087E-4
END: th(14504.965105039946)=6.563172159083087E-4; dx=-1.2567463402441822E-8 evalInputDelta=2.52901453092154E-4
Fitness changed from 9.092186690004627E-4 to 6.563172159083087E-4
Iteration 14 complete. Error: 6.563172159083087E-4 Total: 0.1365; Orientation: 0.0033; Line Search: 0.1238
th(0)=6.563172159083087E-4;dx=-1.4721344398917206E-8
New Minimum: 6.563172159083087E-4 > 4.7933999728245536E-4
WOLF (strong): th(31250.00000000003)=4.7933999728245536E-4; dx=7.945170737770432E-9 evalInputDelta=1.7697721862585335E-4
END: th(15625.000000000015)=4.80996518
...skipping 59866 bytes...
114174E-7 > 2.571320325077863E-7
END: th(29620.689138896112)=2.571320325077863E-7; dx=-2.5042171679113936E-13 evalInputDelta=8.227524208759085E-9
Fitness changed from 2.6535955671654537E-7 to 2.571320325077863E-7
Iteration 117 complete. Error: 2.571320325077863E-7 Total: 0.3390; Orientation: 0.0040; Line Search: 0.2383
Low gradient: 6.538833054824065E-7
th(0)=2.571320325077863E-7;dx=-4.2756337718859815E-13
Armijo: th(63815.840223488434)=2.7716497858543675E-7; dx=1.059383701115299E-12 evalInputDelta=-2.003294607765047E-8
New Minimum: 2.571320325077863E-7 > 2.552872201601282E-7
WOLF (strong): th(31907.920111744217)=2.552872201601282E-7; dx=3.1292203998452735E-13 evalInputDelta=1.844812347658063E-9
New Minimum: 2.552872201601282E-7 > 2.538930095699527E-7
END: th(10635.973370581405)=2.538930095699527E-7; dx=-1.8139606440528873E-13 evalInputDelta=3.239022937833576E-9
Fitness changed from 2.571320325077863E-7 to 2.538930095699527E-7
Iteration 118 complete. Error: 2.538930095699527E-7 Total: 0.2622; Orientation: 0.0041; Line Search: 0.2500
Low gradient: 5.920433579298881E-7
th(0)=2.538930095699527E-7;dx=-3.5051533766889756E-13
New Minimum: 2.538930095699527E-7 > 2.488550741921531E-7
END: th(22914.50999183592)=2.488550741921531E-7; dx=-8.929614244308877E-14 evalInputDelta=5.0379353777995885E-9
Fitness changed from 2.538930095699527E-7 to 2.488550741921531E-7
Iteration 119 complete. Error: 2.488550741921531E-7 Total: 0.0447; Orientation: 0.0038; Line Search: 0.0314
Low gradient: 8.773914683846237E-7
th(0)=2.488550741921531E-7;dx=-7.69815788794126E-13
Armijo: th(49367.81523149352)=3.082839167352073E-7; dx=3.19123962587217E-12 evalInputDelta=-5.942884254305419E-8
Armijo: th(24683.90761574676)=2.5412601240492706E-7; dx=1.2003399864913889E-12 evalInputDelta=-5.270938212773943E-9
New Minimum: 2.488550741921531E-7 > 2.452117515551735E-7
END: th(8227.96920524892)=2.452117515551735E-7; dx=-1.153961638162921E-13 evalInputDelta=3.6433226369796074E-9
Fitness changed from 2.488550741921531E-7 to 2.452117515551735E-7
Iteration 120 complete. Error: 2.452117515551735E-7 Total: 0.0688; Orientation: 0.0030; Line Search: 0.0576
Low gradient: 5.648288891041887E-7
th(0)=2.452117515551735E-7;dx=-3.190316739666719E-13
New Minimum: 2.452117515551735E-7 > 2.408093421610474E-7
END: th(17726.622284302342)=2.408093421610474E-7; dx=-1.776873335034292E-13 evalInputDelta=4.402409394126088E-9
Fitness changed from 2.452117515551735E-7 to 2.408093421610474E-7
Iteration 121 complete. Error: 2.408093421610474E-7 Total: 0.0414; Orientation: 0.0049; Line Search: 0.0244
Low gradient: 6.548854266214304E-7
th(0)=2.408093421610474E-7;dx=-4.288749220011329E-13
Armijo: th(38190.8499863932)=2.43708597183282E-7; dx=5.824127262885899E-13 evalInputDelta=-2.8992550222345655E-9
New Minimum: 2.408093421610474E-7 > 2.374312371443116E-7
WOLF (strong): th(19095.4249931966)=2.374312371443116E-7; dx=7.548791717605051E-14 evalInputDelta=3.3781050167358384E-9
END: th(6365.141664398867)=2.386134952716055E-7; dx=-2.610379150257948E-13 evalInputDelta=2.195846889441901E-9
Fitness changed from 2.408093421610474E-7 to 2.374312371443116E-7
Iteration 122 complete. Error: 2.374312371443116E-7 Total: 0.2935; Orientation: 0.0030; Line Search: 0.2833
Low gradient: 9.755813816086715E-7
th(0)=2.374312371443116E-7;dx=-9.517590321414842E-13
New Minimum: 2.374312371443116E-7 > 2.3481118226627452E-7
WOLF (strong): th(13713.282008748201)=2.3481118226627452E-7; dx=5.679392766213003E-13 evalInputDelta=2.62005487803707E-9
New Minimum: 2.3481118226627452E-7 > 2.3351620433710298E-7
END: th(6856.641004374101)=2.3351620433710298E-7; dx=-1.9063453798980337E-13 evalInputDelta=3.9150328072086106E-9
Fitness changed from 2.374312371443116E-7 to 2.3351620433710298E-7
Iteration 123 complete. Error: 2.3351620433710298E-7 Total: 0.2456; Orientation: 0.0036; Line Search: 0.2347
Low gradient: 5.134367858625068E-7
th(0)=2.3351620433710298E-7;dx=-2.6361733307682173E-13
New Minimum: 2.3351620433710298E-7 > 2.2975211368456535E-7
WOLFE (weak): th(14772.18523691862)=2.2975211368456535E-7; dx=-2.459743027975474E-13 evalInputDelta=3.764090652537628E-9
New Minimum: 2.2975211368456535E-7 > 2.2624985615546415E-7
END: th(29544.37047383724)=2.2624985615546415E-7; dx=-2.2816723167204987E-13 evalInputDelta=7.266348181638829E-9
Fitness changed from 2.3351620433710298E-7 to 2.2624985615546415E-7
Iteration 124 complete. Error: 2.2624985615546415E-7 Total: 0.2568; Orientation: 0.0031; Line Search: 0.1837
Low gradient: 5.825864352136217E-7
th(0)=2.2624985615546415E-7;dx=-3.3940695449491537E-13
Armijo: th(63651.416643988676)=2.3431589771018007E-7; dx=5.915608916112824E-13 evalInputDelta=-8.066041554715925E-9
New Minimum: 2.2624985615546415E-7 > 2.2287575937808302E-7
WOLF (strong): th(31825.708321994338)=2.2287575937808302E-7; dx=1.270446744311896E-13 evalInputDelta=3.3740967773811287E-9
END: th(10608.56944066478)=2.2347531402256277E-7; dx=-1.8370522887535193E-13 evalInputDelta=2.774542132901378E-9
Fitness changed from 2.2624985615546415E-7 to 2.2287575937808302E-7
Iteration 125 complete. Error: 2.2287575937808302E-7 Total: 0.1981; Orientation: 0.0035; Line Search: 0.1874
Low gradient: 1.3232330197046053E-6
th(0)=2.2287575937808302E-7;dx=-1.7509456244365689E-12
Armijo: th(22855.47001458034)=2.4605667703277047E-7; dx=3.795203279075385E-12 evalInputDelta=-2.3180917654687453E-8
New Minimum: 2.2287575937808302E-7 > 2.1862127589994662E-7
WOLF (strong): th(11427.73500729017)=2.1862127589994662E-7; dx=1.0102946509252441E-12 evalInputDelta=4.254483478136399E-9
New Minimum: 2.1862127589994662E-7 > 2.1795318665114912E-7
END: th(3809.2450024300565)=2.1795318665114912E-7; dx=-8.331571302463825E-13 evalInputDelta=4.922572726933899E-9
Fitness changed from 2.2287575937808302E-7 to 2.1795318665114912E-7
Iteration 126 complete. Error: 2.1795318665114912E-7 Total: 0.3230; Orientation: 0.0037; Line Search: 0.2335
Low gradient: 6.869890493072938E-7
th(0)=2.1795318665114912E-7;dx=-4.719539538681395E-13
New Minimum: 2.1795318665114912E-7 > 2.1534129521703968E-7
END: th(8206.7695760659)=2.1534129521703968E-7; dx=-1.6443588319169057E-13 evalInputDelta=2.61189143410944E-9
Fitness changed from 2.1795318665114912E-7 to 2.1534129521703968E-7
Iteration 127 complete. Error: 2.1534129521703968E-7 Total: 0.2275; Orientation: 0.0033; Line Search: 0.0790
Low gradient: 5.084029357115381E-7
th(0)=2.1534129521703968E-7;dx=-2.5847354504011033E-13
New Minimum: 2.1534129521703968E-7 > 2.1135774216403947E-7
END: th(17680.949067774633)=2.1135774216403947E-7; dx=-1.9210862528803201E-13 evalInputDelta=3.983553053000212E-9
Fitness changed from 2.1534129521703968E-7 to 2.1135774216403947E-7
Iteration 128 complete. Error: 2.1135774216403947E-7 Total: 0.1409; Orientation: 0.0033; Line Search: 0.0238
Low gradient: 5.53115102443103E-7
th(0)=2.1135774216403947E-7;dx=-3.059363165506444E-13
New Minimum: 2.1135774216403947E-7 > 2.085402397867934E-7
WOLF (strong): th(38092.45002430057)=2.085402397867934E-7; dx=1.5867890354683567E-13 evalInputDelta=2.8175023772460673E-9
New Minimum: 2.085402397867934E-7 > 2.077367071920781E-7
END: th(19046.225012150284)=2.077367071920781E-7; dx=-7.413292273610725E-14 evalInputDelta=3.6210349719613543E-9
Fitness changed from 2.1135774216403947E-7 to 2.077367071920781E-7
Iteration 129 complete. Error: 2.077367071920781E-7 Total: 0.2333; Orientation: 0.0033; Line Search: 0.2226
Low gradient: 7.345816205870387E-7
th(0)=2.077367071920781E-7;dx=-5.396101573042801E-13
Armijo: th(41033.84788032951)=2.2793769654412456E-7; dx=1.519610153455984E-12 evalInputDelta=-2.020098935204645E-8
New Minimum: 2.077367071920781E-7 > 2.0727501137649992E-7
WOLF (strong): th(20516.923940164754)=2.0727501137649992E-7; dx=4.934509528796233E-13 evalInputDelta=4.616958155781933E-10
New Minimum: 2.0727501137649992E-7 > 2.0522690758381547E-7
END: th(6838.974646721585)=2.0522690758381547E-7; dx=-1.944876881757327E-13 evalInputDelta=2.509799608262644E-9
Fitness changed from 2.077367071920781E-7 to 2.0522690758381547E-7
Iteration 130 complete. Error: 2.0522690758381547E-7 Total: 0.2892; Orientation: 0.0039; Line Search: 0.2765
Final threshold in iteration 130: 2.0522690758381547E-7 (> 0.0) after 30.135s (< 30.000s)
Returns
2.0522690758381547E-7
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 30.08 seconds (0.114 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2282118293972
Reset training subject: 2282125451974
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.17726094106397533}, derivative=-2.4378975770333734E-5}
New Minimum: 0.17726094106397533 > 0.1772609410639729
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.1772609410639729}, derivative=-2.4378975770333345E-5}, evalInputDelta = -2.4424906541753444E-15
New Minimum: 0.1772609410639729 > 0.17726094106395818
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.17726094106395818}, derivative=-2.4378975770331014E-5}, evalInputDelta = -1.715294573045867E-14
New Minimum: 0.17726094106395818 > 0.17726094106385554
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.17726094106385554}, derivative=-2.437897577031467E-5}, evalInputDelta = -1.197930643570544E-13
New Minimum: 0.17726094106385554 > 0.1772609410631394
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.1772609410631394}, derivative=-2.437897577020028E-5}, evalInputDelta = -8.359424263915116E-13
New Minimum: 0.1772609410631394 > 0.1772609410581218
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.1772609410581218}, derivative=-2.437897576939957E-5}, evalInputDelta = -5.853539875033675E-12
New Minimum: 0.1772609410581218 > 0.17726094102300122
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.17726094102300122}, derivative=-2.4378975763794572E-5}, evalInputDelta = -4.097411299142095E-11
New Minimum: 0.17726094102300122 > 0.1772609407771592
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.1772609407771592}, derivative=-2.4378975724559596E-5}, evalInputDelta = -2.8681612640468757E-10
New Minimum: 0.1772609407771592 > 0.17726093905626222
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.17726093905626222}, derivative=-2.4378975449914757E-5}, evalInputDelta = -2.007713106877418E-9
New Minimum: 0.17726093905626222 > 0.1772609270099817
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.1772609270099817}, derivative=-2.4378973527400956E-5}, evalInputDelta = -1.4053993635521067E-8
New Minimum: 0.1772609270099817 > 0.17726084268604564
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.17726084268604564}, derivative=-2.4378960069807447E-5}, evalInputDelta = -9.83779296914733E-8
New Minimum: 0.17726084268604564 > 0.17726025241980253
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.17726025241980253}, derivative=-2.4378865866804E-5}, evalInputDelta = -6.88644172797126E-7
New Minimum: 0.17726025241980253 > 0.17725612061995935
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.17725612061995935}, derivative=-2.4378206453184653E-5}, evalInputDelta = -4.820444015979364E-6
New Minimum: 0.17725612061995935 > 0.17722720115020713
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.17722720115020713}, derivative=-2.4373590920673496E-5}, evalInputDelta = -3.373991376820484E-5
New Minimum: 0.17722720115020713 > 0.1770249181330061
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.1770249181330061}, derivative=-2.4341299967824315E-5}, evalInputDelta = -2.3602293096922078E-4
New Minimum: 0.1770249181330061 > 0.1756164276820548
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.1756164276820548}, derivative=-2.4116133004097767E-5}, evalInputDelta = -0.001644513381920537
New Minimum: 0.1756164276820548 > 0.16611736126434035
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.16611736126434035}, derivative=-2.2582149226641513E-5}, evalInputDelta = -0.011143579799634984
New Minimum: 0.16611736126434035 > 0.1151146446984376
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.1151146446984376}, derivative=-1.3763618086030074E-5}, evalInputDelta = -0.06214629636553773
New Minimum: 0.1151146446984376 > 0.06831170781272985
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.06831170781272985}, derivative=4.581589435493533E-6}, evalInputDelta = -0.10894923325124548
0.06831170781272985 <= 0.17726094106397533
New Minimum: 0.06831170781272985 > 0.05797421436067287
F(19582.81415997109) = LineSearchPoint{point=PointSample{avg=0.05797421436067287}, derivative=1.4126749937749685E-6}, evalInputDelta = -0.11928672670330245
Right bracket at 19582.81415997109
New Minimum: 0.05797421436067287 > 0.056774718138952754
F(18510.21310296428) = LineSearchPoint{point=PointSample{avg=0.056774718138952754}, derivative=8.42769209820135E-7}, evalInputDelta = -0.12048622292502258
Right bracket at 18510.21310296428
New Minimum: 0.056774718138952754 > 0.05634245643960819
F(17891.70563321298) = LineSearchPoint{point=PointSample{avg=0.05634245643960819}, derivative=5.596953171583875E-7}, evalInputDelta = -0.12091848462436713
Right bracket at 17891.70563321298
New Minimum: 0.05634245643960819 > 0.056152176577152144
F(17490.164435458137) = LineSearchPoint{point=PointSample{avg=0.056152176577152144}, derivative=3.8964096106073965E-7}, evalInputDelta = -0.12110876448682319
Right bracket at 17490.164435458137
New Minimum: 0.056152176577152144 > 0.056060387471539554
F(17215.022526903173) = LineSearchPoint{point=PointSample{avg=0.056060387471539554}, derivative=2.7819463891752173E-7}, evalInputDelta = -0.12120055359243578
Right bracket at 17215.022526903173
New Minimum: 0.056060387471539554 > 0.056013815961278655
F(17020.793955808414) = LineSearchPoint{point=PointSample{avg=0.056013815961278655}, derivative=2.016267971397426E-7}, evalInputDelta = -0.12124712510269667
Right bracket at 17020.793955808414
Converged to right
Fitness changed from 0.17726094106397533 to 0.056013815961278655
Iteration 1 complete. Error: 0.056013815961278655 Total: 2.5388; Orientation: 0.0042; Line Search: 2.5126
F(0.0) = LineSearchPoint{point=PointSample{avg=0.056013815961278655}, derivative=-3.0517202376222833E-6}
New Minimum: 0.056013815961278655 > 0.03619991806210166
F(17020.793955808414) = LineSearchPoint{point=PointSample{avg=0.03619991806210166}, derivative=7.644065756971946E-7}, evalInputDelta = -0.019813897899176994
0.03619991806210166 <= 0.056013815961278655
New Minimum: 0.03619991806210166 > 0.0349940575206432
F(13611.366685730483) = LineSearchPoint{point=PointSample{avg=0.0349940575206432}, derivative=-4.6442916046012756E-8}, evalInputDelta = -0.021019758440635458
Left bracket at 13611.366685730483
New Minimum: 0.0349940575206432 > 0.03498936297662122
F(13806.647987725011) = LineSearchPoint{point=PointSample{avg=0.03498936297662122}, derivative=-1.6090725514924808E-9}, evalInputDelta = -0.021024452984657438
Left bracket at 13806.647987725011
Converged to left
Fitness changed from 0.056013815961278655 to 0.03498936297662122
Iteration 2 complete. Error: 0.03498936297662122 Total: 0.4015; Orientation: 0.0031; Line Search: 0.3907
F(0.0) = LineSearchPoint{point=PointSample{avg=0.03498936297662122}, derivative=-1.9620402039918E-6}
New Minimum: 0.03498936297662122 > 0.019915851360032478
F(13806.647987725011) = LineSearchPoint{point=PointSample{avg=0.019915851360032478}, derivative=1.2481014567931374E-7}, evalInputDelta = -0.01507351161658874
0.019915851360032478 <= 0.03498936297662122
New Minimum: 0.019915851360032478 > 0.01989059559547691
F(12980.901308303299) = LineSearchPoint{point=PointSample{avg=0.01989059559547691}, derivative=-6.21472620160488E-8}, evalInputDelta = -0.015098767381144306
Left bracket at 12980.901308303299
New Minimum: 0.01989059559547691 > 0.01988190756252765
F(13255.391075130423) = LineSearchPoint{point=PointSample{avg=0.01988190756252765}, derivative=-9.921925917684317E-10}, evalInputDelta = -0.015107455414093567
Left bracket at 13255.391075130423
Converged to left
Fitness changed from 0.03498936297662122 to 0.01988190756252765
Iteration 3 complete. Error: 0.01988190756252765 Total: 0.6670; Orientation: 0.0027; Line Search: 0.6573
F(0.0) = LineSearchPoint{point=PointSample{avg=0.01988190756252765}, derivative=-1.0587601232070121E-6}
New Minimum: 0.01988190756252765 > 0.012869599581136986
F(13255.391075130423) = LineSearchPoint{point=PointSample{avg=0.012869599581136986}, derivative=6.117889376877894E-8}, evalInputDelta = -0.007012307981390664
0.012869599581136986
...skipping 57617 bytes...
9716E-5 > 1.6014543678054894E-5
F(12417.47035751882) = LineSearchPoint{point=PointSample{avg=1.6014543678054894E-5}, derivative=1.644544594301871E-12}, evalInputDelta = -9.17748786942266E-7
1.6014543678054894E-5 <= 1.693229246499716E-5
New Minimum: 1.6014543678054894E-5 > 1.6014432562349478E-5
F(12282.320832754542) = LineSearchPoint{point=PointSample{avg=1.6014432562349478E-5}, derivative=-2.0613496959296035E-16}, evalInputDelta = -9.178599026476818E-7
Left bracket at 12282.320832754542
Converged to left
Fitness changed from 1.693229246499716E-5 to 1.6014432562349478E-5
Iteration 52 complete. Error: 1.6014432562349478E-5 Total: 0.2962; Orientation: 0.0032; Line Search: 0.2772
F(0.0) = LineSearchPoint{point=PointSample{avg=1.6014432562349478E-5}, derivative=-1.3746766757908988E-10}
New Minimum: 1.6014432562349478E-5 > 1.5162471356062722E-5
F(12282.320832754542) = LineSearchPoint{point=PointSample{avg=1.5162471356062722E-5}, derivative=-1.1123926023614574E-12}, evalInputDelta = -8.51961206286756E-7
F(85976.24582928179) = LineSearchPoint{point=PointSample{avg=4.5763621956129884E-5}, derivative=8.379524064879782E-10}, evalInputDelta = 2.9749189393780406E-5
F(6613.557371483214) = LineSearchPoint{point=PointSample{avg=1.534756192410038E-5}, derivative=-6.415721859013047E-11}, evalInputDelta = -6.668706382490971E-7
F(46294.9016003825) = LineSearchPoint{point=PointSample{avg=2.1608911771210792E-5}, derivative=3.814320561647946E-10}, evalInputDelta = 5.594479208861314E-6
F(3561.1462769525) = LineSearchPoint{point=PointSample{avg=1.559509903804859E-5}, derivative=-9.802478418017888E-11}, evalInputDelta = -4.1933352430088686E-7
F(24928.023938667502) = LineSearchPoint{point=PointSample{avg=1.60410958988995E-5}, derivative=1.402393681509582E-10}, evalInputDelta = 2.6663336550021777E-8
F(1917.5403029744232) = LineSearchPoint{point=PointSample{avg=1.5771183334804154E-5}, derivative=-1.1623840149946818E-10}, evalInputDelta = -2.4324922754532364E-7
F(13422.782120820963) = LineSearchPoint{point=PointSample{avg=1.5168447950957304E-5}, derivative=1.1594731678537862E-11}, evalInputDelta = -8.459846113921739E-7
1.5168447950957304E-5 <= 1.6014432562349478E-5
New Minimum: 1.5162471356062722E-5 > 1.5162415878491087E-5
F(12378.698852032234) = LineSearchPoint{point=PointSample{avg=1.5162415878491087E-5}, derivative=-3.8847414730470814E-14}, evalInputDelta = -8.520166838583909E-7
Left bracket at 12378.698852032234
Converged to left
Fitness changed from 1.6014432562349478E-5 to 1.5162415878491087E-5
Iteration 53 complete. Error: 1.5162415878491087E-5 Total: 0.6804; Orientation: 0.0031; Line Search: 0.6700
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5162415878491087E-5}, derivative=-1.2851563534567878E-10}
New Minimum: 1.5162415878491087E-5 > 1.4372680676620693E-5
F(12378.698852032234) = LineSearchPoint{point=PointSample{avg=1.4372680676620693E-5}, derivative=9.227548902376559E-13}, evalInputDelta = -7.897352018703936E-7
1.4372680676620693E-5 <= 1.5162415878491087E-5
Converged to right
Fitness changed from 1.5162415878491087E-5 to 1.4372680676620693E-5
Iteration 54 complete. Error: 1.4372680676620693E-5 Total: 0.0757; Orientation: 0.0030; Line Search: 0.0570
F(0.0) = LineSearchPoint{point=PointSample{avg=1.4372680676620693E-5}, derivative=-1.2021489364273908E-10}
New Minimum: 1.4372680676620693E-5 > 1.3638714087354102E-5
F(12378.698852032234) = LineSearchPoint{point=PointSample{avg=1.3638714087354102E-5}, derivative=1.7543616376880668E-12}, evalInputDelta = -7.339665892665917E-7
1.3638714087354102E-5 <= 1.4372680676620693E-5
New Minimum: 1.3638714087354102E-5 > 1.3638558383591017E-5
F(12200.648126539418) = LineSearchPoint{point=PointSample{avg=1.3638558383591017E-5}, derivative=-5.353114473154484E-15}, evalInputDelta = -7.341222930296764E-7
Left bracket at 12200.648126539418
Converged to left
Fitness changed from 1.4372680676620693E-5 to 1.3638558383591017E-5
Iteration 55 complete. Error: 1.3638558383591017E-5 Total: 0.4381; Orientation: 0.0031; Line Search: 0.3678
F(0.0) = LineSearchPoint{point=PointSample{avg=1.3638558383591017E-5}, derivative=-1.0927838759679922E-10}
New Minimum: 1.3638558383591017E-5 > 1.295731874368918E-5
F(12200.648126539418) = LineSearchPoint{point=PointSample{avg=1.295731874368918E-5}, derivative=-2.3924121093198067E-12}, evalInputDelta = -6.812396399018375E-7
F(85404.53688577592) = LineSearchPoint{point=PointSample{avg=3.628783433182457E-5}, derivative=6.40481324441095E-10}, evalInputDelta = 2.2649275948233552E-5
F(6569.579760444301) = LineSearchPoint{point=PointSample{avg=1.3109692510650196E-5}, derivative=-5.172586142271294E-11}, evalInputDelta = -5.288658729408207E-7
F(45987.05832311011) = LineSearchPoint{point=PointSample{avg=1.7879103319249474E-5}, derivative=2.938237763464057E-10}, evalInputDelta = 4.240544935658457E-6
F(3537.466024854624) = LineSearchPoint{point=PointSample{avg=1.3306802182119424E-5}, derivative=-7.828874213571934E-11}, evalInputDelta = -3.31756201471593E-7
F(24762.262173982366) = LineSearchPoint{point=PointSample{avg=1.36185846203384E-5}, derivative=1.0768320438791119E-10}, evalInputDelta = -1.9973763252616173E-8
1.36185846203384E-5 <= 1.3638558383591017E-5
New Minimum: 1.295731874368918E-5 > 1.2956992113409432E-5
F(12472.161818450795) = LineSearchPoint{point=PointSample{avg=1.2956992113409432E-5}, derivative=-1.3580633636869027E-14}, evalInputDelta = -6.815662701815846E-7
Left bracket at 12472.161818450795
Converged to left
Fitness changed from 1.3638558383591017E-5 to 1.2956992113409432E-5
Iteration 56 complete. Error: 1.2956992113409432E-5 Total: 0.5997; Orientation: 0.0031; Line Search: 0.5793
F(0.0) = LineSearchPoint{point=PointSample{avg=1.2956992113409432E-5}, derivative=-1.0391854717383103E-10}
New Minimum: 1.2956992113409432E-5 > 1.2322077656378943E-5
F(12472.161818450795) = LineSearchPoint{point=PointSample{avg=1.2322077656378943E-5}, derivative=2.2061593435501262E-12}, evalInputDelta = -6.349144570304888E-7
1.2322077656378943E-5 <= 1.2956992113409432E-5
New Minimum: 1.2322077656378943E-5 > 1.2321792463201666E-5
F(12212.885941673338) = LineSearchPoint{point=PointSample{avg=1.2321792463201666E-5}, derivative=-6.1941838059117435E-15}, evalInputDelta = -6.351996502077667E-7
Left bracket at 12212.885941673338
Converged to left
Fitness changed from 1.2956992113409432E-5 to 1.2321792463201666E-5
Iteration 57 complete. Error: 1.2321792463201666E-5 Total: 0.3794; Orientation: 0.0035; Line Search: 0.3658
Low gradient: 9.740947708072627E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=1.2321792463201666E-5}, derivative=-9.488606225140537E-11}
New Minimum: 1.2321792463201666E-5 > 1.172977033032361E-5
F(12212.885941673338) = LineSearchPoint{point=PointSample{avg=1.172977033032361E-5}, derivative=-2.0643648750909025E-12}, evalInputDelta = -5.920221328780554E-7
F(85490.20159171337) = LineSearchPoint{point=PointSample{avg=3.200436662179375E-5}, derivative=5.559155230478194E-10}, evalInputDelta = 1.9682574158592082E-5
F(6576.169353208721) = LineSearchPoint{point=PointSample{avg=1.186214792041571E-5}, derivative=-4.490517030190643E-11}, evalInputDelta = -4.596445427859549E-7
F(46033.185472461046) = LineSearchPoint{point=PointSample{avg=1.6007891422881736E-5}, derivative=2.55111234571334E-10}, evalInputDelta = 3.6860989596800704E-6
F(3541.014267112388) = LineSearchPoint{point=PointSample{avg=1.2033449390571776E-5}, derivative=-6.797308346444722E-11}, evalInputDelta = -2.8834307262988994E-7
F(24787.099869786718) = LineSearchPoint{point=PointSample{avg=1.230470891764968E-5}, derivative=9.351573117528025E-11}, evalInputDelta = -1.708354555198526E-8
1.230470891764968E-5 <= 1.2321792463201666E-5
New Minimum: 1.172977033032361E-5 > 1.1729489981463923E-5
F(12483.693804069964) = LineSearchPoint{point=PointSample{avg=1.1729489981463923E-5}, derivative=-6.097388249759103E-15}, evalInputDelta = -5.923024817377422E-7
Left bracket at 12483.693804069964
Converged to left
Fitness changed from 1.2321792463201666E-5 to 1.1729489981463923E-5
Iteration 58 complete. Error: 1.1729489981463923E-5 Total: 0.6218; Orientation: 0.0030; Line Search: 0.6105
Final threshold in iteration 58: 1.1729489981463923E-5 (> 0.0) after 30.081s (< 30.000s)
Returns
1.1729489981463923E-5
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.10832563822031188 ], [ 0.3531756069605953 ], [ 0.13284211387454095 ], [ -0.6594909300029155 ], [ -0.5887227436916922 ], [ -0.14117722349343775 ], [ 0.852822828466834 ], [ -0.7788890598027733 ], ... ],
[ [ -0.11364819409938164 ], [ 0.9561130362882329 ], [ -0.009243142403874052 ], [ -0.3602933686735659 ], [ 0.8046453184816705 ], [ 0.12890299271170563 ], [ -0.3366246416034421 ], [ -0.3402271334484298 ], ... ],
[ [ -0.6337307812545718 ], [ -0.8902171595137027 ], [ -0.5628663424611302 ], [ -0.8795880103623205 ], [ 0.4782943258672219 ], [ -0.09021431214417526 ], [ -0.8022677820278079 ], [ -0.08886018790675604 ], ... ],
[ [ -0.728577800261027 ], [ -0.7753697945274831 ], [ -0.14600549145766445 ], [ -0.5408592789247528 ], [ -0.2811637046867294 ], [ -0.40761657871702517 ], [ -0.872000805731147 ], [ -0.3157503359952842 ], ... ],
[ [ 0.09822302139146424 ], [ -0.9197683434216566 ], [ -0.21895980905737522 ], [ 0.7453156088951773 ], [ 0.3196818483943744 ], [ -0.0912775038388895 ], [ 0.1747479125764629 ], [ -0.37960592266709625 ], ... ],
[ [ 0.459103174381405 ], [ 0.5845878326095819 ], [ -0.3834365690771001 ], [ -0.5423891260370208 ], [ 0.8098343244099748 ], [ -0.2982872181569829 ], [ 0.4419652171446424 ], [ 0.5063713155679466 ], ... ],
[ [ 0.13882754758585625 ], [ -0.5803775148819643 ], [ 0.09115339585035666 ], [ -0.6760271575099701 ], [ 0.18105211240586236 ], [ 0.8054165949621275 ], [ 0.8947402937382618 ], [ -0.15127630358998662 ], ... ],
[ [ 0.2340680407806806 ], [ 0.10261124505092986 ], [ 0.3330231435876974 ], [ -0.2475210833209716 ], [ -0.22626388474728237 ], [ 0.10521597508986479 ], [ -0.3022563295602243 ], [ -0.11723233809266533 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.011734443895837894 ], [ 0.12473300935198489 ], [ 0.017647027218656505 ], [ 0.43492828675611034 ], [ 0.3465944689398739 ], [ 0.019931008433316074 ], [ 0.727306776754171 ], [ 0.6066681674804483 ], ... ],
[ [ 0.012915912022050722 ], [ 0.9141521381603037 ], [ 8.543568149829458E-5 ], [ 0.12981131151014605 ], [ 0.647454088554469 ], [ 0.016615981530034036 ], [ 0.11331614933464583 ], [ 0.11575450233453566 ], ... ],
[ [ 0.40161470310952996 ], [ 0.7924865910926453 ], [ 0.3168185194755703 ], [ 0.7736750679731456 ], [ 0.22876546215678029 ], [ 0.008138622115646688 ], [ 0.6436335940798183 ], [ 0.007896132994823992 ], ... ],
[ [ 0.530825611033197 ], [ 0.6011983182655913 ], [ 0.021317603535794125 ], [ 0.2925287595990036 ], [ 0.07905302883316638 ], [ 0.16615127524497278 ], [ 0.7603854051957696 ], [ 0.09969827468113486 ], ... ],
[ [ 0.009647761931268043 ], [ 0.8459738055606184 ], [ 0.04794339798244222 ], [ 0.555495356862789 ], [ 0.10219648419284379 ], [ 0.008331582707058487 ], [ 0.030536832949831123 ], [ 0.14410065652393747 ], ... ],
[ [ 0.21077572472708278 ], [ 0.3417429340351686 ], [ 0.14702360250561777 ], [ 0.29418596404320324 ], [ 0.6558316329925603 ], [ 0.0889752645158315 ], [ 0.19533325316571087 ], [ 0.25641190923001295 ], ... ],
[ [ 0.01927308796870318 ], [ 0.3368380597805647 ], [ 0.008308941575051819 ], [ 0.45701271769100993 ], [ 0.032779867406625016 ], [ 0.6486958914403878 ], [ 0.800560193238831 ], [ 0.022884520027849797 ], ... ],
[ [ 0.05478784771490636 ], [ 0.010529067610901978 ], [ 0.11090441416503213 ], [ 0.06126668668838737 ], [ 0.05119534554093148 ], [ 0.011070401414111046 ], [ 0.09135888875921892 ], [ 0.013743421094672989 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 31.32 seconds (0.078 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2312223145927
Reset training subject: 2312232250490
Adding measurement 304b420c to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.17726094106397533 < 0.17726094106397533. Total: 1
th(0)=0.17726094106397533;dx=-2.4378975770333734E-5
Adding measurement 1c0a03d0 to history. Total: 1
New Minimum: 0.17726094106397533 > 0.17720842718171878
WOLFE (weak): th(2.154434690031884)=0.17720842718171878; dx=-2.437059447699295E-5 evalInputDelta=5.2513882256549627E-5
Adding measurement 1bac3ab0 to history. Total: 2
New Minimum: 0.17720842718171878 > 0.1771559313541557
WOLFE (weak): th(4.308869380063768)=0.1771559313541557; dx=-2.4362215277266792E-5 evalInputDelta=1.0500970981963675E-4
Adding measurement 6da32746 to history. Total: 3
New Minimum: 0.1771559313541557 > 0.1769461285006471
WOLFE (weak): th(12.926608140191302)=0.1769461285006471; dx=-2.4328719408740067E-5 evalInputDelta=3.148125633282428E-4
Adding measurement 42c040c0 to history. Total: 4
New Minimum: 0.1769461285006471 > 0.17600558186316975
WOLFE (weak): th(51.70643256076521)=0.17600558186316975; dx=-2.417840198403685E-5 evalInputDelta=0.001255359200805578
Adding measurement 57ed4bb0 to history. Total: 5
New Minimum: 0.17600558186316975 > 0.17108692415388357
WOLFE (weak): th(258.53216280382605)=0.17108692415388357; dx=-2.3388087979078938E-5 evalInputDelta=0.006174016910091756
Adding measurement 475933c0 to history. Total: 6
New Minimum: 0.17108692415388357 > 0.14385171737406613
END: th(1551.1929768229563)=0.14385171737406613; dx=-1.8868488674881193E-5 evalInputDelta=0.0334092236899092
Fitness changed from 0.17726094106397533 to 0.14385171737406613
Iteration 1 complete. Error: 0.14385171737406613 Total: 0.4356; Orientation: 0.0133; Line Search: 0.3961
Non-optimal measurement 0.14385171737406613 < 0.14385171737406613. Total: 7
Rejected: LBFGS Orientation magnitude: 3.127e+01, gradient 3.852e-03, dot -0.969; [b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.14385171737406613, 0.17108692415388357, 0.17600558186316975, 0.1769461285006471, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 3.128e+01, gradient 3.852e-03, dot -0.969; [558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.14385171737406613, 0.17108692415388357, 0.17600558186316975, 0.1769461285006471, 0.1771559313541557, 0.17720842718171878
Rejected: LBFGS Orientation magnitude: 3.130e+01, gradient 3.852e-03, dot -0.969; [36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.14385171737406613, 0.17108692415388357, 0.17600558186316975, 0.1769461285006471, 0.1771559313541557
Rejected: LBFGS Orientation magnitude: 3.138e+01, gradient 3.852e-03, dot -0.969; [36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.14385171737406613, 0.17108692415388357, 0.17600558186316975, 0.1769461285006471
LBFGS Accumulation History: 3 points
Removed measurement 475933c0 to history. Total: 6
Removed measurement 57ed4bb0 to history. Total: 5
Removed measurement 42c040c0 to history. Total: 4
Removed measurement 6da32746 to history. Total: 3
Adding measurement 65376f35 to history. Total: 3
th(0)=0.14385171737406613;dx=-1.4837876634737385E-5
Adding measurement 3bc12fdc to history. Total: 4
New Minimum: 0.14385171737406613 > 0.1030957617931636
END: th(3341.943960201201)=0.1030957617931636; dx=-9.801173154234549E-6 evalInputDelta=0.04075595558090253
Fitness changed from 0.14385171737406613 to 0.1030957617931636
Iteration 2 complete. Error: 0.1030957617931636 Total: 4.6209; Orientation: 4.5763; Line Search: 0.0353
Non-optimal measurement 0.1030957617931636 < 0.1030957617931636. Total: 5
Rejected: LBFGS Orientation magnitude: 2.424e+01, gradient 2.635e-03, dot -0.987; [36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.1030957617931636, 0.14385171737406613, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 2.424e+01, gradient 2.635e-03, dot -0.987; [ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.1030957617931636, 0.14385171737406613, 0.1771559313541557, 0.17720842718171878
LBFGS Accumulation History: 3 points
Removed measurement 3bc12fdc to history. Total: 4
Removed measurement 65376f35 to history. Total: 3
Adding measurement 21ddfb70 to history. Total: 3
th(0)=0.1030957617931636;dx=-6.943877812472072E-6
Adding measurement 3104f714 to history. Total: 4
New Minimum: 0.1030957617931636 > 0.06367898243254494
END: th(7200.000000000001)=0.06367898243254494; dx=-4.060728971028692E-6 evalInputDelta=0.03941677936061866
Fitness changed from 0.1030957617931636 to 0.06367898243254494
Iteration 3 complete. Error: 0.06367898243254494 Total: 1.5666; Orientation: 1.5315; Line Search: 0.0262
Non-optimal measurement 0.06367898243254494 < 0.06367898243254494. Total: 5
Rejected: LBFGS Orientation magnitude: 2.083e+01, gradient 1.729e-03, dot -0.990; [558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06367898243254494, 0.1030957617931636, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 2.084e+01, gradient 1.729e-03, dot -0.990; [558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.06367898243254494, 0.1030957617931636, 0.1771559313541557, 0.17720842718171878
LBFGS Accumulation History: 3 points
Removed measurement 3104f714 to history. Total: 4
Removed measurement 21ddfb70 to history. Total: 3
Adding measurement 350d99e3 to history. Total: 3
th(0)=0.06367898243254494;dx=-2.98934935755973E-6
Adding measurement 7d10d484 to history. Total: 4
New Minimum: 0.06367898243254494 > 0.032121464852960024
END: th(15511.929768229566)=0.032121464852960024; dx=-5.866196439732105E-7 evalInputDelta=0.031557517579584915
Fitness changed from 0.06367898243254494 to 0.032121464852960024
Iteration 4 complete. Error: 0.032121464852960024 Total: 2.6796; Orientation: 2.4171; Line Search: 0.2554
Non-optimal measurement 0.032121464852960024 < 0.032121464852
...skipping 12276 bytes...
9e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00217748775628272, 0.003323798792179633, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 4.377e+00, gradient 2.753e-04, dot -0.698; [1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.00217748775628272, 0.003323798792179633, 0.1771559313541557, 0.17720842718171878
LBFGS Accumulation History: 3 points
Removed measurement 11d27063 to history. Total: 4
Removed measurement 54269fec to history. Total: 3
Adding measurement 3c43fb5e to history. Total: 3
th(0)=0.00217748775628272;dx=-7.57879915503662E-8
Non-optimal measurement 0.0027690719528446817 < 0.00217748775628272. Total: 4
Armijo: th(34811.91625209586)=0.0027690719528446817; dx=1.2270157043214643E-7 evalInputDelta=-5.915841965619616E-4
Adding measurement 28b9b54b to history. Total: 4
New Minimum: 0.00217748775628272 > 0.0016308665010736898
WOLF (strong): th(17405.95812604793)=0.0016308665010736898; dx=1.3762195582509528E-8 evalInputDelta=5.466212552090303E-4
Non-optimal measurement 0.0018236480617418566 < 0.0016308665010736898. Total: 5
END: th(5801.986042015978)=0.0018236480617418566; dx=-4.6278353823498616E-8 evalInputDelta=3.538396945408635E-4
Fitness changed from 0.00217748775628272 to 0.0016308665010736898
Iteration 11 complete. Error: 0.0016308665010736898 Total: 1.7876; Orientation: 1.3116; Line Search: 0.4689
Non-optimal measurement 0.0016308665010736898 < 0.0016308665010736898. Total: 5
Rejected: LBFGS Orientation magnitude: 2.103e+00, gradient 2.975e-04, dot -0.777; [b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0016308665010736898, 0.00217748775628272, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 2.103e+00, gradient 2.975e-04, dot -0.777; [1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0016308665010736898, 0.00217748775628272, 0.1771559313541557, 0.17720842718171878
LBFGS Accumulation History: 3 points
Removed measurement 28b9b54b to history. Total: 4
Removed measurement 3c43fb5e to history. Total: 3
Adding measurement 27058f33 to history. Total: 3
th(0)=0.0016308665010736898;dx=-8.850134991249001E-8
Adding measurement fe24829 to history. Total: 4
New Minimum: 0.0016308665010736898 > 0.0011672176436779722
WOLF (strong): th(12500.00000000001)=0.0011672176436779722; dx=2.096121842384057E-8 evalInputDelta=4.636488573957177E-4
Non-optimal measurement 0.0012285035704977139 < 0.0011672176436779722. Total: 5
END: th(6250.000000000005)=0.0012285035704977139; dx=-3.875283001532425E-8 evalInputDelta=4.0236293057597594E-4
Fitness changed from 0.0016308665010736898 to 0.0011672176436779722
Iteration 12 complete. Error: 0.0011672176436779722 Total: 2.1003; Orientation: 1.8490; Line Search: 0.1821
Non-optimal measurement 0.0011672176436779722 < 0.0011672176436779722. Total: 5
Rejected: LBFGS Orientation magnitude: 9.921e-01, gradient 2.424e-04, dot -0.975; [b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0011672176436779722, 0.0016308665010736898, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 9.923e-01, gradient 2.424e-04, dot -0.975; [558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0011672176436779722, 0.0016308665010736898, 0.1771559313541557, 0.17720842718171878
LBFGS Accumulation History: 3 points
Removed measurement fe24829 to history. Total: 4
Removed measurement 27058f33 to history. Total: 3
Adding measurement 5ddd47f6 to history. Total: 3
th(0)=0.0011672176436779722;dx=-5.875757650471927E-8
Adding measurement 18ee4226 to history. Total: 4
New Minimum: 0.0011672176436779722 > 9.166616821343402E-4
WOLF (strong): th(13465.216812699284)=9.166616821343402E-4; dx=2.0230111478134265E-8 evalInputDelta=2.50555961543632E-4
Adding measurement 1ab476ff to history. Total: 5
New Minimum: 9.166616821343402E-4 > 9.092186690004627E-4
END: th(6732.608406349642)=9.092186690004627E-4; dx=-1.827959982245862E-8 evalInputDelta=2.5799897467750944E-4
Fitness changed from 0.0011672176436779722 to 9.092186690004627E-4
Iteration 13 complete. Error: 9.092186690004627E-4 Total: 2.1361; Orientation: 2.0892; Line Search: 0.0392
Non-optimal measurement 9.092186690004627E-4 < 9.092186690004627E-4. Total: 6
Rejected: LBFGS Orientation magnitude: 1.714e+00, gradient 1.438e-04, dot -0.908; [1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00]
Orientation rejected. Popping history element from 9.092186690004627E-4, 9.166616821343402E-4, 0.0011672176436779722, 0.1771559313541557, 0.17720842718171878, 0.17726094106397533
Rejected: LBFGS Orientation magnitude: 1.714e+00, gradient 1.438e-04, dot -0.908; [ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00, 1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 9.092186690004627E-4, 9.166616821343402E-4, 0.0011672176436779722, 0.1771559313541557, 0.17720842718171878
Rejected: LBFGS Orientation magnitude: 3.546e+00, gradient 1.438e-04, dot -0.965; [1f06ce08-2831-499f-a476-e4d317c39d3f = 1.000/1.000e+00, ed28b353-d3d9-4f94-9cc0-88c8a1e834ed = 1.000/1.000e+00, 558bedb7-6dc0-407c-b045-10e24b1f0893 = 1.000/1.000e+00, b99e0066-d632-4543-810b-962e2987d4ce = 1.000/1.000e+00, 36bb9318-5116-420e-9a80-70f8e47c78d1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 9.092186690004627E-4, 9.166616821343402E-4, 0.0011672176436779722, 0.1771559313541557
LBFGS Accumulation History: 3 points
Removed measurement 1ab476ff to history. Total: 5
Removed measurement 18ee4226 to history. Total: 4
Removed measurement 5ddd47f6 to history. Total: 3
Adding measurement 72e442e7 to history. Total: 3
th(0)=9.092186690004627E-4;dx=-2.0678925477216505E-8
Adding measurement 6a18db84 to history. Total: 4
New Minimum: 9.092186690004627E-4 > 6.563172159083087E-4
END: th(14504.965105039946)=6.563172159083087E-4; dx=-1.2567463402441822E-8 evalInputDelta=2.52901453092154E-4
Fitness changed from 9.092186690004627E-4 to 6.563172159083087E-4
Iteration 14 complete. Error: 6.563172159083087E-4 Total: 3.0848; Orientation: 2.8554; Line Search: 0.0845
Final threshold in iteration 14: 6.563172159083087E-4 (> 0.0) after 31.323s (< 30.000s)
Returns
6.563172159083087E-4
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.18488645335346668 ], [ 0.35553871649877655 ], [ -0.21767960414170368 ], [ -0.6594909299974905 ], [ -0.5887226925145878 ], [ -0.17711134679803203 ], [ 0.8528587055740904 ], [ -0.778888856170041 ], ... ],
[ [ -0.18673437080964927 ], [ 0.9560813522313006 ], [ -0.004423898374903002 ], [ -0.36137461936632836 ], [ 0.8046491839345351 ], [ 0.16878012960586347 ], [ -0.34016532993921395 ], [ -0.34277313232917106 ], ... ],
[ [ -0.6337307812823628 ], [ -0.8902297414475315 ], [ -0.5628663673097806 ], [ -0.8795960721810545 ], [ 0.4783175391711465 ], [ -0.1645378289507312 ], [ -0.802270421184976 ], [ -0.16832635683711944 ], ... ],
[ [ -0.7285786111583459 ], [ -0.7753702352412742 ], [ -0.18712893370761896 ], [ -0.5408595033694052 ], [ 0.2919824766053809 ], [ -0.3539506356683101 ], [ -0.8720410153539161 ], [ -0.3200484447420934 ], ... ],
[ [ -0.1819549185608333 ], [ -0.9197280528115762 ], [ -0.24509183367385484 ], [ 0.7453156062837193 ], [ 0.32477544146024984 ], [ 0.17842830785935465 ], [ -0.22036518100374836 ], [ -0.38070247071487584 ], ... ],
[ [ 0.45678191202802143 ], [ 0.5845878319458604 ], [ -0.38446386900608437 ], [ -0.5423488668259425 ], [ 0.8087216928039079 ], [ -0.30633016739761154 ], [ 0.44142069831625436 ], [ 0.5063439879492425 ], ... ],
[ [ 0.19703679364763788 ], [ -0.5803775147494648 ], [ -0.17865489137294016 ], [ -0.6760271575167073 ], [ 0.22556120892701206 ], [ 0.8054156362552244 ], [ 0.8947560144866678 ], [ -0.20645180951097528 ], ... ],
[ [ 0.22453248305037699 ], [ 0.1813875669060769 ], [ 0.33502945914737536 ], [ -0.11570182447273943 ], [ -0.2251852027840539 ], [ 0.18969078429791808 ], [ 0.3096792198920763 ], [ -0.1909340536856101 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.03418300063362361 ], [ 0.1264077789295974 ], [ 0.04738441005928882 ], [ 0.4349282867489549 ], [ 0.34659440868162583 ], [ 0.03136842916461277 ], [ 0.727367971673513 ], [ 0.6066678502658748 ], ... ],
[ [ 0.03486972524167559 ], [ 0.9140915520844324 ], [ 1.957087683146942E-5 ], [ 0.1305916155221587 ], [ 0.6474603092065133 ], [ 0.02848673214977207 ], [ 0.11571245169265429 ], [ 0.11749342024675141 ], ... ],
[ [ 0.401614703144754 ], [ 0.7925089925577389 ], [ 0.3168185474485088 ], [ 0.7736892501963388 ], [ 0.22878766827874128 ], [ 0.027072697155820082 ], [ 0.6436378287083189 ], [ 0.028333762406057266 ], ... ],
[ [ 0.5308267926374242 ], [ 0.6011990016981089 ], [ 0.035017237830550454 ], [ 0.2925290023849997 ], [ 0.0852537666446118 ], [ 0.1252810524900008 ], [ 0.7604555324594889 ], [ 0.1024310069818328 ], ... ],
[ [ 0.033107592388479476 ], [ 0.8458996911285735 ], [ 0.06007000693361252 ], [ 0.555495352970068 ], [ 0.10547908737570016 ], [ 0.03183666104555264 ], [ 0.048560812998814776 ], [ 0.1449343712084109 ], ... ],
[ [ 0.2086497151559751 ], [ 0.3417429332591615 ], [ 0.1478124665711276 ], [ 0.29414229334738395 ], [ 0.6540307764116184 ], [ 0.09383817145784872 ], [ 0.19485223290200965 ], [ 0.2563842341323426 ], ... ],
[ [ 0.03882349805094183 ], [ 0.33683805962676516 ], [ 0.03191757021147705 ], [ 0.457012717700119 ], [ 0.05087785897261519 ], [ 0.6486943471244079 ], [ 0.800588325460066 ], [ 0.04262234965035602 ], ... ],
[ [ 0.05041483594476783 ], [ 0.03290144942810653 ], [ 0.11224473849658285 ], [ 0.013386912186320604 ], [ 0.05070837555289548 ], [ 0.035982593647559284 ], [ 0.09590121923296495 ], [ 0.03645581285681944 ], ... ],
...
]
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -6.687765698876658], [130.0, -0.8420849489170119]; valueStats=DoubleSummaryStatistics{count=202, sum=0.953023, min=0.000000, average=0.004718, max=0.143852}
Plotting 130 points for GD
Plotting 58 points for CjGD
Plotting 14 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -6.687765698876658], [30.887, -0.8420849489170119]; valueStats=DoubleSummaryStatistics{count=202, sum=0.953023, min=0.000000, average=0.004718, max=0.143852}
Plotting 130 points for GD
Plotting 58 points for CjGD
Plotting 14 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 6.563172159083087E-4 }, "CjGD": { "type": "NonConverged", "value": 1.1729489981463923E-5 }, "GD": { "type": "NonConverged", "value": 2.0522690758381547E-7 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 6.563172159083087E-4 }, "CjGD": { "type": "NonConverged", "value": 1.1729489981463923E-5 }, "GD": { "type": "NonConverged", "value": 2.0522690758381547E-7 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "92.330",
"gc_time": "0.661"
},
"created_on": 1586736839744,
"file_name": "trainingTest",
"report": {
"simpleName": "Basic",
"canonicalName": "com.simiacryptus.mindseye.layers.java.SqActivationLayerTest.Basic",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/SqActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 6.563172159083087E-4
},
"CjGD": {
"type": "NonConverged",
"value": 1.1729489981463923E-5
},
"GD": {
"type": "NonConverged",
"value": 2.0522690758381547E-7
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/SqActivationLayer/Basic/trainingTest/202004131359",
"id": "36d68179-8517-4284-b1bc-394b1139a2eb",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "SqActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.SqActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/SqActivationLayer.java",
"javaDoc": ""
}
}