Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 5232848441000379392
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.8108619228504756 ], [ 0.23950959790603488 ], [ -0.7427184455577929 ], [ -0.27398483644635063 ], [ 0.9586317940479918 ], [ 0.41921777337411026 ], [ -0.12263873362345468 ], [ 0.3468846131879024 ], ... ],
[ [ -0.22189092206298788 ], [ 0.2555124335451213 ], [ -0.576270931698359 ], [ -0.38118045547563506 ], [ -0.9851041066452195 ], [ -0.2851965341154652 ], [ -0.5668037694171806 ], [ -0.40713189442535813 ], ... ],
[ [ -0.3705167621094796 ], [ -0.8489993310684223 ], [ 0.5870372664465657 ], [ -0.883703671593477 ], [ -0.25702518485801185 ], [ 0.33826061795304097 ], [ 0.41150716853038105 ], [ -0.7921741841158878 ], ... ],
[ [ 0.6919367001284729 ], [ 0.7195486172564809 ], [ -0.5152576766542327 ], [ 0.3082505074754889 ], [ 0.3382223359334704 ], [ 0.5155018294973608 ], [ -0.8713553514819807 ], [ 0.2726999853566372 ], ... ],
[ [ 0.6003938989989539 ], [ 0.9229050436665769 ], [ 0.5455429247115813 ], [ -0.8049803861416107 ], [ -0.23152115789021988 ], [ -0.8250791589490516 ], [ -0.16934857652897262 ], [ -0.31227734595891876 ], ... ],
[ [ 0.5152954639584681 ], [ -0.42825427228482527 ], [ 0.7414436459015081 ], [ 0.41967285876266414 ], [ 0.2759535639045876 ], [ 0.026671880544114845 ], [ -0.8180001181285563 ], [ -0.7850783286168355 ], ... ],
[ [ 0.34714500238201174 ], [ 0.716010304427543 ], [ 0.32432770391566623 ], [ -0.18271434790997465 ], [ -0.5265613358328931 ], [ -0.3631531849592735 ], [ -0.3656755607476858 ], [ -0.39538291420757776 ], ... ],
[ [ 0.669699568622157 ], [ -0.5531993563073145 ], [ 0.28007078895624593 ], [ 0.5288203649245022 ], [ 0.7346021419210038 ], [ 0.647862491206429 ], [ 0.07095456817229384 ], [ -0.010256803416679006 ], ... ],
...
]
[
[ [ 0.05667925870250534 ], [ -0.20873576748897293 ], [ 0.9278399227329786 ], [ -0.09242774092286088 ], [ -0.43818460164186646 ], [ 0.6844538193178789 ], [ 0.3028850563875083 ], [ 0.423095062129619 ], ... ],
[ [ 0.11322130012952258 ], [ 0.5957369334851046 ], [ -0.3660581002404155 ], [ -0.3538162462787233 ], [ -0.8981292743490681 ], [ -0.8121552721940746 ], [ 0.3525482795230048 ], [ 0.30666018025264696 ], ... ],
[ [ -0.2811405861977261 ], [ -0.8144755406005504 ], [ 0.6580023564689148 ], [ 0.6569412318593046 ], [ 0.6977848936068058 ], [ 0.5493802726949619 ], [ -0.18569773380259735 ], [ 0.9979504755668194 ], ... ],
[ [ -0.22839995608154728 ], [ 0.673651579793666 ], [ 0.02263208717596532 ], [ 0.34393887078013896 ], [ -0.31918432491876203 ], [ 0.15692855112207194 ], [ -0.08201279001749895 ], [ 0.6492502313730446 ], ... ],
[ [ 0.4586610965100064 ], [ -0.3946258622554859 ], [ 0.43274618005137655 ], [ -0.10448878646934778 ], [ 0.6145872986882854 ], [ -0.48321201839247596 ], [ -0.3692446948240198 ], [ -0.3866064810939349 ], ... ],
[ [ -0.045392262567677255 ], [ -0.768075507023114 ], [ 0.36365764804877365 ], [ 0.09508410892811492 ], [ -0.8736755949090742 ], [ -0.6816707776553153 ], [ 0.2288697800242827 ], [ -0.565840759558738 ], ... ],
[ [ -0.8425487005600785 ], [ -0.538276274807189 ], [ -0.6611441138038554 ], [ -0.28711383968889614 ], [ 0.4590884438002778 ], [ -0.2668472173814602 ], [ -0.23967073051564092 ], [ -0.6683344882305606 ], ... ],
[ [ 0.9971716560624162 ], [ 0.004260788773500002 ], [ -0.19256785524836517 ], [ -0.6771262764217612 ], [ 0.2710333309904962 ], [ -0.18161078303343237 ], [ -0.9797972937212887 ], [ -0.3307064972237612 ], ... ],
...
]
[
[ [ -0.48158146202836694 ], [ 0.8200447642765836 ], [ 0.8497004253184168 ], [ -0.245089720113004 ], [ -0.7755637813009149 ], [ 0.6682319461389155 ], [ -0.6283267781777535 ], [ -0.15517242138885368 ], ... ],
[ [ -0.5687783504013242 ], [ -0.8892922183304166 ], [ 0.002117500607013767 ], [ 0.14202845189881863 ], [ 0.6633940847471529 ], [ -0.2504783158749666 ], [ 0.032495176117016245 ], [ 0.7725294636557334 ], ... ],
[ [ 0.5251014194223644 ], [ -0.0520100300856281 ], [ 0.12525413952452902 ], [ 0.20766516558770443 ], [ -0.39466222959806463 ], [ 0.7144234722315141 ], [ -0.9111872126206173 ], [ 0.7942129917113232 ], ... ],
[ [ -0.8818608256232103 ], [ 0.8553027137160218 ], [ -0.44179047944740946 ], [ -0.04752541703569779 ], [ -0.19912294398683583 ], [ 0.9157730892697613 ], [ -0.8788895824754059 ], [ -0.7563038323297671 ], ... ],
[ [ 0.5590122982914001 ], [ -0.7253477908538954 ], [ -0.031176287946629078 ], [ -0.026927957839445904 ], [ 0.7672376378512249 ], [ -0.5659441838938075 ], [ 0.8034246020492395 ], [ 0.5559340257056313 ], ... ],
[ [ 0.18281885943053822 ], [ 0.410090327616349 ], [ 0.13236896238903073 ], [ -0.03130866233091112 ], [ 0.8193273519502706 ], [ 0.7482877991782415 ], [ -0.16720082196718167 ], [ -0.556815602585337 ], ... ],
[ [ -0.4784376349486428 ], [ -0.32211332908472556 ], [ -0.6124978998062738 ], [ 0.6910664349642306 ], [ -0.09137322549242088 ], [ 0.8980682821067967 ], [ 0.27962908055244684 ], [ -0.3060745364321371 ], ... ],
[ [ -0.0384288455426407 ], [ 0.1878537239863225 ], [ 0.45581251597940087 ], [ -0.10574695452075589 ], [ -0.5337527974427076 ], [ 0.10981317223095655 ], [ 0.4454457849545357 ], [ 0.7883519463609279 ], ... ],
...
]
[
[ [ -0.325209186941003 ], [ 0.29138750195264573 ], [ 0.9328393657207685 ], [ -0.11024727336475748 ], [ 0.5796209029627388 ], [ -0.805257149443394 ], [ -0.36869769388997087 ], [ -0.6343159346534009 ], ... ],
[ [ 0.5831348415800259 ], [ 0.790363932071861 ], [ -0.799185940731592 ], [ -0.6645637145670102 ], [ -0.931915457678365 ], [ 0.7040132091498124 ], [ -0.6492804614938137 ], [ -0.32456938647837275 ], ... ],
[ [ 0.05260710071095254 ], [ 0.399211327479178 ], [ 0.8798121620357588 ], [ 0.5470029784207933 ], [ 0.857669901888386 ], [ 0.24374776744336613 ], [ -0.21747372107722263 ], [ -0.7258149820912931 ], ... ],
[ [ 0.9129019979015978 ], [ -0.9548456785720254 ], [ -0.38031045989218937 ], [ 0.6400465665293267 ], [ 0.9365405814771182 ], [ -0.3939483976773981 ], [ -0.24677848931259083 ], [ -0.5044766234696634 ], ... ],
[ [ 0.5923837597959414 ], [ 0.5176840383036283 ], [ -0.2551435450280588 ], [ 0.06964549410496002 ], [ 0.2522169164021808 ], [ 0.428247092121391 ], [ -0.2256224407515921 ], [ 0.3929410039887149 ], ... ],
[ [ -0.6192094626897657 ], [ -0.09994741062537815 ], [ 0.614372419789303 ], [ 0.9056694301266524 ], [ 0.9131744704099181 ], [ -0.5393770631465038 ], [ 0.8686278266754199 ], [ 0.1234841737919723 ], ... ],
[ [ 0.20938676151272584 ], [ 0.38971698061478643 ], [ -0.7479516142106482 ], [ -0.5336454633553831 ], [ 0.2305926913510663 ], [ 0.6156471813462698 ], [ -0.08630781119601472 ], [ -0.12226886330703479 ], ... ],
[ [ 0.5802036746799029 ], [ 0.16396748335050382 ], [ 0.3853515707134072 ], [ -0.07867153704723573 ], [ -0.7124606978964914 ], [ -0.5133280765897403 ], [ 0.770808623257963 ], [ -0.6115898076926314 ], ... ],
...
]
[
[ [ -0.6070264068484219 ], [ -0.0612952521445167 ], [ -0.14780541628116328 ], [ 0.32917122511013197 ], [ 0.43515645994178187 ], [ 0.2067003790098374 ], [ 0.37191088374107406 ], [ -0.19833230460544837 ], ... ],
[ [ 0.6828302628103333 ], [ 0.014579373949045848 ], [ -0.4744276967470582 ], [ -0.9418981573760514 ], [ 0.15400933597672406 ], [ -0.7803845464219155 ], [ -0.887853371067856 ], [ 0.999768620326478 ], ... ],
[ [ -0.2134664947835374 ], [ 0.6768220248683012 ], [ -0.9389361188568373 ], [ 0.4145190568940158 ], [ 0.9272473257408977 ], [ -0.5800963840574793 ], [ 0.5448746742748698 ], [ -0.8492524143983635 ], ... ],
[ [ 0.5603549779374317 ], [ 0.24735853097879446 ], [ 0.6354377395754072 ], [ -0.9434600093320846 ], [ 0.6707017035346758 ], [ 0.43575123479558875 ], [ 0.5114777755681998 ], [ -0.05645874490062286 ], ... ],
[ [ 0.8599949530273128 ], [ -0.4516853894735817 ], [ 0.25374325649123874 ], [ -0.22216385495620128 ], [ -0.11563256630354646 ], [ -0.06649019361829289 ], [ 0.36632830571327735 ], [ -0.3001962245064056 ], ... ],
[ [ -0.8654216659221758 ], [ -0.81535695519886 ], [ -0.2791037346717429 ], [ 0.2832443733300263 ], [ 0.7985786980052891 ], [ 0.0458176447366776 ], [ 0.20559225032422734 ], [ 0.973638181423998 ], ... ],
[ [ -0.16767475558341083 ], [ -0.13249572421011724 ], [ -0.37095825493114654 ], [ 0.3694747585348659 ], [ 0.4485158372216451 ], [ 0.046716874992764224 ], [ -0.5488433091824276 ], [ 0.6212206842469141 ], ... ],
[ [ 0.4470241109824238 ], [ -0.7844286022301514 ], [ -0.39540279887232055 ], [ -0.7547752799706364 ], [ 0.6524503975567566 ], [ -0.4849912424983358 ], [ 0.42757665580142334 ], [ 0.06879504506632061 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.10 seconds (0.170 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2898419896338
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 2898485097014
Constructing line search parameters: GD
th(0)=0.10865262101983993;dx=-2.7568298031681143E-5
New Minimum: 0.10865262101983993 > 0.10859325696455376
WOLFE (weak): th(2.154434690031884)=0.10859325696455376; dx=-2.75404133465621E-5 evalInputDelta=5.9364055286170725E-5
New Minimum: 0.10859325696455376 > 0.10853395295817445
WOLFE (weak): th(4.308869380063768)=0.10853395295817445; dx=-2.751255356201757E-5 evalInputDelta=1.186680616654806E-4
New Minimum: 0.10853395295817445 > 0.10829733634975405
WOLFE (weak): th(12.926608140191302)=0.10829733634975405; dx=-2.740136307050584E-5 evalInputDelta=3.5528467008587306E-4
New Minimum: 0.10829733634975405 > 0.10724434905968175
WOLFE (weak): th(51.70643256076521)=0.10724434905968175; dx=-2.6905901893223584E-5 evalInputDelta=0.0014082719601581778
New Minimum: 0.10724434905968175 > 0.10194293687170992
END: th(258.53216280382605)=0.10194293687170992; dx=-2.4394957653865473E-5 evalInputDelta=0.006709684148130002
Fitness changed from 0.10865262101983993 to 0.10194293687170992
Iteration 1 complete. Error: 0.10194293687170992 Total: 0.3565; Orientation: 0.0128; Line Search: 0.2295
th(0)=0.10194293687170992;dx=-2.162268609013506E-5
New Minimum: 0.10194293687170992 > 0.09118841654485552
END: th(556.9906600335335)=0.09118841654485552; dx=-1.7147047548929363E-5 evalInputDelta=0.010754520326854408
Fitness changed from 0.10194293687170992 to 0.09118841654485552
Iteration 2 complete. Error: 0.09118841654485552 Total: 0.0792; Orientation: 0.0048; Line Search: 0.0576
th(0)=0.09118841654485552;dx=-1.368603116856748E-5
New Minimum: 0.09118841654485552 > 0.07757193458257992
END: th(1200.0)=0.07757193458257992; dx=-9.262678580960757E-6 evalInputDelta=0.013616481962275592
Fitness changed from 0.09118841654485552 to 0.07757193458257992
Iteration 3 complete. Error: 0.07757193458257992 Total: 0.2386; Orientation: 0.0073; Line Search: 0.0947
th(0)=0.07757193458257992;dx=-6.410304719306324E-6
New Minimum: 0.07757193458257992 > 0.0647585908614881
END: th(2585.3216280382608)=0.0647585908614881; dx=-3.7108288577908756E-6 evalInputDelta=0.01281334372109183
Fitness changed from 0.07757193458257992 to 0.0647585908614881
Iteration 4 complete. Error: 0.0647585908614881 Total: 0.0597; Orientation: 0.0047; Line Search: 0.0393
th(0)=0.0647585908614881;dx=-2.3003122715767832E-6
New Minimum: 0.0647585908614881 > 0.05526640228665584
END: th(5569.9066003353355)=0.05526640228665584; dx=-1.1386164322722797E-6 evalInputDelta=0.009492188574832257
Fitness changed from 0.0647585908614881 to 0.05526640228665584
Iteration 5 complete. Error: 0.05526640228665584 Total: 0.1736; Orientation: 0.0037; Line Search: 0.1576
th(0)=0.05526640228665584;dx=-7.977086888282303E-7
New Minimum: 0.05526640228665584 > 0.0506003008408916
WOLF (strong): th(12000.000000000002)=0.0506003008408916; dx=2.7149384583091394E-7 evalInputDelta=0.004666101445764237
END: th(6000.000000000001)=0.05141342679761481; dx=-4.5057508676807647E-7 evalInputDelta=0.0038529754890410245
Fitness changed from 0.05526640228665584 to 0.0506003008408916
Iteration 6 complete. Error: 0.0506003008408916 Total: 0.2246; Orientation: 0.0042; Line Search: 0.2064
th(0)=0.0506003008408916;dx=-2.540350496754964E-6
Armijo: th(12926.608140191305)=0.052938166925708866; dx=1.1527335822003642E-6 evalInputDelta=-0.0023378660848172653
New Minimum: 0.0506003008408916 > 0.04909393824548925
WOLF (strong): th(6463.304070095653)=0.04909393824548925; dx=2.41066962761795E-7 evalInputDelta=0.0015063625954023496
New Minimum: 0.04909393824548925 > 0.04841539673929719
END: th(2154.434690031884)=0.04841539673929719; dx=-1.2035281066327076E-7 evalInputDelta=0.002184904101594408
Fitness changed from 0.0506003008408916 to 0.04841539673929719
Iteration 7 complete. Error: 0.04841539673929719 Total: 0.3337; Orientation: 0.0071; Line Search: 0.3143
th(0)=0.04841539673929719;dx=-3.977736548272524E-7
New Minimum: 0.04841539673929719 > 0.04694505532495576
END: th(4641.58883361278)=0.04694505532495576; dx=-2.0179257377871863E-7 evalInputDelta=0.0014703414143414292
Fitness changed from 0.04841539673929719 to 0.04694505532495576
Iteration 8 complete. Error: 0.04694505532495576 Total: 0.2844; Orientation: 0.0042; Line Search: 0.2673
th(0)=0.04694505532495576;dx=-2.3781177934833354E-7
New Minimum: 0.04694505532495576 > 0.04548830015361836
WOLF (strong): th(10000.000000000004)=0.04548830015361836; dx=1.7543665376549417E-8 evalInputDelta=0.0014567551713373997
END: th(5000.000000000002)=0.04591207653875355; dx=-1.632216224127278E-7 evalInputDelta=0.0010329787862022113
Fitness changed from 0.04694505532495576 to 0.04548830015361836
Iteration 9 complete. Error: 0.04548830015361836 Total: 0.3619; Orientation: 0.0039; Line Search: 0.3463
th(0)=0.04548830015361836;dx=-4.94833079185323E-7
Armijo: th(10772.173450159424)=0.04576235839156307; dx=3.3739840750669694E-7 evalInputDelta=-2.7405823794470624E-4
New Minimum: 0.04548830015361836 > 0.044756457819270214
WOLF (strong): th(5386.086725079712)=0.044756457819270214; dx=6.39303603723412E-8 evalInputDelta=7.318423343481498E-4
END: th(1795.3622416932374)=0.04490214338766442; dx=-1.9025623276821844E-7 evalInputDelta=5.861567659539435E-4
Fitness changed from 0.04548830015361836 to 0.044756457819270214
Iteration 10 complete. Error: 0.044756457819270214 Total: 0.3486; Orientation: 0.0041; Line Search: 0.3315
th(0)=0.044756457819270214;dx=-2.4093804726271303E-7
New Minimum: 0.044756457819270214 > 0.044036723802033204
END: th(3867.990694677318)=0.044036723802033204; dx=-1.2536341817216134E-7 evalInputDelta=7.197340172370101E-4
Fitness changed from 0.044756457819270214 to 0.044036723802033204
Iteration 11 complete. Error: 0.044036723802033204 Total: 0.1515; Orientation: 0.0037; Line Search: 0.1370
th(0)=0.044036723802033204;dx=-1.5315154100722408E-7
New Minimum: 0.044036723802033204 > 0.043249586122233945
WOLF (strong): th(8333.33333333334)=0.043249586122233945; dx=6.0225857857372025E-9 evalInputDelta=7.871376797992588E-4
END: th(4166.66666666667)=0.043482816055300286; dx=-1.0481511364124287E-7 evalInputDelta=5.539077467329179E-4
Fitness changed from 0.044036723802033204 to 0.043249586122233945
Iteration 12 complete. Error: 0.043249586122233945 Total: 0.3749; Orientation: 0.0037; Line Search: 0.3604
th(0)=0.043249586122233945;dx=-2.7240725135320297E-7
New Minimum: 0.043249586122233945 > 0.04317368067409795
WOLF (strong): th(8976.81120846619)=0.04317368067409795; dx=1.3829404859467205E-7 evalInputDelta=7.59054481359947E-5
New Minimum: 0.04317368067409795 > 0.04279014734250651
WOLF (strong): th(4488.405604233095)=0.04279014734250651; dx=1.9878194371578154E-8 evalInputDelta=4.594387797274349E-4
END: th(1496.1352014110316)=0.04294528961653616; dx=-1.4145526751837263E-7 evalInputDelta=3.0429650569778477E-4
Fitness changed from 0.043249586122233945 to 0.04279014734250651
Iteration 13 complete. Error: 0.04279014734250651 Total: 0.5192; Orientation: 0.0035; Line Search: 0.5054
th(0)=0.04279014734250651;dx=-1.5722463480059958E-7
New Minimum: 0.04279014734250651 > 0.042370484335324664
END: th(3223.325578897766)=0.042370484335324664; dx=-9.66551297955287E-8 evalInputDelta=4.1966300718184607E-4
Fitness changed from 0.04279014734250651 to 0.042370484335324664
Iteration 14 complete. Error: 0.042370484335324664 Total: 0.2946; Orientation: 0.0033; Line Search: 0.0854
th(0)=0.042370484335324664;dx=-9.341372353314854E-8
New Minimum: 0.042370484335324664 > 0.041814001339213605
END: th(6944.444444444451)=0.041814001339213605; dx=-5.350578768095869E-8 evalInputDelta=5.564829961110587E-4
Fitness changed from 0.042370484335324664 to 0.041814001339213605
Iteration 15 complete. Error: 0.041814001339213605 Total: 0.4283; Orientation: 0.0043; Line Search: 0.4131
th(0)=0.041814001339213605;dx=-9.435815767573387E-8
New Minimum: 0.041814001339213605 > 0.0418017585278437
WOLF (strong): th(14961.352014110318)=0.0418017585278437; dx=1.4881717347236713E-7 evalInputDelta=1.2242811369904405E-5
New Minimum: 0
...skipping 48509 bytes...
; Line Search: 0.3096
th(0)=0.03616141383759762;dx=-2.3961800850472305E-9
New Minimum: 0.03616141383759762 > 0.036145904168692344
END: th(6874.35299755077)=0.036145904168692344; dx=-1.9115384908440087E-9 evalInputDelta=1.5509668905275276E-5
Fitness changed from 0.03616141383759762 to 0.036145904168692344
Iteration 110 complete. Error: 0.036145904168692344 Total: 0.0677; Orientation: 0.0035; Line Search: 0.0541
th(0)=0.036145904168692344;dx=-2.47836524346076E-9
New Minimum: 0.036145904168692344 > 0.036131730722991515
WOLF (strong): th(14810.344569448045)=0.036131730722991515; dx=3.0317256529226852E-9 evalInputDelta=1.4173445700829679E-5
New Minimum: 0.036131730722991515 > 0.03612956958511565
END: th(7405.172284724023)=0.03612956958511565; dx=-1.5603252192523798E-9 evalInputDelta=1.6334583576693706E-5
Fitness changed from 0.036145904168692344 to 0.03612956958511565
Iteration 111 complete. Error: 0.03612956958511565 Total: 0.2529; Orientation: 0.0670; Line Search: 0.1756
th(0)=0.03612956958511565;dx=-2.2763495282071696E-9
New Minimum: 0.03612956958511565 > 0.036108777826530866
WOLF (strong): th(15953.960055872096)=0.036108777826530866; dx=1.0684253065093698E-9 evalInputDelta=2.0791758584784437E-5
END: th(7976.980027936048)=0.036113180368834784; dx=-1.6430933685684668E-9 evalInputDelta=1.6389216280866814E-5
Fitness changed from 0.03612956958511565 to 0.036108777826530866
Iteration 112 complete. Error: 0.036108777826530866 Total: 0.0599; Orientation: 0.0035; Line Search: 0.0466
th(0)=0.036108777826530866;dx=-7.2495578135252054E-9
Armijo: th(17185.88249387693)=0.03629423332895573; dx=3.355405833304057E-8 evalInputDelta=-1.8545550242486664E-4
Armijo: th(8592.941246938464)=0.0361196527831979; dx=9.584880866098204E-9 evalInputDelta=-1.087495666703131E-5
New Minimum: 0.036108777826530866 > 0.036096589412290464
END: th(2864.313748979488)=0.036096589412290464; dx=-1.3972892334283314E-9 evalInputDelta=1.2188414240402179E-5
Fitness changed from 0.036108777826530866 to 0.036096589412290464
Iteration 113 complete. Error: 0.036096589412290464 Total: 0.6208; Orientation: 0.0084; Line Search: 0.4825
th(0)=0.036096589412290464;dx=-1.3850199638591546E-9
New Minimum: 0.036096589412290464 > 0.03608822565937779
WOLFE (weak): th(6170.9769039366865)=0.03608822565937779; dx=-1.2964507928707783E-9 evalInputDelta=8.363752912673994E-6
New Minimum: 0.03608822565937779 > 0.03608116147877093
END: th(12341.953807873373)=0.03608116147877093; dx=-9.259709238106614E-10 evalInputDelta=1.5427933519530923E-5
Fitness changed from 0.036096589412290464 to 0.03608116147877093
Iteration 114 complete. Error: 0.03608116147877093 Total: 0.2843; Orientation: 0.0032; Line Search: 0.2331
th(0)=0.03608116147877093;dx=-4.159712802140527E-9
Armijo: th(26589.9334264535)=0.03659770844465545; dx=7.642479506283216E-8 evalInputDelta=-5.165469658845179E-4
Armijo: th(13294.96671322675)=0.03611541150418718; dx=1.1527780974905045E-8 evalInputDelta=-3.4250025416245056E-5
New Minimum: 0.03608116147877093 > 0.036071950818694506
END: th(4431.655571075583)=0.036071950818694506; dx=-6.462005752046489E-11 evalInputDelta=9.210660076426624E-6
Fitness changed from 0.03608116147877093 to 0.036071950818694506
Iteration 115 complete. Error: 0.036071950818694506 Total: 0.1603; Orientation: 0.0030; Line Search: 0.1480
th(0)=0.036071950818694506;dx=-2.025119954895405E-9
Armijo: th(9547.712496598295)=0.03607787716782288; dx=4.198264142935429E-9 evalInputDelta=-5.926349128375363E-6
New Minimum: 0.036071950818694506 > 0.036067565847603775
WOLF (strong): th(4773.856248299147)=0.036067565847603775; dx=3.8840582056228905E-10 evalInputDelta=4.384971090731771E-6
END: th(1591.2854160997158)=0.036069247910268075; dx=-1.3519421384107325E-9 evalInputDelta=2.7029084264315606E-6
Fitness changed from 0.036071950818694506 to 0.036067565847603775
Iteration 116 complete. Error: 0.036067565847603775 Total: 0.1653; Orientation: 0.0038; Line Search: 0.1451
th(0)=0.036067565847603775;dx=-1.2987414610372606E-9
New Minimum: 0.036067565847603775 > 0.036064858042391935
END: th(3428.3205021870485)=0.036064858042391935; dx=-3.177164464107131E-10 evalInputDelta=2.707805211839276E-6
Fitness changed from 0.036067565847603775 to 0.036064858042391935
Iteration 117 complete. Error: 0.036064858042391935 Total: 0.1520; Orientation: 0.0035; Line Search: 0.1381
th(0)=0.036064858042391935;dx=-7.132742406882099E-10
New Minimum: 0.036064858042391935 > 0.03606062080219977
END: th(7386.092618459306)=0.03606062080219977; dx=-4.193236661702146E-10 evalInputDelta=4.237240192161984E-6
Fitness changed from 0.036064858042391935 to 0.03606062080219977
Iteration 118 complete. Error: 0.03606062080219977 Total: 0.1059; Orientation: 0.0033; Line Search: 0.0927
th(0)=0.03606062080219977;dx=-1.1006890922739178E-9
New Minimum: 0.03606062080219977 > 0.036059619108285104
WOLF (strong): th(15912.85416099716)=0.036059619108285104; dx=6.883117361364641E-10 evalInputDelta=1.001693914669255E-6
New Minimum: 0.036059619108285104 > 0.036056616130468545
WOLF (strong): th(7956.42708049858)=0.036056616130468545; dx=8.491553221744116E-12 evalInputDelta=4.004671731228093E-6
END: th(2652.142360166193)=0.036058283029893834; dx=-6.728490824430035E-10 evalInputDelta=2.3377723059395117E-6
Fitness changed from 0.03606062080219977 to 0.036056616130468545
Iteration 119 complete. Error: 0.036056616130468545 Total: 0.2696; Orientation: 0.0045; Line Search: 0.2539
th(0)=0.036056616130468545;dx=-2.258454421247836E-9
New Minimum: 0.036056616130468545 > 0.03605148312166808
WOLF (strong): th(5713.867503645081)=0.03605148312166808; dx=7.647014996993793E-10 evalInputDelta=5.1330088004647445E-6
END: th(2856.9337518225407)=0.03605190024632902; dx=-9.739919043788171E-10 evalInputDelta=4.715884139527837E-6
Fitness changed from 0.036056616130468545 to 0.03605148312166808
Iteration 120 complete. Error: 0.03605148312166808 Total: 0.0590; Orientation: 0.0035; Line Search: 0.0457
th(0)=0.03605148312166808;dx=-3.058650570930798E-9
New Minimum: 0.03605148312166808 > 0.03604793586128768
WOLF (strong): th(6155.077182049422)=0.03604793586128768; dx=1.3427380970386463E-9 evalInputDelta=3.547260380397377E-6
New Minimum: 0.03604793586128768 > 0.03604635015358757
END: th(3077.538591024711)=0.03604635015358757; dx=-4.357552703489785E-10 evalInputDelta=5.132968080509559E-6
Fitness changed from 0.03605148312166808 to 0.03604635015358757
Iteration 121 complete. Error: 0.03604635015358757 Total: 0.2713; Orientation: 0.0035; Line Search: 0.1966
th(0)=0.03604635015358757;dx=-1.3080107861609554E-9
New Minimum: 0.03604635015358757 > 0.03603911602598062
END: th(6630.355900415484)=0.03603911602598062; dx=-8.284084965035887E-10 evalInputDelta=7.234127606947638E-6
Fitness changed from 0.03604635015358757 to 0.03603911602598062
Iteration 122 complete. Error: 0.03603911602598062 Total: 0.3164; Orientation: 0.1604; Line Search: 0.1309
th(0)=0.03603911602598062;dx=-1.7824709209686516E-9
New Minimum: 0.03603911602598062 > 0.03603440951405065
WOLF (strong): th(14284.668759112705)=0.03603440951405065; dx=9.510159418653578E-10 evalInputDelta=4.706511929970791E-6
New Minimum: 0.03603440951405065 > 0.03603201178774825
END: th(7142.334379556352)=0.03603201178774825; dx=-2.861977394626668E-10 evalInputDelta=7.104238232372162E-6
Fitness changed from 0.03603911602598062 to 0.03603201178774825
Iteration 123 complete. Error: 0.03603201178774825 Total: 0.2591; Orientation: 0.0032; Line Search: 0.2465
th(0)=0.03603201178774825;dx=-2.5116912128843844E-9
Armijo: th(15387.692955123557)=0.036078864944109286; dx=1.203265105309293E-8 evalInputDelta=-4.685315636103471E-5
New Minimum: 0.03603201178774825 > 0.03602824292620764
WOLF (strong): th(7693.846477561779)=0.03602824292620764; dx=2.193468587163203E-9 evalInputDelta=3.7688615406109016E-6
New Minimum: 0.03602824292620764 > 0.03602695389038974
END: th(2564.615492520593)=0.03602695389038974; dx=-1.3714522014048465E-9 evalInputDelta=5.0578973585097E-6
Fitness changed from 0.03603201178774825 to 0.03602695389038974
Iteration 124 complete. Error: 0.03602695389038974 Total: 0.2718; Orientation: 0.0031; Line Search: 0.2589
Final threshold in iteration 124: 0.03602695389038974 (> 0.0) after 30.085s (< 30.000s)
Returns
0.03602695389038974
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.8108619228504756 ], [ -0.1267906201054001 ], [ 0.2305728522936061 ], [ 0.0056659439836565725 ], [ 0.9586317940479918 ], [ 0.4192272079959039 ], [ -0.7697358829734293 ], [ 0.17410229118732007 ], ... ],
[ [ -0.13412817208966965 ], [ -0.9985336908190898 ], [ 0.09478382823694617 ], [ -0.2409587440511085 ], [ 0.11903053764881925 ], [ 0.15432517910452917 ], [ 0.24217427190608337 ], [ 0.2421792819994192 ], ... ],
[ [ -0.9716928429424472 ], [ 0.24216137212574074 ], [ -0.26006905030270744 ], [ 0.2420800243319142 ], [ -0.35221163799381805 ], [ -0.6789646273454881 ], [ -0.7442865409627577 ], [ -0.9318518487575018 ], ... ],
[ [ 0.6919367001284729 ], [ -0.9549029136179537 ], [ 0.24155385892137565 ], [ -0.020027441790561262 ], [ -0.6763435806477174 ], [ 0.5155018296177144 ], [ 0.1637417261108316 ], [ -0.7818685724083476 ], ... ],
[ [ -0.413601797183482 ], [ -0.3058333917293983 ], [ 0.5455325051071519 ], [ -0.8370564486484813 ], [ 0.24195163414004417 ], [ -0.8761926701304987 ], [ 0.24169201036897411 ], [ 0.09172477232463369 ], ... ],
[ [ -0.28814023972097247 ], [ 0.2421817485030779 ], [ -0.4495636080218268 ], [ 0.41686513114643414 ], [ 0.28885606250250767 ], [ 0.2420975779926643 ], [ -0.5670274712933507 ], [ 0.23978478794266003 ], ... ],
[ [ 0.34913904195200557 ], [ -0.23709827193250987 ], [ -0.7163364580171203 ], [ -0.9187047114195044 ], [ -0.9225574886039023 ], [ -0.3895176037132879 ], [ -0.40298765498145506 ], [ -0.10640320308111306 ], ... ],
[ [ 0.669699568622157 ], [ 0.24218147760933798 ], [ 0.03432348686950868 ], [ 0.5288203649323465 ], [ 0.7346021419210038 ], [ 0.647862491206429 ], [ -0.07543045364552214 ], [ -0.7847549433002523 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.5175451840957728 ], [ 0.0 ], [ 0.009958704233184058 ], [ 8.743729012545765E-8 ], [ 0.8757043760264851 ], [ 0.06514633670164965 ], [ 0.0 ], [ 0.00412018723310178 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 6.099803193455717E-4 ], [ 0.0 ], [ 0.0012476569931182601 ], [ 0.002820973331418724 ], [ 0.011619329237640646 ], [ 0.011620084431352479 ], ... ],
[ [ 0.0 ], [ 0.011617384950196314 ], [ 0.0 ], [ 0.011605129114306549 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.31445149988683774 ], [ 0.0 ], [ 0.011526069976376865 ], [ 0.0 ], [ 0.0 ], [ 0.1247224246225701 ], [ 0.0033978862175471337 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.14900389919367357 ], [ 0.0 ], [ 0.011585803821524812 ], [ 0.0 ], [ 0.011546792303781318 ], [ 5.50244433688562E-4 ], ... ],
[ [ 0.0 ], [ 0.011620456230739406 ], [ 0.0 ], [ 0.06400013264432235 ], [ 0.020215334009890215 ], [ 0.011607772997817482 ], [ 0.0 ], [ 0.011262950669766373 ], ... ],
[ [ 0.03666799644662126 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.283782720282616 ], [ 0.011620415395971606 ], [ 2.5085612109756393E-5 ], [ 0.13512850886210132 ], [ 0.3794814445430804 ], [ 0.2557148001023676 ], [ 0.0 ], [ 0.0 ], ... ],
...
]
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 30.07 seconds (0.160 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2928530381971
Reset training subject: 2928542070355
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.10865262101983993}, derivative=-2.7568298031681143E-5}
New Minimum: 0.10865262101983993 > 0.10865262101983716
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.10865262101983716}, derivative=-2.7568298031679848E-5}, evalInputDelta = -2.761679773755077E-15
New Minimum: 0.10865262101983716 > 0.10865262101982069
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.10865262101982069}, derivative=-2.756829803167208E-5}, evalInputDelta = -1.9234613901630837E-14
New Minimum: 0.10865262101982069 > 0.10865262101970492
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.10865262101970492}, derivative=-2.7568298031617693E-5}, evalInputDelta = -1.3500311979441904E-13
New Minimum: 0.10865262101970492 > 0.10865262101889439
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.10865262101889439}, derivative=-2.7568298031237002E-5}, evalInputDelta = -9.455353167098224E-13
New Minimum: 0.10865262101889439 > 0.10865262101322078
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.10865262101322078}, derivative=-2.756829802857216E-5}, evalInputDelta = -6.619149672815183E-12
New Minimum: 0.10865262101322078 > 0.10865262097350585
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.10865262097350585}, derivative=-2.756829800991825E-5}, evalInputDelta = -4.63340754652819E-11
New Minimum: 0.10865262097350585 > 0.10865262069550179
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.10865262069550179}, derivative=-2.7568297879340912E-5}, evalInputDelta = -3.2433813967891467E-10
New Minimum: 0.10865262069550179 > 0.10865261874947214
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.10865261874947214}, derivative=-2.756829696529954E-5}, evalInputDelta = -2.2703677826640956E-9
New Minimum: 0.10865261874947214 > 0.10865260512726682
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.10865260512726682}, derivative=-2.7568290567010703E-5}, evalInputDelta = -1.5892573104747676E-8
New Minimum: 0.10865260512726682 > 0.10865250977191868
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.10865250977191868}, derivative=-2.7568245779025547E-5}, evalInputDelta = -1.1124792125005722E-7
New Minimum: 0.10865250977191868 > 0.10865184228882088
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.10865184228882088}, derivative=-2.756793226492784E-5}, evalInputDelta = -7.78731019043799E-7
New Minimum: 0.10865184228882088 > 0.10864717011967276
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.10864717011967276}, derivative=-2.7565737754362997E-5}, evalInputDelta = -5.450900167167094E-6
New Minimum: 0.10864717011967276 > 0.10861447534804292
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.10861447534804292}, derivative=-2.755038049758042E-5}, evalInputDelta = -3.814567179701023E-5
New Minimum: 0.10861447534804292 > 0.10838612147380508
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.10838612147380508}, derivative=-2.744309101353023E-5}, evalInputDelta = -2.664995460348457E-4
New Minimum: 0.10838612147380508 > 0.10681237903084757
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.10681237903084757}, derivative=-2.6702341108200736E-5}, evalInputDelta = -0.0018402419889923527
New Minimum: 0.10681237903084757 > 0.0969316522851276
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0969316522851276}, derivative=-2.199468181859579E-5}, evalInputDelta = -0.011720968734712331
New Minimum: 0.0969316522851276 > 0.06348584339741478
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.06348584339741478}, derivative=-5.182467243463914E-6}, evalInputDelta = -0.045166777622425144
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.09342041567444709}, derivative=7.778273342706397E-6}, evalInputDelta = -0.015232205345392838
0.09342041567444709 <= 0.10865262101983993
F(18143.84561640843) = LineSearchPoint{point=PointSample{avg=0.06584171008234356}, derivative=3.397906826851993E-6}, evalInputDelta = -0.042810910937496366
Right bracket at 18143.84561640843
New Minimum: 0.06348584339741478 > 0.06024375815228716
F(16152.93012750718) = LineSearchPoint{point=PointSample{avg=0.06024375815228716}, derivative=2.27026802722331E-6}, evalInputDelta = -0.04840886286755276
Right bracket at 16152.93012750718
New Minimum: 0.06024375815228716 > 0.05781374438065048
F(14923.934044315436) = LineSearchPoint{point=PointSample{avg=0.05781374438065048}, derivative=1.6986534981948665E-6}, evalInputDelta = -0.05083887663918945
Right bracket at 14923.934044315436
New Minimum: 0.05781374438065048 > 0.05649828645255446
F(14057.74910034108) = LineSearchPoint{point=PointSample{avg=0.05649828645255446}, derivative=1.3450553781747441E-6}, evalInputDelta = -0.052154334567285465
Right bracket at 14057.74910034108
New Minimum: 0.05649828645255446 > 0.05569916342903336
F(13403.779608652872) = LineSearchPoint{point=PointSample{avg=0.05569916342903336}, derivative=1.1021128592362254E-6}, evalInputDelta = -0.052953457590806566
Right bracket at 13403.779608652872
New Minimum: 0.05569916342903336 > 0.055177694027652525
F(12888.527911519208) = LineSearchPoint{point=PointSample{avg=0.055177694027652525}, derivative=9.238560036632074E-7}, evalInputDelta = -0.0534749269921874
Right bracket at 12888.527911519208
New Minimum: 0.055177694027652525 > 0.054820431062452844
F(12470.618339829154) = LineSearchPoint{point=PointSample{avg=0.054820431062452844}, derivative=7.870007128641024E-7}, evalInputDelta = -0.05383218995738708
Right bracket at 12470.618339829154
New Minimum: 0.054820431062452844 > 0.0545669433676258
F(12124.496593353473) = LineSearchPoint{point=PointSample{avg=0.0545669433676258}, derivative=6.784143402003453E-7}, evalInputDelta = -0.05408567765221413
Right bracket at 12124.496593353473
New Minimum: 0.0545669433676258 > 0.05438231213494814
F(11833.296957503839) = LineSearchPoint{point=PointSample{avg=0.05438231213494814}, derivative=5.901000478548959E-7}, evalInputDelta = -0.05427030888489179
Right bracket at 11833.296957503839
New Minimum: 0.05438231213494814 > 0.05424509030046866
F(11585.313067185201) = LineSearchPoint{point=PointSample{avg=0.05424509030046866}, derivative=5.168913654460494E-7}, evalInputDelta = -0.054407530719371267
Right bracket at 11585.313067185201
New Minimum: 0.05424509030046866 > 0.05414146720880954
F(11372.092205266077) = LineSearchPoint{point=PointSample{avg=0.05414146720880954}, derivative=4.5528370122312845E-7}, evalInputDelta = -0.05451115381103039
Right bracket at 11372.092205266077
New Minimum: 0.05414146720880954 > 0.05406221054246996
F(11187.336085252162) = LineSearchPoint{point=PointSample{avg=0.05406221054246996}, derivative=4.0280990450202554E-7}, evalInputDelta = -0.05459041047736997
Loops = 12
Fitness changed from 0.10865262101983993 to 0.05406221054246996
Iteration 1 complete. Error: 0.05406221054246996 Total: 3.5018; Orientation: 0.0047; Line Search: 3.4622
F(0.0) = LineSearchPoint{point=PointSample{avg=0.05406221054246996}, derivative=-2.8581527760603585E-6}
New Minimum: 0.05406221054246996 > 0.05361411918354018
F(11187.336085252162) = LineSearchPoint{point=PointSample{avg=0.05361411918354018}, derivative=4.7626619142827877E-7}, evalInputDelta = -4.480913589297794E-4
0.05361411918354018 <= 0.05406221054246996
New Minimum: 0.05361411918354018 > 0.05299805012062967
F(9589.411528829618) = LineSearchPoint{point=PointSample{avg=0.05299805012062967}, derivative=3.066901185918888E-7}, evalInputDelta = -0.0010641604218402864
Right bracket at 9589.411528829618
New Minimum: 0.05299805012062967 > 0.05274552510451652
F(8660.146520455059) = LineSearchPoint{point=PointSample{avg=0.05274552510451652}, derivative=2.4086893885897566E-7}, evalInputDelta = -0.0013166854379534354
Right bracket at 8660.146520455059
New Minimum: 0.05274552510451652 > 0.05259497947904308
F(7987.043685227086) = LineSearchPoint{point=PointSample{avg=0.05259497947904308}, derivative=2.0863569220731696E-7}, evalInputDelta = -0.0014672310634268743
Right bracket
...skipping 48648 bytes...
85893 to 0.03805906425469684
Iteration 36 complete. Error: 0.03805906425469684 Total: 0.3535; Orientation: 0.0033; Line Search: 0.2847
F(0.0) = LineSearchPoint{point=PointSample{avg=0.03805906425469684}, derivative=-2.2233874694895515E-8}
New Minimum: 0.03805906425469684 > 0.03799415828603493
F(4916.258712258692) = LineSearchPoint{point=PointSample{avg=0.03799415828603493}, derivative=-3.5922261701768345E-9}, evalInputDelta = -6.490596866190468E-5
F(34413.810985810844) = LineSearchPoint{point=PointSample{avg=0.04184675744163861}, derivative=3.9360679874098545E-7}, evalInputDelta = 0.0037876931869417763
F(2647.2162296777574) = LineSearchPoint{point=PointSample{avg=0.038012736872077214}, derivative=-1.263166955835237E-8}, evalInputDelta = -4.632738261962227E-5
F(18530.513607744302) = LineSearchPoint{point=PointSample{avg=0.0384685932118502}, derivative=8.71564298077151E-8}, evalInputDelta = 4.095289571533628E-4
F(1425.4241236726386) = LineSearchPoint{point=PointSample{avg=0.038030959668874695}, derivative=-1.716497535867591E-8}, evalInputDelta = -2.8104585822141814E-5
F(9977.96886570847) = LineSearchPoint{point=PointSample{avg=0.038035038494411755}, derivative=2.0964360256805852E-8}, evalInputDelta = -2.4025760285081854E-5
0.038035038494411755 <= 0.03805906425469684
New Minimum: 0.03799415828603493 > 0.037993471665337966
F(5135.60125124958) = LineSearchPoint{point=PointSample{avg=0.037993471665337966}, derivative=-2.6667986021797445E-9}, evalInputDelta = -6.559258935887047E-5
Left bracket at 5135.60125124958
New Minimum: 0.037993471665337966 > 0.03799265371536424
F(5682.067011246328) = LineSearchPoint{point=PointSample{avg=0.03799265371536424}, derivative=-3.1575457593124167E-10}, evalInputDelta = -6.641053933259533E-5
Left bracket at 5682.067011246328
New Minimum: 0.03799265371536424 > 0.03799264246328836
F(5745.809649806303) = LineSearchPoint{point=PointSample{avg=0.03799264246328836}, derivative=-3.713614792993942E-11}, evalInputDelta = -6.642179140847748E-5
Left bracket at 5745.809649806303
Converged to left
Fitness changed from 0.03805906425469684 to 0.03799264246328836
Iteration 37 complete. Error: 0.03799264246328836 Total: 1.0608; Orientation: 0.0031; Line Search: 1.0481
F(0.0) = LineSearchPoint{point=PointSample{avg=0.03799264246328836}, derivative=-2.840210168455205E-8}
New Minimum: 0.03799264246328836 > 0.03792567168988308
F(5745.809649806303) = LineSearchPoint{point=PointSample{avg=0.03792567168988308}, derivative=5.860708728217048E-9}, evalInputDelta = -6.697077340527685E-5
0.03792567168988308 <= 0.03799264246328836
New Minimum: 0.03792567168988308 > 0.037923021998693955
F(4762.9796846163) = LineSearchPoint{point=PointSample{avg=0.037923021998693955}, derivative=-4.209909626169155E-10}, evalInputDelta = -6.962046459440402E-5
Left bracket at 4762.9796846163
New Minimum: 0.037923021998693955 > 0.03792300784260587
F(4828.8476110531265) = LineSearchPoint{point=PointSample{avg=0.03792300784260587}, derivative=-8.650695789532377E-12}, evalInputDelta = -6.963462068249049E-5
Left bracket at 4828.8476110531265
Converged to left
Fitness changed from 0.03799264246328836 to 0.03792300784260587
Iteration 38 complete. Error: 0.03792300784260587 Total: 0.6935; Orientation: 0.0029; Line Search: 0.5832
F(0.0) = LineSearchPoint{point=PointSample{avg=0.03792300784260587}, derivative=-2.3008395176146705E-8}
New Minimum: 0.03792300784260587 > 0.03786041235973832
F(4828.8476110531265) = LineSearchPoint{point=PointSample{avg=0.03786041235973832}, derivative=-1.882905388576407E-9}, evalInputDelta = -6.259548286755012E-5
F(33801.933277371885) = LineSearchPoint{point=PointSample{avg=0.04319528666317969}, derivative=5.676851968973226E-7}, evalInputDelta = 0.0052722788205738225
F(2600.148713643991) = LineSearchPoint{point=PointSample{avg=0.03787662943558502}, derivative=-1.2410118303090411E-8}, evalInputDelta = -4.637840702084728E-5
F(18201.040995507938) = LineSearchPoint{point=PointSample{avg=0.03850001261467739}, derivative=1.1907474157518872E-7}, evalInputDelta = 5.770047720715238E-4
F(1400.0800765775336) = LineSearchPoint{point=PointSample{avg=0.03789460820209708}, derivative=-1.7492742408459878E-8}, evalInputDelta = -2.8399640508790236E-5
F(9800.560536042736) = LineSearchPoint{point=PointSample{avg=0.03792245129427524}, derivative=2.8792815606052688E-8}, evalInputDelta = -5.565483306296803E-7
0.03792245129427524 <= 0.03792300784260587
F(4353.086855616646) = LineSearchPoint{point=PointSample{avg=0.03786187785708599}, derivative=-4.264575196953429E-9}, evalInputDelta = -6.112998551988102E-5
Left bracket at 4353.086855616646
New Minimum: 0.03786041235973832 > 0.03786011678791332
F(5055.839268293123) = LineSearchPoint{point=PointSample{avg=0.03786011678791332}, derivative=-7.182234065998188E-10}, evalInputDelta = -6.289105469255152E-5
Left bracket at 5055.839268293123
New Minimum: 0.03786011678791332 > 0.03786006843177701
F(5171.313680116735) = LineSearchPoint{point=PointSample{avg=0.03786006843177701}, derivative=-1.1847314823645996E-10}, evalInputDelta = -6.293941082886195E-5
Left bracket at 5171.313680116735
Converged to left
Fitness changed from 0.03792300784260587 to 0.03786006843177701
Iteration 39 complete. Error: 0.03786006843177701 Total: 1.0885; Orientation: 0.0703; Line Search: 1.0088
F(0.0) = LineSearchPoint{point=PointSample{avg=0.03786006843177701}, derivative=-2.186744420483432E-8}
New Minimum: 0.03786006843177701 > 0.037806096533708665
F(5171.313680116735) = LineSearchPoint{point=PointSample{avg=0.037806096533708665}, derivative=1.4007628205759756E-9}, evalInputDelta = -5.3971898068341795E-5
0.037806096533708665 <= 0.03786006843177701
New Minimum: 0.037806096533708665 > 0.037805893368878615
F(4859.99687221947) = LineSearchPoint{point=PointSample{avg=0.037805893368878615}, derivative=-9.244016557098608E-11}, evalInputDelta = -5.417506289839141E-5
Left bracket at 4859.99687221947
Converged to left
Fitness changed from 0.03786006843177701 to 0.037805893368878615
Iteration 40 complete. Error: 0.037805893368878615 Total: 0.0976; Orientation: 0.0035; Line Search: 0.0777
F(0.0) = LineSearchPoint{point=PointSample{avg=0.037805893368878615}, derivative=-1.772667171389847E-8}
New Minimum: 0.037805893368878615 > 0.03775711773998491
F(4859.99687221947) = LineSearchPoint{point=PointSample{avg=0.03775711773998491}, derivative=-1.5629711878141363E-9}, evalInputDelta = -4.8775628893701906E-5
F(34019.97810553629) = LineSearchPoint{point=PointSample{avg=0.041824520570108006}, derivative=4.313399483529932E-7}, evalInputDelta = 0.004018627201229391
F(2616.921392733561) = LineSearchPoint{point=PointSample{avg=0.03776986875162578}, derivative=-9.610814427474946E-9}, evalInputDelta = -3.6024617252834734E-5
F(18318.449749134925) = LineSearchPoint{point=PointSample{avg=0.03824377636132879}, derivative=9.004696901068539E-8}, evalInputDelta = 4.3788299245017814E-4
F(1409.111519164225) = LineSearchPoint{point=PointSample{avg=0.037783854369108236}, derivative=-1.3502292737070183E-8}, evalInputDelta = -2.2038999770379597E-5
F(9863.780634149574) = LineSearchPoint{point=PointSample{avg=0.037804031456440665}, derivative=2.177437132327964E-8}, evalInputDelta = -1.8619124379504193E-6
0.037804031456440665 <= 0.037805893368878615
F(4426.516054143414) = LineSearchPoint{point=PointSample{avg=0.03775815446914434}, derivative=-3.2121914541166354E-9}, evalInputDelta = -4.7738899734278695E-5
Left bracket at 4426.516054143414
New Minimum: 0.03775711773998491 > 0.037756839686346064
F(5125.513150033162) = LineSearchPoint{point=PointSample{avg=0.037756839686346064}, derivative=-5.282981724497888E-10}, evalInputDelta = -4.90536825325516E-5
Left bracket at 5125.513150033162
New Minimum: 0.037756839686346064 > 0.03775680522742538
F(5237.7516447531) = LineSearchPoint{point=PointSample{avg=0.03775680522742538}, derivative=-8.515380347393501E-11}, evalInputDelta = -4.908814145323376E-5
Left bracket at 5237.7516447531
Converged to left
Fitness changed from 0.037805893368878615 to 0.03775680522742538
Iteration 41 complete. Error: 0.03775680522742538 Total: 0.8784; Orientation: 0.0033; Line Search: 0.8651
Final threshold in iteration 41: 0.03775680522742538 (> 0.0) after 30.065s (< 30.000s)
Returns
0.03775680522742538
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.8108619228504756 ], [ -0.1267906201054001 ], [ 0.279358853681073 ], [ 0.005665944077476445 ], [ 0.958636354423723 ], [ 0.4216486419897587 ], [ -0.7697358829734293 ], [ 0.13328960742401164 ], ... ],
[ [ -0.13412817208966965 ], [ -0.9985336908190898 ], [ 0.09505890202754316 ], [ -0.2409587440511085 ], [ 0.11995897166499692 ], [ 0.15815279977961227 ], [ 0.15627495121507293 ], [ -0.01457191086138121 ], ... ],
[ [ -0.9716928429424472 ], [ 0.27606655747310616 ], [ -0.26006905030270744 ], [ 0.31456175707506817 ], [ -0.35221163799381805 ], [ -0.6789646273454881 ], [ -0.7442865409627577 ], [ -0.9318518487575018 ], ... ],
[ [ 0.6919367001093645 ], [ -0.9549029136179537 ], [ 0.316385020560211 ], [ -0.020027441790561262 ], [ -0.6763435806477174 ], [ 0.50779603333862 ], [ 0.1690818281061105 ], [ -0.7818685724083476 ], ... ],
[ [ -0.413601797183482 ], [ -0.3058333917293983 ], [ 0.17477990198304985 ], [ -0.8370564486484813 ], [ 0.316915167217549 ], [ -0.8761926701304987 ], [ 0.3168030566076313 ], [ 0.09195583590856078 ], ... ],
[ [ -0.28814023972097247 ], [ -0.1947988942750658 ], [ -0.4495636080218268 ], [ 0.28317899623698306 ], [ 0.317447427647218 ], [ 0.3152147340440819 ], [ -0.5670274712933507 ], [ 0.3095100466939961 ], ... ],
[ [ 0.3747344606439272 ], [ -0.23709827193250987 ], [ -0.7163364580171203 ], [ -0.9187047114195044 ], [ -0.9225574886039023 ], [ -0.3895176037132879 ], [ -0.40298765498145506 ], [ -0.10640320308111306 ], ... ],
[ [ 0.6678479239271271 ], [ -0.16827532571082715 ], [ 0.033419114689252946 ], [ 0.5278880803840535 ], [ 0.7346021419208676 ], [ 0.6478613807640122 ], [ -0.07543045364552214 ], [ -0.7847549433002523 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.5175451840957728 ], [ 0.0 ], [ 0.018199857603558665 ], [ 8.743729467396744E-8 ], [ 0.8757174635779358 ], [ 0.06633578808151773 ], [ 0.0 ], [ 0.0017801903833315111 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 6.155589763340953E-4 ], [ 0.0 ], [ 0.0012784860788342693 ], [ 0.0030466728658035003 ], [ 0.002934463913072621 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.01753448680502431 ], [ 0.0 ], [ 0.026423881601320018 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.3144514998595567 ], [ 0.0 ], [ 0.026908035072960086 ], [ 0.0 ], [ 0.0 ], [ 0.11895855488877954 ], [ 0.0037583307814756215 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.004170775699681433 ], [ 0.0 ], [ 0.02704993788758666 ], [ 0.0 ], [ 0.027019887137576776 ], [ 5.546108125859669E-4 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.018993236779084682 ], [ 0.027192918823565716 ], [ 0.026596586026242954 ], [ 0.0 ], [ 0.025113517081616184 ], ... ],
[ [ 0.04579449731759572 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.28132503018655547 ], [ 0.0 ], [ 2.306711863380678E-5 ], [ 0.13438151487932512 ], [ 0.37948144454285937 ], [ 0.2557134231524443 ], [ 0.0 ], [ 0.0 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 30.06 seconds (0.066 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2958612995372
Reset training subject: 2958623724565
Adding measurement 4367d7e1 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.10865262101983993 < 0.10865262101983993. Total: 1
th(0)=0.10865262101983993;dx=-2.7568298031681143E-5
Adding measurement 630ca4fb to history. Total: 1
New Minimum: 0.10865262101983993 > 0.10859325696455376
WOLFE (weak): th(2.154434690031884)=0.10859325696455376; dx=-2.75404133465621E-5 evalInputDelta=5.9364055286170725E-5
Adding measurement 243adaaf to history. Total: 2
New Minimum: 0.10859325696455376 > 0.10853395295817445
WOLFE (weak): th(4.308869380063768)=0.10853395295817445; dx=-2.751255356201757E-5 evalInputDelta=1.186680616654806E-4
Adding measurement 5eaeca64 to history. Total: 3
New Minimum: 0.10853395295817445 > 0.10829733634975405
WOLFE (weak): th(12.926608140191302)=0.10829733634975405; dx=-2.740136307050584E-5 evalInputDelta=3.5528467008587306E-4
Adding measurement 7a3f1dea to history. Total: 4
New Minimum: 0.10829733634975405 > 0.10724434905968175
WOLFE (weak): th(51.70643256076521)=0.10724434905968175; dx=-2.6905901893223584E-5 evalInputDelta=0.0014082719601581778
Adding measurement 56add49d to history. Total: 5
New Minimum: 0.10724434905968175 > 0.10194293687170992
END: th(258.53216280382605)=0.10194293687170992; dx=-2.4394957653865473E-5 evalInputDelta=0.006709684148130002
Fitness changed from 0.10865262101983993 to 0.10194293687170992
Iteration 1 complete. Error: 0.10194293687170992 Total: 0.7838; Orientation: 0.0166; Line Search: 0.5796
Non-optimal measurement 0.10194293687170992 < 0.10194293687170992. Total: 6
Rejected: LBFGS Orientation magnitude: 1.142e+01, gradient 4.650e-03, dot -0.969; [2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10194293687170992, 0.10724434905968175, 0.10829733634975405, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 1.143e+01, gradient 4.650e-03, dot -0.969; [4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10194293687170992, 0.10724434905968175, 0.10829733634975405, 0.10853395295817445, 0.10859325696455376
Rejected: LBFGS Orientation magnitude: 1.145e+01, gradient 4.650e-03, dot -0.969; [308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.10194293687170992, 0.10724434905968175, 0.10829733634975405, 0.10853395295817445
LBFGS Accumulation History: 3 points
Removed measurement 56add49d to history. Total: 5
Removed measurement 7a3f1dea to history. Total: 4
Removed measurement 5eaeca64 to history. Total: 3
Adding measurement 5aeb2c08 to history. Total: 3
th(0)=0.10194293687170992;dx=-2.162268609013506E-5
Adding measurement 6ca7c1e3 to history. Total: 4
New Minimum: 0.10194293687170992 > 0.09118841654485552
END: th(556.9906600335335)=0.09118841654485552; dx=-1.7147047548929363E-5 evalInputDelta=0.010754520326854408
Fitness changed from 0.10194293687170992 to 0.09118841654485552
Iteration 2 complete. Error: 0.09118841654485552 Total: 3.7743; Orientation: 3.4601; Line Search: 0.3047
Non-optimal measurement 0.09118841654485552 < 0.09118841654485552. Total: 5
Rejected: LBFGS Orientation magnitude: 9.429e+00, gradient 3.699e-03, dot -0.974; [308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.09118841654485552, 0.10194293687170992, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 9.436e+00, gradient 3.699e-03, dot -0.974; [308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.09118841654485552, 0.10194293687170992, 0.10853395295817445, 0.10859325696455376
LBFGS Accumulation History: 3 points
Removed measurement 6ca7c1e3 to history. Total: 4
Removed measurement 5aeb2c08 to history. Total: 3
Adding measurement 4c7e8902 to history. Total: 3
th(0)=0.09118841654485552;dx=-1.368603116856748E-5
Adding measurement 70d92381 to history. Total: 4
New Minimum: 0.09118841654485552 > 0.07757193458257992
END: th(1200.0)=0.07757193458257992; dx=-9.262678580960756E-6 evalInputDelta=0.013616481962275592
Fitness changed from 0.09118841654485552 to 0.07757193458257992
Iteration 3 complete. Error: 0.07757193458257992 Total: 2.0719; Orientation: 1.8205; Line Search: 0.1310
Non-optimal measurement 0.07757193458257992 < 0.07757193458257992. Total: 5
Rejected: LBFGS Orientation magnitude: 6.866e+00, gradient 2.532e-03, dot -0.981; [1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.07757193458257992, 0.09118841654485552, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 6.871e+00, gradient 2.532e-03, dot -0.981; [b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.07757193458257992, 0.09118841654485552, 0.10853395295817445, 0.10859325696455376
LBFGS Accumulation History: 3 points
Removed measurement 70d92381 to history. Total: 4
Removed measurement 4c7e8902 to history. Total: 3
Adding measurement a6e2756 to history. Total: 3
th(0)=0.07757193458257992;dx=-6.410304719306324E-6
Adding measurement 293cf792 to history. Total: 4
New Minimum: 0.07757193458257992 > 0.0647585908614881
END: th(2585.3216280382608)=0.0647585908614881; dx=-3.7108288577908756E-6 evalInputDelta=0.01281334372109183
Fitness changed from 0.07757193458257992 to 0.0647585908614881
Iteration 4 complete. Error: 0.0647585908614881 Total: 2.6309; Orientation: 2.3179; Line Search: 0.1689
Non-optimal measurement 0.0647585908614881 < 0.0647585908614881. Total: 5
Rejected: LBFGS Orientation magnitude: 4.577e+00, gradient 1.517e-03, dot -0.986; [4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0647585908614881, 0.07757193458257992, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 4.581e+00, gradient 1.517e-03, dot -0.986; [2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 =
...skipping 8823 bytes...
16224127278E-7 evalInputDelta=0.0010329787862022113
Fitness changed from 0.04694505532495576 to 0.04548830015361836
Iteration 9 complete. Error: 0.04548830015361836 Total: 2.1460; Orientation: 2.0122; Line Search: 0.1217
Non-optimal measurement 0.04548830015361836 < 0.04548830015361836. Total: 5
Rejected: LBFGS Orientation magnitude: 1.358e+00, gradient 7.034e-04, dot -0.804; [b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.04548830015361836, 0.04694505532495576, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 1.360e+00, gradient 7.034e-04, dot -0.804; [308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.04548830015361836, 0.04694505532495576, 0.10853395295817445, 0.10859325696455376
LBFGS Accumulation History: 3 points
Removed measurement 23a32608 to history. Total: 4
Removed measurement 35ebb79 to history. Total: 3
Adding measurement 5e502d94 to history. Total: 3
th(0)=0.04548830015361836;dx=-4.94833079185323E-7
Non-optimal measurement 0.04576235839156307 < 0.04548830015361836. Total: 4
Armijo: th(10772.173450159424)=0.04576235839156307; dx=3.3739840750669694E-7 evalInputDelta=-2.7405823794470624E-4
Adding measurement 533d1b3a to history. Total: 4
New Minimum: 0.04548830015361836 > 0.044756457819270214
WOLF (strong): th(5386.086725079712)=0.044756457819270214; dx=6.39303603723412E-8 evalInputDelta=7.318423343481498E-4
Non-optimal measurement 0.04490214338766442 < 0.044756457819270214. Total: 5
END: th(1795.3622416932374)=0.04490214338766442; dx=-1.9025623276821844E-7 evalInputDelta=5.861567659539435E-4
Fitness changed from 0.04548830015361836 to 0.044756457819270214
Iteration 10 complete. Error: 0.044756457819270214 Total: 1.8051; Orientation: 1.5817; Line Search: 0.2124
Non-optimal measurement 0.044756457819270214 < 0.044756457819270214. Total: 5
Rejected: LBFGS Orientation magnitude: 1.299e+00, gradient 4.909e-04, dot -0.874; [2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.044756457819270214, 0.04548830015361836, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 1.300e+00, gradient 4.909e-04, dot -0.874; [308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.044756457819270214, 0.04548830015361836, 0.10853395295817445, 0.10859325696455376
LBFGS Accumulation History: 3 points
Removed measurement 533d1b3a to history. Total: 4
Removed measurement 5e502d94 to history. Total: 3
Adding measurement 5133dd1b to history. Total: 3
th(0)=0.044756457819270214;dx=-2.4093804726271303E-7
Adding measurement 692d851b to history. Total: 4
New Minimum: 0.044756457819270214 > 0.044036723802033204
END: th(3867.990694677318)=0.044036723802033204; dx=-1.2536341817216134E-7 evalInputDelta=7.197340172370101E-4
Fitness changed from 0.044756457819270214 to 0.044036723802033204
Iteration 11 complete. Error: 0.044036723802033204 Total: 2.0440; Orientation: 1.9121; Line Search: 0.1221
Non-optimal measurement 0.044036723802033204 < 0.044036723802033204. Total: 5
Rejected: LBFGS Orientation magnitude: 3.428e+00, gradient 3.913e-04, dot -0.857; [4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.044036723802033204, 0.044756457819270214, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 3.430e+00, gradient 3.913e-04, dot -0.857; [4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.044036723802033204, 0.044756457819270214, 0.10853395295817445, 0.10859325696455376
LBFGS Accumulation History: 3 points
Removed measurement 692d851b to history. Total: 4
Removed measurement 5133dd1b to history. Total: 3
Adding measurement 2da3d001 to history. Total: 3
th(0)=0.044036723802033204;dx=-1.5315154100722408E-7
Adding measurement c2faede to history. Total: 4
New Minimum: 0.044036723802033204 > 0.043249586122233945
WOLF (strong): th(8333.33333333334)=0.043249586122233945; dx=6.0225857857372025E-9 evalInputDelta=7.871376797992588E-4
Non-optimal measurement 0.043482816055300286 < 0.043249586122233945. Total: 5
END: th(4166.66666666667)=0.043482816055300286; dx=-1.0481511364124287E-7 evalInputDelta=5.539077467329179E-4
Fitness changed from 0.044036723802033204 to 0.043249586122233945
Iteration 12 complete. Error: 0.043249586122233945 Total: 2.3310; Orientation: 2.1204; Line Search: 0.2008
Non-optimal measurement 0.043249586122233945 < 0.043249586122233945. Total: 5
Rejected: LBFGS Orientation magnitude: 1.239e+00, gradient 5.219e-04, dot -0.699; [b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.043249586122233945, 0.044036723802033204, 0.10853395295817445, 0.10859325696455376, 0.10865262101983993
Rejected: LBFGS Orientation magnitude: 1.241e+00, gradient 5.219e-04, dot -0.699; [4af74b9d-1128-4405-a4c4-45250a1e12b5 = 1.000/1.000e+00, b4df16d4-ce6c-463e-9e4c-a302e79623f7 = 1.000/1.000e+00, 1ea8b1fa-152f-4f6e-8977-6143443957f5 = 1.000/1.000e+00, 2c92a18f-663b-4106-be47-c83ea836d67f = 1.000/1.000e+00, 308e54ee-3773-4d9a-9fc7-95457780661d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.043249586122233945, 0.044036723802033204, 0.10853395295817445, 0.10859325696455376
LBFGS Accumulation History: 3 points
Removed measurement c2faede to history. Total: 4
Removed measurement 2da3d001 to history. Total: 3
Adding measurement 50b72361 to history. Total: 3
th(0)=0.043249586122233945;dx=-2.7240725135320297E-7
Adding measurement 22e376f7 to history. Total: 4
New Minimum: 0.043249586122233945 > 0.04317368067409795
WOLF (strong): th(8976.81120846619)=0.04317368067409795; dx=1.3829404859467205E-7 evalInputDelta=7.59054481359947E-5
Adding measurement 1208a690 to history. Total: 5
New Minimum: 0.04317368067409795 > 0.04279014734250651
WOLF (strong): th(4488.405604233095)=0.04279014734250651; dx=1.9878194371578154E-8 evalInputDelta=4.594387797274349E-4
Non-optimal measurement 0.04294528961653616 < 0.04279014734250651. Total: 6
END: th(1496.1352014110316)=0.04294528961653616; dx=-1.4145526751837263E-7 evalInputDelta=3.0429650569778477E-4
Fitness changed from 0.043249586122233945 to 0.04279014734250651
Iteration 13 complete. Error: 0.04279014734250651 Total: 2.3385; Orientation: 2.0125; Line Search: 0.3171
Final threshold in iteration 13: 0.04279014734250651 (> 0.0) after 30.060s (< 30.000s)
Returns
0.04279014734250651
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.8108177510287379 ], [ -0.1267906201054001 ], [ 0.3119765931025024 ], [ 0.005665944105195876 ], [ 0.1614649665039717 ], [ 0.4322573454358207 ], [ -0.7697358829734293 ], [ 0.12454981650837531 ], ... ],
[ [ -0.13412817208966965 ], [ -0.9985336908190898 ], [ 0.09514098073296123 ], [ -0.2409587440511085 ], [ 0.12024067728053318 ], [ 0.1593841852364767 ], [ 0.41637606563405843 ], [ 0.41646860815066483 ], ... ],
[ [ -0.9716928429424472 ], [ 0.4161382670464939 ], [ -0.26006905030270744 ], [ 0.4146542519577021 ], [ -0.35221163799381805 ], [ -0.6789646273454881 ], [ -0.7442865409627577 ], [ -0.9318518487575018 ], ... ],
[ [ 0.6919169367031899 ], [ -0.9549029136179537 ], [ 0.4056522812939734 ], [ -0.020027441790561262 ], [ -0.6763435806477174 ], [ 0.5313073475868773 ], [ 0.17084708868939258 ], [ -0.7818685724083476 ], ... ],
[ [ -0.413601797183482 ], [ -0.3058333917293983 ], [ 0.1251624611396231 ], [ -0.8370564486484813 ], [ 0.4123650692977845 ], [ -0.8761926701304987 ], [ 0.4079219887970291 ], [ 0.09202469255419414 ], ... ],
[ [ -0.28814023972097247 ], [ 0.41651420558168095 ], [ -0.4495636080218268 ], [ 0.22718780477440506 ], [ 0.34201936529537885 ], [ 0.41440630283574226 ], [ -0.5670274712933507 ], [ 0.3810988419631078 ], ... ],
[ [ 0.42971481474565265 ], [ -0.23709827193250987 ], [ -0.7163364580171203 ], [ -0.9187047114195044 ], [ -0.9225574886039023 ], [ -0.3895176037132879 ], [ -0.40298765498145506 ], [ -0.10640320308111306 ], ... ],
[ [ 0.22786356895984503 ], [ 0.41650919642900636 ], [ 0.033161511257603034 ], [ 0.5411848458002316 ], [ 0.7271980766258973 ], [ 0.5540695577580017 ], [ -0.07543045364552214 ], [ -0.7847549433002523 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.51745661724246 ], [ 0.0 ], [ 0.025747639895307452 ], [ 8.743729601784163E-8 ], [ 0.003251656240645654 ], [ 0.07172176085529561 ], [ 0.0 ], [ 0.0014385882340893059 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 6.172302929545698E-4 ], [ 0.0 ], [ 0.0012879419234675309 ], [ 0.0031218194588957197 ], [ 0.06376454281851843 ], [ 0.06380907644131956 ], ... ],
[ [ 0.0 ], [ 0.06365020575843139 ], [ 0.0 ], [ 0.06293982527468384 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.314423284493391 ], [ 0.0 ], [ 0.05874612731939195 ], [ 0.0 ], [ 0.0 ], [ 0.1371350445275787 ], [ 0.0038829841621591645 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.0014609361819994072 ], [ 0.0 ], [ 0.061854649266588894 ], [ 0.0 ], [ 0.059784958147262424 ], [ 5.559165412957829E-4 ], ... ],
[ [ 0.0 ], [ 0.06383102678626665 ], [ 0.0 ], [ 0.009506570907392415 ], [ 0.034369806875115705 ], [ 0.0628216643848503 ], [ 0.0 ], [ 0.048282627212995824 ], ... ],
[ [ 0.07040475779489691 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.009595689133890003 ], [ 0.06382861515691318 ], [ 2.2513117071659678E-5 ], [ 0.1453050073716571 ], [ 0.3675946706158843 ], [ 0.156452841558557 ], [ 0.0 ], [ 0.0 ], ... ],
...
]
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -1.4433724562931949], [124.0, -0.9916428589874025]; valueStats=DoubleSummaryStatistics{count=178, sum=7.331228, min=0.036027, average=0.041187, max=0.101943}
Plotting 124 points for GD
Plotting 41 points for CjGD
Plotting 13 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -1.4433724562931949], [29.727, -0.9916428589874025]; valueStats=DoubleSummaryStatistics{count=178, sum=7.331228, min=0.036027, average=0.041187, max=0.101943}
Plotting 124 points for GD
Plotting 41 points for CjGD
Plotting 13 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.04279014734250651 }, "CjGD": { "type": "NonConverged", "value": 0.03775680522742538 }, "GD": { "type": "NonConverged", "value": 0.03602695389038974 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.04279014734250651 }, "CjGD": { "type": "NonConverged", "value": 0.03775680522742538 }, "GD": { "type": "NonConverged", "value": 0.03602695389038974 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "91.051",
"gc_time": "0.632"
},
"created_on": 1586737486183,
"file_name": "trainingTest",
"report": {
"simpleName": "NthPowerTest",
"canonicalName": "com.simiacryptus.mindseye.layers.java.NthPowerActivationLayerTest.NthPowerTest",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 0.04279014734250651
},
"CjGD": {
"type": "NonConverged",
"value": 0.03775680522742538
},
"GD": {
"type": "NonConverged",
"value": 0.03602695389038974
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayer/NthPowerTest/trainingTest/202004132446",
"id": "7f87da4e-7196-4809-a8da-17c35e24a537",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "NthPowerActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.NthPowerActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayer.java",
"javaDoc": ""
}
}