Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 6283239966981862400
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ -0.6518438990887327 ], [ -0.7888819351414185 ], [ 0.8093483043296383 ], [ -0.8037358696749217 ], [ -0.23158448085817018 ], [ -0.0072231813449228355 ], [ 0.4872436242429634 ], [ -0.8593258042948764 ], ... ],
[ [ -0.34328464303513395 ], [ 0.6130388607439166 ], [ 0.7277548700222518 ], [ -0.5122175221930463 ], [ -0.3317112382914085 ], [ -0.5594454616166173 ], [ 0.7704379081636379 ], [ 0.2044204686564428 ], ... ],
[ [ 0.19289866506251152 ], [ -0.32499519088360573 ], [ -0.38032273268506445 ], [ 0.47854833099406546 ], [ 0.06795454420195535 ], [ 0.9058447403172789 ], [ -0.007866627887558097 ], [ -0.50284010298311 ], ... ],
[ [ -0.9805614231128823 ], [ -0.8168215786359354 ], [ 0.4893259538203629 ], [ 0.5252580361789763 ], [ -0.9077281490023252 ], [ -0.059164449419868426 ], [ 0.2873123165179081 ], [ 0.43277501245570793 ], ... ],
[ [ 0.4550001822098244 ], [ -0.8673530862201424 ], [ 0.9212767317556412 ], [ 0.021914437803396858 ], [ 0.8542107844888474 ], [ 0.37474723705041657 ], [ 0.02131184226653149 ], [ -0.27074847417119585 ], ... ],
[ [ 0.6248228276019556 ], [ 0.5134618640688442 ], [ -0.542428007258869 ], [ -0.7484663180542714 ], [ -0.9572531339501762 ], [ 0.4890210055053563 ], [ 0.12825491629881336 ], [ -0.16721849223022445 ], ... ],
[ [ -0.20100432452641126 ], [ 0.027663203017254334 ], [ -0.08738401571709087 ], [ 0.31747485986466306 ], [ 0.9615415149226465 ], [ 0.7389853865260159 ], [ 0.7903748146411951 ], [ -0.7927714991536952 ], ... ],
[ [ -0.44455526262063105 ], [ 0.12804423709494195 ], [ -0.3746301985796969 ], [ -0.5446597694042123 ], [ 0.5675178886668002 ], [ 0.9614250815086449 ], [ 0.19197077251745798 ], [ -0.12575403024685738 ], ... ],
...
]
[
[ [ 0.6205959294554897 ], [ 0.2424913011962999 ], [ 0.19479733395020205 ], [ 0.19936140875218422 ], [ -0.5322935682440426 ], [ -0.014998763148523109 ], [ 0.5273632050592083 ], [ -0.6121052438981891 ], ... ],
[ [ 0.2930939830408985 ], [ 0.5742877166382288 ], [ -0.22911453635332468 ], [ -0.0295298327819411 ], [ -0.4145981697405894 ], [ 0.009610199601139513 ], [ 0.6043158408085714 ], [ 0.46122633153873815 ], ... ],
[ [ -0.8089774309470621 ], [ -0.6432444143733893 ], [ -0.05261469017394238 ], [ -0.20652840955726304 ], [ 0.9395992984889601 ], [ 0.8725596139847904 ], [ -0.6573646514456799 ], [ -0.8432875723733386 ], ... ],
[ [ -0.2548776808197193 ], [ -0.9896405860273323 ], [ 0.09821107975742538 ], [ -0.5774483592663036 ], [ -0.44929163525329163 ], [ 0.18258554258486814 ], [ -0.24962639110542661 ], [ -0.06397758700450007 ], ... ],
[ [ 0.8231236274258862 ], [ 0.4634775594410483 ], [ -0.9523984782670105 ], [ 0.7518191749099039 ], [ 0.32524167716098606 ], [ 0.33120630719413313 ], [ 0.24637548540446463 ], [ -0.5973355120862738 ], ... ],
[ [ 0.4582998471356168 ], [ 0.22467987231694386 ], [ -0.2644079312523502 ], [ -0.3085895419635205 ], [ -0.5140714053156874 ], [ 0.5154599166096321 ], [ 0.15262084830416733 ], [ 0.8587336484293626 ], ... ],
[ [ -0.13052574734733557 ], [ 0.9707966152266623 ], [ -0.41252166880732855 ], [ 0.16534854944071076 ], [ -0.3281345683390946 ], [ 0.22381283696665477 ], [ -0.9064379553222945 ], [ -0.8613629224021533 ], ... ],
[ [ -0.1408628137404384 ], [ 0.8429625989144385 ], [ 0.6131951894141423 ], [ -0.35271698534334384 ], [ -0.7838558107305946 ], [ -0.9288285286599791 ], [ -0.9934794242163292 ], [ 0.5255040280693115 ], ... ],
...
]
[
[ [ 0.840529446500516 ], [ 0.6120364902753954 ], [ -0.4418968776887606 ], [ -0.08691013031947303 ], [ 0.500120373611775 ], [ -0.5561984308181278 ], [ -0.5973482974013558 ], [ 0.5857029418433701 ], ... ],
[ [ 0.7213525538014345 ], [ -0.09519417687933962 ], [ -0.19907984233668974 ], [ -0.42130922527766823 ], [ -0.6987139543717316 ], [ 0.19915057548077453 ], [ -0.033383716482978 ], [ 0.8821315065534614 ], ... ],
[ [ -0.6383831679266514 ], [ -0.9548886528710612 ], [ 0.3080754152269003 ], [ -0.44611286418424156 ], [ -0.6590244474558025 ], [ 0.07747077386243606 ], [ -0.6351336530028864 ], [ 0.2834785185246753 ], ... ],
[ [ -0.7458507176712048 ], [ 0.2282400170731551 ], [ 0.4690025121496364 ], [ 0.9351819848893379 ], [ 0.07319283372714325 ], [ 0.9356242747535655 ], [ 0.9156922939089269 ], [ -0.6286208576918859 ], ... ],
[ [ 0.1279568935574784 ], [ 0.39634180119653606 ], [ 0.03999595868651329 ], [ 0.6753828229714776 ], [ -0.32165706522065607 ], [ 0.8769863737924897 ], [ 0.7401858461500141 ], [ 0.5858996008885831 ], ... ],
[ [ -0.6884972006433285 ], [ -0.5992893516104261 ], [ -0.07056241829952259 ], [ 0.3050203146745176 ], [ 0.389177795950969 ], [ -0.06754509132393727 ], [ -0.21800853150776378 ], [ -0.06188390692140233 ], ... ],
[ [ 0.835628921733804 ], [ -0.5045850933583029 ], [ -0.7405969851738541 ], [ -0.261654254186624 ], [ -0.46971408092645905 ], [ 0.8882083922100068 ], [ 0.2987278469821575 ], [ -0.819387115813659 ], ... ],
[ [ -0.7750759856443117 ], [ 0.8766518027409902 ], [ 0.44745643078301023 ], [ -0.5801178424993272 ], [ 0.5572740207628064 ], [ 0.07122806162996365 ], [ 0.6987496949490684 ], [ 0.14450323798200992 ], ... ],
...
]
[
[ [ -0.6349114246584904 ], [ -0.7479510647388325 ], [ 0.3393608858336894 ], [ 0.8455440617832326 ], [ 0.7740459388656751 ], [ 0.5517806678658104 ], [ 0.3800041428051988 ], [ -0.1489481006104716 ], ... ],
[ [ -0.6849214105486483 ], [ 0.5388644522201351 ], [ -0.21155029785852109 ], [ -0.7483450458296725 ], [ -0.711481090657091 ], [ -0.9869056111992542 ], [ -0.7504176709119659 ], [ 0.014476683164389659 ], ... ],
[ [ 0.12015653411827354 ], [ -0.8720020168723093 ], [ 0.630808012274203 ], [ -0.18899736349023888 ], [ -0.504188758991156 ], [ 0.5185910614881468 ], [ -0.30868824032474396 ], [ -0.40494242069234665 ], ... ],
[ [ -0.7201806092943601 ], [ -0.0983581983098838 ], [ -0.46009780307792125 ], [ 0.2736455859304523 ], [ -0.6132323554504862 ], [ -0.853590483883756 ], [ 0.3489101729979682 ], [ 0.8836207431264567 ], ... ],
[ [ 0.2935795151648024 ], [ -0.8895362086815555 ], [ -0.6138072997737938 ], [ -0.30444482966877473 ], [ 0.16126574505353286 ], [ -0.2483680243871058 ], [ 0.11801094103920673 ], [ 0.0353483169032236 ], ... ],
[ [ 0.22331824074752982 ], [ 0.791107106689521 ], [ -0.28614511451495694 ], [ -0.9294735501667781 ], [ -0.6232245657960145 ], [ 0.7433645426423272 ], [ 0.5728026920525415 ], [ 0.47126319024042584 ], ... ],
[ [ 0.8849241508091616 ], [ 0.46466911767433383 ], [ -0.8179706167628158 ], [ -0.3539937897505805 ], [ -0.19970318301752532 ], [ -0.01265331484526544 ], [ 0.3621581872458792 ], [ -0.9712805950350025 ], ... ],
[ [ 0.48802876210882107 ], [ -0.4811992003549901 ], [ -0.18092603767400584 ], [ -0.3113576082129612 ], [ 0.0442172045597633 ], [ 0.9940803391357571 ], [ 0.5131202084057165 ], [ 0.9001952905637944 ], ... ],
...
]
[
[ [ 0.18218675650783922 ], [ -0.24343427709016563 ], [ -0.6425974218357222 ], [ 0.48971095704899237 ], [ 0.38120447039443106 ], [ -0.33127490096364665 ], [ -0.6041150587093367 ], [ 0.164510328139299 ], ... ],
[ [ 0.4809583575394911 ], [ 0.9524549442858565 ], [ 0.03809741429751634 ], [ -0.3332649979243465 ], [ -0.39809006136446845 ], [ 0.6497291047229228 ], [ 0.24815410559494522 ], [ -0.05905572219282784 ], ... ],
[ [ 0.09014605982995949 ], [ 0.6769108584230685 ], [ -0.958649229948699 ], [ -0.18340663251802347 ], [ -0.7238899085904174 ], [ 0.18688999241148374 ], [ -0.8578080420885643 ], [ 0.49478535935723444 ], ... ],
[ [ -0.046562381007829945 ], [ -0.19207925484225186 ], [ 0.4739297382069867 ], [ -0.8385809793048316 ], [ -0.1263786004747003 ], [ -0.009159867217311657 ], [ 0.9296592954139447 ], [ -0.895559363437882 ], ... ],
[ [ 0.9274189806748239 ], [ 0.6893172855094185 ], [ -0.28676430163009714 ], [ -0.19305607753225984 ], [ 0.9861764995589988 ], [ -0.8446050934031382 ], [ 0.7787782769963638 ], [ -0.8562010822450148 ], ... ],
[ [ -0.6444705200494583 ], [ 0.9707876954935704 ], [ 0.028241146170399967 ], [ -0.7650305411821869 ], [ 0.2391730576469686 ], [ -0.26379763228962294 ], [ 0.5339463839852678 ], [ 0.8562075075647593 ], ... ],
[ [ -0.27554642247296024 ], [ 0.34088332041187597 ], [ 0.903436511308088 ], [ -0.9435761767091484 ], [ -0.40996668584342766 ], [ 0.8822281966690917 ], [ 0.03537239838221651 ], [ 0.2445084184138253 ], ... ],
[ [ -0.7050357490869414 ], [ 0.8553275995100007 ], [ 0.10101559042867359 ], [ 0.9159381032652736 ], [ 0.09910473985133916 ], [ -0.10175997330719722 ], [ -0.009295396086438812 ], [ 0.8202303288984831 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.19 seconds (0.164 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4168259434736
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 4168351031077
Constructing line search parameters: GD
th(0)=1.9743119500970525;dx=-1606.3692576195074
New Minimum: 1.9743119500970525 > 1.7057421230205698
END: th(2.154434690031884)=1.7057421230205698; dx=-0.028914023653969216 evalInputDelta=0.2685698270764827
Fitness changed from 1.9743119500970525 to 1.7057421230205698
Iteration 1 complete. Error: 1.7057421230205698 Total: 0.2561; Orientation: 0.0152; Line Search: 0.0964
th(0)=1.7057421230205698;dx=-5.709461465987502
New Minimum: 1.7057421230205698 > 1.6413426193805818
END: th(4.641588833612779)=1.6413426193805818; dx=-0.004439874023207853 evalInputDelta=0.06439950363998803
Fitness changed from 1.7057421230205698 to 1.6413426193805818
Iteration 2 complete. Error: 1.6413426193805818 Total: 0.0794; Orientation: 0.0039; Line Search: 0.0597
th(0)=1.6413426193805818;dx=-3.53603453299379
New Minimum: 1.6413426193805818 > 1.5667804346759415
END: th(10.000000000000002)=1.5667804346759415; dx=-0.006737690956850565 evalInputDelta=0.07456218470464027
Fitness changed from 1.6413426193805818 to 1.5667804346759415
Iteration 3 complete. Error: 1.5667804346759415 Total: 0.0622; Orientation: 0.0047; Line Search: 0.0421
th(0)=1.5667804346759415;dx=-0.02375363504332467
New Minimum: 1.5667804346759415 > 1.4809047283063945
END: th(21.544346900318843)=1.4809047283063945; dx=-0.0031453975340821824 evalInputDelta=0.085875706369547
Fitness changed from 1.5667804346759415 to 1.4809047283063945
Iteration 4 complete. Error: 1.4809047283063945 Total: 0.3124; Orientation: 0.0035; Line Search: 0.2930
th(0)=1.4809047283063945;dx=-0.011679794597003492
New Minimum: 1.4809047283063945 > 1.3780347277380751
END: th(46.4158883361278)=1.3780347277380751; dx=-0.008439348856899368 evalInputDelta=0.10287000056831941
Fitness changed from 1.4809047283063945 to 1.3780347277380751
Iteration 5 complete. Error: 1.3780347277380751 Total: 0.0796; Orientation: 0.0049; Line Search: 0.0434
th(0)=1.3780347277380751;dx=-29.280568260349092
New Minimum: 1.3780347277380751 > 1.2705408038420904
END: th(100.00000000000003)=1.2705408038420904; dx=-0.0011029316710938933 evalInputDelta=0.10749392389598467
Fitness changed from 1.3780347277380751 to 1.2705408038420904
Iteration 6 complete. Error: 1.2705408038420904 Total: 0.0536; Orientation: 0.0047; Line Search: 0.0365
th(0)=1.2705408038420904;dx=-0.4961158236329222
New Minimum: 1.2705408038420904 > 1.1511241140996145
END: th(215.44346900318845)=1.1511241140996145; dx=-0.0011256707464330034 evalInputDelta=0.11941668974247599
Fitness changed from 1.2705408038420904 to 1.1511241140996145
Iteration 7 complete. Error: 1.1511241140996145 Total: 0.2698; Orientation: 0.1895; Line Search: 0.0701
th(0)=1.1511241140996145;dx=-2.529283059596939
New Minimum: 1.1511241140996145 > 1.043576878453147
END: th(464.1588833612781)=1.043576878453147; dx=-6.709169177749347E-5 evalInputDelta=0.10754723564646751
Fitness changed from 1.1511241140996145 to 1.043576878453147
Iteration 8 complete. Error: 1.043576878453147 Total: 0.1112; Orientation: 0.0038; Line Search: 0.0950
th(0)=1.043576878453147;dx=-0.6648918949763938
New Minimum: 1.043576878453147 > 0.9850858226415122
WOLF (strong): th(1000.0000000000005)=0.9850858226415122; dx=0.002218731987504415 evalInputDelta=0.05849105581163472
END: th(500.0000000000002)=0.9877337032347944; dx=-5.780226813978425E-4 evalInputDelta=0.05584317521835258
Fitness changed from 1.043576878453147 to 0.9850858226415122
Iteration 9 complete. Error: 0.9850858226415122 Total: 0.3524; Orientation: 0.0037; Line Search: 0.3374
th(0)=0.9850858226415122;dx=-268.69378804096857
Armijo: th(1077.2173450159423)=1.0177012435820543; dx=0.0017228146813326522 evalInputDelta=-0.03261542094054204
New Minimum: 0.9850858226415122 > 0.9838945228884093
Armijo: th(538.6086725079712)=0.9838945228884093; dx=0.006185426011931313 evalInputDelta=0.0011912997531029479
New Minimum: 0.9838945228884093 > 0.9685389211509416
Armijo: th(179.53622416932373)=0.9685389211509416; dx=9.194369066515512E-4 evalInputDelta=0.016546901490570587
WOLF (strong): th(44.88405604233093)=0.9686586354927889; dx=0.0035405811990372148 evalInputDelta=0.016427187148723377
New Minimum: 0.9685389211509416 > 0.9659680357314896
END: th(8.976811208466186)=0.9659680357314896; dx=-1.4292223281004349E-4 evalInputDelta=0.01911778691002264
Fitness changed from 0.9850858226415122 to 0.9659680357314896
Iteration 10 complete. Error: 0.9659680357314896 Total: 0.2516; Orientation: 0.0033; Line Search: 0.2334
th(0)=0.9659680357314896;dx=-0.002482933089200057
New Minimum: 0.9659680357314896 > 0.9556739224780545
END: th(19.33995347338659)=0.9556739224780545; dx=-5.737110892782212E-4 evalInputDelta=0.0102941132534351
Fitness changed from 0.9659680357314896 to 0.9556739224780545
Iteration 11 complete. Error: 0.9556739224780545 Total: 0.0515; Orientation: 0.0043; Line Search: 0.0365
th(0)=0.9556739224780545;dx=-0.011954882512255328
New Minimum: 0.9556739224780545 > 0.9453994298011681
WOLFE (weak): th(41.66666666666669)=0.9453994298011681; dx=-0.013559917790188005 evalInputDelta=0.010274492676886404
New Minimum: 0.9453994298011681 > 0.9356675689202639
END: th(83.33333333333339)=0.9356675689202639; dx=-0.00536746832657548 evalInputDelta=0.020006353557790568
Fitness changed from 0.9556739224780545 to 0.9356675689202639
Iteration 12 complete. Error: 0.9356675689202639 Total: 0.0946; Orientation: 0.0043; Line Search: 0.0759
th(0)=0.9356675689202639;dx=-41.48278383797709
New Minimum: 0.9356675689202639 > 0.8924715592756283
END: th(179.53622416932376)=0.8924715592756283; dx=-6.652865007505832E-5 evalInputDelta=0.04319600964463566
Fitness changed from 0.9356675689202639 to 0.8924715592756283
Iteration 13 complete. Error: 0.8924715592756283 Total: 0.1702; Orientation: 0.0036; Line Search: 0.1126
th(0)=0.8924715592756283;dx=-0.04942799163342897
New Minimum: 0.8924715592756283 > 0.8864074512624992
WOLF (strong): th(386.79906946773184)=0.8864074512624992; dx=8.498716114134074E-4 evalInputDelta=0.006064108013129066
New Minimum: 0.8864074512624992 > 0.8744114401326094
WOLF (strong): th(193.39953473386592)=0.8744114401326094; dx=0.0010891184236647361 evalInputDelta=0.01806011914301886
WOLF (strong): th(64.4665115779553)=0.876602826272569; dx=1.7979035805720597E-4 evalInputDelta=0.015868733003059243
END: th(16.116627894488825)=0.8838187581834538; dx=-0.020175196432269827 evalInputDelta=0.008652801092174478
Fitness changed from 0.8924715592756283 to 0.8744114401326094
Iteration 14 complete. Error: 0.8744114401326094 Total: 0.4638; Orientation: 0.0030; Line Search: 0.4507
th(0)=0.8744114401326094;dx=-1.8562504269959237
New Minimum: 0.8744114401326094 > 0.8703339278050702
WOLF (strong): th(34.72222222222224)=0.8703339278050702; dx=0.14143232690710889 evalInputDelta=0.004077512327539168
New Minimum: 0.8703339278050702 > 0.8683763106665164
WOLF (strong): th(17.36111111111112)=0.8683763106665164; dx=9.310633500979023E-4 evalInputDelta=0.00603512946609297
New Minimum: 0.8683763106665164 > 0.8681748610604995
END: th(5.787037037037041)=0.8681748610604995; dx=-2.315929389540735E-4 evalInputDelta=0.006236579072109949
Fitness changed from 0.8744114401326094 to 0.8681748610604995
Iteration 15 complete. Error: 0.8681748610604995 Total: 0.2370; Orientation: 0.0026; Line Search: 0.2252
th(0)=0.8681748610604995;dx=-0.006493534904602635
New Minimum: 0.8681748610604995 > 0.8630508153264802
END: th(12.467793345091929)=0.8630508153264802; dx=-5.013888214586978E-4 evalInputDelta=0.005124045734019256
Fitness changed from 0.8681748610604995 to 0.8630508153264802
Iteration 16 complete. Error: 0.8630508153264802 Total: 0.1275; Orientation: 0.0553; Line Search: 0.0618
th(0)=0.8630508153264802;dx=-0.050254059237078895
New Minimum: 0.8630508153264802 > 0.8540236536425931
WOLF (strong): th(26.86104649081471)=0.8540236536425931; dx=0.0019332153009525882 evalInputDelta=0.009027161683887064
WOLF (strong): th(13.430523245407356)=0.8573446064873071; dx=3.4645033897157606E-4 evalInputDelta=
...skipping 48921 bytes...
18
New Minimum: 0.8273946409345841 > 0.8273923924718545
END: th(0.0019204139869376934)=0.8273923924718545; dx=-0.001013867218236976 evalInputDelta=2.2484627296703863E-6
Fitness changed from 0.8273946409345841 to 0.8273923924718545
Iteration 100 complete. Error: 0.8273923924718545 Total: 0.1991; Orientation: 0.0026; Line Search: 0.1895
Reset training subject: 4194428761482
th(0)=0.8273923924718545;dx=-9.105588169084191E-4
New Minimum: 0.8273923924718545 > 0.827388844193357
END: th(0.004137406512681003)=0.827388844193357; dx=-7.987304024962235E-4 evalInputDelta=3.5482784974671944E-6
Fitness changed from 0.8273923924718545 to 0.827388844193357
Iteration 101 complete. Error: 0.827388844193357 Total: 0.1277; Orientation: 0.0026; Line Search: 0.0250
th(0)=0.827388844193357;dx=-9.086418071386232E-4
New Minimum: 0.827388844193357 > 0.8273817624886307
END: th(0.008913772117683795)=0.8273817624886307; dx=-7.028442438325711E-4 evalInputDelta=7.081704726297211E-6
Fitness changed from 0.827388844193357 to 0.8273817624886307
Iteration 102 complete. Error: 0.8273817624886307 Total: 0.2982; Orientation: 0.0028; Line Search: 0.2865
th(0)=0.8273817624886307;dx=-0.0012407527202494806
Armijo: th(0.019204139869376937)=0.8274390452264034; dx=-0.03734135127695133 evalInputDelta=-5.7282737772723635E-5
New Minimum: 0.8273817624886307 > 0.8273817022107686
WOLF (strong): th(0.009602069934688469)=0.8273817022107686; dx=0.002662176087761294 evalInputDelta=6.027786214612973E-8
New Minimum: 0.8273817022107686 > 0.8273785173534698
END: th(0.0032006899782294897)=0.8273785173534698; dx=-7.335276375670893E-4 evalInputDelta=3.2451351609230628E-6
Fitness changed from 0.8273817624886307 to 0.8273785173534698
Iteration 103 complete. Error: 0.8273785173534698 Total: 0.4720; Orientation: 0.0021; Line Search: 0.3633
th(0)=0.8273785173534698;dx=-8.797198320907464E-4
New Minimum: 0.8273785173534698 > 0.827372834260423
END: th(0.006895677521135008)=0.827372834260423; dx=-7.755208802905651E-4 evalInputDelta=5.6830930468265706E-6
Fitness changed from 0.8273785173534698 to 0.827372834260423
Iteration 104 complete. Error: 0.827372834260423 Total: 0.2093; Orientation: 0.0026; Line Search: 0.1989
th(0)=0.827372834260423;dx=-9.888914185713312E-4
New Minimum: 0.827372834260423 > 0.8273713231596229
WOLF (strong): th(0.01485628686280633)=0.8273713231596229; dx=0.0018201571112441955 evalInputDelta=1.5111008000445736E-6
New Minimum: 0.8273713231596229 > 0.8273675482165442
END: th(0.007428143431403165)=0.8273675482165442; dx=-3.272611942744618E-4 evalInputDelta=5.28604387872722E-6
Fitness changed from 0.827372834260423 to 0.8273675482165442
Iteration 105 complete. Error: 0.8273675482165442 Total: 0.2622; Orientation: 0.0026; Line Search: 0.2498
th(0)=0.8273675482165442;dx=-0.002655509173185195
Armijo: th(0.01600344989114745)=0.8273795774871824; dx=0.0013303200829613477 evalInputDelta=-1.2029270638147516E-5
Armijo: th(0.008001724945573725)=0.8273689786222823; dx=0.0012140850781647908 evalInputDelta=-1.4304057380787327E-6
New Minimum: 0.8273675482165442 > 0.8273647250782542
WOLF (strong): th(0.002667241648524575)=0.8273647250782542; dx=8.14623534713013E-6 evalInputDelta=2.823138290031224E-6
END: th(6.668104121311438E-4)=0.8273661302361175; dx=-0.0016539031928319331 evalInputDelta=1.4179804267433482E-6
Fitness changed from 0.8273675482165442 to 0.8273647250782542
Iteration 106 complete. Error: 0.8273647250782542 Total: 0.3329; Orientation: 0.0025; Line Search: 0.3223
th(0)=0.8273647250782542;dx=-0.0012340592025250308
New Minimum: 0.8273647250782542 > 0.8273630821444558
END: th(0.0014365994835697934)=0.8273630821444558; dx=-0.0010444721903875049 evalInputDelta=1.6429337984513737E-6
Fitness changed from 0.8273647250782542 to 0.8273630821444558
Iteration 107 complete. Error: 0.8273630821444558 Total: 0.2618; Orientation: 0.0025; Line Search: 0.1334
th(0)=0.8273630821444558;dx=-9.454849692936208E-4
New Minimum: 0.8273630821444558 > 0.8273603765578826
END: th(0.003095059763084652)=0.8273603765578826; dx=-7.945762554038133E-4 evalInputDelta=2.7055865731151485E-6
Fitness changed from 0.8273630821444558 to 0.8273603765578826
Iteration 108 complete. Error: 0.8273603765578826 Total: 0.1067; Orientation: 0.0033; Line Search: 0.0959
th(0)=0.8273603765578826;dx=-8.536939919692806E-4
New Minimum: 0.8273603765578826 > 0.8273548716395629
WOLFE (weak): th(0.0066681041213114385)=0.8273548716395629; dx=-7.998969967128186E-4 evalInputDelta=5.504918319743091E-6
New Minimum: 0.8273548716395629 > 0.8273496810836491
END: th(0.013336208242622877)=0.8273496810836491; dx=-7.587016615034384E-4 evalInputDelta=1.069547423349082E-5
Fitness changed from 0.8273603765578826 to 0.8273496810836491
Iteration 109 complete. Error: 0.8273496810836491 Total: 0.3210; Orientation: 0.0086; Line Search: 0.3032
th(0)=0.8273496810836491;dx=-0.0011625811085474865
New Minimum: 0.8273496810836491 > 0.8273288063643502
WOLFE (weak): th(0.028731989671395872)=0.8273288063643502; dx=-0.002404048661913829 evalInputDelta=2.0874719298902278E-5
New Minimum: 0.8273288063643502 > 0.8273184371618892
WOLF (strong): th(0.057463979342791745)=0.8273184371618892; dx=1.1322481111093909E-4 evalInputDelta=3.124392175990476E-5
New Minimum: 0.8273184371618892 > 0.8273177250897147
END: th(0.04309798450709381)=0.8273177250897147; dx=-8.811635394311589E-5 evalInputDelta=3.195599393446269E-5
Fitness changed from 0.8273496810836491 to 0.8273177250897147
Iteration 110 complete. Error: 0.8273177250897147 Total: 0.3120; Orientation: 0.0034; Line Search: 0.2996
th(0)=0.8273177250897147;dx=-0.0024412180207576016
Armijo: th(0.09285179289253959)=0.8273577150222889; dx=3.952430429042818E-4 evalInputDelta=-3.998993257425809E-5
Armijo: th(0.04642589644626979)=0.8273271336490433; dx=0.0010085569971585591 evalInputDelta=-9.408559328627497E-6
New Minimum: 0.8273177250897147 > 0.8273061050293131
WOLFE (weak): th(0.015475298815423264)=0.8273061050293131; dx=-0.005546705082778869 evalInputDelta=1.1620060401562604E-5
WOLF (strong): th(0.030950597630846528)=0.8273093189447728; dx=0.0012591199227888219 evalInputDelta=8.406144941908167E-6
New Minimum: 0.8273061050293131 > 0.8273002103452874
WOLF (strong): th(0.023212948223134897)=0.8273002103452874; dx=9.371693078505119E-4 evalInputDelta=1.7514744427238327E-5
New Minimum: 0.8273002103452874 > 0.8272982037667435
END: th(0.01934412351927908)=0.8272982037667435; dx=-1.3925427191418573E-4 evalInputDelta=1.952132297122766E-5
Fitness changed from 0.8273177250897147 to 0.8272982037667435
Iteration 111 complete. Error: 0.8272982037667435 Total: 0.5163; Orientation: 0.0027; Line Search: 0.5045
th(0)=0.8272982037667435;dx=-0.0010910487459840001
New Minimum: 0.8272982037667435 > 0.8272653883570079
END: th(0.04167565075819649)=0.8272653883570079; dx=-3.484283416495257E-4 evalInputDelta=3.2815409735542644E-5
Fitness changed from 0.8272982037667435 to 0.8272653883570079
Iteration 112 complete. Error: 0.8272653883570079 Total: 0.2197; Orientation: 0.0026; Line Search: 0.1481
th(0)=0.8272653883570079;dx=-0.0015510756237791014
Armijo: th(0.08978746772311211)=0.827271506248953; dx=2.5006890497546115E-4 evalInputDelta=-6.117891945045528E-6
New Minimum: 0.8272653883570079 > 0.8272523995779864
WOLF (strong): th(0.044893733861556054)=0.8272523995779864; dx=6.022932739867345E-4 evalInputDelta=1.2988779021561037E-5
Armijo: th(0.014964577953852018)=0.8273275800657934; dx=-0.052591427491712014 evalInputDelta=-6.21917087855195E-5
END: th(0.0037411444884630045)=0.8272612248017628; dx=-4.997379827457201E-4 evalInputDelta=4.163555245129125E-6
Fitness changed from 0.8272653883570079 to 0.8272523995779864
Iteration 113 complete. Error: 0.8272523995779864 Total: 0.3830; Orientation: 0.0028; Line Search: 0.3032
th(0)=0.8272523995779864;dx=-0.0034096623763524847
New Minimum: 0.8272523995779864 > 0.8272258137790433
END: th(0.008060051466366283)=0.8272258137790433; dx=-0.0028362322874082744 evalInputDelta=2.6585798943101757E-5
Fitness changed from 0.8272523995779864 to 0.8272258137790433
Iteration 114 complete. Error: 0.8272258137790433 Total: 0.1840; Orientation: 0.0026; Line Search: 0.1736
Final threshold in iteration 114: 0.8272258137790433 (> 0.0) after 30.176s (< 30.000s)
Returns
0.8272258137790433
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.6849728261263854 ], [ -0.7677262942997048 ], [ -0.9345605750818599 ], [ 0.4835265508081751 ], [ 0.394605000791696 ], [ -1.1483293630781954 ], [ -0.3792144745007994 ], [ -0.7904612401788131 ], ... ],
[ [ 0.31024952855466126 ], [ 0.4266703542862974 ], [ 0.6456709211708459 ], [ -0.3891801834983882 ], [ 0.46784431758568445 ], [ -0.9311423407526476 ], [ 0.49012881249732054 ], [ -0.20510097752941658 ], ... ],
[ [ -0.25854342019250204 ], [ 0.549002753082526 ], [ 0.33088919531956 ], [ -0.42428765329882184 ], [ -0.09400211945970222 ], [ 0.5144971229832966 ], [ -0.055575594903035166 ], [ -0.7368243012302759 ], ... ],
[ [ -0.6359054255349509 ], [ -0.5377062714163499 ], [ -0.8280621839794062 ], [ 0.4522888491771144 ], [ 0.49226490439614473 ], [ -0.06103717179300503 ], [ 0.5304954542217724 ], [ 0.38254129136567866 ], ... ],
[ [ -0.6286258344096344 ], [ -0.9856963144002724 ], [ -0.49589717407223505 ], [ 0.02191443803668815 ], [ 0.756968993090258 ], [ 0.3452957722186102 ], [ 0.02197566270307827 ], [ 0.559707201806055 ], ... ],
[ [ -0.4309386833179386 ], [ -0.895911237290305 ], [ 0.5075876967465127 ], [ -0.746792659399292 ], [ -0.8121688233260775 ], [ 0.8452084537103327 ], [ 0.2430837924063558 ], [ 0.44351927033284044 ], ... ],
[ [ -0.6526985894388102 ], [ -0.03772775278319229 ], [ -0.4350112260230601 ], [ -0.2947436425674972 ], [ 0.5958691162763677 ], [ 0.5427926878069292 ], [ -0.6450044741218218 ], [ 0.47334471179060955 ], ... ],
[ [ -0.6742093989437712 ], [ -0.1253573282603397 ], [ 0.8353766272980684 ], [ -0.8226819410962457 ], [ -0.6138683690973147 ], [ -0.8349689738787535 ], [ -0.6693810816196364 ], [ -0.1272330131649973 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ -0.378376111395271 ], [ -0.2643219969862217 ], [ -0.0676788333399378 ], [ -0.7266490518257078 ], [ -0.9298700124082987 ], [ 0.13830815838502922 ], [ -0.969653338177873 ], [ -0.23513865556658164 ], ... ],
[ [ -1.170378374464534 ], [ -0.8517435679154236 ], [ -0.4374653149451164 ], [ -0.9437128459521917 ], [ -0.7596196931377924 ], [ -0.07134312321687958 ], [ -0.7130870397774605 ], [ -1.584252847803914 ], ... ],
[ [ -1.352691629342087 ], [ -0.5996518227629105 ], [ -1.1059717169915402 ], [ -0.8573436261865846 ], [ -2.3644379495226824 ], [ -0.6645653155928672 ], [ -2.8900111146811143 ], [ -0.3054058124591567 ], ... ],
[ [ -0.4527054286883228 ], [ -0.6204428318230472 ], [ -0.18866702599244448 ], [ -0.7934342564013726 ], [ -0.7087382838183904 ], [ -2.7962722267457387 ], [ -0.6339438898210377 ], [ -0.9609186802590469 ], ... ],
[ [ -0.4642190571341433 ], [ -0.014406969386139912 ], [ -0.7013866840872038 ], [ -3.820609588301743 ], [ -0.27843298663423216 ], [ -1.0633539184531815 ], [ -3.817819678728946 ], [ -0.5803414859064746 ], ... ],
[ [ -0.8417894650828376 ], [ -0.10991393643873293 ], [ -0.678085781523688 ], [ -0.29196769676928985 ], [ -0.20804704993075634 ], [ -0.1681719912666719 ], [ -1.4143490703624544 ], [ -0.8130140275950756 ], ... ],
[ [ -0.42663983446049036 ], [ -3.277359307256992 ], [ -0.8323834412768831 ], [ -1.221649308694649 ], [ -0.5177342395963219 ], [ -0.6110278223595279 ], [ -0.4384980255874433 ], [ -0.7479313783496268 ], ... ],
[ [ -0.39421453539567863 ], [ -2.0765869936986623 ], [ -0.17987260516768164 ], [ -0.19518561582458643 ], [ -0.48797475639588767 ], [ -0.18036071185305413 ], [ -0.4014017523018654 ], [ -2.061735124289465 ], ... ],
...
]
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 30.51 seconds (0.158 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4198458540078
Reset training subject: 4198467125302
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.9743119500970525}, derivative=-1606.3692576195074}
New Minimum: 1.9743119500970525 > 1.9743117894740743
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.9743117894740743}, derivative=-1606.0903331445857}, evalInputDelta = -1.6062297825136795E-7
New Minimum: 1.9743117894740743 > 1.9743108263214417
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.9743108263214417}, derivative=-1604.4190948655312}, evalInputDelta = -1.123775610789579E-6
New Minimum: 1.9743108263214417 > 1.9743041121648734
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.9743041121648734}, derivative=-1592.8301458469587}, evalInputDelta = -7.837932179066698E-6
New Minimum: 1.9743041121648734 > 1.9742584223232367
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.9742584223232367}, derivative=-1516.7348181280904}, evalInputDelta = -5.3527773815842394E-5
New Minimum: 1.9742584223232367 > 1.9739879584123248
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.9739879584123248}, derivative=-1150.2617413959026}, evalInputDelta = -3.239916847277424E-4
New Minimum: 1.9739879584123248 > 1.9729658733476039
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.9729658733476039}, derivative=-486.5342635074685}, evalInputDelta = -0.0013460767494486436
New Minimum: 1.9729658733476039 > 1.97066428907191
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.97066428907191}, derivative=-134.55407627631683}, evalInputDelta = -0.003647661025142579
New Minimum: 1.97066428907191 > 1.9665700983227468
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.9665700983227468}, derivative=-33.5561590732182}, evalInputDelta = -0.0077418517743057524
New Minimum: 1.9665700983227468 > 1.9589066867017095
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.9589066867017095}, derivative=-9.52773115122217}, evalInputDelta = -0.015405263395342983
New Minimum: 1.9589066867017095 > 1.9431171533978628
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.9431171533978628}, derivative=-2.8355194787631453}, evalInputDelta = -0.031194796699189675
New Minimum: 1.9431171533978628 > 1.9110393506246628
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.9110393506246628}, derivative=-0.7986192360988692}, evalInputDelta = -0.06327259947238972
New Minimum: 1.9110393506246628 > 1.8497866679814936
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=1.8497866679814936}, derivative=-0.21354965610075377}, evalInputDelta = -0.12452528211555891
New Minimum: 1.8497866679814936 > 1.7394225457305084
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=1.7394225457305084}, derivative=-0.05533593600321874}, evalInputDelta = -0.23488940436654415
New Minimum: 1.7394225457305084 > 1.5657925343158152
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=1.5657925343158152}, derivative=-0.010524840795456398}, evalInputDelta = -0.4085194157812373
New Minimum: 1.5657925343158152 > 1.3662007251145962
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=1.3662007251145962}, derivative=-0.0018319147347999499}, evalInputDelta = -0.6081112249824563
New Minimum: 1.3662007251145962 > 1.2899582731373822
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=1.2899582731373822}, derivative=0.001338604141187352}, evalInputDelta = -0.6843536769596703
1.2899582731373822 <= 1.9743119500970525
Converged to right
Fitness changed from 1.9743119500970525 to 1.2899582731373822
Iteration 1 complete. Error: 1.2899582731373822 Total: 1.5748; Orientation: 0.0027; Line Search: 1.3774
F(0.0) = LineSearchPoint{point=PointSample{avg=1.2899582731373822}, derivative=-6.665610638817688}
New Minimum: 1.2899582731373822 > 1.225214697429757
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=1.225214697429757}, derivative=0.004235029395172624}, evalInputDelta = -0.06474357570762512
1.225214697429757 <= 1.2899582731373822
Converged to right
Fitness changed from 1.2899582731373822 to 1.225214697429757
Iteration 2 complete. Error: 1.225214697429757 Total: 0.1248; Orientation: 0.0025; Line Search: 0.1148
F(0.0) = LineSearchPoint{point=PointSample{avg=1.225214697429757}, derivative=-38.15212992957096}
New Minimum: 1.225214697429757 > 1.185244907743256
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=1.185244907743256}, derivative=-0.012284766478339568}, evalInputDelta = -0.039969789686501
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=1.3902926700080314}, derivative=-3.108303835316395E-4}, evalInputDelta = 0.16507797257827428
F(255.63792745846925) = LineSearchPoint{point=PointSample{avg=1.1938994449474534}, derivative=0.0011972209696304943}, evalInputDelta = -0.03131525248230371
1.1938994449474534 <= 1.225214697429757
Converged to right
Fitness changed from 1.225214697429757 to 1.185244907743256
Iteration 3 complete. Error: 1.185244907743256 Total: 0.1295; Orientation: 0.0028; Line Search: 0.1182
F(0.0) = LineSearchPoint{point=PointSample{avg=1.185244907743256}, derivative=-1500.2711191512194}
New Minimum: 1.185244907743256 > 1.1763386496603259
F(255.63792745846925) = LineSearchPoint{point=PointSample{avg=1.1763386496603259}, derivative=0.003436490358343934}, evalInputDelta = -0.008906258082930218
1.1763386496603259 <= 1.185244907743256
Converged to right
Fitness changed from 1.185244907743256 to 1.1763386496603259
Iteration 4 complete. Error: 1.1763386496603259 Total: 0.3793; Orientation: 0.0027; Line Search: 0.2879
F(0.0) = LineSearchPoint{point=PointSample{avg=1.1763386496603259}, derivative=-133.4509518489494}
F(255.63792745846925) = LineSearchPoint{point=PointSample{avg=1.1851126589604626}, derivative=0.0012793522322995469}, evalInputDelta = 0.00877400930013672
New Minimum: 1.1763386496603259 > 1.1700740083440795
F(19.664455958343787) = LineSearchPoint{point=PointSample{avg=1.1700740083440795}, derivative=-4.920429470137836E-4}, evalInputDelta = -0.006264641316246378
F(137.65119170840651) = LineSearchPoint{point=PointSample{avg=1.1809894360038886}, derivative=-5.129379001755975E-4}, evalInputDelta = 0.004650786343562752
New Minimum: 1.1700740083440795 > 1.168653197624105
F(10.588553208338963) = LineSearchPoint{point=PointSample{avg=1.168653197624105}, derivative=-1.2349505573398725E-5}, evalInputDelta = -0.007685452036220841
F(74.11987245837274) = LineSearchPoint{point=PointSample{avg=1.176685156376917}, derivative=-1.1424080002643447E-4}, evalInputDelta = 3.4650671659108845E-4
New Minimum: 1.168653197624105 > 1.166736353727342
F(5.701528650644057) = LineSearchPoint{point=PointSample{avg=1.166736353727342}, derivative=4.5944621063380127E-4}, evalInputDelta = -0.009602295932983784
1.166736353727342 <= 1.1763386496603259
Converged to right
Fitness changed from 1.1763386496603259 to 1.166736353727342
Iteration 5 complete. Error: 1.166736353727342 Total: 0.3836; Orientation: 0.0024; Line Search: 0.3729
F(0.0) = LineSearchPoint{point=PointSample{avg=1.166736353727342}, derivative=-0.007284827745850383}
New Minimum: 1.166736353727342 > 1.1623412715817862
F(5.701528650644057) = LineSearchPoint{point=PointSample{avg=1.1623412715817862}, derivative=-5.526056286985362E-4}, evalInputDelta = -0.00439508214555584
New Minimum: 1.1623412715817862 > 1.1493373685238955
F(39.910700554508395) = LineSearchPoint{point=PointSample{avg=1.1493373685238955}, derivative=0.008578206350847143}, evalInputDelta = -0.017398985203446626
1.1493373685238955 <= 1.166736353727342
F(18.32830825323245) = LineSearchPoint{point=PointSample{avg=1.1568624412231356}, derivative=6.423696318490921E-4}, evalInputDelta = -0.009873912504206439
1.1568624412231356 > 1.1493373685238955
Fitness changed from 1.166736353727342 to 1.1493373685238955
Iteration 6 complete. Error: 1.1493373685238955 Total: 0.6507; Orientation: 0.0027; Line Search: 0.5352
F(0.0) = LineSearchPoint{point=PointSample{avg=1.1493373685238955}, derivative=-42.40777136157923}
New Minimum: 1.1493373685238955 > 1.1428261093249656
F(39.910700554508395) = LineSearchPoint{point=PointSample{avg=1
...skipping 50773 bytes...
= -2.3252839964627015E-7
Right bracket at 3.125907908906179E-5
New Minimum: 1.0135573014860149 > 1.0135573014192476
F(3.078694740915753E-5) = LineSearchPoint{point=PointSample{avg=1.0135573014192476}, derivative=3.7135187399183454E-5}, evalInputDelta = -2.3259516690465887E-7
Right bracket at 3.078694740915753E-5
Converged to right
Fitness changed from 1.0135575340144145 to 1.0135573014192476
Iteration 51 complete. Error: 1.0135573014192476 Total: 0.8490; Orientation: 0.0025; Line Search: 0.7108
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0135573014192476}, derivative=-0.0026227236543252285}
New Minimum: 1.0135573014192476 > 1.0135572265384698
F(3.078694740915753E-5) = LineSearchPoint{point=PointSample{avg=1.0135572265384698}, derivative=-0.0022384003156371256}, evalInputDelta = -7.48807778006011E-8
New Minimum: 1.0135572265384698 > 1.0135570568680816
F(2.155086318641027E-4) = LineSearchPoint{point=PointSample{avg=1.0135570568680816}, derivative=5.524790723779424E-4}, evalInputDelta = -2.4455116598431914E-7
1.0135570568680816 <= 1.0135573014192476
New Minimum: 1.0135570568680816 > 1.013557048340879
F(1.7801055086902125E-4) = LineSearchPoint{point=PointSample{avg=1.013557048340879}, derivative=-9.037482228095996E-5}, evalInputDelta = -2.5307836870602785E-7
Left bracket at 1.7801055086902125E-4
New Minimum: 1.013557048340879 > 1.0135570480954026
F(1.8328217235869996E-4) = LineSearchPoint{point=PointSample{avg=1.0135570480954026}, derivative=-2.6180774943646793E-6}, evalInputDelta = -2.533238450119768E-7
Left bracket at 1.8328217235869996E-4
Converged to left
Fitness changed from 1.0135573014192476 to 1.0135570480954026
Iteration 52 complete. Error: 1.0135570480954026 Total: 0.7394; Orientation: 0.0025; Line Search: 0.7293
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0135570480954026}, derivative=-0.015517483877009861}
F(1.8328217235869996E-4) = LineSearchPoint{point=PointSample{avg=1.0135600639111169}, derivative=0.03493235677089633}, evalInputDelta = 3.01581571426901E-6
New Minimum: 1.0135570480954026 > 1.0135568848433907
F(1.4098628642976919E-5) = LineSearchPoint{point=PointSample{avg=1.0135568848433907}, derivative=-0.00783086423315436}, evalInputDelta = -1.632520119088099E-7
F(9.869040050083843E-5) = LineSearchPoint{point=PointSample{avg=1.0135576176419332}, derivative=0.02129208251969388}, evalInputDelta = 5.695465306043701E-7
F(7.591569269295264E-6) = LineSearchPoint{point=PointSample{avg=1.0135569467574193}, derivative=-0.01123739699042388}, evalInputDelta = -1.0133798333811228E-7
F(5.314098488506685E-5) = LineSearchPoint{point=PointSample{avg=1.0135569186711961}, derivative=0.008513501735400187}, evalInputDelta = -1.2942420646311348E-7
1.0135569186711961 <= 1.0135570480954026
New Minimum: 1.0135568848433907 > 1.0135568231762957
F(3.431462984758325E-5) = LineSearchPoint{point=PointSample{avg=1.0135568231762957}, derivative=0.0014158042958240787}, evalInputDelta = -2.2491910689481642E-7
Right bracket at 3.431462984758325E-5
New Minimum: 1.0135568231762957 > 1.0135568208287684
F(3.144555918322387E-5) = LineSearchPoint{point=PointSample{avg=1.0135568208287684}, derivative=2.1495084061891843E-4}, evalInputDelta = -2.2726663417849124E-7
Right bracket at 3.144555918322387E-5
New Minimum: 1.0135568208287684 > 1.0135568207756518
F(3.1015921336222864E-5) = LineSearchPoint{point=PointSample{avg=1.0135568207756518}, derivative=3.21676036814457E-5}, evalInputDelta = -2.27319750800703E-7
Right bracket at 3.1015921336222864E-5
Converged to right
Fitness changed from 1.0135570480954026 to 1.0135568207756518
Iteration 53 complete. Error: 1.0135568207756518 Total: 0.7820; Orientation: 0.0025; Line Search: 0.7713
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0135568207756518}, derivative=-0.0026316134362472975}
New Minimum: 1.0135568207756518 > 1.013556745283154
F(3.1015921336222864E-5) = LineSearchPoint{point=PointSample{avg=1.013556745283154}, derivative=-0.002232842027921236}, evalInputDelta = -7.549249780858247E-8
New Minimum: 1.013556745283154 > 1.0135565854290776
F(2.1711144935356003E-4) = LineSearchPoint{point=PointSample{avg=1.0135565854290776}, derivative=6.760707372929252E-4}, evalInputDelta = -2.353465742288563E-7
1.0135565854290776 <= 1.0135568207756518
New Minimum: 1.0135565854290776 > 1.0135565730751304
F(1.727351758225551E-4) = LineSearchPoint{point=PointSample{avg=1.0135565730751304}, derivative=-1.0858831992259239E-4}, evalInputDelta = -2.477005214274186E-7
Left bracket at 1.727351758225551E-4
New Minimum: 1.0135565730751304 > 1.0135565727294868
F(1.788763717252154E-4) = LineSearchPoint{point=PointSample{avg=1.0135565727294868}, derivative=-3.7811006377936815E-6}, evalInputDelta = -2.480461649412291E-7
Left bracket at 1.788763717252154E-4
Converged to left
Fitness changed from 1.0135568207756518 to 1.0135565727294868
Iteration 54 complete. Error: 1.0135565727294868 Total: 0.6534; Orientation: 0.0910; Line Search: 0.5542
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0135565727294868}, derivative=-0.015084578420644985}
F(1.788763717252154E-4) = LineSearchPoint{point=PointSample{avg=1.0135593477736768}, derivative=0.033545867006464884}, evalInputDelta = 2.7750441899954126E-6
New Minimum: 1.0135565727294868 > 1.0135564162393293
F(1.3759720901939647E-5) = LineSearchPoint{point=PointSample{avg=1.0135564162393293}, derivative=-0.00783338089407016}, evalInputDelta = -1.5649015749907846E-7
F(9.631804631357753E-5) = LineSearchPoint{point=PointSample{avg=1.013557070010459}, derivative=0.020099492289665732}, evalInputDelta = 4.972809721781601E-7
F(7.40908048565981E-6) = LineSearchPoint{point=PointSample{avg=1.0135564760965636}, derivative=-0.011052282886801261}, evalInputDelta = -9.663292321882011E-8
F(5.186356339961867E-5) = LineSearchPoint{point=PointSample{avg=1.0135564328346187}, derivative=0.007741602301247015}, evalInputDelta = -1.3989486813414942E-7
1.0135564328346187 <= 1.0135565727294868
New Minimum: 1.0135564162393293 > 1.0135563520319049
F(3.427380159683563E-5) = LineSearchPoint{point=PointSample{avg=1.0135563520319049}, derivative=0.0012647713797824885}, evalInputDelta = -2.2069758198739464E-7
Right bracket at 3.427380159683563E-5
New Minimum: 1.0135563520319049 > 1.0135563500972489
F(3.162241032653034E-5) = LineSearchPoint{point=PointSample{avg=1.0135563500972489}, derivative=1.8995478278126818E-4}, evalInputDelta = -2.2263223797835963E-7
Right bracket at 3.162241032653034E-5
New Minimum: 1.0135563500972489 > 1.0135563500543407
F(3.122915260764641E-5) = LineSearchPoint{point=PointSample{avg=1.0135563500543407}, derivative=2.815720254104124E-5}, evalInputDelta = -2.2267514609985994E-7
Right bracket at 3.122915260764641E-5
Converged to right
Fitness changed from 1.0135565727294868 to 1.0135563500543407
Iteration 55 complete. Error: 1.0135563500543407 Total: 1.0161; Orientation: 0.0173; Line Search: 0.9907
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0135563500543407}, derivative=-0.0026399285426444196}
New Minimum: 1.0135563500543407 > 1.0135562739949777
F(3.122915260764641E-5) = LineSearchPoint{point=PointSample{avg=1.0135562739949777}, derivative=-0.002227394819658033}, evalInputDelta = -7.605936303001215E-8
New Minimum: 1.0135562739949777 > 1.0135561237439144
F(2.1860406825352488E-4) = LineSearchPoint{point=PointSample{avg=1.0135561237439144}, derivative=7.947495591628674E-4}, evalInputDelta = -2.26310426398868E-7
1.0135561237439144 <= 1.0135563500543407
New Minimum: 1.0135561237439144 > 1.0135561071850285
F(1.6802131152174245E-4) = LineSearchPoint{point=PointSample{avg=1.0135561071850285}, derivative=-1.255191095245107E-4}, evalInputDelta = -2.428693122169534E-7
Left bracket at 1.6802131152174245E-4
New Minimum: 1.0135561071850285 > 1.0135561067337253
F(1.7492049520396465E-4) = LineSearchPoint{point=PointSample{avg=1.0135561067337253}, derivative=-5.053363229919755E-6}, evalInputDelta = -2.433206154339729E-7
Left bracket at 1.7492049520396465E-4
Converged to left
Fitness changed from 1.0135563500543407 to 1.0135561067337253
Iteration 56 complete. Error: 1.0135561067337253 Total: 0.5930; Orientation: 0.0025; Line Search: 0.5829
Final threshold in iteration 56: 1.0135561067337253 (> 0.0) after 30.511s (< 30.000s)
Returns
1.0135561067337253
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.6844055006851989 ], [ -0.7680081328373601 ], [ -0.9332780478793051 ], [ 0.4884063909886193 ], [ 0.3837977787410277 ], [ 0.003623084850394547 ], [ -1.1587040090896874 ], [ -0.7913019565895454 ], ... ],
[ [ 0.33216727916809574 ], [ 0.43192251002532445 ], [ 0.6470510502709362 ], [ -0.4263499163975722 ], [ 0.46222383165538494 ], [ -0.9264734816799872 ], [ 0.49424120755633966 ], [ -0.20965990788354863 ], ... ],
[ [ -0.25145108046696424 ], [ 0.5411487613626926 ], [ 0.3578514349762905 ], [ -0.42569240749523496 ], [ -0.07144709291952063 ], [ 0.5196101708594009 ], [ 6.45578010851466 ], [ -0.732520952745322 ], ... ],
[ [ -0.6405220267154429 ], [ -0.5420951632840914 ], [ -0.8226686411547673 ], [ 0.454075153792714 ], [ 0.7842214398007535 ], [ 6.464503743970497 ], [ 0.5208454475759481 ], [ 0.383642145045007 ], ... ],
[ [ -0.6244791173881897 ], [ -0.9846104986174005 ], [ -0.5355746429931687 ], [ 0.4599312201499649 ], [ 0.7582202766946873 ], [ 0.34576931056009574 ], [ -0.021308560345108203 ], [ 0.548633190791063 ], ... ],
[ [ -0.4357398349948055 ], [ -0.8906028161286774 ], [ 0.508453922440325 ], [ -0.7468164534634623 ], [ -0.8137735378630091 ], [ 0.8396969688813201 ], [ 0.22036343713209755 ], [ 0.42377582987313395 ], ... ],
[ [ -0.63648065225374 ], [ -0.19427481269306704 ], [ -0.3985669536212785 ], [ -0.29594128503210904 ], [ 0.6009022978079491 ], [ 0.5462924326456123 ], [ -0.6473040249115053 ], [ 0.4820571398318113 ], ... ],
[ [ -0.66910996802901 ], [ -0.12828637685667205 ], [ 0.8270525860450587 ], [ -0.8184675663714575 ], [ -0.6128902248647301 ], [ -0.8363197353749154 ], [ -0.6525278714874968 ], [ -0.12894730420549377 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ -0.37920469975911053 ], [ -0.2639549562585553 ], [ -0.06905210765640891 ], [ -0.716607451268919 ], [ -0.9576394829682043 ], [ -5.620429447416708 ], [ 0.1473021470042676 ], [ -0.2340756437432987 ], ... ],
[ [ -1.1021165840409568 ], [ -0.8395090817690142 ], [ -0.4353300845476488 ], [ -0.8524948698828224 ], [ -0.7717060211223632 ], [ -0.07636985570629001 ], [ -0.704731606555827 ], [ -1.562268547318922 ], ... ],
[ [ -1.3805068194278225 ], [ -0.614061063141887 ], [ -1.027637364811745 ], [ -0.85403824164832 ], [ -2.6387980623546308 ], [ -0.6546764199750346 ], [ 1.8649758703681776 ], [ -0.3112633340695398 ], ... ],
[ [ -0.44547176836087726 ], [ -0.6123137149395779 ], [ -0.19520178252075698 ], [ -0.7894925576521676 ], [ -0.2430638497874101 ], [ 1.866326248870861 ], [ -0.6523019269643643 ], [ -0.9580450748414717 ], ... ],
[ [ -0.47083738890563864 ], [ -0.015509148889980366 ], [ -0.62441500946514 ], [ -0.7766783220915571 ], [ -0.27678133305690567 ], [ -1.0619834587746328 ], [ -3.8486463928680394 ], [ -0.6003252014136192 ], ... ],
[ [ -0.8307099224621192 ], [ -0.11585672404173351 ], [ -0.6763806822692404 ], [ -0.2919358356029815 ], [ -0.206073160700406 ], [ -0.17471420356295508 ], [ -1.512477108695229 ], [ -0.8585506666923617 ], ... ],
[ [ -0.4518012585342808 ], [ -1.638481562020267 ], [ -0.9198797807585625 ], [ -1.2175942057123068 ], [ -0.5093229237055669 ], [ -0.6046008556888299 ], [ -0.43493919544984155 ], [ -0.7296926245888177 ], ... ],
[ [ -0.40180685563730456 ], [ -2.053490194941568 ], [ -0.18988699946842397 ], [ -0.20032150864762674 ], [ -0.4895694376064936 ], [ -0.17874428043364168 ], [ -0.42690142577417817 ], [ -2.0483514526081406 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 33.80 seconds (0.064 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 4228987022978
Reset training subject: 4228995323606
Adding measurement 6e1a9dba to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.9743119500970525 < 1.9743119500970525. Total: 1
th(0)=1.9743119500970525;dx=-1606.3692576195074
Adding measurement 45a8bf1c to history. Total: 1
New Minimum: 1.9743119500970525 > 1.7057421230205698
END: th(2.154434690031884)=1.7057421230205698; dx=-0.028914023653969216 evalInputDelta=0.2685698270764827
Fitness changed from 1.9743119500970525 to 1.7057421230205698
Iteration 1 complete. Error: 1.7057421230205698 Total: 0.1492; Orientation: 0.0191; Line Search: 0.0461
Non-optimal measurement 1.7057421230205698 < 1.7057421230205698. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 1.7057421230205698 < 1.7057421230205698. Total: 2
th(0)=1.7057421230205698;dx=-5.709461465987502
Adding measurement 44aa53ec to history. Total: 2
New Minimum: 1.7057421230205698 > 1.6413426193805818
END: th(4.641588833612779)=1.6413426193805818; dx=-0.004439874023207853 evalInputDelta=0.06439950363998803
Fitness changed from 1.7057421230205698 to 1.6413426193805818
Iteration 2 complete. Error: 1.6413426193805818 Total: 0.1974; Orientation: 0.0227; Line Search: 0.1636
Non-optimal measurement 1.6413426193805818 < 1.6413426193805818. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 1.6413426193805818 < 1.6413426193805818. Total: 3
th(0)=1.6413426193805818;dx=-3.53603453299379
Adding measurement 366a6b50 to history. Total: 3
New Minimum: 1.6413426193805818 > 1.5667804346759415
END: th(10.000000000000002)=1.5667804346759415; dx=-0.006737690956850565 evalInputDelta=0.07456218470464027
Fitness changed from 1.6413426193805818 to 1.5667804346759415
Iteration 3 complete. Error: 1.5667804346759415 Total: 0.0692; Orientation: 0.0135; Line Search: 0.0424
Non-optimal measurement 1.5667804346759415 < 1.5667804346759415. Total: 4
Rejected: LBFGS Orientation magnitude: 3.566e-01, gradient 1.541e-01, dot -0.942; [908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5667804346759415, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
LBFGS Accumulation History: 3 points
Removed measurement 366a6b50 to history. Total: 3
Adding measurement 744a4618 to history. Total: 3
th(0)=1.5667804346759415;dx=-0.02375363504332467
Adding measurement 706e47d1 to history. Total: 4
New Minimum: 1.5667804346759415 > 1.4809047283063945
END: th(21.544346900318843)=1.4809047283063945; dx=-0.0031453975340821824 evalInputDelta=0.085875706369547
Fitness changed from 1.5667804346759415 to 1.4809047283063945
Iteration 4 complete. Error: 1.4809047283063945 Total: 0.8050; Orientation: 0.6034; Line Search: 0.1916
Non-optimal measurement 1.4809047283063945 < 1.4809047283063945. Total: 5
Rejected: LBFGS Orientation magnitude: 7.792e-01, gradient 1.081e-01, dot -0.510; [908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.4809047283063945, 1.5667804346759415, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
Rejected: LBFGS Orientation magnitude: 8.769e-01, gradient 1.081e-01, dot -0.568; [908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.4809047283063945, 1.5667804346759415, 1.6413426193805818, 1.7057421230205698
LBFGS Accumulation History: 3 points
Removed measurement 706e47d1 to history. Total: 4
Removed measurement 744a4618 to history. Total: 3
Adding measurement 773ebb7b to history. Total: 3
th(0)=1.4809047283063945;dx=-0.01167979459700349
Adding measurement 45eac36d to history. Total: 4
New Minimum: 1.4809047283063945 > 1.3780347277380751
END: th(46.4158883361278)=1.3780347277380751; dx=-0.008439348856899368 evalInputDelta=0.10287000056831941
Fitness changed from 1.4809047283063945 to 1.3780347277380751
Iteration 5 complete. Error: 1.3780347277380751 Total: 1.7532; Orientation: 1.6152; Line Search: 0.1283
Non-optimal measurement 1.3780347277380751 < 1.3780347277380751. Total: 5
Rejected: LBFGS Orientation magnitude: 7.641e+03, gradient 5.411e+00, dot -0.020; [a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.3780347277380751, 1.4809047283063945, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
Rejected: LBFGS Orientation magnitude: 1.020e+04, gradient 5.411e+00, dot -0.020; [908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.3780347277380751, 1.4809047283063945, 1.6413426193805818, 1.7057421230205698
LBFGS Accumulation History: 3 points
Removed measurement 45eac36d to history. Total: 4
Removed measurement 773ebb7b to history. Total: 3
Adding measurement 79ce989d to history. Total: 3
th(0)=1.3780347277380751;dx=-29.28056826034909
Adding measurement 37c7aaa4 to history. Total: 4
New Minimum: 1.3780347277380751 > 1.2705408038420904
END: th(100.00000000000003)=1.2705408038420904; dx=-0.0011029316710938933 evalInputDelta=0.10749392389598467
Fitness changed from 1.3780347277380751 to 1.2705408038420904
Iteration 6 complete. Error: 1.2705408038420904 Total: 2.5719; Orientation: 2.3396; Line Search: 0.2241
Non-optimal measurement 1.2705408038420904 < 1.2705408038420904. Total: 5
Rejected: LBFGS Orientation magnitude: 1.103e+01, gradient 7.044e-01, dot -0.147; [5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2705408038420904, 1.3780347277380751, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
Rejected: LBFGS Orientation magnitude: 1.136e+01, gradient 7.044e-01, dot -0.187; [39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2705408038420904, 1.3780347277380751, 1.6413426193805818, 1.7057421230205698
LBFGS Accumulation History: 3 points
Removed measurement 37c7aaa4 to history. Total: 4
Removed measurement 79ce989d to history. Total: 3
Adding measurement 52489cee to history. Total: 3
th(0)=1.2705408038420904;dx=-0.4961158236329222
Adding measurement 406853ad to history. Total: 4
New Minimum: 1.2705408038420904 > 1.1511241140996145
END: th(215.44346900318845)=1.1511241140996145; dx=-0.0011256707464330034 evalInputDelta=0.11941668974247599
Fitness changed from 1.2705408038420904 to 1.1511241140996145
Iteration 7 complete. Error: 1.1511241140996145 Total: 2.2577; Orientation: 2.1286; Line Search: 0.1207
Non-optimal measurement 1.1511241140996145 < 1.1511241140996145. Total: 5
Rejected:
...skipping 9920 bytes...
98
LBFGS Accumulation History: 3 points
Removed measurement 153944db to history. Total: 4
Removed measurement 5248b428 to history. Total: 3
Adding measurement 7eb1bec8 to history. Total: 3
th(0)=0.9556739224780545;dx=-0.011954882512255328
Adding measurement 18f99459 to history. Total: 4
New Minimum: 0.9556739224780545 > 0.9453994298011681
WOLFE (weak): th(41.66666666666669)=0.9453994298011681; dx=-0.013559917790188005 evalInputDelta=0.010274492676886404
Adding measurement fb15c35 to history. Total: 5
New Minimum: 0.9453994298011681 > 0.9356675689202639
END: th(83.33333333333339)=0.9356675689202639; dx=-0.00536746832657548 evalInputDelta=0.020006353557790568
Fitness changed from 0.9556739224780545 to 0.9356675689202639
Iteration 12 complete. Error: 0.9356675689202639 Total: 1.7910; Orientation: 1.6026; Line Search: 0.1575
Non-optimal measurement 0.9356675689202639 < 0.9356675689202639. Total: 6
Rejected: LBFGS Orientation magnitude: 5.060e+05, gradient 6.441e+00, dot -0.002; [908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.9356675689202639, 0.9453994298011681, 0.9556739224780545, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
Rejected: LBFGS Orientation magnitude: 6.757e+05, gradient 6.441e+00, dot -0.002; [908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.9356675689202639, 0.9453994298011681, 0.9556739224780545, 1.6413426193805818, 1.7057421230205698
Rejected: LBFGS Orientation magnitude: 2.550e+06, gradient 6.441e+00, dot -0.002; [39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.9356675689202639, 0.9453994298011681, 0.9556739224780545, 1.6413426193805818
LBFGS Accumulation History: 3 points
Removed measurement fb15c35 to history. Total: 5
Removed measurement 18f99459 to history. Total: 4
Removed measurement 7eb1bec8 to history. Total: 3
Adding measurement 67045b4 to history. Total: 3
th(0)=0.9356675689202639;dx=-41.48278383797709
Adding measurement 4a9b3797 to history. Total: 4
New Minimum: 0.9356675689202639 > 0.8924715592756283
END: th(179.53622416932376)=0.8924715592756283; dx=-6.652865007505832E-5 evalInputDelta=0.04319600964463566
Fitness changed from 0.9356675689202639 to 0.8924715592756283
Iteration 13 complete. Error: 0.8924715592756283 Total: 3.8828; Orientation: 3.7443; Line Search: 0.1304
Non-optimal measurement 0.8924715592756283 < 0.8924715592756283. Total: 5
Rejected: LBFGS Orientation magnitude: 7.719e+02, gradient 2.223e-01, dot -0.001; [5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.8924715592756283, 0.9356675689202639, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
Rejected: LBFGS Orientation magnitude: 7.733e+02, gradient 2.223e-01, dot -0.001; [a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.8924715592756283, 0.9356675689202639, 1.6413426193805818, 1.7057421230205698
LBFGS Accumulation History: 3 points
Removed measurement 4a9b3797 to history. Total: 4
Removed measurement 67045b4 to history. Total: 3
Adding measurement 1bf2390b to history. Total: 3
th(0)=0.8924715592756283;dx=-0.04942799163342897
Adding measurement 57308ef8 to history. Total: 4
New Minimum: 0.8924715592756283 > 0.8864074512624992
WOLF (strong): th(386.79906946773184)=0.8864074512624992; dx=8.498716114134075E-4 evalInputDelta=0.006064108013129066
Adding measurement 5b3e2af3 to history. Total: 5
New Minimum: 0.8864074512624992 > 0.8744114401326094
WOLF (strong): th(193.39953473386592)=0.8744114401326094; dx=0.0010891184236647364 evalInputDelta=0.01806011914301886
Non-optimal measurement 0.876602826272569 < 0.8744114401326094. Total: 6
WOLF (strong): th(64.4665115779553)=0.876602826272569; dx=1.7979035805720605E-4 evalInputDelta=0.015868733003059243
Non-optimal measurement 0.8838187581834538 < 0.8744114401326094. Total: 6
END: th(16.116627894488825)=0.8838187581834538; dx=-0.020175196432269824 evalInputDelta=0.008652801092174478
Fitness changed from 0.8924715592756283 to 0.8744114401326094
Iteration 14 complete. Error: 0.8744114401326094 Total: 2.9373; Orientation: 2.5872; Line Search: 0.3420
Non-optimal measurement 0.8744114401326094 < 0.8744114401326094. Total: 6
Rejected: LBFGS Orientation magnitude: 5.505e+01, gradient 1.362e+00, dot -0.054; [39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.8744114401326094, 0.8864074512624992, 0.8924715592756283, 1.6413426193805818, 1.7057421230205698, 1.9743119500970525
Rejected: LBFGS Orientation magnitude: 5.868e+01, gradient 1.362e+00, dot -0.067; [39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.8744114401326094, 0.8864074512624992, 0.8924715592756283, 1.6413426193805818, 1.7057421230205698
Rejected: LBFGS Orientation magnitude: 9.718e+01, gradient 1.362e+00, dot -0.149; [39e2fc8d-c85c-46a2-89f6-56c46aa89a2b = 1.000/1.000e+00, a2bca0de-db46-4278-bfab-b2db8877cd4c = 1.000/1.000e+00, 1e6459cf-0b2e-4ee9-a4dd-3f15d3064b80 = 1.000/1.000e+00, 5d93da0c-a8ad-4d2f-a099-ac890683bfc8 = 1.000/1.000e+00, 908aceef-4ab0-4535-9f27-c6a340a9cddf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.8744114401326094, 0.8864074512624992, 0.8924715592756283, 1.6413426193805818
LBFGS Accumulation History: 3 points
Removed measurement 5b3e2af3 to history. Total: 5
Removed measurement 57308ef8 to history. Total: 4
Removed measurement 1bf2390b to history. Total: 3
Adding measurement 31329e96 to history. Total: 3
th(0)=0.8744114401326094;dx=-1.8562504269959237
Adding measurement 33f2b457 to history. Total: 4
New Minimum: 0.8744114401326094 > 0.8703339278050702
WOLF (strong): th(34.72222222222224)=0.8703339278050702; dx=0.14143232690710889 evalInputDelta=0.004077512327539168
Adding measurement 591de976 to history. Total: 5
New Minimum: 0.8703339278050702 > 0.8683763106665164
WOLF (strong): th(17.36111111111112)=0.8683763106665164; dx=9.310633500979024E-4 evalInputDelta=0.00603512946609297
Adding measurement d088d06 to history. Total: 6
New Minimum: 0.8683763106665164 > 0.8681748610604995
END: th(5.787037037037041)=0.8681748610604995; dx=-2.315929389540735E-4 evalInputDelta=0.006236579072109949
Fitness changed from 0.8744114401326094 to 0.8681748610604995
Iteration 15 complete. Error: 0.8681748610604995 Total: 4.1462; Orientation: 3.6131; Line Search: 0.4535
Final threshold in iteration 15: 0.8681748610604995 (> 0.0) after 33.805s (< 30.000s)
Returns
0.8681748610604995
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.6852852640807611 ], [ -0.7675735426405655 ], [ -0.9352233795196302 ], [ 0.4789402278948783 ], [ 0.40044816080135687 ], [ -1.1671542067301313 ], [ -0.3763140929946246 ], [ -0.7900053605066606 ], ... ],
[ [ 0.3088089817288905 ], [ 0.4229542535334754 ], [ 0.6448694888491372 ], [ -0.3860850861665317 ], [ 0.4710246015189922 ], [ -0.9334960749672486 ], [ 0.4861058004412744 ], [ -0.20517471911883203 ], ... ],
[ [ -0.2635331827871483 ], [ 0.553120861076855 ], [ 0.32903517540289123 ], [ -0.4230506564932769 ], [ -0.11129407946203661 ], [ 0.509705334895507 ], [ -0.157987178892359 ], [ -0.7390580782216412 ], ... ],
[ [ -0.632954494942524 ], [ -0.5343258434049301 ], [ -0.8307959380030786 ], [ 0.45084875960420806 ], [ 0.4868373416795259 ], [ -0.06535869778484128 ], [ 0.5354777684095701 ], [ 0.3811297028111763 ], ... ],
[ [ -0.6308439262290897 ], [ -0.9862549654013313 ], [ -0.4904445421041169 ], [ 0.02100053822599411 ], [ 0.7562798615377261 ], [ 0.34425480851961116 ], [ 0.07970369916450123 ], [ 0.5652928351320858 ], ... ],
[ [ -0.4271680565019703 ], [ -0.8985840781732297 ], [ 0.5070201999780092 ], [ -0.7467797256114244 ], [ -0.8112957258349826 ], [ 0.8479938867242813 ], [ 0.254537769378467 ], [ 0.45296958782942537 ], ... ],
[ [ -0.6604429240883438 ], [ -0.09077155094884794 ], [ -0.4507420933449073 ], [ -0.29362779822320734 ], [ 0.5923842075267054 ], [ 0.5403242372008463 ], [ -0.643641260899033 ], [ 0.46858646931268955 ], ... ],
[ [ -0.6768712710550363 ], [ -0.1245016358167074 ], [ 0.839503528260518 ], [ -0.8248400233855717 ], [ -0.6144210311102093 ], [ -0.8342405302806802 ], [ -0.6773864166016441 ], [ -0.12769227554085605 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ -0.37792008349509204 ], [ -0.26452098308127364 ], [ -0.06696986965041497 ], [ -0.7361794745456867 ], [ -0.9151709570526673 ], [ 0.1545684840179005 ], [ -0.9773311305968582 ], [ -0.23571554809262185 ], ... ],
[ [ -1.1750323753448835 ], [ -0.8604912534592396 ], [ -0.43870732552999964 ], [ -0.9516975033252928 ], [ -0.7528449538053905 ], [ -0.06881852065060408 ], [ -0.721328982392839 ], [ -1.5838933744506933 ], ... ],
[ [ -1.3335759880483296 ], [ -0.5921787460468456 ], [ -1.1115906178136756 ], [ -0.860263351802698 ], [ -2.1955792165266823 ], [ -0.6739224949545878 ], [ -1.8452413954985978 ], [ -0.30237877083560144 ], ... ],
[ [ -0.4573567473485241 ], [ -0.6267494324721938 ], [ -0.1853710762517873 ], [ -0.7966233402783157 ], [ -0.7198252123482478 ], [ -2.727864752361408 ], [ -0.6245959055311615 ], [ -0.9646155344617487 ], ... ],
[ [ -0.46069679057089674 ], [ -0.013840372207267904 ], [ -0.7124430703768526 ], [ -3.8632072117779086 ], [ -0.279343784085773 ], [ -1.0663731733674802 ], [ -2.529439280655753 ], [ -0.5704113898335648 ], ... ],
[ [ -0.8505777682377496 ], [ -0.10693500098376638 ], [ -0.6792044340189333 ], [ -0.29198501603415805 ], [ -0.20912264788664744 ], [ -0.16488185226777435 ], [ -1.3683060478004427 ], [ -0.7919302907871414 ], ... ],
[ [ -0.41484457194307883 ], [ -2.399409358009781 ], [ -0.7968599581620255 ], [ -1.2254423061269462 ], [ -0.5235998554065214 ], [ -0.6155858803545177 ], [ -0.4406137563802531 ], [ -0.7580346280314924 ], ... ],
[ [ -0.3902741703020218 ], [ -2.0834364240736725 ], [ -0.17494459966155917 ], [ -0.19256582249842596 ], [ -0.48707486741191247 ], [ -0.1812335125919679 ], [ -0.38951339105572375 ], [ -2.058132006881593 ], ... ],
...
]
TrainingTester.java:432 executed in 0.11 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, -0.08237592177566926], [114.0, 0.23191337441971493]; valueStats=DoubleSummaryStatistics{count=185, sum=177.197907, min=0.827226, average=0.957827, max=1.705742}
Plotting 114 points for GD
Plotting 56 points for CjGD
Plotting 15 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, -0.08237592177566926], [33.655, 0.23191337441971493]; valueStats=DoubleSummaryStatistics{count=185, sum=177.197907, min=0.827226, average=0.957827, max=1.705742}
Plotting 114 points for GD
Plotting 56 points for CjGD
Plotting 15 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.8681748610604995 }, "CjGD": { "type": "NonConverged", "value": 1.0135561067337253 }, "GD": { "type": "NonConverged", "value": 0.8272258137790433 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.8681748610604995 }, "CjGD": { "type": "NonConverged", "value": 1.0135561067337253 }, "GD": { "type": "NonConverged", "value": 0.8272258137790433 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "95.316",
"gc_time": "0.596"
},
"created_on": 1586738756024,
"file_name": "trainingTest",
"report": {
"simpleName": "Basic",
"canonicalName": "com.simiacryptus.mindseye.layers.java.LogActivationLayerTest.Basic",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/LogActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 0.8681748610604995
},
"CjGD": {
"type": "NonConverged",
"value": 1.0135561067337253
},
"GD": {
"type": "NonConverged",
"value": 0.8272258137790433
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/LogActivationLayer/Basic/trainingTest/202004134556",
"id": "27a0941c-8a49-4d77-aa6a-d8f2b306c99b",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "LogActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.LogActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/LogActivationLayer.java",
"javaDoc": ""
}
}