1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 5078651485734767616

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.4342204986479088 ], [ -0.6750827982203931 ], [ 0.2050655758333786 ], [ -0.11886069735303284 ], [ -0.5722144495470372 ], [ 0.3457202605064462 ], [ -0.2784958345146862 ], [ -0.8917772609914059 ], ... ],
    	[ [ -0.13079028490653033 ], [ 0.29940448563769184 ], [ 0.5829928407838301 ], [ 0.48241460876766973 ], [ -0.3475018725750034 ], [ 0.20157372006922625 ], [ 0.870234087415346 ], [ -0.5823529452313754 ], ... ],
    	[ [ 0.23344470325557354 ], [ -0.5298363273829825 ], [ -0.36381626723923843 ], [ -0.8932603697323345 ], [ 0.046623108802519075 ], [ 0.15243973344964346 ], [ -0.5754235881858571 ], [ 0.0651318720263907 ], ... ],
    	[ [ -0.3101855296284417 ], [ -0.5058136089634746 ], [ 0.35612113797782285 ], [ -0.11902909544157203 ], [ -0.863308375248127 ], [ -0.8570489833621218 ], [ -0.46056185468384725 ], [ -0.053870145718739826 ], ... ],
    	[ [ -0.41676228491939016 ], [ 0.751526038389817 ], [ -0.7660981823548669 ], [ 0.24149486456224523 ], [ -0.4138531682053923 ], [ -0.47138201466563956 ], [ 0.19779564490612045 ], [ -0.9234305318597511 ], ... ],
    	[ [ 0.5625507230842948 ], [ 0.5445460298981322 ], [ -0.761352066890822 ], [ -0.25524728147702125 ], [ 0.03995941567202976 ], [ -0.1746245679939682 ], [ 0.09072796233739422 ], [ 0.34801435606369835 ], ... ],
    	[ [ -0.4023616543352231 ], [ -0.6190109572147362 ], [ -0.05786954665168831 ], [ -0.9792900641784381 ], [ 0.050845914061671715 ], [ -0.7268626848522732 ], [ -0.20070327083090955 ], [ -0.33488777986442897 ], ... ],
    	[ [ -0.6344332729026756 ], [ 0.17545989540584883 ], [ -0.5735463047935643 ], [ 0.3952183752305063 ], [ 0.7917363422124288 ], [ -0.36015408458902765 ], [ 0.8990590580663163 ], [ -0.017012634890601674 ], ... ],
    	...
    ]
    [
    	[ [ 0.3503852788878645 ], [ 0.3208255865171603 ], [ 0.4671569850122659 ], [ -0.42169848784215413 ], [ 0.5536989078423105 ], [ 0.9126825607903946 ], [ -0.929647716932331 ], [ -0.14978035620132868 ], ... ],
    	[ [ 0.9997302109969171 ], [ 0.6209525246305541 ], [ -0.6153627416395389 ], [ 0.6797169129544374 ], [ -0.47006129707364375 ], [ -0.5438059845824326 ], [ -0.9475575941072609 ], [ 0.5604473365684848 ], ... ],
    	[ [ -0.04800733047608885 ], [ 0.8299948278128457 ], [ -0.8091977588132166 ], [ -0.31546578956180116 ], [ -0.1834242732870428 ], [ 0.43080826525227023 ], [ 0.9220147568564854 ], [ -0.7184771159512076 ], ... ],
    	[ [ -0.39131930231612255 ], [ -0.0828100310043538 ], [ 0.5528565896392652 ], [ -0.7643447966015076 ], [ 0.7610469955758694 ], [ 0.18593383790922657 ], [ -0.7375827124762759 ], [ -0.5475693621409601 ], ... ],
    	[ [ -0.35654881729441557 ], [ -0.6809000731922288 ], [ 0.5492841496054772 ], [ 0.4357621141688628 ], [ 0.5138059348044663 ], [ -0.30497931923634747 ], [ 0.6095337173662452 ], [ -0.02396672127053745 ], ... ],
    	[ [ 0.6338796869764296 ], [ 0.07988728303519621 ], [ -0.26349324541062824 ], [ -0.47463272980105664 ], [ 0.16278352906626337 ], [ -0.8521350001916428 ], [ 0.07219158203193543 ], [ 0.1475996154778818 ], ... ],
    	[ [ -0.6215430444566006 ], [ 0.3323639664917146 ], [ -0.6874889514362277 ], [ -0.8512791458761559 ], [ -0.0435451797364077 ], [ -0.1759742481236808 ], [ -0.5277911453362765 ], [ 0.179770019663281 ], ... ],
    	[ [ -0.342559899241746 ], [ 0.8491832348816302 ], [ -0.7976413094269694 ], [ 0.35794648846128885 ], [ 0.35212846288049793 ], [ 0.4126719789714417 ], [ 0.19800377391109092 ], [ 0.7046208008892065 ], ... ],
    	...
    ]
    [
    	[ [ -0.22528523468120665 ], [ 0.5661420284475707 ], [ 0.19591509822926056 ], [ -0.6031330570918243 ], [ 0.6887401080658544 ], [ 0.598141542170914 ], [ -0.9406990717083066 ], [ 0.3487207865917388 ], ... ],
    	[ [ 0.08236968594247851 ], [ -0.4937566081486924 ], [ 0.23453594954079238 ], [ 0.9757735670818923 ], [ -0.8424494298638852 ], [ 0.4816767369165489 ], [ -0.6798837396567403 ], [ 0.9322480401681772 ], ... ],
    	[ [ -0.9146119722720987 ], [ -0.130685004177574 ], [ -0.8773850180106939 ], [ -0.1157312911438193 ], [ -0.9157682058886862 ], [ 0.3858047366525936 ], [ -0.26211118714358816 ], [ 0.7490063456742868 ], ... ],
    	[ [ 0.42941642811288805 ], [ -0.34165139952002677 ], [ 0.28649389637874956 ], [ -0.1042352854307731 ], [ 0.809696079612005 ], [ 0.3834724192298604 ], [ -0.8583876945921682 ], [ -0.08330697436800061 ], ... ],
    	[ [ 0.4408353035908237 ], [ -0.5941582936893817 ], [ 0.9123684911253716 ], [ 0.5483616444351771 ], [ 0.08779114200267979 ], [ 0.7868713974145474 ], [ 0.6892709816447409 ], [ -0.2983967806808365 ], ... ],
    	[ [ 0.015834670730212386 ], [ 0.9385006390114019 ], [ -0.22111145004903165 ], [ 0.4454158817532172 ], [ -0.4188405208214723 ], [ 0.3661209256012248 ], [ 0.2760669202098063 ], [ 0.639803425180683 ], ... ],
    	[ [ 0.5168439790437165 ], [ 0.3176801931932316 ], [ -0.48738921484915076 ], [ 0.8939416173639418 ], [ 0.07051281871998194 ], [ 0.8795281698434017 ], [ -0.9281909977147276 ], [ 0.821802320283062 ], ... ],
    	[ [ 0.9586138933281954 ], [ 0.3998349663203653 ], [ -0.3751924160487563 ], [ -0.6609515221149462 ], [ -0.020292199508013198 ], [ -0.3837302597865202 ], [ 0.5266091088278533 ], [ -0.2556367555728283 ], ... ],
    	...
    ]
    [
    	[ [ 0.021906431493501133 ], [ 0.48629952675366894 ], [ -0.09980341232326273 ], [ -0.22572639186032273 ], [ -0.010479267559881933 ], [ -0.9196846213000536 ], [ 0.8938549829067373 ], [ -0.13814895944005734 ], ... ],
    	[ [ 0.5798095347865024 ], [ 0.8826163446224786 ], [ 0.13427859476706683 ], [ 0.7914906126845993 ], [ 0.3037521051831591 ], [ 0.8779347056001992 ], [ 0.24208736382533647 ], [ 0.08432294270500584 ], ... ],
    	[ [ 0.5154635627679287 ], [ 0.5332675993350964 ], [ -0.9960770423698653 ], [ -0.49352064634453785 ], [ 0.5205787120682739 ], [ 0.3711272971728339 ], [ -0.8551826858487503 ], [ 0.6981880320578666 ], ... ],
    	[ [ 0.009982455632171927 ], [ -0.9512483861725252 ], [ -0.9875918471351317 ], [ 0.31391951033783005 ], [ 0.2374211096104979 ], [ -0.13545760063126244 ], [ 0.9794942483377524 ], [ 0.9266120826376014 ], ... ],
    	[ [ -0.34799711486599794 ], [ 0.32565124643252163 ], [ 0.608757490287265 ], [ -0.7273012146739954 ], [ 0.01350122627597905 ], [ 0.4852503370127239 ], [ 0.7333603724639395 ], [ 0.9218056081973163 ], ... ],
    	[ [ -0.9703575368589743 ], [ -0.4378416150569371 ], [ 0.14524754219779257 ], [ 0.8285508317361934 ], [ 0.33149026309596374 ], [ -0.5424585776413868 ], [ -0.868658995262886 ], [ 0.8457997574553546 ], ... ],
    	[ [ 0.6681833090087024 ], [ 0.9960963631818693 ], [ 0.8738239986748211 ], [ 0.6258507833290545 ], [ -0.007275058038220816 ], [ -0.45808094045342007 ], [ -0.0656098647702823 ], [ -0.1440176403367046 ], ... ],
    	[ [ -0.6092523420480125 ], [ -0.2411842488076441 ], [ -0.11690328346675916 ], [ 0.7720786508492925 ], [ 0.41201339160736694 ], [ -0.9145897099606735 ], [ 0.12880207081141415 ], [ 0.4960546616986887 ], ... ],
    	...
    ]
    [
    	[ [ -0.8033575297310298 ], [ 0.9970154603986137 ], [ 0.011653065401375784 ], [ 0.25765937314555587 ], [ 0.05427662137867206 ], [ 0.8466153767822582 ], [ -0.6174491014973316 ], [ 0.36150001551248256 ], ... ],
    	[ [ -0.13157974115812476 ], [ 0.2854510336517533 ], [ -0.6663529196245661 ], [ -0.1024558699549778 ], [ -0.33463963689652987 ], [ -0.25541624730845136 ], [ -0.5335044977292787 ], [ 0.7507847360066184 ], ... ],
    	[ [ 0.10560091956303341 ], [ -0.8055167831715841 ], [ -0.1604659323866593 ], [ -0.511063346266974 ], [ -0.021102812333598786 ], [ 0.4487836849607727 ], [ -0.2286730278902458 ], [ 0.6253867772409358 ], ... ],
    	[ [ -0.9641452160990533 ], [ 0.672067471633969 ], [ 0.8172310426039928 ], [ 0.671911855718822 ], [ -0.1890446618846453 ], [ 0.40333783655079447 ], [ -0.9268668903166724 ], [ 0.1956006827091996 ], ... ],
    	[ [ -0.7186246244941561 ], [ -0.48397098942910466 ], [ -0.7771595870091585 ], [ 0.5461711990035056 ], [ -0.15382239072814152 ], [ -0.021799782198556272 ], [ 0.14413138864909758 ], [ 0.342677234335663 ], ... ],
    	[ [ 0.6071337261842555 ], [ 0.7879198591774514 ], [ -0.11856603259510834 ], [ -0.7620196384084654 ], [ -0.7194298740214802 ], [ -0.5459474343816464 ], [ -0.5054092957023104 ], [ 0.7519655378561736 ], ... ],
    	[ [ -0.2529891145535168 ], [ 0.46672448538094047 ], [ 0.6730067183778639 ], [ 0.13022894444847544 ], [ -0.6841039787324337 ], [ 0.22995197104387377 ], [ -0.07668918647672629 ], [ 0.8125675599509083 ], ... ],
    	[ [ 0.7154092008261059 ], [ -0.6668567494328068 ], [ -0.17791382216122464 ], [ -0.46403463541749956 ], [ 0.9972568112697062 ], [ 0.07194887318045096 ], [ 0.8787439579597662 ], [ -0.13071737744159262 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 3.37 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3889917479575
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 3889995317102
Constructing line search parameters: GD
th(0)=3703.9375980032173;dx=-79.00208198552897
New Minimum: 3703.9375980032173 > 3546.725410306643
END: th(2.154434690031884)=3546.725410306643; dx=-66.94078499850956 evalInputDelta=157.21218769657435
Fitness changed from 3703.9375980032173 to 3546.725410306643
Iteration 1 complete. Error: 3546.725410306643 Total: 0.2584; Orientation: 0.0170; Line Search: 0.1252
th(0)=3546.725410306643;dx=-56.748782520152744
New Minimum: 3546.725410306643 > 3326.4016088191374
END: th(4.641588833612779)=3326.4016088191374; dx=-38.185865672727054 evalInputDelta=220.32380148750553
Fitness changed from 3546.725410306643 to 3326.4016088191374
Iteration 2 complete. Error: 3326.4016088191374 Total: 0.5648; Orientation: 0.1012; Line Search: 0.4399
th(0)=3326.4016088191374;dx=-25.791556045620904
New Minimum: 3326.4016088191374 > 3157.931486913457
END: th(10.000000000000002)=3157.931486913457; dx=-7.902468335514637 evalInputDelta=168.47012190568057
Fitness changed from 3326.4016088191374 to 3157.931486913457
Iteration 3 complete. Error: 3157.931486913457 Total: 0.1668; Orientation: 0.0111; Line Search: 0.1232
th(0)=3157.931486913457;dx=-2.643424214252218
New Minimum: 3157.931486913457 > 3140.065911239202
WOLF (strong): th(21.544346900318843)=3140.065911239202; dx=0.9849310831383707 evalInputDelta=17.86557567425507
New Minimum: 3140.065911239202 > 3139.2273809257513
END: th(10.772173450159421)=3139.2273809257513; dx=-0.8292465655569237 evalInputDelta=18.70410598770559
Fitness changed from 3157.931486913457 to 3139.2273809257513
Iteration 4 complete. Error: 3139.2273809257513 Total: 0.3124; Orientation: 0.0131; Line Search: 0.2715
th(0)=3139.2273809257513;dx=-0.28876136919752887
New Minimum: 3139.2273809257513 > 3137.042220242793
WOLF (strong): th(23.2079441680639)=3137.042220242793; dx=0.10044993005023896 evalInputDelta=2.1851606829582124
New Minimum: 3137.042220242793 > 3137.0057013214578
END: th(11.60397208403195)=3137.0057013214578; dx=-0.09415571957364496 evalInputDelta=2.221679604293513
Fitness changed from 3139.2273809257513 to 3137.0057013214578
Iteration 5 complete. Error: 3137.0057013214578 Total: 0.3586; Orientation: 0.0103; Line Search: 0.3151
th(0)=3137.0057013214578;dx=-0.033045447170755124
New Minimum: 3137.0057013214578 > 3136.7433035882364
WOLF (strong): th(25.000000000000007)=3136.7433035882364; dx=0.012053628513141094 evalInputDelta=0.26239773322140536
New Minimum: 3136.7433035882364 > 3136.7335678433346
END: th(12.500000000000004)=3136.7335678433346; dx=-0.010495909328807014 evalInputDelta=0.2721334781231235
Fitness changed from 3137.0057013214578 to 3136.7335678433346
Iteration 6 complete. Error: 3136.7335678433346 Total: 0.4488; Orientation: 0.0139; Line Search: 0.4083
th(0)=3136.7335678433346;dx=-0.003482241155481328
New Minimum: 3136.7335678433346 > 3136.7063636243174
WOLF (strong): th(26.930433625398557)=3136.7063636243174; dx=0.0014619083676814847 evalInputDelta=0.027204219017221476
New Minimum: 3136.7063636243174 > 3136.7033222225036
END: th(13.465216812699278)=3136.7033222225036; dx=-0.0010101663938999216 evalInputDelta=0.030245620831010456
Fitness changed from 3136.7335678433346 to 3136.7033222225036
Iteration 7 complete. Error: 3136.7033222225036 Total: 0.1051; Orientation: 0.0037; Line Search: 0.0836
th(0)=3136.7033222225036;dx=-3.022565867063353E-4
New Minimum: 3136.7033222225036 > 3136.7011130842566
WOLF (strong): th(29.00993021007988)=3136.7011130842566; dx=1.4995437637218808E-4 evalInputDelta=0.002209138247053488
New Minimum: 3136.7011130842566 > 3136.700577827319
END: th(14.50496510503994)=3136.700577827319; dx=-7.615110516707361E-5 evalInputDelta=0.0027443951844361436
Fitness changed from 3136.7033222225036 to 3136.700577827319
Iteration 8 complete. Error: 3136.700577827319 Total: 0.1084; Orientation: 0.0041; Line Search: 0.0894
th(0)=3136.700577827319;dx=-1.9698523797223748E-5
New Minimum: 3136.700577827319 > 3136.700450644962
WOLF (strong): th(31.250000000000014)=3136.700450644962; dx=1.1558852917460278E-5 evalInputDelta=1.2718235711872694E-4
New Minimum: 3136.700450644962 > 3136.7003921370065
END: th(15.625000000000007)=3136.7003921370065; dx=-4.069835439881734E-6 evalInputDelta=1.8569031271908898E-4
Fitness changed from 3136.700577827319 to 3136.7003921370065
Iteration 9 complete. Error: 3136.7003921370065 Total: 0.1039; Orientation: 0.0038; Line Search: 0.0854
th(0)=3136.7003921370065;dx=-8.629969689196574E-7
New Minimum: 3136.7003921370065 > 3136.700387623428
WOLF (strong): th(33.6630420317482)=3136.700387623428; dx=5.948343988525603E-7 evalInputDelta=4.513578460318968E-6
New Minimum: 3136.700387623428 > 3136.700383745838
END: th(16.8315210158741)=3136.700383745838; dx=-1.3408128503354854E-7 evalInputDelta=8.391168648813618E-6
Fitness changed from 3136.7003921370065 to 3136.700383745838
Iteration 10 complete. Error: 3136.700383745838 Total: 0.1474; Orientation: 0.0035; Line Search: 0.1311
th(0)=3136.700383745838;dx=-2.147222479516458E-8
New Minimum: 3136.700383745838 > 3136.700383668908
WOLF (strong): th(36.26241276259985)=3136.700383668908; dx=1.722918384390755E-8 evalInputDelta=7.692960934946314E-8
New Minimum: 3136.700383668908 > 3136.7003835319456
END: th(18.131206381299926)=3136.7003835319456; dx=-2.1215204756285126E-9 evalInputDelta=2.1389223547885194E-7
Fitness changed from 3136.700383745838 to 3136.7003835319456
Iteration 11 complete. Error: 3136.7003835319456 Total: 0.0968; Orientation: 0.0035; Line Search: 0.0817
th(0)=3136.7003835319456;dx=-2.196249211750886E-10
New Minimum: 3136.7003835319456 > 3136.7003835316254
WOLF (strong): th(39.06250000000002)=3136.7003835316254; dx=2.032366504729845E-10 evalInputDelta=3.2014213502407074E-10
New Minimum: 3136.7003835316254 > 3136.700383529721
END: th(19.53125000000001)=3136.700383529721; dx=-8.194135351052056E-12 evalInputDelta=2.2246240405365825E-9
Fitness changed from 3136.7003835319456 to 3136.700383529721
Iteration 12 complete. Error: 3136.700383529721 Total: 0.1051; Orientation: 0.0034; Line Search: 0.0856
Low gradient: 6.011001748045638E-7
th(0)=3136.700383529721;dx=-3.613214201500772E-13
Armijo: th(42.078802539685256)=3136.700383529722; dx=3.8255330840312863E-13 evalInputDelta=-9.094947017729282E-13
New Minimum: 3136.700383529721 > 3136.7003835297164
WOLF (strong): th(21.039401269842628)=3136.7003835297164; dx=1.061594412652574E-14 evalInputDelta=4.547473508864641E-12
END: th(7.013133756614209)=3136.7003835297182; dx=-2.3734229872454287E-13 evalInputDelta=2.7284841053187847E-12
Fitness changed from 3136.700383529721 to 3136.7003835297164
Iteration 13 complete. Error: 3136.7003835297164 Total: 0.1005; Orientation: 0.0042; Line Search: 0.0789
Low gradient: 2.225631529388711E-8
th(0)=3136.7003835297164;dx=-4.953435704609133E-16
END: th(15.109338651083275)=3136.7003835297164; dx=-6.158164734969548E-17 evalInputDelta=0.0
Fitness changed from 3136.7003835297164 to 3136.7003835297164
Static Iteration Total: 0.4777; Orientation: 0.0026; Line Search: 0.2973
Iteration 14 failed. Error: 3136.7003835297164
Previous Error: 0.0 -> 3136.7003835297164
Optimization terminated 14
Final threshold in iteration 14: 3136.7003835297164 (> 0.0) after 3.355s (< 30.000s)

Returns

    3136.7003835297164

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.01 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 5.055592033505019E-10 ], [ -1.1572668946546784E-10 ], [ 4.2551460759347085E-11 ], [ 4.715724476219627E-10 ], [ 4.76653994551654E-10 ], [ 2.2948646272792136E-10 ], [ -2.023697912957748E-9 ], [ -7.0667449093335E-10 ], ... ],
    	[ [ 9.650127495979636E-10 ], [ 1.4250433144660112E-11 ], [ 3.8416785785871116E-11 ], [ -1.0865027350449591E-9 ], [ -1.3980034164124166E-10 ], [ -3.686776335530228E-10 ], [ -7.531984957577562E-11 ], [ 5.268837186585856E-10 ], ... ],
    	[ [ -8.916261650928137E-10 ], [ 6.948914659334782E-11 ], [ -3.3086087122707144E-10 ], [ 1.9868115145296733E-9 ], [ -3.300840863251831E-10 ], [ -1.3292000736891143E-9 ], [ 5.920317471343252E-10 ], [ 6.497139515508578E-10 ], ... ],
    	[ [ 4.9383081169939995E-11 ], [ -3.8313464407384724E-10 ], [ 4.423466727058452E-10 ], [ -2.1687413163476986E-10 ], [ -2.197071313844589E-10 ], [ -3.983911682549185E-12 ], [ -2.1767232486625911E-10 ], [ 1.7232651863537762E-10 ], ... ],
    	[ [ -1.8177400331316712E-10 ], [ 1.0945009015177354E-10 ], [ 5.823668953733057E-11 ], [ -1.3691054561597938E-9 ], [ -3.1375924980053504E-10 ], [ 5.111032132088251E-10 ], [ 7.699437430962938E-10 ], [ 2.7212417515707946E-9 ], ... ],
    	[ [ 1.8057117014795931E-10 ], [ 7.793477468718436E-11 ], [ 3.8276755744007515E-11 ], [ 1.4993399393704627E-9 ], [ -4.899945689840394E-10 ], [ 1.5935888406443231E-9 ], [ -3.975843385358685E-10 ], [ -3.3609157436327463E-10 ], ... ],
    	[ [ 5.463154580959835E-10 ], [ 1.899556248549124E-10 ], [ -2.05579445538643E-10 ], [ 7.96906352937169E-9 ], [ 5.658469417278522E-10 ], [ 6.210701608355076E-11 ], [ 8.193151716440217E-10 ], [ -9.044820163527577E-10 ], ... ],
    	[ [ 3.4203345613284535E-11 ], [ 2.7560632668730494E-10 ], [ 2.3602006741581E-10 ], [ 2.6240054361892575E-11 ], [ 2.0949284262299603E-10 ], [ -4.550050573966829E-10 ], [ 2.9540233560995053E-9 ], [ 2.109623541738631E-10 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 6.68 seconds (0.053 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3893301677575
Reset training subject: 3893318463670
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=3703.9375980032173}, derivative=-79.00208198552897}
New Minimum: 3703.9375980032173 > 3703.937597995314
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=3703.937597995314}, derivative=-79.00208198496914}, evalInputDelta = -7.903508958406746E-9
New Minimum: 3703.937597995314 > 3703.937597947913
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=3703.937597947913}, derivative=-79.00208198161012}, evalInputDelta = -5.5304099078057334E-8
New Minimum: 3703.937597947913 > 3703.9375976161054
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=3703.9375976161054}, derivative=-79.00208195809702}, evalInputDelta = -3.8711186789441854E-7
New Minimum: 3703.9375976161054 > 3703.937595293445
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=3703.937595293445}, derivative=-79.00208179350528}, evalInputDelta = -2.7097721613245085E-6
New Minimum: 3703.937595293445 > 3703.937579034814
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=3703.937579034814}, derivative=-79.00208064136318}, evalInputDelta = -1.8968403310282156E-5
New Minimum: 3703.937579034814 > 3703.9374652244187
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=3703.9374652244187}, derivative=-79.00207257636845}, evalInputDelta = -1.3277879861561814E-4
New Minimum: 3703.9374652244187 > 3703.936668552008
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=3703.936668552008}, derivative=-79.00201612140528}, evalInputDelta = -9.29451209231047E-4
New Minimum: 3703.936668552008 > 3703.9310918610295
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=3703.9310918610295}, derivative=-79.00162093666309}, evalInputDelta = -0.0065061421878453984
New Minimum: 3703.9310918610295 > 3703.8920558053433
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=3703.8920558053433}, derivative=-78.9988546434678}, evalInputDelta = -0.045542197874056
New Minimum: 3703.8920558053433 > 3703.6188416885634
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=3703.6188416885634}, derivative=-78.97949059110071}, evalInputDelta = -0.3187563146539105
New Minimum: 3703.6188416885634 > 3701.7082182535933
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=3701.7082182535933}, derivative=-78.84394222453112}, evalInputDelta = -2.2293797496240586
New Minimum: 3701.7082182535933 > 3688.42574794944
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=3688.42574794944}, derivative=-77.895103658544}, evalInputDelta = -15.51185005377738
New Minimum: 3688.42574794944 > 3599.9512491115247
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=3599.9512491115247}, derivative=-71.25323369663417}, evalInputDelta = -103.98634889169261
New Minimum: 3599.9512491115247 > 3201.2666285129135
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=3201.2666285129135}, derivative=-24.76014396326534}, evalInputDelta = -502.67096949030383
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=11221.680976394404}, derivative=300.69148417031647}, evalInputDelta = 7517.7433783911865
F(5.217100560376924) = LineSearchPoint{point=PointSample{avg=3367.9642347579997}, derivative=-49.79488458892547}, evalInputDelta = -335.97336324521757
F(36.51970392263847) = LineSearchPoint{point=PointSample{avg=4552.038659288749}, derivative=125.44829979069551}, evalInputDelta = 848.1010612855316
F(2.809207994049113) = LineSearchPoint{point=PointSample{avg=3504.0944580072537}, derivative=-63.27512954120401}, evalInputDelta = -199.8431399959636
F(19.66445595834379) = LineSearchPoint{point=PointSample{avg=3232.821509132696}, derivative=31.08658512474575}, evalInputDelta = -471.11608887052125
3232.821509132696 <= 3703.9375980032173
New Minimum: 3201.2666285129135 > 3146.5126530080315
F(14.111652021962819) = LineSearchPoint{point=PointSample{avg=3146.5126530080315}, derivative=3.6741443221188774E-15}, evalInputDelta = -557.4249449951858
Right bracket at 14.111652021962819
Converged to right
Fitness changed from 3703.9375980032173 to 3146.5126530080315
Iteration 1 complete. Error: 3146.5126530080315 Total: 1.3190; Orientation: 0.0035; Line Search: 1.2690
F(0.0) = LineSearchPoint{point=PointSample{avg=3146.5126530080315}, derivative=-1.196516881264909}
New Minimum: 3146.5126530080315 > 3137.143930138323
F(14.111652021962819) = LineSearchPoint{point=PointSample{avg=3137.143930138323}, derivative=-0.13128270664014433}, evalInputDelta = -9.368722869708563
F(98.78156415373974) = LineSearchPoint{point=PointSample{avg=3396.6080867984215}, derivative=6.260122341108445}, evalInputDelta = 250.09543379038996
F(7.59858185797998) = LineSearchPoint{point=PointSample{avg=3139.6000478303895}, derivative=-0.6229292487746511}, evalInputDelta = -6.912605177642035
F(53.19007300585986) = LineSearchPoint{point=PointSample{avg=3189.6519209067587}, derivative=2.818596546166897}, evalInputDelta = 43.13926789872721
F(4.091544077373835) = LineSearchPoint{point=PointSample{avg=3142.248898124548}, derivative=-0.8876620022316932}, evalInputDelta = -4.263754883483671
F(28.640808541616845) = LineSearchPoint{point=PointSample{avg=3143.203929196281}, derivative=0.9654672719676017}, evalInputDelta = -3.3087238117504967
3143.203929196281 <= 3146.5126530080315
New Minimum: 3137.143930138323 > 3137.0297691247515
F(15.850815031129079) = LineSearchPoint{point=PointSample{avg=3137.0297691247515}, derivative=-6.461844948013606E-17}, evalInputDelta = -9.48288388328001
Left bracket at 15.850815031129079
Converged to left
Fitness changed from 3146.5126530080315 to 3137.0297691247515
Iteration 2 complete. Error: 3137.0297691247515 Total: 0.8006; Orientation: 0.0029; Line Search: 0.6254
F(0.0) = LineSearchPoint{point=PointSample{avg=3137.0297691247515}, derivative=-0.042617668892396314}
New Minimum: 3137.0297691247515 > 3136.7163442437623
F(15.850815031129079) = LineSearchPoint{point=PointSample{avg=3136.7163442437623}, derivative=0.003070821568609318}, evalInputDelta = -0.3134248809892597
3136.7163442437623 <= 3137.0297691247515
New Minimum: 3136.7163442437623 > 3136.714708467279
F(14.785447710246125) = LineSearchPoint{point=PointSample{avg=3136.714708467279}, derivative=-1.3417001884508117E-18}, evalInputDelta = -0.3150606574727135
Left bracket at 14.785447710246125
Converged to left
Fitness changed from 3137.0297691247515 to 3136.714708467279
Iteration 3 complete. Error: 3136.714708467279 Total: 0.5738; Orientation: 0.0741; Line Search: 0.4899
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.714708467279}, derivative=-0.0017020374904757347}
New Minimum: 3136.714708467279 > 3136.701136500569
F(14.785447710246125) = LineSearchPoint{point=PointSample{avg=3136.701136500569}, derivative=-1.338171930720978E-4}, evalInputDelta = -0.013571966709605476
F(103.49813397172286) = LineSearchPoint{point=PointSample{avg=3137.1066283236046}, derivative=0.009275504591349723}, evalInputDelta = 0.3919198563257851
F(7.961394920901759) = LineSearchPoint{point=PointSample{avg=3136.7045192803353}, derivative=-8.57611176489161E-4}, evalInputDelta = -0.010189186943534878
F(55.72976444631231) = LineSearchPoint{point=PointSample{avg=3136.78456319736}, derivative=0.004208946707430281}, evalInputDelta = 0.06985473008126064
F(4.286904957408639) = LineSearchPoint{point=PointSample{avg=3136.7083866030775}, derivative=-0.001247346398329118}, evalInputDelta = -0.006321864201254357
F(30.008334701860477) = LineSearchPoint{point=PointSample{avg=3136.711388985272}, derivative=0.0014808001545505815}, evalInputDelta = -0.0033194820066455577
3136.711388985272 <= 3136.714708467279
New Minimum: 3136.701136500569 > 3136.70105208526
F(16.0470989681562) = LineSearchPoint{point=PointSample{avg=3136.70105208526}, derivative=-1.861354901591325E-19}, evalInputDelta = -0.013656382018780278
Left bracket at 16.0470989681562
Converged to left
Fitness changed from 3136.714708467279 to 3136.70105208526
Iteration 4 complete. Error: 3136.70105208526 Total: 0.4786; Orientation: 0.0767; Line Search: 0.3919
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.70105208526},

...skipping 2668 bytes...

33742522) = LineSearchPoint{point=PointSample{avg=3136.700383615843}, derivative=1.442396193139342E-23}, evalInputDelta = -1.5770756363053806E-6
Right bracket at 14.929729633742522
Converged to right
Fitness changed from 3136.7003851929185 to 3136.700383615843
Iteration 7 complete. Error: 3136.700383615843 Total: 0.1766; Orientation: 0.0027; Line Search: 0.1639
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.700383615843}, derivative=-1.0103103521064498E-8}
New Minimum: 3136.700383615843 > 3136.700383534713
F(14.929729633742522) = LineSearchPoint{point=PointSample{avg=3136.700383534713}, derivative=-7.649209630385581E-10}, evalInputDelta = -8.112965588225052E-8
F(104.50810743619765) = LineSearchPoint{point=PointSample{avg=3136.70038597569}, derivative=5.526417438511708E-8}, evalInputDelta = 2.3598472580488306E-6
F(8.039085187399818) = LineSearchPoint{point=PointSample{avg=3136.700383554836}, derivative=-5.074851374435146E-9}, evalInputDelta = -6.10066308581736E-8
F(56.27359631179873) = LineSearchPoint{point=PointSample{avg=3136.7003840376537}, derivative=2.5094661505340963E-8}, evalInputDelta = 4.218109097564593E-7
F(4.3287381778306715) = LineSearchPoint{point=PointSample{avg=3136.700383577963}, derivative=-7.395583134417924E-9}, evalInputDelta = -3.7879999581491575E-8
F(30.3011672448147) = LineSearchPoint{point=PointSample{avg=3136.7003835968476}, derivative=8.84953918546152E-9}, evalInputDelta = -1.8995251593878493E-8
3136.7003835968476 <= 3136.700383615843
New Minimum: 3136.700383534713 > 3136.700383534244
F(16.15267243855333) = LineSearchPoint{point=PointSample{avg=3136.700383534244}, derivative=-1.7933017186208217E-25}, evalInputDelta = -8.159895514836535E-8
Left bracket at 16.15267243855333
Converged to left
Fitness changed from 3136.700383615843 to 3136.700383534244
Iteration 8 complete. Error: 3136.700383534244 Total: 0.3340; Orientation: 0.0028; Line Search: 0.3216
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.700383534244}, derivative=-5.730846212929875E-10}
New Minimum: 3136.700383534244 > 3136.7003835299865
F(16.15267243855333) = LineSearchPoint{point=PointSample{avg=3136.7003835299865}, derivative=4.604861146680526E-11}, evalInputDelta = -4.257344698999077E-9
3136.7003835299865 <= 3136.700383534244
New Minimum: 3136.7003835299865 > 3136.700383529958
F(14.9513023651719) = LineSearchPoint{point=PointSample{avg=3136.700383529958}, derivative=7.310531780864161E-26}, evalInputDelta = -4.285993782104924E-9
Right bracket at 14.9513023651719
Converged to right
Fitness changed from 3136.700383534244 to 3136.700383529958
Iteration 9 complete. Error: 3136.700383529958 Total: 0.0891; Orientation: 0.0034; Line Search: 0.0759
Low gradient: 5.310573929179225E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.700383529958}, derivative=-2.820219545727807E-11}
New Minimum: 3136.700383529958 > 3136.7003835297314
F(14.9513023651719) = LineSearchPoint{point=PointSample{avg=3136.7003835297314}, derivative=-2.128807792835843E-12}, evalInputDelta = -2.2646418074145913E-10
F(104.6591165562033) = LineSearchPoint{point=PointSample{avg=3136.7003835365585}, derivative=1.5431151819381753E-10}, evalInputDelta = 6.600657798117027E-9
F(8.0507012735541) = LineSearchPoint{point=PointSample{avg=3136.7003835297864}, derivative=-1.4162679022578408E-11}, evalInputDelta = -1.7143975128419697E-10
F(56.354908914878706) = LineSearchPoint{point=PointSample{avg=3136.700383531138}, derivative=7.007441958561957E-11}, evalInputDelta = 1.1800693755503744E-9
F(4.334992993452208) = LineSearchPoint{point=PointSample{avg=3136.7003835298515}, derivative=-2.0642455838593638E-11}, evalInputDelta = -1.064108801074326E-10
F(30.34495095416546) = LineSearchPoint{point=PointSample{avg=3136.700383529907}, derivative=2.471598187351297E-11}, evalInputDelta = -5.093170329928398E-11
3136.700383529907 <= 3136.700383529958
New Minimum: 3136.7003835297314 > 3136.700383529731
F(16.17202785729616) = LineSearchPoint{point=PointSample{avg=3136.700383529731}, derivative=-1.8175355256292112E-27}, evalInputDelta = -2.269189280923456E-10
Left bracket at 16.17202785729616
Converged to left
Fitness changed from 3136.700383529958 to 3136.700383529731
Iteration 10 complete. Error: 3136.700383529731 Total: 0.2498; Orientation: 0.0030; Line Search: 0.2378
Low gradient: 1.2794910659615705E-6
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.700383529731}, derivative=-1.6370973878754756E-12}
New Minimum: 3136.700383529731 > 3136.7003835297173
F(16.17202785729616) = LineSearchPoint{point=PointSample{avg=3136.7003835297173}, derivative=1.3211337972474672E-13}, evalInputDelta = -1.3642420526593924E-11
3136.7003835297173 <= 3136.700383529731
New Minimum: 3136.7003835297173 > 3136.700383529717
F(14.964403928956534) = LineSearchPoint{point=PointSample{avg=3136.700383529717}, derivative=1.165541987464045E-28}, evalInputDelta = -1.4097167877480388E-11
Right bracket at 14.964403928956534
Converged to right
Fitness changed from 3136.700383529731 to 3136.700383529717
Iteration 11 complete. Error: 3136.700383529717 Total: 0.3154; Orientation: 0.0029; Line Search: 0.3025
Low gradient: 2.8639502281175844E-7
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.700383529717}, derivative=-8.202210909134765E-14}
New Minimum: 3136.700383529717 > 3136.7003835297164
F(14.964403928956534) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=-6.182923415912745E-15}, evalInputDelta = -4.547473508864641E-13
F(104.75082750269574) = LineSearchPoint{point=PointSample{avg=3136.7003835297373}, derivative=4.4885219063669675E-13}, evalInputDelta = 2.0463630789890885E-11
F(8.057755961745826) = LineSearchPoint{point=PointSample{avg=3136.700383529717}, derivative=-4.11856244968827E-14}, evalInputDelta = 0.0
F(56.404291732220784) = LineSearchPoint{point=PointSample{avg=3136.700383529721}, derivative=2.0383328306990703E-13}, evalInputDelta = 4.092726157978177E-12
F(4.338791671709291) = LineSearchPoint{point=PointSample{avg=3136.700383529717}, derivative=-6.003323277125114E-14}, evalInputDelta = 0.0
F(30.37154170196504) = LineSearchPoint{point=PointSample{avg=3136.7003835297173}, derivative=7.190002514932795E-14}, evalInputDelta = 4.547473508864641E-13
F(2.336272438612695) = LineSearchPoint{point=PointSample{avg=3136.700383529717}, derivative=-7.018194491898799E-14}, evalInputDelta = 0.0
F(16.353907070288866) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=8.590401151699688E-16}, evalInputDelta = -4.547473508864641E-13
3136.7003835297164 <= 3136.700383529717
F(16.184403361090354) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=-8.689795909075208E-30}, evalInputDelta = -4.547473508864641E-13
Left bracket at 16.184403361090354
Converged to left
Fitness changed from 3136.700383529717 to 3136.7003835297164
Iteration 12 complete. Error: 3136.7003835297164 Total: 0.5475; Orientation: 0.0032; Line Search: 0.5067
Low gradient: 6.781165034210955E-8
F(0.0) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=-4.598419922120526E-15}
F(16.184403361090354) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=-3.3950408385497367E-16}, evalInputDelta = 0.0
F(113.29082352763248) = LineSearchPoint{point=PointSample{avg=3136.700383529718}, derivative=2.5213990945738346E-14}, evalInputDelta = 1.3642420526593924E-12
F(8.714678732894807) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=-2.305157547669844E-15}, evalInputDelta = 0.0
F(61.00275113026365) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=1.145441669903425E-14}, evalInputDelta = 0.0
3136.7003835297164 <= 3136.7003835297164
F(17.47456058525472) = LineSearchPoint{point=PointSample{avg=3136.7003835297164}, derivative=6.008901426488176E-31}, evalInputDelta = 0.0
Right bracket at 17.47456058525472
Converged to right
Fitness changed from 3136.7003835297164 to 3136.7003835297164
Static Iteration Total: 0.4668; Orientation: 0.0026; Line Search: 0.4555
Iteration 13 failed. Error: 3136.7003835297164
Previous Error: 0.0 -> 3136.7003835297164
Optimization terminated 13
Final threshold in iteration 13: 3136.7003835297164 (> 0.0) after 6.675s (< 30.000s)

Returns

    3136.7003835297164

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 4.2701917227981197E-13 ], [ 2.1531391640379945E-18 ], [ -3.511167723766207E-11 ], [ -1.9211420557875504E-9 ], [ 6.082810697295371E-20 ], [ -1.685102185790563E-11 ], [ 4.1590589718654615E-10 ], [ -2.1140674348382671E-10 ], ... ],
    	[ [ -3.156650156077382E-9 ], [ -1.9943905636261017E-12 ], [ -1.6211078223786332E-22 ], [ -4.903673537201186E-15 ], [ 1.0181465173615746E-11 ], [ 3.2480763922009655E-10 ], [ -2.163295133399052E-11 ], [ -3.0674007439602598E-21 ], ... ],
    	[ [ 4.3060037215763563E-10 ], [ -1.6450388418087395E-16 ], [ 2.7532831342675124E-11 ], [ 6.171347881326927E-10 ], [ 4.777068541663864E-9 ], [ 2.9174699466952536E-9 ], [ -5.056263442178804E-21 ], [ -6.973793823311535E-9 ], ... ],
    	[ [ -5.728754642362641E-12 ], [ 1.5894142617898662E-15 ], [ -3.248287189963247E-11 ], [ 8.807937451818907E-10 ], [ -1.2269400477334597E-10 ], [ 2.7604066093084533E-12 ], [ -2.0345727836537843E-14 ], [ -2.2292392083260326E-9 ], ... ],
    	[ [ -5.825456951482308E-13 ], [ 1.4327364392298977E-15 ], [ 4.6331707532825047E-14 ], [ 5.674577753287771E-10 ], [ -1.2616487285964279E-12 ], [ 1.5036496833788495E-14 ], [ -7.28057426691579E-10 ], [ 3.3548864904938893E-9 ], ... ],
    	[ [ 1.2130175646443759E-18 ], [ 8.631747997831052E-17 ], [ 4.7638663932065205E-15 ], [ -4.784172362602253E-10 ], [ 7.802697470994563E-9 ], [ -2.3203395523802934E-9 ], [ 2.710621977076432E-9 ], [ 2.4431634925721385E-11 ], ... ],
    	[ [ 5.689481762929384E-12 ], [ 2.569870310873316E-19 ], [ 2.4924517041698474E-9 ], [ -2.1352281004138608E-8 ], [ -7.677737856365275E-9 ], [ 2.2217162493155442E-18 ], [ -7.336925224773326E-10 ], [ 7.298270144357743E-11 ], ... ],
    	[ [ 3.7306452610953293E-19 ], [ -3.9512330411920435E-10 ], [ 1.0202975074793223E-20 ], [ 5.429097864180708E-13 ], [ -2.038216773628093E-13 ], [ 3.495859179625447E-11 ], [ 1.0902236594701753E-9 ], [ -4.291272516849875E-9 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.39894228040143265 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 32.79 seconds (0.068 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3899996176130
Reset training subject: 3900006143656
Adding measurement 14cadfc9 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 3703.9375980032173 < 3703.9375980032173. Total: 1
th(0)=3703.9375980032173;dx=-79.00208198552897
Adding measurement 7cefe9b7 to history. Total: 1
New Minimum: 3703.9375980032173 > 3546.725410306643
END: th(2.154434690031884)=3546.725410306643; dx=-66.94078499850956 evalInputDelta=157.21218769657435
Fitness changed from 3703.9375980032173 to 3546.725410306643
Iteration 1 complete. Error: 3546.725410306643 Total: 0.2935; Orientation: 0.0147; Line Search: 0.0954
Non-optimal measurement 3546.725410306643 < 3546.725410306643. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 3546.725410306643 < 3546.725410306643. Total: 2
th(0)=3546.725410306643;dx=-56.748782520152744
Adding measurement 6fce61c6 to history. Total: 2
New Minimum: 3546.725410306643 > 3326.4016088191374
END: th(4.641588833612779)=3326.4016088191374; dx=-38.185865672727054 evalInputDelta=220.32380148750553
Fitness changed from 3546.725410306643 to 3326.4016088191374
Iteration 2 complete. Error: 3326.4016088191374 Total: 0.0586; Orientation: 0.0069; Line Search: 0.0384
Non-optimal measurement 3326.4016088191374 < 3326.4016088191374. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 3326.4016088191374 < 3326.4016088191374. Total: 3
th(0)=3326.4016088191374;dx=-25.791556045620904
Adding measurement 4a161e4f to history. Total: 3
New Minimum: 3326.4016088191374 > 3157.931486913457
END: th(10.000000000000002)=3157.931486913457; dx=-7.902468335514637 evalInputDelta=168.47012190568057
Fitness changed from 3326.4016088191374 to 3157.931486913457
Iteration 3 complete. Error: 3157.931486913457 Total: 0.0518; Orientation: 0.0058; Line Search: 0.0319
Non-optimal measurement 3157.931486913457 < 3157.931486913457. Total: 4
Rejected: LBFGS Orientation magnitude: 2.529e+01, gradient 1.626e+00, dot -0.997; [d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3157.931486913457, 3326.4016088191374, 3546.725410306643, 3703.9375980032173
LBFGS Accumulation History: 3 points
Removed measurement 4a161e4f to history. Total: 3
Adding measurement 6ce9f11e to history. Total: 3
th(0)=3157.931486913457;dx=-2.643424214252218
Adding measurement d789677 to history. Total: 4
New Minimum: 3157.931486913457 > 3140.065911239202
WOLF (strong): th(21.544346900318843)=3140.065911239202; dx=0.9849310831383707 evalInputDelta=17.86557567425507
Adding measurement 58b253b1 to history. Total: 5
New Minimum: 3140.065911239202 > 3139.2273809257513
END: th(10.772173450159421)=3139.2273809257513; dx=-0.8292465655569237 evalInputDelta=18.70410598770559
Fitness changed from 3157.931486913457 to 3139.2273809257513
Iteration 4 complete. Error: 3139.2273809257513 Total: 0.8750; Orientation: 0.7263; Line Search: 0.1387
Non-optimal measurement 3139.2273809257513 < 3139.2273809257513. Total: 6
Rejected: LBFGS Orientation magnitude: 8.983e+00, gradient 5.374e-01, dot -0.998; [dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3139.2273809257513, 3140.065911239202, 3157.931486913457, 3326.4016088191374, 3546.725410306643, 3703.9375980032173
Rejected: LBFGS Orientation magnitude: 9.013e+00, gradient 5.374e-01, dot -0.998; [bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3139.2273809257513, 3140.065911239202, 3157.931486913457, 3326.4016088191374, 3546.725410306643
Rejected: LBFGS Orientation magnitude: 9.093e+00, gradient 5.374e-01, dot -0.998; [d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3139.2273809257513, 3140.065911239202, 3157.931486913457, 3326.4016088191374
LBFGS Accumulation History: 3 points
Removed measurement 58b253b1 to history. Total: 5
Removed measurement d789677 to history. Total: 4
Removed measurement 6ce9f11e to history. Total: 3
Adding measurement 17b8fe86 to history. Total: 3
th(0)=3139.2273809257513;dx=-0.28876136919752887
Adding measurement 1ff1cb03 to history. Total: 4
New Minimum: 3139.2273809257513 > 3137.042220242793
WOLF (strong): th(23.2079441680639)=3137.042220242793; dx=0.10044993005023896 evalInputDelta=2.1851606829582124
Adding measurement 4b532cb3 to history. Total: 5
New Minimum: 3137.042220242793 > 3137.0057013214578
END: th(11.60397208403195)=3137.0057013214578; dx=-0.09415571957364496 evalInputDelta=2.221679604293513
Fitness changed from 3139.2273809257513 to 3137.0057013214578
Iteration 5 complete. Error: 3137.0057013214578 Total: 3.4331; Orientation: 3.1707; Line Search: 0.2526
Non-optimal measurement 3137.0057013214578 < 3137.0057013214578. Total: 6
Rejected: LBFGS Orientation magnitude: 3.270e+00, gradient 1.818e-01, dot -0.999; [dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3137.0057013214578, 3137.042220242793, 3139.2273809257513, 3326.4016088191374, 3546.725410306643, 3703.9375980032173
Rejected: LBFGS Orientation magnitude: 3.277e+00, gradient 1.818e-01, dot -0.999; [dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3137.0057013214578, 3137.042220242793, 3139.2273809257513, 3326.4016088191374, 3546.725410306643
Rejected: LBFGS Orientation magnitude: 3.323e+00, gradient 1.818e-01, dot -0.999; [bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3137.0057013214578, 3137.042220242793, 3139.2273809257513, 3326.4016088191374
LBFGS Accumulation History: 3 points
Removed measurement 4b532cb3 to history. Total: 5
Removed measurement 1ff1cb03 to history. Total: 4
Removed measurement 17b8fe86 to history. Total: 3
Adding measurement 3083c17d to history. Total: 3
th(0)=3137.0057013214578;dx=-0.033045447170755124
Adding measurement 660462c9 to history. Total: 4
New Minimum: 3137.0057013214578 > 3136.7433035882364
WOLF (strong): th(25.000000000000007)=3136.7433035882364; dx=0.012053628513141094 evalInputDelta=0.26239773322140536
Adding measurement 409db3e8 to history. Total: 5
New Minimum: 3136.7433035882364 > 3136.7335678433346
END: th(12.500000000000004)=3136.7335678433346; dx=-0.010495909328807014 evalInputDelta=0.2721334781231235
Fitness changed from 3137.0057013214578 to 3136.7335678433346
Iteration 6 complete. Error: 3136.7335678433346 Total: 3.6199; Orientation: 3.4690; Line Search: 0.1390
Non-optimal meas

...skipping 9444 bytes...

WOLF (strong): th(33.6630420317482)=3136.700387623428; dx=5.948343988525603E-7 evalInputDelta=4.513578460318968E-6
Adding measurement 73d51487 to history. Total: 5
New Minimum: 3136.700387623428 > 3136.700383745838
END: th(16.8315210158741)=3136.700383745838; dx=-1.3408128503354854E-7 evalInputDelta=8.391168648813618E-6
Fitness changed from 3136.7003921370065 to 3136.700383745838
Iteration 10 complete. Error: 3136.700383745838 Total: 3.5072; Orientation: 3.1442; Line Search: 0.3497
Non-optimal measurement 3136.700383745838 < 3136.700383745838. Total: 6
Rejected: LBFGS Orientation magnitude: 2.902e-03, gradient 1.465e-04, dot -0.999; [dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.700383745838, 3136.700387623428, 3136.7003921370065, 3326.4016088191374, 3546.725410306643, 3703.9375980032173
Rejected: LBFGS Orientation magnitude: 2.903e-03, gradient 1.465e-04, dot -0.999; [d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.700383745838, 3136.700387623428, 3136.7003921370065, 3326.4016088191374, 3546.725410306643
Rejected: LBFGS Orientation magnitude: 2.915e-03, gradient 1.465e-04, dot -0.999; [dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.700383745838, 3136.700387623428, 3136.7003921370065, 3326.4016088191374
LBFGS Accumulation History: 3 points
Removed measurement 73d51487 to history. Total: 5
Removed measurement eb52c7d to history. Total: 4
Removed measurement 18171ce4 to history. Total: 3
Adding measurement 10e913cc to history. Total: 3
th(0)=3136.700383745838;dx=-2.147222479516458E-8
Adding measurement f4a897a to history. Total: 4
New Minimum: 3136.700383745838 > 3136.700383668908
WOLF (strong): th(36.26241276259985)=3136.700383668908; dx=1.722918384390755E-8 evalInputDelta=7.692960934946314E-8
Adding measurement 76d6f5d5 to history. Total: 5
New Minimum: 3136.700383668908 > 3136.7003835319456
END: th(18.131206381299926)=3136.7003835319456; dx=-2.1215204756285126E-9 evalInputDelta=2.1389223547885194E-7
Fitness changed from 3136.700383745838 to 3136.7003835319456
Iteration 11 complete. Error: 3136.7003835319456 Total: 4.2428; Orientation: 3.9582; Line Search: 0.2755
Non-optimal measurement 3136.7003835319456 < 3136.7003835319456. Total: 6
Rejected: LBFGS Orientation magnitude: 2.949e-04, gradient 1.482e-05, dot -0.998; [dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.7003835319456, 3136.700383668908, 3136.700383745838, 3326.4016088191374, 3546.725410306643, 3703.9375980032173
Rejected: LBFGS Orientation magnitude: 2.949e-04, gradient 1.482e-05, dot -0.998; [dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.7003835319456, 3136.700383668908, 3136.700383745838, 3326.4016088191374, 3546.725410306643
Rejected: LBFGS Orientation magnitude: 2.959e-04, gradient 1.482e-05, dot -0.998; [dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.7003835319456, 3136.700383668908, 3136.700383745838, 3326.4016088191374
LBFGS Accumulation History: 3 points
Removed measurement 76d6f5d5 to history. Total: 5
Removed measurement f4a897a to history. Total: 4
Removed measurement 10e913cc to history. Total: 3
Adding measurement 2bb7e63c to history. Total: 3
th(0)=3136.7003835319456;dx=-2.196249211750886E-10
Adding measurement 6eef962c to history. Total: 4
New Minimum: 3136.7003835319456 > 3136.7003835316254
WOLF (strong): th(39.06250000000002)=3136.7003835316254; dx=2.032366504729845E-10 evalInputDelta=3.2014213502407074E-10
Adding measurement 2b00e0f3 to history. Total: 5
New Minimum: 3136.7003835316254 > 3136.700383529721
END: th(19.53125000000001)=3136.700383529721; dx=-8.194135351052056E-12 evalInputDelta=2.2246240405365825E-9
Fitness changed from 3136.7003835319456 to 3136.700383529721
Iteration 12 complete. Error: 3136.700383529721 Total: 2.9602; Orientation: 2.8419; Line Search: 0.1087
Non-optimal measurement 3136.700383529721 < 3136.700383529721. Total: 6
Rejected: LBFGS Orientation magnitude: 1.170e-05, gradient 6.011e-07, dot -0.993; [d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.700383529721, 3136.7003835316254, 3136.7003835319456, 3326.4016088191374, 3546.725410306643, 3703.9375980032173
Rejected: LBFGS Orientation magnitude: 1.171e-05, gradient 6.011e-07, dot -0.994; [dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.700383529721, 3136.7003835316254, 3136.7003835319456, 3326.4016088191374, 3546.725410306643
Rejected: LBFGS Orientation magnitude: 1.177e-05, gradient 6.011e-07, dot -0.995; [d62f17cd-8213-4d0e-87d0-569531a2d8b3 = 1.000/1.000e+00, dacfcbc6-f7d5-4d52-9663-5aa70381fa6f = 1.000/1.000e+00, bf07d518-9fe9-48e4-9074-2e617c592512 = 1.000/1.000e+00, d779b7cb-5aa1-4068-9b65-a2f4af0081fd = 1.000/1.000e+00, dfb8c3f9-5f85-46e5-8586-ecff03e0a6e7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 3136.700383529721, 3136.7003835316254, 3136.7003835319456, 3326.4016088191374
LBFGS Accumulation History: 3 points
Removed measurement 2b00e0f3 to history. Total: 5
Removed measurement 6eef962c to history. Total: 4
Removed measurement 2bb7e63c to history. Total: 3
Adding measurement 21a5dc4b to history. Total: 3
th(0)=3136.700383529721;dx=-3.613214201500772E-13
Non-optimal measurement 3136.700383529722 < 3136.700383529721. Total: 4
Armijo: th(42.078802539685256)=3136.700383529722; dx=3.8255330840312863E-13 evalInputDelta=-9.094947017729282E-13
Adding measurement 4bc85119 to history. Total: 4
New Minimum: 3136.700383529721 > 3136.7003835297164
WOLF (strong): th(21.039401269842628)=3136.7003835297164; dx=1.061594412652574E-14 evalInputDelta=4.547473508864641E-12
Non-optimal measurement 3136.7003835297182 < 3136.7003835297164. Total: 5
END: th(7.013133756614209)=3136.7003835297182; dx=-2.3734229872454287E-13 evalInputDelta=2.7284841053187847E-12
Fitness changed from 3136.700383529721 to 3136.7003835297164
Iteration 13 complete. Error: 3136.7003835297164 Total: 3.5489; Orientation: 3.3215; Line Search: 0.0801
Final threshold in iteration 13: 3136.7003835297164 (> 0.0) after 32.795s (< 30.000s)

Returns

    3136.7003835297164

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 5.055592033505019E-10 ], [ -1.1572668946546784E-10 ], [ 4.2551460759347085E-11 ], [ 4.715724476219627E-10 ], [ 4.76653994551654E-10 ], [ 2.2948646272792136E-10 ], [ -2.023697912957748E-9 ], [ -7.0667449093335E-10 ], ... ],
    	[ [ 9.650127495979636E-10 ], [ 1.4250433144660112E-11 ], [ 3.8416785785871116E-11 ], [ -1.0865027350449591E-9 ], [ -1.3980034164124166E-10 ], [ -3.686776335530228E-10 ], [ -7.531984957577562E-11 ], [ 5.268837186585856E-10 ], ... ],
    	[ [ -8.916261650928137E-10 ], [ 6.948914659334782E-11 ], [ -3.3086087122707144E-10 ], [ 1.9868115145296733E-9 ], [ -3.300840863251831E-10 ], [ -1.3292000736891143E-9 ], [ 5.920317471343252E-10 ], [ 6.497139515508578E-10 ], ... ],
    	[ [ 4.9383081169939995E-11 ], [ -3.8313464407384724E-10 ], [ 4.423466727058452E-10 ], [ -2.1687413163476986E-10 ], [ -2.197071313844589E-10 ], [ -3.983911682549185E-12 ], [ -2.1767232486625911E-10 ], [ 1.7232651863537762E-10 ], ... ],
    	[ [ -1.8177400331316712E-10 ], [ 1.0945009015177354E-10 ], [ 5.823668953733057E-11 ], [ -1.3691054561597938E-9 ], [ -3.1375924980053504E-10 ], [ 5.111032132088251E-10 ], [ 7.699437430962938E-10 ], [ 2.7212417515707946E-9 ], ... ],
    	[ [ 1.8057117014795931E-10 ], [ 7.793477468718436E-11 ], [ 3.8276755744007515E-11 ], [ 1.4993399393704627E-9 ], [ -4.899945689840394E-10 ], [ 1.5935888406443231E-9 ], [ -3.975843385358685E-10 ], [ -3.3609157436327463E-10 ], ... ],
    	[ [ 5.463154580959835E-10 ], [ 1.899556248549124E-10 ], [ -2.05579445538643E-10 ], [ 7.96906352937169E-9 ], [ 5.658469417278522E-10 ], [ 6.210701608355076E-11 ], [ 8.193151716440217E-10 ], [ -9.044820163527577E-10 ], ... ],
    	[ [ 3.4203345613284535E-11 ], [ 2.7560632668730494E-10 ], [ 2.3602006741581E-10 ], [ 2.6240054361892575E-11 ], [ 2.0949284262299603E-10 ], [ -4.550050573966829E-10 ], [ 2.9540233560995053E-9 ], [ 2.109623541738631E-10 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	[ [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], [ 0.3989422804014327 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, 3.4964730370292316], [13.0, 3.5498275664237875]; valueStats=DoubleSummaryStatistics{count=38, sum=120452.423251, min=3136.700384, average=3169.800612, max=3546.725410}
Plotting 13 points for GD
Plotting 12 points for CjGD
Plotting 13 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, 3.4964730370292316], [32.501, 3.5498275664237875]; valueStats=DoubleSummaryStatistics{count=38, sum=120452.423251, min=3136.700384, average=3169.800612, max=3546.725410}
Plotting 13 points for GD
Plotting 12 points for CjGD
Plotting 13 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 3136.7003835297164 }, "CjGD": { "type": "NonConverged", "value": 3136.7003835297164 }, "GD": { "type": "NonConverged", "value": 3136.7003835297164 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 3136.7003835297164 }, "CjGD": { "type": "NonConverged", "value": 3136.7003835297164 }, "GD": { "type": "NonConverged", "value": 3136.7003835297164 } }, "model":null, "complete":null}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "43.552",
      "gc_time": "0.334"
    },
    "created_on": 1586738477681,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Basic",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.GaussianActivationLayerTest.Basic",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/GaussianActivationLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 3136.7003835297164
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 3136.7003835297164
        },
        "GD": {
          "type": "NonConverged",
          "value": 3136.7003835297164
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/GaussianActivationLayer/Basic/trainingTest/202004134117",
    "id": "b911b72f-e054-4b93-8aee-23a99de8ca8a",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "GaussianActivationLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.GaussianActivationLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/GaussianActivationLayer.java",
      "javaDoc": ""
    }
  }