1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 6374017303959833600

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.02 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.6167486119470178 ], [ -0.40883050032757473 ], [ -0.9092626216241029 ], [ -0.6677610908518865 ], [ 0.1770692580353732 ], [ -0.598664763805099 ], [ -0.9834120514804483 ], [ -0.6540177582566513 ], ... ],
    	[ [ -0.0026235872746109834 ], [ -0.6487869038335323 ], [ -0.6001167052407923 ], [ 0.6660944781631033 ], [ 0.4996200558737567 ], [ -0.9538557762889095 ], [ 0.8754178457909554 ], [ 0.8207888019856846 ], ... ],
    	[ [ -0.24737610661486364 ], [ -0.2299042823974653 ], [ -0.33238841734561975 ], [ -0.12171062054299764 ], [ 0.5247669269821706 ], [ 0.2653588999539145 ], [ -0.795186249795701 ], [ -0.8824656648700344 ], ... ],
    	[ [ -0.9961462886841508 ], [ 0.8396565641247227 ], [ -0.9433686915248862 ], [ 0.47527817463282584 ], [ 0.13915065067339905 ], [ -0.8020189693700008 ], [ 0.949781613186351 ], [ 0.5988672127551908 ], ... ],
    	[ [ -0.015371131348543177 ], [ -0.6732045248724086 ], [ 0.37884435286199003 ], [ -0.09134696583510138 ], [ 0.6169612953548309 ], [ -0.45789627689501256 ], [ 0.13471946568658089 ], [ -0.7370018513842866 ], ... ],
    	[ [ 0.8887127883732608 ], [ -0.9871654567197878 ], [ 0.39379231475945775 ], [ 0.2502648058480499 ], [ -0.03125952508819818 ], [ 0.5906977034334271 ], [ 0.7889911481172813 ], [ 0.7233980690138941 ], ... ],
    	[ [ -0.4087654728919692 ], [ -0.9594141530952702 ], [ 0.910213664805853 ], [ -0.20113205172143434 ], [ -0.047848457510939246 ], [ 0.4723645694910352 ], [ 0.26317588710569484 ], [ 0.9355010302488795 ], ... ],
    	[ [ 0.7082568569903163 ], [ 0.39153080497953185 ], [ -0.26026752163145717 ], [ 0.22902435236550844 ], [ -0.27417515709014095 ], [ -0.4028379140149134 ], [ -0.8649063657056704 ], [ 0.5962786429202114 ], ... ],
    	...
    ]
    [
    	[ [ -0.06857565210522143 ], [ 0.4733639736460111 ], [ 0.9800244026164335 ], [ 0.1790884535844135 ], [ 0.8142139702451252 ], [ 0.5161033719727004 ], [ -0.558854096996173 ], [ 0.15498814592674837 ], ... ],
    	[ [ 0.5284002931413683 ], [ -0.4589137375743544 ], [ -0.39276174073503967 ], [ 0.25254116008747207 ], [ 0.2907208229603775 ], [ 0.09937271309268003 ], [ -0.5237556750742229 ], [ 0.2295973244145697 ], ... ],
    	[ [ -0.5516532425323595 ], [ 0.9863631051378332 ], [ 0.48369837630357826 ], [ 0.4177978308385506 ], [ 0.9538197395690295 ], [ 0.9030130432090973 ], [ 0.5906656332283835 ], [ -0.9544502505300094 ], ... ],
    	[ [ -0.7804981109545346 ], [ 0.477413214434129 ], [ 0.12566895364403297 ], [ -0.3982932829879946 ], [ -0.7495693911229304 ], [ -0.7243269147638676 ], [ -0.302443334574543 ], [ -0.9007368366101249 ], ... ],
    	[ [ -0.6342343757667805 ], [ -0.9692669495480322 ], [ -0.5212411504920689 ], [ 0.4752040205686239 ], [ -0.8205443739591782 ], [ -0.3336032879763169 ], [ -0.8301447443310062 ], [ 0.09765393199597217 ], ... ],
    	[ [ -0.40497333056777585 ], [ -0.2327117798496685 ], [ -0.05552480079136046 ], [ -0.1267535831565254 ], [ 0.9694889413183694 ], [ -0.08942237233295058 ], [ 0.3020465296954655 ], [ -0.3398129164406405 ], ... ],
    	[ [ -0.7311426985206038 ], [ -0.8416604485756203 ], [ 0.9344010490337504 ], [ 0.8723626499058632 ], [ -0.3256936362349867 ], [ -0.28381034865103993 ], [ 0.6013193310266616 ], [ -0.33937168371810156 ], ... ],
    	[ [ -0.7849226479068376 ], [ -0.8847929670310382 ], [ 0.9783161524074588 ], [ 0.326151814311872 ], [ -0.973347210674175 ], [ -0.27223314990902203 ], [ 0.7413199451536447 ], [ 0.1545871614954588 ], ... ],
    	...
    ]
    [
    	[ [ 0.40921703627240813 ], [ -0.9407430859159986 ], [ 0.7200890916748541 ], [ 0.26971745217987064 ], [ -0.7144940573303344 ], [ 0.6860384505724486 ], [ -0.5027595594512806 ], [ 0.759180950535653 ], ... ],
    	[ [ -0.0893917485745721 ], [ 0.10928537762436386 ], [ 0.20397984945119574 ], [ -0.9396661562230395 ], [ -0.4465880474843309 ], [ -8.533917275552749E-4 ], [ 0.7436298073792362 ], [ 0.6654395922214575 ], ... ],
    	[ [ 0.16485092843681248 ], [ 0.8357227936230041 ], [ -0.8736673774610717 ], [ -0.3902038663425611 ], [ 0.5284418346704218 ], [ 0.4708549725594864 ], [ 0.709712492194579 ], [ -0.22360155643323898 ], ... ],
    	[ [ -0.7455214852430669 ], [ 0.5692941292960092 ], [ -0.529464465935902 ], [ 0.841577232896173 ], [ 0.6705212914405903 ], [ -0.7532463588297864 ], [ 0.3665737112088283 ], [ -0.4420319145635987 ], ... ],
    	[ [ 0.730879768173404 ], [ -0.8025654624431866 ], [ -0.43013529091653857 ], [ -0.10526605350462237 ], [ 0.8134502547916909 ], [ 0.26004347777962344 ], [ 0.33917411576861034 ], [ 0.2230570110133332 ], ... ],
    	[ [ 0.6698861704065202 ], [ 0.40114052271528533 ], [ -0.38371087335066956 ], [ -0.29872662970379493 ], [ -0.6079669729269923 ], [ -0.7732716280883465 ], [ 0.3138287699747888 ], [ -0.4908532820344709 ], ... ],
    	[ [ -0.5631583886483367 ], [ -0.5935108844716832 ], [ 0.4602684344930954 ], [ 0.1709026671112508 ], [ -0.7096507271933394 ], [ 0.7968790665519523 ], [ 0.7262985177573016 ], [ 0.8418200330913623 ], ... ],
    	[ [ -0.40475058534826824 ], [ -0.8324448201639283 ], [ 0.21889550175893402 ], [ -0.16747810719910938 ], [ 0.2787313285344346 ], [ 0.35515501174414 ], [ -0.5306648051291143 ], [ -0.2821977581806041 ], ... ],
    	...
    ]
    [
    	[ [ 0.014073533539620042 ], [ 0.46585050227134484 ], [ 0.8730235948170866 ], [ -0.950440690813366 ], [ 0.5402225720145999 ], [ 0.06154653173083857 ], [ -0.16999561459995283 ], [ 0.33286312734450174 ], ... ],
    	[ [ -0.3006035782210792 ], [ -0.01627713719702184 ], [ -0.9672994717541925 ], [ 0.7869802096946146 ], [ -0.5207959038238621 ], [ 0.694701037844768 ], [ 0.8236703177510227 ], [ 0.26160542363268435 ], ... ],
    	[ [ -0.4124936692204957 ], [ 0.5094160044820311 ], [ 0.1592842345619343 ], [ -0.882774559248934 ], [ 0.17940490494017802 ], [ 0.9851939780984114 ], [ -0.03741266529244758 ], [ -0.6680664574811912 ], ... ],
    	[ [ 0.8615100865094718 ], [ -0.2346529234578112 ], [ -0.7198097978358637 ], [ -0.5746876032864949 ], [ 0.14810420564413707 ], [ -0.4325174883104004 ], [ -0.614204042210418 ], [ 0.7058610206611979 ], ... ],
    	[ [ 0.525580056367634 ], [ -0.6161111936058443 ], [ -0.841803332027971 ], [ -0.07651004317633081 ], [ 0.18460330934089986 ], [ -0.5651245112728831 ], [ 0.7595377656282434 ], [ -0.8884177230196464 ], ... ],
    	[ [ -0.22219260639965066 ], [ 0.1571923818577512 ], [ -0.3909683264237682 ], [ -0.5000459564783559 ], [ 0.33248995038007423 ], [ -0.5628611422121332 ], [ -0.12199167456529114 ], [ 0.5166653939510879 ], ... ],
    	[ [ 0.3353874234403167 ], [ -0.09561444099244132 ], [ -0.2470005497679657 ], [ 0.5954525192297433 ], [ -0.7427282352600524 ], [ -0.08997332528339497 ], [ -0.5257547619645608 ], [ -0.34092699946720795 ], ... ],
    	[ [ -0.026034800581773654 ], [ 0.36114692141780136 ], [ -0.33192462522543664 ], [ -0.9390808583905905 ], [ 0.35694597231746816 ], [ -0.5207099963350659 ], [ 0.10249435823663511 ], [ 0.2196024195506816 ], ... ],
    	...
    ]
    [
    	[ [ -0.26292354020878994 ], [ 0.9477339154328965 ], [ 0.4341154980507067 ], [ 0.34763689659563335 ], [ -0.28046562256316854 ], [ 0.35426566675510607 ], [ -0.22069824179508557 ], [ 0.34585274560569923 ], ... ],
    	[ [ 0.497313000563401 ], [ 0.7618538172180553 ], [ -0.5977572167935594 ], [ 0.3953865817264286 ], [ -0.34527850371372426 ], [ -0.1812089991529806 ], [ 0.3212045534914101 ], [ -0.29970867416089875 ], ... ],
    	[ [ -0.9381243085003368 ], [ 0.5302178639409347 ], [ 0.702420655564602 ], [ -0.10996422487539492 ], [ 0.8148732612623615 ], [ 0.04271350989745715 ], [ 0.29900155039528875 ], [ -0.42936138192695594 ], ... ],
    	[ [ -0.7924214216440648 ], [ 0.2534868651683493 ], [ 0.7434497068997765 ], [ -0.5383133696732938 ], [ 0.1833356211086563 ], [ -0.6085286622054813 ], [ 0.47829472020727026 ], [ 0.8936888014217013 ], ... ],
    	[ [ -0.8132999562409302 ], [ 0.562824426138258 ], [ 0.369828990910378 ], [ 0.12496858927632748 ], [ -0.3609322587960835 ], [ 0.7456244106335324 ], [ -0.654601283604854 ], [ 0.18329606544843835 ], ... ],
    	[ [ 0.19730731559929238 ], [ 0.22410906999252767 ], [ 0.5276693579481786 ], [ 0.3549369314549844 ], [ 0.40557971399417325 ], [ -0.2312197711985784 ], [ -0.3724422698684968 ], [ 0.5268207887162828 ], ... ],
    	[ [ 0.5168194008444389 ], [ -0.5005255884167252 ], [ -0.14681021202138833 ], [ -0.04582089904887976 ], [ 0.10865560220182124 ], [ 0.9879270208242419 ], [ -0.4780501050161232 ], [ 0.13397346538324495 ], ... ],
    	[ [ -0.7111799408590402 ], [ -0.11939630680532876 ], [ -0.9943393348357108 ], [ -0.9818920850544257 ], [ -0.751304629067113 ], [ 0.407060452409993 ], [ -0.2205061836298685 ], [ 0.42955261910394804 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 30.08 seconds (0.132 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2642042294712
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 2642118704848
Constructing line search parameters: GD
th(0)=744357.0818131719;dx=-5.926313173012783E22
New Minimum: 744357.0818131719 > 64468.84127145627
Armijo: th(2.154434690031884)=64468.84127145627; dx=303436.045184706 evalInputDelta=679888.2405417156
New Minimum: 64468.84127145627 > 63866.190916583524
Armijo: th(1.077217345015942)=63866.190916583524; dx=200879.96741117246 evalInputDelta=680490.8908965883
New Minimum: 63866.190916583524 > 63445.00257550504
Armijo: th(0.3590724483386473)=63445.00257550504; dx=-397.9849470766299 evalInputDelta=680912.0792376668
Armijo: th(0.08976811208466183)=63464.52297251872; dx=-16886.082537306087 evalInputDelta=680892.5588406532
New Minimum: 63445.00257550504 > 63441.49691079435
Armijo: th(0.017953622416932366)=63441.49691079435; dx=9111.541076240646 evalInputDelta=680915.5849023776
Armijo: th(0.002992270402822061)=63458.969825109736; dx=-14956.8058493398 evalInputDelta=680898.1119880622
Armijo: th(4.2746720040315154E-4)=63486.440966088245; dx=-39218.043235531404 evalInputDelta=680870.6408470836
Armijo: th(5.343340005039394E-5)=63533.390734134126; dx=-529007.2489979756 evalInputDelta=680823.6910790377
Armijo: th(5.9370444500437714E-6)=63622.66683712414; dx=-8720226.354778072 evalInputDelta=680734.4149760477
Armijo: th(5.937044450043771E-7)=63773.70369274631; dx=-1.3887515237506703E8 evalInputDelta=680583.3781204255
Armijo: th(5.397313136403428E-8)=64052.060270710346; dx=-2.949666899621247E9 evalInputDelta=680305.0215424615
Armijo: th(4.4977609470028565E-9)=64618.21266908829; dx=-7.028133976920047E10 evalInputDelta=679738.8691440836
Armijo: th(3.4598161130791205E-10)=65786.77524346588; dx=-1.8034511646451465E12 evalInputDelta=678570.306569706
Armijo: th(2.4712972236279432E-11)=68357.00097581353; dx=-6.2763124305643984E13 evalInputDelta=676000.0808373584
END: th(1.6475314824186289E-12)=74188.90572298915; dx=-1.2967981928542085E15 evalInputDelta=670168.1760901827
Fitness changed from 744357.0818131719 to 63441.49691079435
Iteration 1 complete. Error: 63441.49691079435 Total: 1.0801; Orientation: 0.0077; Line Search: 0.9500
th(0)=63441.49691079435;dx=-2.3941032436191073E8
New Minimum: 63441.49691079435 > 63441.49606103935
WOLFE (weak): th(3.549498978642349E-12)=63441.49606103935; dx=-2.3939244003992206E8 evalInputDelta=8.497550006723031E-4
New Minimum: 63441.49606103935 > 63441.495211347894
WOLFE (weak): th(7.098997957284698E-12)=63441.495211347894; dx=-2.3937455765764728E8 evalInputDelta=0.001699446453130804
New Minimum: 63441.495211347894 > 63441.49181321661
WOLFE (weak): th(2.1296993871854094E-11)=63441.49181321661; dx=-2.3930304752041677E8 evalInputDelta=0.005097577733977232
New Minimum: 63441.49181321661 > 63441.47653418102
WOLFE (weak): th(8.518797548741638E-11)=63441.47653418102; dx=-2.389816354621681E8 evalInputDelta=0.02037661332724383
New Minimum: 63441.47653418102 > 63441.39539149867
WOLFE (weak): th(4.2593987743708185E-10)=63441.39539149867; dx=-2.3727798160667405E8 evalInputDelta=0.10151929567655316
New Minimum: 63441.39539149867 > 63440.90110125164
WOLFE (weak): th(2.555639264622491E-9)=63440.90110125164; dx=-2.2701982811344638E8 evalInputDelta=0.5958095427049557
New Minimum: 63440.90110125164 > 63437.89491768637
END: th(1.7889474852357437E-8)=63437.89491768637; dx=-1.6925877761793292E8 evalInputDelta=3.6019931079790695
Fitness changed from 63441.49691079435 to 63437.89491768637
Iteration 2 complete. Error: 63437.89491768637 Total: 0.9896; Orientation: 0.0037; Line Search: 0.9721
th(0)=63437.89491768637;dx=-1.219281600215784E8
New Minimum: 63437.89491768637 > 63434.01893827193
END: th(3.8541705208371876E-8)=63434.01893827193; dx=-8.342482417621462E7 evalInputDelta=3.875979414435278
Fitness changed from 63437.89491768637 to 63434.01893827193
Iteration 3 complete. Error: 63434.01893827193 Total: 0.1851; Orientation: 0.0034; Line Search: 0.1684
th(0)=63434.01893827193;dx=-6.045574768685583E7
New Minimum: 63434.01893827193 > 63429.60565911565
END: th(8.30355867138989E-8)=63429.60565911565; dx=-4.797182843038876E7 evalInputDelta=4.413279156280623
Fitness changed from 63434.01893827193 to 63429.60565911565
Iteration 4 complete. Error: 63429.60565911565 Total: 0.0594; Orientation: 0.0034; Line Search: 0.0375
th(0)=63429.60565911565;dx=-4.330593199874306E7
New Minimum: 63429.60565911565 > 63420.76878702689
WOLFE (weak): th(1.7889474852357439E-7)=63420.76878702689; dx=-5.90427953253801E7 evalInputDelta=8.836872088759264
New Minimum: 63420.76878702689 > 63406.41481518412
WOLFE (weak): th(3.5778949704714877E-7)=63406.41481518412; dx=-1.1156689936001892E8 evalInputDelta=23.190843931530253
Armijo: th(1.0733684911414464E-6)=63500.27246505844; dx=-1.5959110233819813E8 evalInputDelta=-70.66680594279023
Armijo: th(7.155789940942975E-7)=103054.37349515157; dx=-1.744323147012257E13 evalInputDelta=-39624.76783603591
New Minimum: 63406.41481518412 > 63372.56383095762
WOLFE (weak): th(5.366842455707232E-7)=63372.56383095762; dx=-3.155653142272843E8 evalInputDelta=57.04182815803506
New Minimum: 63372.56383095762 > 63338.99919616563
WOLFE (weak): th(6.261316198325104E-7)=63338.99919616563; dx=-2.1584341950199622E8 evalInputDelta=90.60646295002516
Armijo: th(6.708553069634039E-7)=63430.84713181645; dx=9.607893715934313E9 evalInputDelta=-1.2414727007999318
WOLF (strong): th(6.484934633979572E-7)=63344.08391410725; dx=1.0134415733241404E9 evalInputDelta=85.52174500840192
New Minimum: 63338.99919616563 > 63338.31665169826
WOLF (strong): th(6.373125416152338E-7)=63338.31665169826; dx=1.4542940744369847E8 evalInputDelta=91.28900741739199
New Minimum: 63338.31665169826 > 63338.160662043956
WOLFE (weak): th(6.317220807238721E-7)=63338.160662043956; dx=-7.38359343444925E7 evalInputDelta=91.44499707169598
mu ~= nu (6.317220807238721E-7): th(6.317220807238721E-7)=63338.160662043956
Fitness changed from 63429.60565911565 to 63338.160662043956
Iteration 5 complete. Error: 63338.160662043956 Total: 1.2507; Orientation: 0.0034; Line Search: 1.2349
th(0)=63338.160662043956;dx=-1.6967906680693328E8
Armijo: th(1.3670261066094402E-6)=63453.53393686836; dx=-4517607.896779662 evalInputDelta=-115.37327482440742
Armijo: th(6.835130533047201E-7)=63459.71655256055; dx=-1.841166099179653E7 evalInputDelta=-121.55589051659626
Armijo: th(2.278376844349067E-7)=63488.88217617181; dx=-2.244108290250388E8 evalInputDelta=-150.72151412785752
Armijo: th(5.695942110872668E-8)=63974.564303122796; dx=-3.177225124382664E10 evalInputDelta=-636.4036410788394
Armijo: th(1.1391884221745336E-8)=63362.83796709499; dx=6.944143524323733E9 evalInputDelta=-24.677305051031
Armijo: th(1.898647370290889E-9)=63338.195400116194; dx=2.294017234577372E8 evalInputDelta=-0.03473807223781478
New Minimum: 63338.160662043956 > 63338.12121035638
END: th(2.712353386129842E-10)=63338.12121035638; dx=-1.2081902348340857E8 evalInputDelta=0.039451687574910466
Fitness changed from 63338.160662043956 to 63338.12121035638
Iteration 6 complete. Error: 63338.12121035638 Total: 0.7129; Orientation: 0.0029; Line Search: 0.6987
th(0)=63338.12121035638;dx=-8.607319367324677E7
New Minimum: 63338.12121035638 > 63338.08727028625
END: th(5.843588226703577E-10)=63338.08727028625; dx=-2.9367852437358674E7 evalInputDelta=0.033940070134121925
Fitness changed from 63338.12121035638 to 63338.08727028625
Iteration 7 complete. Error: 63338.08727028625 Total: 0.1532; Orientation: 0.0036; Line Search: 0.1371
th(0)=63338.08727028625;dx=-1.025480717215682E7
New Minimum: 63338.08727028625 > 63338.08360387884
END: th(1.2589629189872085E-9)=63338.08360387884; dx=4567074.874953526 evalInputDelta=0.0036664074068539776
Fitness changed from 63338.08727028625 to 63338.08360387884
Iteration 8 complete. Error: 63338.08360387884 Total: 0.0999; Orientation: 0.0028; Line Search: 0.0860
th(0)=63338.08360387884;dx=-3217747.428201792
Armijo: th(2.712353386129842E-9)=63338.08690560877; dx=5560683.136166658 evalInputDelta=-0.00330172992835287
New Minimum: 63338.08360387884 > 63338.0822787109

...skipping 36565 bytes...

2; dx=6180376.908587385 evalInputDelta=-7.469154588761739E-4
New Minimum: 63337.980141796565 > 63337.977906930166
END: th(6.351019449808512E-10)=63337.977906930166; dx=-1537171.771841222 evalInputDelta=0.002234866398794111
Fitness changed from 63337.980141796565 to 63337.977906930166
Iteration 79 complete. Error: 63337.977906930166 Total: 0.3138; Orientation: 0.0026; Line Search: 0.2000
th(0)=63337.977906930166;dx=-695695.8959326858
New Minimum: 63337.977906930166 > 63337.9772141181
END: th(1.3682856619734666E-9)=63337.9772141181; dx=-317553.63032985746 evalInputDelta=6.928120637894608E-4
Fitness changed from 63337.977906930166 to 63337.9772141181
Iteration 80 complete. Error: 63337.9772141181 Total: 0.3732; Orientation: 0.0104; Line Search: 0.1863
th(0)=63337.9772141181;dx=-572051.8014960869
New Minimum: 63337.9772141181 > 63337.976073339305
END: th(2.9478820960288765E-9)=63337.976073339305; dx=-201090.91149689863 evalInputDelta=0.0011407787969801575
Fitness changed from 63337.9772141181 to 63337.976073339305
Iteration 81 complete. Error: 63337.976073339305 Total: 0.2859; Orientation: 0.1068; Line Search: 0.1681
th(0)=63337.976073339305;dx=-1169183.2677579527
Armijo: th(6.351019449808512E-9)=63337.985657353405; dx=4121182.389790872 evalInputDelta=-0.009584014100255445
Armijo: th(3.175509724904256E-9)=63337.97666593484; dx=1525575.8760630954 evalInputDelta=-5.925955338170752E-4
New Minimum: 63337.976073339305 > 63337.97531811197
END: th(1.058503241634752E-9)=63337.97531811197; dx=-259682.90839846295 evalInputDelta=7.552273382316343E-4
Fitness changed from 63337.976073339305 to 63337.97531811197
Iteration 82 complete. Error: 63337.97531811197 Total: 0.3810; Orientation: 0.0024; Line Search: 0.3684
th(0)=63337.97531811197;dx=-530642.0775940651
New Minimum: 63337.97531811197 > 63337.97430621196
END: th(2.280476103289111E-9)=63337.97430621196; dx=-356571.9613571007 evalInputDelta=0.0010119000071426854
Fitness changed from 63337.97531811197 to 63337.97430621196
Iteration 83 complete. Error: 63337.97430621196 Total: 0.0480; Orientation: 0.0032; Line Search: 0.0315
th(0)=63337.97430621196;dx=-667342.6543372833
New Minimum: 63337.97430621196 > 63337.97395868708
END: th(4.913136826714794E-9)=63337.97395868708; dx=519751.94379219844 evalInputDelta=3.4752488136291504E-4
Fitness changed from 63337.97430621196 to 63337.97395868708
Iteration 84 complete. Error: 63337.97395868708 Total: 0.1356; Orientation: 0.0032; Line Search: 0.1215
th(0)=63337.97395868708;dx=-5344645.796905895
Armijo: th(1.0585032416347521E-8)=63338.2818682723; dx=6.752667632970849E7 evalInputDelta=-0.3079095852226601
Armijo: th(5.292516208173761E-9)=63338.031717086655; dx=2.808978146386357E7 evalInputDelta=-0.057758399576414376
New Minimum: 63337.97395868708 > 63337.97374010689
END: th(1.7641720693912536E-9)=63337.97374010689; dx=5193381.167202318 evalInputDelta=2.1858018590137362E-4
Fitness changed from 63337.97395868708 to 63337.97374010689
Iteration 85 complete. Error: 63337.97374010689 Total: 0.2923; Orientation: 0.0027; Line Search: 0.1852
th(0)=63337.97374010689;dx=-7031607.640958929
Armijo: th(3.8007935054818524E-9)=63338.00405303965; dx=2.2310709906568743E7 evalInputDelta=-0.03031293275853386
Armijo: th(1.9003967527409262E-9)=63337.97496151838; dx=8143314.1049105 evalInputDelta=-0.001221411490405444
New Minimum: 63337.97374010689 > 63337.97093103366
END: th(6.334655842469754E-10)=63337.97093103366; dx=-1857045.4571319125 evalInputDelta=0.0028090732303098775
Fitness changed from 63337.97374010689 to 63337.97093103366
Iteration 86 complete. Error: 63337.97093103366 Total: 0.2449; Orientation: 0.0526; Line Search: 0.1817
th(0)=63337.97093103366;dx=-760486.5494344706
New Minimum: 63337.97093103366 > 63337.970229788705
END: th(1.3647602296429985E-9)=63337.970229788705; dx=-268016.57514956174 evalInputDelta=7.012449568719603E-4
Fitness changed from 63337.97093103366 to 63337.970229788705
Iteration 87 complete. Error: 63337.970229788705 Total: 0.1651; Orientation: 0.0023; Line Search: 0.0305
th(0)=63337.970229788705;dx=-597554.899282034
New Minimum: 63337.970229788705 > 63337.969172789
END: th(2.940286782318756E-9)=63337.969172789; dx=-120223.5776393732 evalInputDelta=0.0010569997029961087
Fitness changed from 63337.970229788705 to 63337.969172789
Iteration 88 complete. Error: 63337.969172789 Total: 0.0434; Orientation: 0.0026; Line Search: 0.0308
th(0)=63337.969172789;dx=-1362589.2023072771
Armijo: th(6.334655842469754E-9)=63337.98219438315; dx=5378679.670541493 evalInputDelta=-0.01302159414626658
Armijo: th(3.167327921234877E-9)=63337.97034642855; dx=2079392.9426423274 evalInputDelta=-0.001173639546323102
New Minimum: 63337.969172789 > 63337.96834987266
END: th(1.055775973744959E-9)=63337.96834987266; dx=-199032.18545711046 evalInputDelta=8.229163431678899E-4
Fitness changed from 63337.969172789 to 63337.96834987266
Iteration 89 complete. Error: 63337.96834987266 Total: 0.4437; Orientation: 0.0027; Line Search: 0.4056
th(0)=63337.96834987266;dx=-542537.3893078619
New Minimum: 63337.96834987266 > 63337.96736388933
END: th(2.274600382738331E-9)=63337.96736388933; dx=-324086.82426777156 evalInputDelta=9.85983329883311E-4
Fitness changed from 63337.96834987266 to 63337.96736388933
Iteration 90 complete. Error: 63337.96736388933 Total: 0.2795; Orientation: 0.0025; Line Search: 0.2200
th(0)=63337.96736388933;dx=-712845.2662611783
Armijo: th(4.900477970531261E-9)=63337.96751018715; dx=764060.2637645209 evalInputDelta=-1.4629781799158081E-4
New Minimum: 63337.96736388933 > 63337.96653236779
END: th(2.4502389852656304E-9)=63337.96653236779; dx=31976.885584199277 evalInputDelta=8.315215382026508E-4
Fitness changed from 63337.96736388933 to 63337.96653236779
Iteration 91 complete. Error: 63337.96653236779 Total: 0.2618; Orientation: 0.0024; Line Search: 0.1693
th(0)=63337.96653236779;dx=-1446405.4582988448
Armijo: th(5.278879868724796E-9)=63337.97587229895; dx=5065145.69880854 evalInputDelta=-0.00933993115904741
Armijo: th(2.639439934362398E-9)=63337.96690611078; dx=1749269.0859792172 evalInputDelta=-3.7374298699432984E-4
New Minimum: 63337.96653236779 > 63337.96572169578
END: th(8.798133114541327E-10)=63337.96572169578; dx=-394263.5098442928 evalInputDelta=8.106720124487765E-4
Fitness changed from 63337.96653236779 to 63337.96572169578
Iteration 92 complete. Error: 63337.96572169578 Total: 0.3995; Orientation: 0.1128; Line Search: 0.2760
th(0)=63337.96572169578;dx=-467384.3964019062
New Minimum: 63337.96572169578 > 63337.96484648691
WOLFE (weak): th(1.8955003189486097E-9)=63337.96484648691; dx=-456078.3801201736 evalInputDelta=8.752088688197546E-4
New Minimum: 63337.96484648691 > 63337.96399268929
WOLFE (weak): th(3.791000637897219E-9)=63337.96399268929; dx=-444792.93537455925 evalInputDelta=0.001729006486129947
New Minimum: 63337.96399268929 > 63337.960790831086
WOLFE (weak): th(1.1373001913691658E-8)=63337.960790831086; dx=-399856.1178014415 evalInputDelta=0.004930864692141768
New Minimum: 63337.960790831086 > 63337.950548040426
END: th(4.549200765476663E-8)=63337.950548040426; dx=-201642.14513126487 evalInputDelta=0.015173655352555215
Fitness changed from 63337.96572169578 to 63337.950548040426
Iteration 93 complete. Error: 63337.950548040426 Total: 0.2444; Orientation: 0.0030; Line Search: 0.2311
th(0)=63337.950548040426;dx=-1.4061127315849606E7
Armijo: th(9.800955941062523E-8)=66519.66416350765; dx=4.1679788873028564E11 evalInputDelta=-3181.713615467226
Armijo: th(4.9004779705312616E-8)=63394.864374224235; dx=4.0605554152343946E9 evalInputDelta=-56.91382618380885
Armijo: th(1.6334926568437537E-8)=63340.60447102911; dx=3.971585621034206E8 evalInputDelta=-2.6539229886839166
Armijo: th(4.083731642109384E-9)=63338.03587111014; dx=5.84141250625997E7 evalInputDelta=-0.08532306971756043
New Minimum: 63337.950548040426 > 63337.94445226671
END: th(8.167463284218769E-10)=63337.94445226671; dx=-771694.0917436223 evalInputDelta=0.006095773715060204
Fitness changed from 63337.950548040426 to 63337.94445226671
Iteration 94 complete. Error: 63337.94445226671 Total: 0.6047; Orientation: 0.1058; Line Search: 0.4837
Final threshold in iteration 94: 63337.94445226671 (> 0.0) after 30.073s (< 30.000s)

Returns

    63337.94445226671

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.7119426265942062 ], [ 0.4569749109144618 ], [ 0.33901458639010495 ], [ 0.1437644171747467 ], [ -0.5043441756063168 ], [ -0.9389953834617626 ], [ -0.1991411858679741 ], [ 0.4944629335217967 ], ... ],
    	[ [ -0.6274653138174322 ], [ 0.7788559554222886 ], [ 0.6187721649323235 ], [ 0.9203149180820754 ], [ 0.9560491814745782 ], [ -0.5175723396156393 ], [ 0.8074432445553917 ], [ 0.24565468341969768 ], ... ],
    	[ [ -1.6301137597477033 ], [ 0.44222003257492903 ], [ -0.2750252798487586 ], [ -0.6255125592928638 ], [ 0.7582084050510021 ], [ 0.7992892183093417 ], [ 0.05615892275043044 ], [ 0.18611471756385006 ], ... ],
    	[ [ -0.47685079505807165 ], [ 0.9653668847196287 ], [ 0.6240051749801301 ], [ 0.6944778894442147 ], [ -0.22153149306614064 ], [ 0.12752283616456986 ], [ 0.8378239141251248 ], [ -0.2997585670661287 ], ... ],
    	[ [ -0.6570959396986128 ], [ 0.7704789084938857 ], [ -0.6416002957939666 ], [ 0.34979446585189966 ], [ -0.9319479627022729 ], [ -0.2482708964580252 ], [ -0.6196226081756246 ], [ 0.6998202373537095 ], ... ],
    	[ [ -0.8452974331012438 ], [ -0.9668881508238157 ], [ -0.5455902108753237 ], [ -0.6681247305766919 ], [ 0.8050258225685086 ], [ -0.9491426300824924 ], [ -0.9266049338528457 ], [ 0.11828202556353498 ], ... ],
    	[ [ 0.4259893756454615 ], [ -0.5488070790846465 ], [ 0.6767624345045479 ], [ -0.209115287758515 ], [ 0.32428232833795534 ], [ 0.9225355803820438 ], [ 0.2758370256474022 ], [ -32.4599217474102 ], ... ],
    	[ [ -0.9886926335768572 ], [ -0.12723277471008015 ], [ -0.3430596699553007 ], [ -0.8287551928417227 ], [ 0.12746376904536133 ], [ 0.9444829251903826 ], [ -0.8817881087515346 ], [ 0.8210848052532141 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -1.4046075661796005 ], [ 2.188303944299436 ], [ 2.9497255874686683 ], [ 6.955824115952787 ], [ -1.9827729720439646 ], [ -1.064967962156889 ], [ -5.021562946114905 ], [ 2.022396285354559 ], ... ],
    	[ [ -1.5937135933715705 ], [ 1.2839344593029518 ], [ 1.6161037239116474 ], [ 1.0865845813778476 ], [ 1.0459712945495476 ], [ -1.9320970682912115 ], [ 1.2384771397160392 ], [ 4.070754874603851 ], ... ],
    	[ [ -0.61345411878173 ], [ 2.2613177294960325 ], [ -3.636029388098135 ], [ -1.5986889234174464 ], [ 1.318898594816729 ], [ 1.2511115840086047 ], [ 17.806609368986432 ], [ 5.373030210020503 ], ... ],
    	[ [ -2.097092026192844 ], [ 1.0358755990375925 ], [ 1.6025508122297905 ], [ 1.4399306517883417 ], [ -4.514030877322887 ], [ 7.8417327443179445 ], [ 1.193568222559299 ], [ -3.3360180821100385 ], ... ],
    	[ [ -1.5218477844478318 ], [ 1.2978940617008932 ], [ -1.5586027727161837 ], [ 2.85882167279168 ], [ -1.073021284472154 ], [ -4.0278583364646146 ], [ -1.6138855923032458 ], [ 1.428938385350767 ], ... ],
    	[ [ -1.183015540850728 ], [ -1.0342457906304592 ], [ -1.8328774601649083 ], [ -1.4967265156265808 ], [ 1.2421961780175055 ], [ -1.053582431455099 ], [ -1.079208585520882 ], [ 8.454369928445736 ], ... ],
    	[ [ 2.3474763859657166 ], [ -1.8221339303200983 ], [ 1.4776233860144612 ], [ -4.782051138962129 ], [ 3.083732638547717 ], [ 1.0839690319433277 ], [ 3.625329114729808 ], [ -0.030807221526336075 ], ... ],
    	[ [ -1.0114366852135181 ], [ -7.859610090864221 ], [ -2.9149447970095 ], [ -1.2066289401712165 ], [ 7.845366628411276 ], [ 1.0587803901255564 ], [ -1.1340592939224752 ], [ 1.2179009934200526 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 30.90 seconds (0.132 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2672144315883
Reset training subject: 2672155126100
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=744357.0818131719}, derivative=-5.926313173012783E22}
New Minimum: 744357.0818131719 > 66708.50394351342
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=66708.50394351342}, derivative=-8.896455803504672E12}, evalInputDelta = -677648.5778696585
New Minimum: 66708.50394351342 > 65384.188363889334
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=65384.188363889334}, derivative=-7.450542076959119E11}, evalInputDelta = -678972.8934492825
New Minimum: 65384.188363889334 > 64591.46188138014
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=64591.46188138014}, derivative=-6.2965363863920906E10}, evalInputDelta = -679765.6199317917
New Minimum: 64591.46188138014 > 64128.8647554599
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=64128.8647554599}, derivative=-5.2486108125890465E9}, evalInputDelta = -680228.2170577119
New Minimum: 64128.8647554599 > 63857.66970783266
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=63857.66970783266}, derivative=-4.356438310922242E8}, evalInputDelta = -680499.4121053392
New Minimum: 63857.66970783266 > 63696.88531554176
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=63696.88531554176}, derivative=-3.946162015468194E7}, evalInputDelta = -680660.1964976301
New Minimum: 63696.88531554176 > 63589.69141959269
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=63589.69141959269}, derivative=-3800148.579380449}, evalInputDelta = -680767.3903935791
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=63672.93029761863}, derivative=8.916261863256405E7}, evalInputDelta = -680684.1515155532
63672.93029761863 <= 744357.0818131719
Converged to right
Fitness changed from 744357.0818131719 to 63589.69141959269
Iteration 1 complete. Error: 63589.69141959269 Total: 0.3037; Orientation: 0.0054; Line Search: 0.2096
F(0.0) = LineSearchPoint{point=PointSample{avg=63589.69141959269}, derivative=-1157853.5942108438}
New Minimum: 63589.69141959269 > 63551.819486540815
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=63551.819486540815}, derivative=-1590204.4732458042}, evalInputDelta = -37.871933051872475
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=63588.62729383968}, derivative=1.0821510768625207E7}, evalInputDelta = -1.0641257530078292
63588.62729383968 <= 63589.69141959269
F(5.5719112931130924E-5) = LineSearchPoint{point=PointSample{avg=63558.75501250215}, derivative=2407879.2830485427}, evalInputDelta = -30.936407090535795
Right bracket at 5.5719112931130924E-5
F(1.809293556031477E-5) = LineSearchPoint{point=PointSample{avg=63571.88247832363}, derivative=-836058.1596816804}, evalInputDelta = -17.808941269060597
Left bracket at 1.809293556031477E-5
F(2.7790308941851085E-5) = LineSearchPoint{point=PointSample{avg=63564.40689584812}, derivative=-708034.0895844523}, evalInputDelta = -25.284523744565377
Left bracket at 2.7790308941851085E-5
F(3.4136616730575836E-5) = LineSearchPoint{point=PointSample{avg=63560.17704096568}, derivative=-623043.0220781603}, evalInputDelta = -29.5143786270055
Left bracket at 3.4136616730575836E-5
F(3.857316188187827E-5) = LineSearchPoint{point=PointSample{avg=63557.57009459498}, derivative=-548382.6625639731}, evalInputDelta = -32.12132499770814
Left bracket at 3.857316188187827E-5
F(4.175371301759049E-5) = LineSearchPoint{point=PointSample{avg=63555.93859265066}, derivative=-472934.90098736883}, evalInputDelta = -33.75282694202906
Left bracket at 4.175371301759049E-5
F(4.404637217692103E-5) = LineSearchPoint{point=PointSample{avg=63554.93867322387}, derivative=-394778.4198967416}, evalInputDelta = -34.75274636881659
Left bracket at 4.404637217692103E-5
F(4.569057801267779E-5) = LineSearchPoint{point=PointSample{avg=63554.35023377118}, derivative=-317135.75649781816}, evalInputDelta = -35.34118582150404
Left bracket at 4.569057801267779E-5
F(4.6857693409023595E-5) = LineSearchPoint{point=PointSample{avg=63554.02047318792}, derivative=-245113.53400135745}, evalInputDelta = -35.670946404767164
Left bracket at 4.6857693409023595E-5
F(4.767641174153218E-5) = LineSearchPoint{point=PointSample{avg=63553.84448413242}, derivative=-182950.76453969142}, evalInputDelta = -35.84693546026392
Left bracket at 4.767641174153218E-5
F(4.824434491529289E-5) = LineSearchPoint{point=PointSample{avg=63553.75455677137}, derivative=-132640.57285521564}, evalInputDelta = -35.936862821319664
Left bracket at 4.824434491529289E-5
Converged to left
Fitness changed from 63589.69141959269 to 63551.819486540815
Iteration 2 complete. Error: 63551.819486540815 Total: 1.8291; Orientation: 0.0027; Line Search: 1.8138
F(0.0) = LineSearchPoint{point=PointSample{avg=63551.819486540815}, derivative=-1.5019176488194916E8}
New Minimum: 63551.819486540815 > 63532.03255015189
F(4.824434491529289E-5) = LineSearchPoint{point=PointSample{avg=63532.03255015189}, derivative=-124790.32805010972}, evalInputDelta = -19.786936388925824
New Minimum: 63532.03255015189 > 63507.04999384567
F(3.377104144070502E-4) = LineSearchPoint{point=PointSample{avg=63507.04999384567}, derivative=-42264.12456416968}, evalInputDelta = -44.769492695144436
F(0.0023639729008493515) = LineSearchPoint{point=PointSample{avg=63527.73436574392}, derivative=-1565844.486463963}, evalInputDelta = -24.085120796895353
New Minimum: 63507.04999384567 > 63445.99937960146
F(0.01654781030594546) = LineSearchPoint{point=PointSample{avg=63445.99937960146}, derivative=-7745.889902068387}, evalInputDelta = -105.82010693935445
F(0.11583467214161822) = LineSearchPoint{point=PointSample{avg=63449.13348021447}, derivative=-1067.2372831400412}, evalInputDelta = -102.6860063263448
F(0.8108427049913275) = LineSearchPoint{point=PointSample{avg=63458.372006565485}, derivative=-1348.8698620096834}, evalInputDelta = -93.44747997532977
F(5.675898934939292) = LineSearchPoint{point=PointSample{avg=64941.53425225841}, derivative=-724431.0636668758}, evalInputDelta = 1389.7147657175956
F(0.43660761037994555) = LineSearchPoint{point=PointSample{avg=63449.269685303254}, derivative=-1489.7842375442362}, evalInputDelta = -102.54980123756104
F(3.0562532726596188) = LineSearchPoint{point=PointSample{avg=63529.314444272684}, derivative=-20943.618818901195}, evalInputDelta = -22.505042268130637
F(21.393772908617333) = LineSearchPoint{point=PointSample{avg=74990.92079751212}, derivative=-6787013.637348636}, evalInputDelta = 11439.10131097131
F(1.6456748391244103) = LineSearchPoint{point=PointSample{avg=64040.83849543902}, derivative=-42415.3585099314}, evalInputDelta = 489.01900889820536
New Minimum: 63445.99937960146 > 63440.66775591415
F(0.12659037224033926) = LineSearchPoint{point=PointSample{avg=63440.66775591415}, derivative=-686.2440778950422}, evalInputDelta = -111.15173062666145
F(0.8861326056823748) = LineSearchPoint{point=PointSample{avg=63464.30562905589}, derivative=5271.53213064266}, evalInputDelta = -87.51385748492612
63464.30562905589 <= 63551.819486540815
Converged to right
Fitness changed from 63551.819486540815 to 63440.66775591415
Iteration 3 complete. Error: 63440.66775591415 Total: 1.0351; Orientation: 0.0099; Line Search: 1.0135
F(0.0) = LineSearchPoint{point=PointSample{avg=63440.66775591415}, derivative=-28379.620331458584}
F(0.8861326056823748) = LineSearchPoint{point=PointSample{avg=63546.40504219616}, derivative=-91630.27532182346}, evalInputDelta = 105.73728628200479
F(0.0681640465909519) = LineSearchPoint{point=PointSample{avg=63697.89822134711}, derivative=823379.1166363503}, evalInputDelta = 257.2304654329564
New Minimum: 63440.66775591415 > 63439.876919087474
F(0.0052433881993039924) = LineSearchPoint{point=PointSample{avg=63439.876919087474}, derivative=345.0585451608473}, evalInputDelta = -0.7908368266798789
63439.876919087474 <= 63440.66775591415
New Minimum: 63439.876919087474 > 63439.86251147352
F(0.005180401388849584) = LineSearchPoint{point=PointSample{avg=63439.86251147352}, derivative=120.28344068449955}, evalInputDelta = -0.8052444406348513
Right bracket at 0.005180401388849584
Conve

...skipping 45766 bytes...

eration 18 complete. Error: 63360.44730350608 Total: 0.7250; Orientation: 0.0198; Line Search: 0.6855
F(0.0) = LineSearchPoint{point=PointSample{avg=63360.44730350608}, derivative=-3074451.2590363612}
F(2.770604185447819E-8) = LineSearchPoint{point=PointSample{avg=63360.93002905056}, derivative=3.445816168813277E7}, evalInputDelta = 0.4827255444834009
New Minimum: 63360.44730350608 > 63360.44470416609
F(2.131233988806015E-9) = LineSearchPoint{point=PointSample{avg=63360.44470416609}, derivative=609003.3895631636}, evalInputDelta = -0.0025993399904109538
63360.44470416609 <= 63360.44730350608
New Minimum: 63360.44470416609 > 63360.444594846325
F(1.7788667556086234E-9) = LineSearchPoint{point=PointSample{avg=63360.444594846325}, derivative=10781.219523842887}, evalInputDelta = -0.002708659754716791
Right bracket at 1.7788667556086234E-9
Converged to right
Fitness changed from 63360.44730350608 to 63360.444594846325
Iteration 19 complete. Error: 63360.444594846325 Total: 0.5870; Orientation: 0.0023; Line Search: 0.4984
F(0.0) = LineSearchPoint{point=PointSample{avg=63360.444594846325}, derivative=-193309.38159641525}
New Minimum: 63360.444594846325 > 63360.44426311789
F(1.7788667556086234E-9) = LineSearchPoint{point=PointSample{avg=63360.44426311789}, derivative=-179651.3016823725}, evalInputDelta = -3.3172843541251495E-4
New Minimum: 63360.44426311789 > 63360.44278557622
F(1.2452067289260363E-8) = LineSearchPoint{point=PointSample{avg=63360.44278557622}, derivative=-97023.43788958451}, evalInputDelta = -0.0018092701066052541
F(8.716447102482254E-8) = LineSearchPoint{point=PointSample{avg=63360.45803251847}, derivative=515273.64829149074}, evalInputDelta = 0.013437672147119883
F(6.704959309601734E-9) = LineSearchPoint{point=PointSample{avg=63360.44347161037}, derivative=-141660.51338525052}, evalInputDelta = -0.0011232359538553283
F(4.6934715167212134E-8) = LineSearchPoint{point=PointSample{avg=63360.44414655962}, derivative=178054.99004323455}, evalInputDelta = -4.4828670797869563E-4
63360.44414655962 <= 63360.444594846325
New Minimum: 63360.44278557622 > 63360.44218568625
F(2.4431317210961474E-8) = LineSearchPoint{point=PointSample{avg=63360.44218568625}, derivative=-2882.4401507212588}, evalInputDelta = -0.0024091600789688528
Left bracket at 2.4431317210961474E-8
New Minimum: 63360.44218568625 > 63360.44218516203
F(2.4789809634155635E-8) = LineSearchPoint{point=PointSample{avg=63360.44218516203}, derivative=-42.07776293549718}, evalInputDelta = -0.0024096842971630394
Left bracket at 2.4789809634155635E-8
Converged to left
Fitness changed from 63360.444594846325 to 63360.44218516203
Iteration 20 complete. Error: 63360.44218516203 Total: 1.2844; Orientation: 0.0027; Line Search: 1.1924
F(0.0) = LineSearchPoint{point=PointSample{avg=63360.44218516203}, derivative=-2604378.3819259624}
F(2.4789809634155635E-8) = LineSearchPoint{point=PointSample{avg=63360.770774002725}, derivative=2.6881820588214938E7}, evalInputDelta = 0.3285888406971935
New Minimum: 63360.44218516203 > 63360.43987985301
F(1.9069084333965875E-9) = LineSearchPoint{point=PointSample{avg=63360.43987985301}, derivative=170355.57806208683}, evalInputDelta = -0.0023053090189932846
63360.43987985301 <= 63360.44218516203
New Minimum: 63360.43987985301 > 63360.43986971442
F(1.7898332495529647E-9) = LineSearchPoint{point=PointSample{avg=63360.43986971442}, derivative=2781.7043897193507}, evalInputDelta = -0.0023154476075433195
Right bracket at 1.7898332495529647E-9
Converged to right
Fitness changed from 63360.44218516203 to 63360.43986971442
Iteration 21 complete. Error: 63360.43986971442 Total: 0.3741; Orientation: 0.1032; Line Search: 0.2607
F(0.0) = LineSearchPoint{point=PointSample{avg=63360.43986971442}, derivative=-182981.4032456122}
New Minimum: 63360.43986971442 > 63360.439554206794
F(1.7898332495529647E-9) = LineSearchPoint{point=PointSample{avg=63360.439554206794}, derivative=-169568.5989347205}, evalInputDelta = -3.1550762651022524E-4
New Minimum: 63360.439554206794 > 63360.438167869186
F(1.2528832746870754E-8) = LineSearchPoint{point=PointSample{avg=63360.438167869186}, derivative=-88428.93005770861}, evalInputDelta = -0.0017018452344927937
F(8.770182922809528E-8) = LineSearchPoint{point=PointSample{avg=63360.45374138302}, derivative=512617.27472037147}, evalInputDelta = 0.013871668597857933
F(6.746294556007329E-9) = LineSearchPoint{point=PointSample{avg=63360.43880610318}, derivative=-132261.13321821328}, evalInputDelta = -0.0010636112419888377
F(4.72240618920513E-8) = LineSearchPoint{point=PointSample{avg=63360.43974946467}, derivative=181642.58288429535}, evalInputDelta = -1.2024975148960948E-4
63360.43974946467 <= 63360.43986971442
New Minimum: 63360.438167869186 > 63360.437657109775
F(2.3698729213295756E-8) = LineSearchPoint{point=PointSample{avg=63360.437657109775}, derivative=-2815.4436929172502}, evalInputDelta = -0.0022126046460471116
Left bracket at 2.3698729213295756E-8
New Minimum: 63360.437657109775 > 63360.43765659652
F(2.405780412450032E-8) = LineSearchPoint{point=PointSample{avg=63360.43765659652}, derivative=-42.44865115766879}, evalInputDelta = -0.0022131178993731737
Left bracket at 2.405780412450032E-8
Converged to left
Fitness changed from 63360.43986971442 to 63360.43765659652
Iteration 22 complete. Error: 63360.43765659652 Total: 0.5991; Orientation: 0.0507; Line Search: 0.5380
F(0.0) = LineSearchPoint{point=PointSample{avg=63360.43765659652}, derivative=-2385248.725443361}
F(2.405780412450032E-8) = LineSearchPoint{point=PointSample{avg=63360.72166972423}, derivative=2.413325161437489E7}, evalInputDelta = 0.28401312771165976
New Minimum: 63360.43765659652 > 63360.435531634095
F(1.8506003172692555E-9) = LineSearchPoint{point=PointSample{avg=63360.435531634095}, derivative=75431.3713810548}, evalInputDelta = -0.0021249624260235578
63360.435531634095 <= 63360.43765659652
New Minimum: 63360.435531634095 > 63360.43552946064
F(1.7938707488909913E-9) = LineSearchPoint{point=PointSample{avg=63360.43552946064}, derivative=1180.1516080434085}, evalInputDelta = -0.002127135878254194
Right bracket at 1.7938707488909913E-9
Converged to right
Fitness changed from 63360.43765659652 to 63360.43552946064
Iteration 23 complete. Error: 63360.43552946064 Total: 0.6507; Orientation: 0.0026; Line Search: 0.5552
F(0.0) = LineSearchPoint{point=PointSample{avg=63360.43552946064}, derivative=-172949.2102907006}
New Minimum: 63360.43552946064 > 63360.43523077088
F(1.7938707488909913E-9) = LineSearchPoint{point=PointSample{avg=63360.43523077088}, derivative=-160057.5206317295}, evalInputDelta = -2.9868975980207324E-4
New Minimum: 63360.43523077088 > 63360.43392670328
F(1.2557095242236939E-8) = LineSearchPoint{point=PointSample{avg=63360.43392670328}, derivative=-82082.45960935908}, evalInputDelta = -0.0016027573656174354
F(8.789966669565857E-8) = LineSearchPoint{point=PointSample{avg=63360.44912849817}, derivative=494896.7649905947}, evalInputDelta = 0.013599037527455948
F(6.761512822742967E-9) = LineSearchPoint{point=PointSample{avg=63360.43452462505}, derivative=-124202.53441479022}, evalInputDelta = -0.0010048355907201767
F(4.7330589759200766E-8) = LineSearchPoint{point=PointSample{avg=63360.43554890811}, derivative=177308.46494307675}, evalInputDelta = 1.9447470549494028E-5
F(3.6408145968615975E-9) = LineSearchPoint{point=PointSample{avg=63360.43494743466}, derivative=-146753.44621003972}, evalInputDelta = -5.820259830215946E-4
New Minimum: 63360.43392670328 > 63360.43347851278
F(2.5485702178031183E-8) = LineSearchPoint{point=PointSample{avg=63360.43347851278}, derivative=13011.652388004833}, evalInputDelta = -0.0020509478636085987
63360.43347851278 <= 63360.43552946064
New Minimum: 63360.43347851278 > 63360.43346709255
F(2.3702471594842818E-8) = LineSearchPoint{point=PointSample{avg=63360.43346709255}, derivative=-198.3618515729031}, evalInputDelta = -0.002062368090264499
Left bracket at 2.3702471594842818E-8
Converged to left
Fitness changed from 63360.43552946064 to 63360.43346709255
Iteration 24 complete. Error: 63360.43346709255 Total: 1.0060; Orientation: 0.0028; Line Search: 0.9932
Final threshold in iteration 24: 63360.43346709255 (> 0.0) after 30.896s (< 30.000s)

Returns

    63360.43346709255

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.7119406575299211 ], [ 0.4570770790295951 ], [ 0.33917680949112705 ], [ 0.1456539831849614 ], [ -0.5044822790898991 ], [ -0.9389922224956442 ], [ -0.19960639810131156 ], [ 0.494529804859785 ], ... ],
    	[ [ -0.6230330174200416 ], [ 0.7788773962594447 ], [ 0.6188116320566219 ], [ 0.9203126640596814 ], [ 0.9560443688108643 ], [ -0.5175875265724844 ], [ 0.8074439235849571 ], [ 0.24587234380274078 ], ... ],
    	[ [ -1.9182577100331766 ], [ 0.44237566992461347 ], [ -0.2750634748214804 ], [ -0.6254347005733225 ], [ 0.7582037067724114 ], [ 0.7992710785889753 ], [ 0.09323765720001884 ], [ 0.18698048171420423 ], ... ],
    	[ [ -0.47687292825904976 ], [ 0.9653661186202245 ], [ 0.6240366542633085 ], [ 0.6944715506037206 ], [ -0.222630274632455 ], [ 0.13011017868752417 ], [ 0.8378248368353587 ], [ -0.30001508606104793 ], ... ],
    	[ [ -0.6564186621452466 ], [ 0.7705004928160993 ], [ -0.6416472457247441 ], [ 0.35031406888089817 ], [ -0.9319622413632138 ], [ -0.24840864497080084 ], [ -0.6197309670929709 ], [ 0.6998464235051082 ], ... ],
    	[ [ -0.8453123046531722 ], [ -0.9668882554365931 ], [ -0.5456578317320337 ], [ -0.6681813747539339 ], [ 0.8052619029767494 ], [ -0.9491566650148202 ], [ -0.9266175159596861 ], [ 0.12062333224577317 ], ... ],
    	[ [ 0.4261109971389179 ], [ -0.5488189985821659 ], [ 0.6767662437447077 ], [ -0.20909530186576286 ], [ 0.32533291735157854 ], [ 0.9225299925477619 ], [ 0.27582647227700324 ], [ -0.3854497905847152 ], ... ],
    	[ [ -0.9887040462738673 ], [ -0.13021280655333123 ], [ -0.3430233990812818 ], [ -0.8287925463761383 ], [ 0.13074214215240423 ], [ 0.9445012000094601 ], [ -0.8817879776980123 ], [ 0.8210816698894785 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -1.4046114510014094 ], [ 2.1878148038467957 ], [ 2.9483147786557624 ], [ 6.865586358391116 ], [ -1.9822301822058637 ], [ -1.0649715472000503 ], [ -5.009859450960301 ], [ 2.0221228127666278 ], ... ],
    	[ [ -1.6050513729448332 ], [ 1.283899115319684 ], [ 1.6160006505962043 ], [ 1.0865872426320877 ], [ 1.0459765598994197 ], [ -1.9320403770587335 ], [ 1.2384760982039673 ], [ 4.067151207548105 ], ... ],
    	[ [ -0.5213063890058364 ], [ 2.260522148902115 ], [ -3.63552449357012 ], [ -1.5988879399932903 ], [ 1.318906767492457 ], [ 1.2511399783980541 ], [ 10.725280214353113 ], [ 5.34815180083063 ], ... ],
    	[ [ -2.096994693430729 ], [ 1.0358764210922142 ], [ 1.6024699721854097 ], [ 1.4399437948619729 ], [ -4.491752083812146 ], [ 7.685793764080708 ], [ 1.19356690806304 ], [ -3.3331657188616077 ], ... ],
    	[ [ -1.5234179917004382 ], [ 1.2978577033028282 ], [ -1.558488728289474 ], [ 2.8545813281052834 ], [ -1.073004844635406 ], [ -4.025624793040294 ], [ -1.613603407121629 ], [ 1.4288849187677546 ], ... ],
    	[ [ -1.1829947280967303 ], [ -1.0342456787299124 ], [ -1.8326503201205557 ], [ -1.4965996326495368 ], [ 1.2418320006241166 ], [ -1.0535668524061683 ], [ -1.079193931451115 ], [ 8.290270061205689 ], ... ],
    	[ [ 2.3468063643379438 ], [ -1.822094356396968 ], [ 1.4776150690772687 ], [ -4.782508220304205 ], [ 3.073774422030977 ], [ 1.0839755976261414 ], [ 3.62546782310196 ], [ -2.5943716261540355 ], ... ],
    	[ [ -1.0114250101116748 ], [ -7.679736167812574 ], [ -2.9152530197015594 ], [ -1.2065745576169324 ], [ 7.648643226560526 ], [ 1.058759904158919 ], [ -1.1340594624691878 ], [ 1.2179056440690057 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 30.61 seconds (0.063 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 2703059852159
Reset training subject: 2703070891446
Adding measurement 69c5440 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 744357.0818131719 < 744357.0818131719. Total: 1
th(0)=744357.0818131719;dx=-5.926313173012783E22
Adding measurement 1a6b14d1 to history. Total: 1
New Minimum: 744357.0818131719 > 64468.84127145627
Armijo: th(2.154434690031884)=64468.84127145627; dx=303436.045184706 evalInputDelta=679888.2405417156
Adding measurement a481dd9 to history. Total: 2
New Minimum: 64468.84127145627 > 63866.190916583524
Armijo: th(1.077217345015942)=63866.190916583524; dx=200879.96741117246 evalInputDelta=680490.8908965883
Adding measurement 26c39190 to history. Total: 3
New Minimum: 63866.190916583524 > 63445.00257550504
Armijo: th(0.3590724483386473)=63445.00257550504; dx=-397.9849470766299 evalInputDelta=680912.0792376668
Non-optimal measurement 63464.52297251872 < 63445.00257550504. Total: 4
Armijo: th(0.08976811208466183)=63464.52297251872; dx=-16886.082537306087 evalInputDelta=680892.5588406532
Adding measurement 727f719d to history. Total: 4
New Minimum: 63445.00257550504 > 63441.49691079435
Armijo: th(0.017953622416932366)=63441.49691079435; dx=9111.541076240648 evalInputDelta=680915.5849023776
Non-optimal measurement 63458.969825109736 < 63441.49691079435. Total: 5
Armijo: th(0.002992270402822061)=63458.969825109736; dx=-14956.805849339798 evalInputDelta=680898.1119880622
Non-optimal measurement 63486.440966088245 < 63441.49691079435. Total: 5
Armijo: th(4.2746720040315154E-4)=63486.440966088245; dx=-39218.043235531404 evalInputDelta=680870.6408470836
Non-optimal measurement 63533.390734134126 < 63441.49691079435. Total: 5
Armijo: th(5.343340005039394E-5)=63533.390734134126; dx=-529007.2489979756 evalInputDelta=680823.6910790377
Non-optimal measurement 63622.66683712414 < 63441.49691079435. Total: 5
Armijo: th(5.9370444500437714E-6)=63622.66683712414; dx=-8720226.354778072 evalInputDelta=680734.4149760477
Non-optimal measurement 63773.70369274631 < 63441.49691079435. Total: 5
Armijo: th(5.937044450043771E-7)=63773.70369274631; dx=-1.3887515237506703E8 evalInputDelta=680583.3781204255
Non-optimal measurement 64052.060270710346 < 63441.49691079435. Total: 5
Armijo: th(5.397313136403428E-8)=64052.060270710346; dx=-2.949666899621247E9 evalInputDelta=680305.0215424615
Non-optimal measurement 64618.21266908829 < 63441.49691079435. Total: 5
Armijo: th(4.4977609470028565E-9)=64618.21266908829; dx=-7.028133976920047E10 evalInputDelta=679738.8691440836
Non-optimal measurement 65786.77524346588 < 63441.49691079435. Total: 5
Armijo: th(3.4598161130791205E-10)=65786.77524346588; dx=-1.8034511646451465E12 evalInputDelta=678570.306569706
Non-optimal measurement 68357.00097581353 < 63441.49691079435. Total: 5
Armijo: th(2.4712972236279432E-11)=68357.00097581353; dx=-6.2763124305643984E13 evalInputDelta=676000.0808373584
Non-optimal measurement 74188.90572298915 < 63441.49691079435. Total: 5
END: th(1.6475314824186289E-12)=74188.90572298915; dx=-1.2967981928542085E15 evalInputDelta=670168.1760901827
Fitness changed from 744357.0818131719 to 63441.49691079435
Iteration 1 complete. Error: 63441.49691079435 Total: 0.9143; Orientation: 0.0177; Line Search: 0.8643
Non-optimal measurement 63441.49691079435 < 63441.49691079435. Total: 5
Rejected: LBFGS Orientation magnitude: 3.334e+04, gradient 1.547e+04, dot -1.000; [6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63441.49691079435, 63445.00257550504, 63866.190916583524, 64468.84127145627, 744357.0818131719
Rejected: LBFGS Orientation magnitude: 4.660e+10, gradient 1.547e+04, dot -0.000; [58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63441.49691079435, 63445.00257550504, 63866.190916583524, 64468.84127145627
LBFGS Accumulation History: 3 points
Removed measurement 727f719d to history. Total: 4
Removed measurement 26c39190 to history. Total: 3
Adding measurement c131fe6 to history. Total: 3
th(0)=63441.49691079435;dx=-2.3941032436191076E8
Adding measurement 7ab01155 to history. Total: 4
New Minimum: 63441.49691079435 > 63441.49606103935
WOLFE (weak): th(3.549498978642349E-12)=63441.49606103935; dx=-2.3939244003992206E8 evalInputDelta=8.497550006723031E-4
Adding measurement 5b1ba230 to history. Total: 5
New Minimum: 63441.49606103935 > 63441.495211347894
WOLFE (weak): th(7.098997957284698E-12)=63441.495211347894; dx=-2.3937455765764728E8 evalInputDelta=0.001699446453130804
Adding measurement 235ca328 to history. Total: 6
New Minimum: 63441.495211347894 > 63441.49181321661
WOLFE (weak): th(2.1296993871854094E-11)=63441.49181321661; dx=-2.3930304752041674E8 evalInputDelta=0.005097577733977232
Adding measurement 7d207098 to history. Total: 7
New Minimum: 63441.49181321661 > 63441.47653418102
WOLFE (weak): th(8.518797548741638E-11)=63441.47653418102; dx=-2.3898163546216807E8 evalInputDelta=0.02037661332724383
Adding measurement cf5ed15 to history. Total: 8
New Minimum: 63441.47653418102 > 63441.39539149867
WOLFE (weak): th(4.2593987743708185E-10)=63441.39539149867; dx=-2.3727798160667408E8 evalInputDelta=0.10151929567655316
Adding measurement 3122eb81 to history. Total: 9
New Minimum: 63441.39539149867 > 63440.90110125164
WOLFE (weak): th(2.555639264622491E-9)=63440.90110125164; dx=-2.2701982811344638E8 evalInputDelta=0.5958095427049557
Adding measurement 62e2b605 to history. Total: 10
New Minimum: 63440.90110125164 > 63437.89491768637
END: th(1.7889474852357437E-8)=63437.89491768637; dx=-1.6925877761793292E8 evalInputDelta=3.6019931079790695
Fitness changed from 63441.49691079435 to 63437.89491768637
Iteration 2 complete. Error: 63437.89491768637 Total: 2.6520; Orientation: 2.1007; Line Search: 0.5392
Non-optimal measurement 63437.89491768637 < 63437.89491768637. Total: 11
Rejected: LBFGS Orientation magnitude: 1.181e+04, gradient 1.104e+04, dot -0.446; [c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63437.89491768637, 63440.90110125164, 63441.39539149867, 63441.47653418102, 63441.49181321661, 63441.495211347894, 63441.49606103935, 63441.49691079435, 63866.190916583524, 64468.84127145627, 744357.0818131719
Rejected: LBFGS Orientation magnitude: 1.887e+10, gradient 1.104e+04, dot -0.000; [760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63437.89491768637, 63440.90110125164, 63441.39539149867, 63441.47653418102, 63441.49181321661, 63441.495211347894, 63441.49606103935, 63441.49691079435, 63866.190916583524, 64468.84127145627
Rejected: LBFGS Orientation magnitude: 9.912e+09, gradient 1.104e+04, dot -0.000; [e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63437.89491768637, 63440.90110125164, 63441.39539149867, 63441.47653418102, 63441.49181321661, 63441.495211347894, 63441.49606103935, 63441.49691079435, 63866.190916583524

...skipping 4625 bytes...

mulation History: 3 points
Removed measurement 75079892 to history. Total: 4
Removed measurement 4b55a976 to history. Total: 3
Adding measurement d6e8c97 to history. Total: 3
th(0)=63434.01893827193;dx=-6.045574768685583E7
Adding measurement 15d9305e to history. Total: 4
New Minimum: 63434.01893827193 > 63429.60565911565
END: th(8.30355867138989E-8)=63429.60565911565; dx=-4.797182843038876E7 evalInputDelta=4.413279156280623
Fitness changed from 63434.01893827193 to 63429.60565911565
Iteration 4 complete. Error: 63429.60565911565 Total: 2.7843; Orientation: 2.6393; Line Search: 0.0574
Non-optimal measurement 63429.60565911565 < 63429.60565911565. Total: 5
Rejected: LBFGS Orientation magnitude: 4.159e+04, gradient 6.581e+03, dot -0.967; [760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63429.60565911565, 63434.01893827193, 63866.190916583524, 64468.84127145627, 744357.0818131719
Rejected: LBFGS Orientation magnitude: 1.455e+11, gradient 6.581e+03, dot -0.000; [c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63429.60565911565, 63434.01893827193, 63866.190916583524, 64468.84127145627
LBFGS Accumulation History: 3 points
Removed measurement 15d9305e to history. Total: 4
Removed measurement d6e8c97 to history. Total: 3
Adding measurement 2332809d to history. Total: 3
th(0)=63429.60565911565;dx=-4.330593199874306E7
Adding measurement 3f0eaa6a to history. Total: 4
New Minimum: 63429.60565911565 > 63420.76878702689
WOLFE (weak): th(1.7889474852357439E-7)=63420.76878702689; dx=-5.90427953253801E7 evalInputDelta=8.836872088759264
Adding measurement 2ae9a32c to history. Total: 5
New Minimum: 63420.76878702689 > 63406.41481518412
WOLFE (weak): th(3.5778949704714877E-7)=63406.41481518412; dx=-1.1156689936001892E8 evalInputDelta=23.190843931530253
Non-optimal measurement 63500.27246505844 < 63406.41481518412. Total: 6
Armijo: th(1.0733684911414464E-6)=63500.27246505844; dx=-1.5959110233819813E8 evalInputDelta=-70.66680594279023
Non-optimal measurement 103054.37349515157 < 63406.41481518412. Total: 6
Armijo: th(7.155789940942975E-7)=103054.37349515157; dx=-1.744323147012257E13 evalInputDelta=-39624.76783603591
Adding measurement 78d41a0e to history. Total: 6
New Minimum: 63406.41481518412 > 63372.56383095762
WOLFE (weak): th(5.366842455707232E-7)=63372.56383095762; dx=-3.155653142272843E8 evalInputDelta=57.04182815803506
Adding measurement 4e723df6 to history. Total: 7
New Minimum: 63372.56383095762 > 63338.99919616563
WOLFE (weak): th(6.261316198325104E-7)=63338.99919616563; dx=-2.1584341950199622E8 evalInputDelta=90.60646295002516
Non-optimal measurement 63430.84713181645 < 63338.99919616563. Total: 8
Armijo: th(6.708553069634039E-7)=63430.84713181645; dx=9.607893715934313E9 evalInputDelta=-1.2414727007999318
Non-optimal measurement 63344.08391410725 < 63338.99919616563. Total: 8
WOLF (strong): th(6.484934633979572E-7)=63344.08391410725; dx=1.0134415733241404E9 evalInputDelta=85.52174500840192
Adding measurement 579cef18 to history. Total: 8
New Minimum: 63338.99919616563 > 63338.31665169826
WOLF (strong): th(6.373125416152338E-7)=63338.31665169826; dx=1.4542940744369847E8 evalInputDelta=91.28900741739199
Adding measurement 5cff6bc1 to history. Total: 9
New Minimum: 63338.31665169826 > 63338.160662043956
WOLFE (weak): th(6.317220807238721E-7)=63338.160662043956; dx=-7.383593434449251E7 evalInputDelta=91.44499707169598
Non-optimal measurement 63338.160662043956 < 63338.160662043956. Total: 10
mu ~= nu (6.317220807238721E-7): th(6.317220807238721E-7)=63338.160662043956
Fitness changed from 63429.60565911565 to 63338.160662043956
Iteration 5 complete. Error: 63338.160662043956 Total: 2.5140; Orientation: 1.8193; Line Search: 0.6841
Non-optimal measurement 63338.160662043956 < 63338.160662043956. Total: 10
Rejected: LBFGS Orientation magnitude: 3.687e+04, gradient 1.303e+04, dot -0.422; [6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63338.160662043956, 63338.31665169826, 63338.99919616563, 63372.56383095762, 63406.41481518412, 63420.76878702689, 63429.60565911565, 63866.190916583524, 64468.84127145627, 744357.0818131719
Rejected: LBFGS Orientation magnitude: 4.339e+11, gradient 1.303e+04, dot -0.000; [e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63338.160662043956, 63338.31665169826, 63338.99919616563, 63372.56383095762, 63406.41481518412, 63420.76878702689, 63429.60565911565, 63866.190916583524, 64468.84127145627
Rejected: LBFGS Orientation magnitude: 1.908e+11, gradient 1.303e+04, dot -0.000; [c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00, 6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 63338.160662043956, 63338.31665169826, 63338.99919616563, 63372.56383095762, 63406.41481518412, 63420.76878702689, 63429.60565911565, 63866.190916583524
Accepted: LBFGS Orientation magnitude: 7.903e-03, gradient 1.303e+04, dot 0.746; [6b83883e-4dc1-41ec-8b41-55c08a02a315 = 1.000/1.000e+00, c6af2418-76a2-4d62-a64f-7852067e46af = 1.000/1.000e+00, e8a7747d-e3da-400f-ae1f-15408cc5cdfd = 1.000/1.000e+00, 760d2088-52de-433a-aaec-5cfaa943bb19 = 1.000/1.000e+00, 58173e81-3381-4216-9fa2-16303775a6c2 = 1.000/1.000e+00]
Overwriting history with 7 points
Constructing line search parameters: LBFGS
Non-optimal measurement 63338.160662043956 < 63338.160662043956. Total: 7
th(0)=63338.160662043956;dx=-76.8399310742878
Non-optimal measurement 63455.30170723567 < 63338.160662043956. Total: 7
Armijo: th(2.154434690031884)=63455.30170723567; dx=-4.139041937894234 evalInputDelta=-117.14104519171087
Non-optimal measurement 63464.67979998923 < 63338.160662043956. Total: 7
Armijo: th(1.077217345015942)=63464.67979998923; dx=-18.346104853209898 evalInputDelta=-126.51913794527354
Non-optimal measurement 63515.192875582085 < 63338.160662043956. Total: 7
Armijo: th(0.3590724483386473)=63515.192875582085; dx=-269.0373066985232 evalInputDelta=-177.03221353812842
Non-optimal measurement 69119.52370104883 < 63338.160662043956. Total: 7
Armijo: th(0.08976811208466183)=69119.52370104883; dx=-821482.6403002867 evalInputDelta=-5781.363039004878
Non-optimal measurement 63345.87041615844 < 63338.160662043956. Total: 7
Armijo: th(0.017953622416932366)=63345.87041615844; dx=1246.8263935368157 evalInputDelta=-7.709754114483076
Adding measurement 570d414f to history. Total: 7
New Minimum: 63338.160662043956 > 63338.090410459394
WOLF (strong): th(0.002992270402822061)=63338.090410459394; dx=34.27882793539564 evalInputDelta=0.07025158456235658
Non-optimal measurement 63338.13085557629 < 63338.090410459394. Total: 8
END: th(4.2746720040315154E-4)=63338.13085557629; dx=-62.53527060007823 evalInputDelta=0.029806467668095138
Fitness changed from 63338.160662043956 to 63338.090410459394
Iteration 6 complete. Error: 63338.090410459394 Total: 8.4979; Orientation: 7.8972; Line Search: 0.5901
Final threshold in iteration 6: 63338.090410459394 (> 0.0) after 30.611s (< 30.000s)

Returns

    63338.090410459394

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.7119426265989717 ], [ 0.45697491066723933 ], [ 0.33901458599759166 ], [ 0.14376441261842132 ], [ -0.5043441752721415 ], [ -0.9389953834694118 ], [ -0.19914118474301898 ], [ 0.4944629333599809 ], ... ],
    	[ [ -0.6274653245571576 ], [ 0.7788559553704042 ], [ 0.6187721648368192 ], [ 0.9203149180875296 ], [ 0.9560491814862242 ], [ -0.5175723395788889 ], [ 0.8074432445537483 ], [ 0.2456546828931346 ], ... ],
    	[ [ -1.6301137597620792 ], [ 0.44222003219833317 ], [ -0.2750252797563408 ], [ -0.6255125594812767 ], [ 0.7582084050623714 ], [ 0.799289218353238 ], [ 0.05615885541104813 ], [ 0.18611471547140004 ], ... ],
    	[ [ -0.47685079500451266 ], [ 0.9653668847214824 ], [ 0.6240051749039549 ], [ 0.6944778894595537 ], [ -0.22153149041018474 ], [ 0.12752282993858174 ], [ 0.8378239141228913 ], [ -0.29975856644551474 ], ... ],
    	[ [ -0.6570959413378604 ], [ 0.7704789084416547 ], [ -0.6416002956803548 ], [ 0.34979446459490726 ], [ -0.93194796266772 ], [ -0.2482708961247622 ], [ -0.6196226079134176 ], [ 0.6998202372903426 ], ... ],
    	[ [ -0.8452974330652566 ], [ -0.9668881508235629 ], [ -0.5455902107116936 ], [ -0.6681247304396223 ], [ 0.8050258219972526 ], [ -0.9491426300485293 ], [ -0.9266049338223984 ], [ 0.11828201993243866 ], ... ],
    	[ [ 0.42598937535117104 ], [ -0.548807079055803 ], [ 0.6767624344953297 ], [ -0.20911528780686842 ], [ 0.3242823257973005 ], [ 0.9225355803955659 ], [ 0.2758370256729383 ], [ -32.4599217474102 ], ... ],
    	[ [ -0.9886926335492401 ], [ -0.12723276754378932 ], [ -0.3430596700430708 ], [ -0.8287551927513321 ], [ 0.12746376116519612 ], [ 0.9444829251461601 ], [ -0.8817881087518509 ], [ 0.8210848052608014 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -1.4046075661701984 ], [ 2.1883039454833035 ], [ 2.94972559088388 ], [ 6.955824336403713 ], [ -1.9827729733577377 ], [ -1.0649679621482138 ], [ -5.02156297448188 ], [ 2.0223962860163995 ], ... ],
    	[ [ -1.5937135660934951 ], [ 1.2839344593884825 ], [ 1.6161037241610845 ], [ 1.086584581371408 ], [ 1.0459712945368063 ], [ -1.9320970684284007 ], [ 1.23847713971856 ], [ 4.070754883329551 ], ... ],
    	[ [ -0.61345411877632 ], [ 2.261317731421777 ], [ -3.636029389319964 ], [ -1.5986889229358994 ], [ 1.3188985947969523 ], [ 1.2511115839398947 ], [ 17.80663072066939 ], [ 5.373030270428393 ], ... ],
    	[ [ -2.097092026428385 ], [ 1.0358755990356037 ], [ 1.6025508124254213 ], [ 1.4399306517565378 ], [ -4.514030931441907 ], [ 7.841733127171233 ], [ 1.1935682225624809 ], [ -3.3360180890168616 ], ... ],
    	[ [ -1.5218477806513004 ], [ 1.2978940617888777 ], [ -1.5586027729921743 ], [ 2.858821683064905 ], [ -1.0730212845119376 ], [ -4.0278583418713545 ], [ -1.6138855929861973 ], [ 1.4289383854801534 ], ... ],
    	[ [ -1.183015540901093 ], [ -1.0342457906307296 ], [ -1.8328774607146139 ], [ -1.4967265159336427 ], [ 1.242196178898983 ], [ -1.0535824314927993 ], [ -1.0792085855563436 ], [ 8.454370330936085 ], ... ],
    	[ [ 2.347476387587447 ], [ -1.822133930415864 ], [ 1.4776233860345878 ], [ -4.782051137856382 ], [ 3.083732662707838 ], [ 1.0839690319274393 ], [ 3.6253291143941873 ], [ -0.030807221526336075 ], ... ],
    	[ [ -1.0114366852417707 ], [ -7.859610533550903 ], [ -2.914944796263726 ], [ -1.206628940302821 ], [ 7.845367113433722 ], [ 1.0587803901751305 ], [ -1.1340592939220684 ], [ 1.2179009934087985 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, 4.801663964821754], [94.0, 4.803386717496693]; valueStats=DoubleSummaryStatistics{count=124, sum=7856158.344609, min=63337.944452, average=63356.115682, max=63589.691420}
Plotting 94 points for GD
Plotting 24 points for CjGD
Plotting 6 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, 4.801663964821754], [30.592, 4.803386717496693]; valueStats=DoubleSummaryStatistics{count=124, sum=7856158.344609, min=63337.944452, average=63356.115682, max=63589.691420}
Plotting 94 points for GD
Plotting 24 points for CjGD
Plotting 6 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 63338.090410459394 }, "CjGD": { "type": "NonConverged", "value": 63360.43346709255 }, "GD": { "type": "NonConverged", "value": 63337.94445226671 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 63338.090410459394 }, "CjGD": { "type": "NonConverged", "value": 63360.43346709255 }, "GD": { "type": "NonConverged", "value": 63337.94445226671 } }, "model":null, "complete":null}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "92.368",
      "gc_time": "0.582"
    },
    "created_on": 1586737229806,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "InvPowerTest",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.NthPowerActivationLayerTest.InvPowerTest",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 63338.090410459394
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 63360.43346709255
        },
        "GD": {
          "type": "NonConverged",
          "value": 63337.94445226671
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayer/InvPowerTest/trainingTest/202004132029",
    "id": "4d41c30b-fb90-49f4-a8d0-82585ba451cb",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "NthPowerActivationLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.NthPowerActivationLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayer.java",
      "javaDoc": ""
    }
  }